Share HTML template renderers and create a watcher framework (#20218)

The recovery, API, Web and package frameworks all create their own HTML
Renderers. This increases the memory requirements of Gitea
unnecessarily with duplicate templates being kept in memory.

Further the reloading framework in dev mode for these involves locking
and recompiling all of the templates on each load. This will potentially
hide concurrency issues and it is inefficient.

This PR stores the templates renderer in the context and stores this
context in the NormalRoutes, it then creates a fsnotify.Watcher
framework to watch files.

The watching framework is then extended to the mailer templates which
were previously not being reloaded in dev.

Then the locales are simplified to a similar structure.

Fix #20210 
Fix #20211
Fix #20217

Signed-off-by: Andrew Thornton <art27@cantab.net>

11 files changed, 73 insertions, 60 deletions

diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go
index cbf041a7e1..0761cacdf2 100644
--- a/routers/api/packages/api.go
+++ b/routers/api/packages/api.go
@@ -5,6 +5,7 @@
 package packages
 
 import (
+	gocontext "context"
 	"net/http"
 	"regexp"
 	"strings"
@@ -38,10 +39,10 @@ func reqPackageAccess(accessMode perm.AccessMode) func(ctx *context.Context) {
 	}
 }
 
-func Routes() *web.Route {
+func Routes(ctx gocontext.Context) *web.Route {
 	r := web.NewRoute()
 
-	r.Use(context.PackageContexter())
+	r.Use(context.PackageContexter(ctx))
 
 	authMethods := []auth.Method{
 		&auth.OAuth2{},
@@ -270,10 +271,10 @@ func Routes() *web.Route {
 	return r
 }
 
-func ContainerRoutes() *web.Route {
+func ContainerRoutes(ctx gocontext.Context) *web.Route {
 	r := web.NewRoute()
 
-	r.Use(context.PackageContexter())
+	r.Use(context.PackageContexter(ctx))
 
 	authMethods := []auth.Method{
 		&auth.Basic{},
diff --git a/routers/api/packages/pypi/pypi.go b/routers/api/packages/pypi/pypi.go
index 3909074504..07cf766c61 100644
--- a/routers/api/packages/pypi/pypi.go
+++ b/routers/api/packages/pypi/pypi.go
@@ -16,7 +16,6 @@ import (
 	packages_module "code.gitea.io/gitea/modules/packages"
 	pypi_module "code.gitea.io/gitea/modules/packages/pypi"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/templates"
 	"code.gitea.io/gitea/modules/validation"
 	"code.gitea.io/gitea/routers/api/packages/helper"
 	packages_service "code.gitea.io/gitea/services/packages"
@@ -58,7 +57,6 @@ func PackageMetadata(ctx *context.Context) {
 	ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi"
 	ctx.Data["PackageDescriptor"] = pds[0]
 	ctx.Data["PackageDescriptors"] = pds
-	ctx.Render = templates.HTMLRenderer()
 	ctx.HTML(http.StatusOK, "api/packages/pypi/simple")
 }
 
diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go
index e1478fa2aa..b413370ad2 100644
--- a/routers/api/v1/api.go
+++ b/routers/api/v1/api.go
@@ -65,6 +65,7 @@
 package v1
 
 import (
+	gocontext "context"
 	"fmt"
 	"net/http"
 	"reflect"
@@ -605,7 +606,7 @@ func buildAuthGroup() *auth.Group {
 }
 
 // Routes registers all v1 APIs routes to web application.
-func Routes() *web.Route {
+func Routes(ctx gocontext.Context) *web.Route {
 	m := web.NewRoute()
 
 	m.Use(securityHeaders())
@@ -623,7 +624,7 @@ func Routes() *web.Route {
 	m.Use(context.APIContexter())
 
 	group := buildAuthGroup()
-	if err := group.Init(); err != nil {
+	if err := group.Init(ctx); err != nil {
 		log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err)
 	}
 
diff --git a/routers/api/v1/misc/markdown_test.go b/routers/api/v1/misc/markdown_test.go
index 9beb88be16..7809fa5cc7 100644
--- a/routers/api/v1/misc/markdown_test.go
+++ b/routers/api/v1/misc/markdown_test.go
@@ -29,7 +29,7 @@ const (
 )
 
 func createContext(req *http.Request) (*context.Context, *httptest.ResponseRecorder) {
-	rnd := templates.HTMLRenderer()
+	_, rnd := templates.HTMLRenderer(req.Context())
 	resp := httptest.NewRecorder()
 	c := &context.Context{
 		Req:    req,
diff --git a/routers/init.go b/routers/init.go
index 612fc5a83d..85a38899e3 100644
--- a/routers/init.go
+++ b/routers/init.go
@@ -27,6 +27,7 @@ import (
 	"code.gitea.io/gitea/modules/ssh"
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/svg"
+	"code.gitea.io/gitea/modules/templates"
 	"code.gitea.io/gitea/modules/translation"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web"
@@ -110,12 +111,12 @@ func GlobalInitInstalled(ctx context.Context) {
 	log.Info("Run Mode: %s", util.ToTitleCase(setting.RunMode))
 
 	// Setup i18n
-	translation.InitLocales()
+	translation.InitLocales(ctx)
 
 	setting.NewServices()
 	mustInit(storage.Init)
 
-	mailer.NewContext()
+	mailer.NewContext(ctx)
 	mustInit(cache.NewContext)
 	notification.NewContext()
 	mustInit(archiver.Init)
@@ -163,18 +164,19 @@ func GlobalInitInstalled(ctx context.Context) {
 }
 
 // NormalRoutes represents non install routes
-func NormalRoutes() *web.Route {
+func NormalRoutes(ctx context.Context) *web.Route {
+	ctx, _ = templates.HTMLRenderer(ctx)
 	r := web.NewRoute()
 	for _, middle := range common.Middlewares() {
 		r.Use(middle)
 	}
 
-	r.Mount("/", web_routers.Routes())
-	r.Mount("/api/v1", apiv1.Routes())
+	r.Mount("/", web_routers.Routes(ctx))
+	r.Mount("/api/v1", apiv1.Routes(ctx))
 	r.Mount("/api/internal", private.Routes())
 	if setting.Packages.Enabled {
-		r.Mount("/api/packages", packages_router.Routes())
-		r.Mount("/v2", packages_router.ContainerRoutes())
+		r.Mount("/api/packages", packages_router.Routes(ctx))
+		r.Mount("/v2", packages_router.ContainerRoutes(ctx))
 	}
 	return r
 }
diff --git a/routers/install/install.go b/routers/install/install.go
index 8060414a11..890725b9a7 100644
--- a/routers/install/install.go
+++ b/routers/install/install.go
@@ -6,6 +6,7 @@
 package install
 
 import (
+	goctx "context"
 	"fmt"
 	"net/http"
 	"os"
@@ -51,39 +52,41 @@ func getSupportedDbTypeNames() (dbTypeNames []map[string]string) {
 }
 
 // Init prepare for rendering installation page
-func Init(next http.Handler) http.Handler {
-	rnd := templates.HTMLRenderer()
+func Init(ctx goctx.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	dbTypeNames := getSupportedDbTypeNames()
-	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		if setting.InstallLock {
-			resp.Header().Add("Refresh", "1; url="+setting.AppURL+"user/login")
-			_ = rnd.HTML(resp, http.StatusOK, string(tplPostInstall), nil)
-			return
-		}
-		locale := middleware.Locale(resp, req)
-		startTime := time.Now()
-		ctx := context.Context{
-			Resp:    context.NewResponse(resp),
-			Flash:   &middleware.Flash{},
-			Locale:  locale,
-			Render:  rnd,
-			Session: session.GetSession(req),
-			Data: map[string]interface{}{
-				"locale":        locale,
-				"Title":         locale.Tr("install.install"),
-				"PageIsInstall": true,
-				"DbTypeNames":   dbTypeNames,
-				"AllLangs":      translation.AllLangs(),
-				"PageStartTime": startTime,
-
-				"PasswordHashAlgorithms": user_model.AvailableHashAlgorithms,
-			},
-		}
-		defer ctx.Close()
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+			if setting.InstallLock {
+				resp.Header().Add("Refresh", "1; url="+setting.AppURL+"user/login")
+				_ = rnd.HTML(resp, http.StatusOK, string(tplPostInstall), nil)
+				return
+			}
+			locale := middleware.Locale(resp, req)
+			startTime := time.Now()
+			ctx := context.Context{
+				Resp:    context.NewResponse(resp),
+				Flash:   &middleware.Flash{},
+				Locale:  locale,
+				Render:  rnd,
+				Session: session.GetSession(req),
+				Data: map[string]interface{}{
+					"locale":        locale,
+					"Title":         locale.Tr("install.install"),
+					"PageIsInstall": true,
+					"DbTypeNames":   dbTypeNames,
+					"AllLangs":      translation.AllLangs(),
+					"PageStartTime": startTime,
+
+					"PasswordHashAlgorithms": user_model.AvailableHashAlgorithms,
+				},
+			}
+			defer ctx.Close()
 
-		ctx.Req = context.WithContext(req, &ctx)
-		next.ServeHTTP(resp, ctx.Req)
-	})
+			ctx.Req = context.WithContext(req, &ctx)
+			next.ServeHTTP(resp, ctx.Req)
+		})
+	}
 }
 
 // Install render installation page
diff --git a/routers/install/routes.go b/routers/install/routes.go
index fdabcb9dc2..7617477827 100644
--- a/routers/install/routes.go
+++ b/routers/install/routes.go
@@ -5,6 +5,7 @@
 package install
 
 import (
+	goctx "context"
 	"fmt"
 	"net/http"
 	"path"
@@ -29,8 +30,8 @@ func (d *dataStore) GetData() map[string]interface{} {
 	return *d
 }
 
-func installRecovery() func(next http.Handler) http.Handler {
-	rnd := templates.HTMLRenderer()
+func installRecovery(ctx goctx.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 			defer func() {
@@ -82,7 +83,7 @@ func installRecovery() func(next http.Handler) http.Handler {
 }
 
 // Routes registers the install routes
-func Routes() *web.Route {
+func Routes(ctx goctx.Context) *web.Route {
 	r := web.NewRoute()
 	for _, middle := range common.Middlewares() {
 		r.Use(middle)
@@ -105,8 +106,8 @@ func Routes() *web.Route {
 		Domain:         setting.SessionConfig.Domain,
 	}))
 
-	r.Use(installRecovery())
-	r.Use(Init)
+	r.Use(installRecovery(ctx))
+	r.Use(Init(ctx))
 	r.Get("/", Install)
 	r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall)
 	r.Get("/api/healthz", healthcheck.Check)
diff --git a/routers/install/routes_test.go b/routers/install/routes_test.go
index 29003c3841..e69d2d15df 100644
--- a/routers/install/routes_test.go
+++ b/routers/install/routes_test.go
@@ -5,13 +5,16 @@
 package install
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestRoutes(t *testing.T) {
-	routes := Routes()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	routes := Routes(ctx)
 	assert.NotNil(t, routes)
 	assert.EqualValues(t, "/", routes.R.Routes()[0].Pattern)
 	assert.Nil(t, routes.R.Routes()[0].SubRoutes)
diff --git a/routers/install/setting.go b/routers/install/setting.go
index cf0a01ce31..c4912f1124 100644
--- a/routers/install/setting.go
+++ b/routers/install/setting.go
@@ -24,7 +24,7 @@ func PreloadSettings(ctx context.Context) bool {
 		log.Info("Log path: %s", setting.LogRootPath)
 		log.Info("Configuration file: %s", setting.CustomConf)
 		log.Info("Prepare to run install page")
-		translation.InitLocales()
+		translation.InitLocales(ctx)
 		if setting.EnableSQLite3 {
 			log.Info("SQLite3 is supported")
 		}
diff --git a/routers/web/base.go b/routers/web/base.go
index 30a24a1275..2441d6d517 100644
--- a/routers/web/base.go
+++ b/routers/web/base.go
@@ -5,6 +5,7 @@
 package web
 
 import (
+	goctx "context"
 	"errors"
 	"fmt"
 	"io"
@@ -123,8 +124,8 @@ func (d *dataStore) GetData() map[string]interface{} {
 
 // Recovery returns a middleware that recovers from any panics and writes a 500 and a log if so.
 // This error will be created with the gitea 500 page.
-func Recovery() func(next http.Handler) http.Handler {
-	rnd := templates.HTMLRenderer()
+func Recovery(ctx goctx.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 			defer func() {
diff --git a/routers/web/web.go b/routers/web/web.go
index 55bce16117..1852ecc2e2 100644
--- a/routers/web/web.go
+++ b/routers/web/web.go
@@ -22,6 +22,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/templates"
 	"code.gitea.io/gitea/modules/validation"
 	"code.gitea.io/gitea/modules/web"
 	"code.gitea.io/gitea/modules/web/routing"
@@ -97,7 +98,7 @@ func buildAuthGroup() *auth_service.Group {
 }
 
 // Routes returns all web routes
-func Routes() *web.Route {
+func Routes(ctx gocontext.Context) *web.Route {
 	routes := web.NewRoute()
 
 	routes.Use(web.WrapWithPrefix(public.AssetsURLPathPrefix, public.AssetsHandlerFunc(&public.Options{
@@ -119,7 +120,9 @@ func Routes() *web.Route {
 	})
 	routes.Use(sessioner)
 
-	routes.Use(Recovery())
+	ctx, _ = templates.HTMLRenderer(ctx)
+
+	routes.Use(Recovery(ctx))
 
 	// We use r.Route here over r.Use because this prevents requests that are not for avatars having to go through this additional handler
 	routes.Route("/avatars/*", "GET, HEAD", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars))
@@ -192,10 +195,10 @@ func Routes() *web.Route {
 	routes.Get("/api/healthz", healthcheck.Check)
 
 	// Removed: toolbox.Toolboxer middleware will provide debug information which seems unnecessary
-	common = append(common, context.Contexter())
+	common = append(common, context.Contexter(ctx))
 
 	group := buildAuthGroup()
-	if err := group.Init(); err != nil {
+	if err := group.Init(ctx); err != nil {
 		log.Error("Could not initialize '%s' auth method, error: %s", group.Name(), err)
 	}