Serve pre-defined files in "public", add "security.txt", add CORS header for ".well-known" (#25974)

Replace #25892

Close  #21942
Close  #25464

Major changes:

1. Serve "robots.txt" and ".well-known/security.txt" in the "public"
custom path
* All files in "public/.well-known" can be served, just like
"public/assets"
3. Add a test for ".well-known/security.txt"
4. Simplify the "FileHandlerFunc" logic, now the paths are consistent so
the code can be simpler
5. Add CORS header for ".well-known" endpoints
6. Add logs to tell users they should move some of their legacy custom
public files

```
2023/07/19 13:00:37 cmd/web.go:178:serveInstalled() [E] Found legacy public asset "img" in CustomPath. Please move it to /work/gitea/custom/public/assets/img
2023/07/19 13:00:37 cmd/web.go:182:serveInstalled() [E] Found legacy public asset "robots.txt" in CustomPath. Please move it to /work/gitea/custom/public/robots.txt
```
This PR is not breaking.

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Giteabot <teabot@gitea.io>

3 files changed, 11 insertions, 11 deletions

diff --git a/routers/install/routes.go b/routers/install/routes.go
index ce6d41b32d..06c9d389a6 100644
--- a/routers/install/routes.go
+++ b/routers/install/routes.go
@@ -20,7 +20,7 @@ import (
 func Routes() *web.Route {
 	base := web.NewRoute()
 	base.Use(common.ProtocolMiddlewares()...)
-	base.Methods("GET, HEAD", "/assets/*", public.AssetsHandlerFunc("/assets/"))
+	base.Methods("GET, HEAD", "/assets/*", public.FileHandlerFunc())
 
 	r := web.NewRoute()
 	r.Use(common.Sessioner(), Contexter())
diff --git a/routers/web/misc/misc.go b/routers/web/misc/misc.go
index 6ed3b5c3ad..54c93763f6 100644
--- a/routers/web/misc/misc.go
+++ b/routers/web/misc/misc.go
@@ -34,9 +34,12 @@ func DummyOK(w http.ResponseWriter, req *http.Request) {
 }
 
 func RobotsTxt(w http.ResponseWriter, req *http.Request) {
-	filePath := util.FilePathJoinAbs(setting.CustomPath, "robots.txt")
+	robotsTxt := util.FilePathJoinAbs(setting.CustomPath, "public/robots.txt")
+	if ok, _ := util.IsExist(robotsTxt); !ok {
+		robotsTxt = util.FilePathJoinAbs(setting.CustomPath, "robots.txt") // the legacy "robots.txt"
+	}
 	httpcache.SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
-	http.ServeFile(w, req, filePath)
+	http.ServeFile(w, req, robotsTxt)
 }
 
 func StaticRedirect(target string) func(w http.ResponseWriter, req *http.Request) {
diff --git a/routers/web/web.go b/routers/web/web.go
index f091bfefb8..455791ad80 100644
--- a/routers/web/web.go
+++ b/routers/web/web.go
@@ -108,7 +108,7 @@ func Routes() *web.Route {
 	routes := web.NewRoute()
 
 	routes.Head("/", misc.DummyOK) // for health check - doesn't need to be passed through gzip handler
-	routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.AssetsHandlerFunc("/assets/"))
+	routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.FileHandlerFunc())
 	routes.Methods("GET, HEAD", "/avatars/*", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars))
 	routes.Methods("GET, HEAD", "/repo-avatars/*", storageHandler(setting.RepoAvatar.Storage, "repo-avatars", storage.RepoAvatars))
 	routes.Methods("GET, HEAD", "/apple-touch-icon.png", misc.StaticRedirect("/assets/img/apple-touch-icon.png"))
@@ -132,15 +132,12 @@ func Routes() *web.Route {
 		routes.Methods("GET,HEAD", "/captcha/*", append(mid, captcha.Captchaer(context.GetImageCaptcha()))...)
 	}
 
-	if setting.HasRobotsTxt {
-		routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
-	}
-
 	if setting.Metrics.Enabled {
 		prometheus.MustRegister(metrics.NewCollector())
 		routes.Get("/metrics", append(mid, Metrics)...)
 	}
 
+	routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
 	routes.Get("/ssh_info", misc.SSHInfo)
 	routes.Get("/api/healthz", healthcheck.Check)
 
@@ -336,8 +333,7 @@ func registerRoutes(m *web.Route) {
 
 	// FIXME: not all routes need go through same middleware.
 	// Especially some AJAX requests, we can reduce middleware number to improve performance.
-	// Routers.
-	// for health check
+
 	m.Get("/", Home)
 	m.Get("/sitemap.xml", sitemapEnabled, ignExploreSignIn, HomeSitemap)
 	m.Group("/.well-known", func() {
@@ -349,7 +345,8 @@ func registerRoutes(m *web.Route) {
 		m.Get("/change-password", func(ctx *context.Context) {
 			ctx.Redirect(setting.AppSubURL + "/user/settings/account")
 		})
-	})
+		m.Any("/*", CorsHandler(), public.FileHandlerFunc())
+	}, CorsHandler())
 
 	m.Group("/explore", func() {
 		m.Get("", func(ctx *context.Context) {