Serve pre-defined files in "public", add "security.txt", add CORS header for ".well...

Replace #25892

Close  #21942
Close  #25464

Major changes:

1. Serve "robots.txt" and ".well-known/security.txt" in the "public"
custom path
* All files in "public/.well-known" can be served, just like
"public/assets"
3. Add a test for ".well-known/security.txt"
4. Simplify the "FileHandlerFunc" logic, now the paths are consistent so
the code can be simpler
5. Add CORS header for ".well-known" endpoints
6. Add logs to tell users they should move some of their legacy custom
public files

```
2023/07/19 13:00:37 cmd/web.go:178:serveInstalled() [E] Found legacy public asset "img" in CustomPath. Please move it to /work/gitea/custom/public/assets/img
2023/07/19 13:00:37 cmd/web.go:182:serveInstalled() [E] Found legacy public asset "robots.txt" in CustomPath. Please move it to /work/gitea/custom/public/robots.txt
```
This PR is not breaking.

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Giteabot <teabot@gitea.io>

diff --git a/cmd/web.go b/cmd/web.go
index d9aafb1fa244db522456213fbf924d0d4d8f0c6e..dfe2091d064b617293de8db8002076490a4fede1 100644 (file)
--- a/cmd/web.go
+++ b/cmd/web.go
@@ -15,9 +15,11 @@ import (
 
        _ "net/http/pprof" // Used for debugging if enabled and a web server is running
 
+       "code.gitea.io/gitea/modules/container"
        "code.gitea.io/gitea/modules/graceful"
        "code.gitea.io/gitea/modules/log"
        "code.gitea.io/gitea/modules/process"
+       "code.gitea.io/gitea/modules/public"
        "code.gitea.io/gitea/modules/setting"
        "code.gitea.io/gitea/routers"
        "code.gitea.io/gitea/routers/install"
@@ -175,6 +177,20 @@ func serveInstalled(ctx *cli.Context) error {
                }
        }
 
+       // in old versions, user's custom web files are placed in "custom/public", and they were served as "http://domain.com/assets/xxx"
+       // now, Gitea only serves pre-defined files in the "custom/public" folder basing on the web root, the user should move their custom files to "custom/public/assets"
+       publicFiles, _ := public.AssetFS().ListFiles(".")
+       publicFilesSet := container.SetOf(publicFiles...)
+       publicFilesSet.Remove(".well-known")
+       publicFilesSet.Remove("assets")
+       publicFilesSet.Remove("robots.txt")
+       for _, fn := range publicFilesSet.Values() {
+               log.Error("Found legacy public asset %q in CustomPath. Please move it to %s/public/assets/%s", fn, setting.CustomPath, fn)
+       }
+       if _, err := os.Stat(filepath.Join(setting.CustomPath, "robots.txt")); err == nil {
+               log.Error(`Found legacy public asset "robots.txt" in CustomPath. Please move it to %s/public/robots.txt`, setting.CustomPath)
+       }
+
        routers.InitWebInstalled(graceful.GetManager().HammerContext())
 
        // We check that AppDataPath exists here (it should have been created during installation)

diff --git a/docs/content/doc/administration/customizing-gitea.en-us.md b/docs/content/doc/administration/customizing-gitea.en-us.md
index 60fcb2314b207bf45e338e64d6750a58983ae10c..ccc5c1bc89af8243b63c89f566d2bd6c2e98930e 100644 (file)
--- a/docs/content/doc/administration/customizing-gitea.en-us.md
+++ b/docs/content/doc/administration/customizing-gitea.en-us.md
@@ -56,7 +56,11 @@ is set under the "Configuration" tab on the site administration page.
 
 To make Gitea serve custom public files (like pages and images), use the folder
 `$GITEA_CUSTOM/public/` as the webroot. Symbolic links will be followed.
-At the moment, only files in the `public/assets/` folder are served.
+At the moment, only the following files are served:
+
+- `public/robots.txt`
+- files in the `public/.well-known/` folder
+- files in the `public/assets/` folder
 
 For example, a file `image.png` stored in `$GITEA_CUSTOM/public/assets/`, can be accessed with
 the url `http://gitea.domain.tld/assets/image.png`.

diff --git a/modules/public/public.go b/modules/public/public.go
index d5f0efb17ac34534213277973a6f09b32069d6ae..5fbfe30a81c7eb80f7580b1679bae1f587fa137a 100644 (file)
--- a/modules/public/public.go
+++ b/modules/public/public.go
@@ -28,27 +28,15 @@ func AssetFS() *assetfs.LayeredFS {
        return assetfs.Layered(CustomAssets(), BuiltinAssets())
 }
 
-// AssetsHandlerFunc implements the static handler for serving custom or original assets.
-func AssetsHandlerFunc(prefix string) http.HandlerFunc {
+// FileHandlerFunc implements the static handler for serving files in "public" assets
+func FileHandlerFunc() http.HandlerFunc {
        assetFS := AssetFS()
-       prefix = strings.TrimSuffix(prefix, "/") + "/"
        return func(resp http.ResponseWriter, req *http.Request) {
-               subPath := req.URL.Path
-               if !strings.HasPrefix(subPath, prefix) {
-                       return
-               }
-               subPath = strings.TrimPrefix(subPath, prefix)
-
                if req.Method != "GET" && req.Method != "HEAD" {
                        resp.WriteHeader(http.StatusNotFound)
                        return
                }
-
-               if handleRequest(resp, req, assetFS, subPath) {
-                       return
-               }
-
-               resp.WriteHeader(http.StatusNotFound)
+               handleRequest(resp, req, assetFS, req.URL.Path)
        }
 }
 
@@ -71,16 +59,17 @@ func setWellKnownContentType(w http.ResponseWriter, file string) {
        }
 }
 
-func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool {
+func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) {
        // actually, fs (http.FileSystem) is designed to be a safe interface, relative paths won't bypass its parent directory, it's also fine to do a clean here
-       f, err := fs.Open(util.PathJoinRelX("assets", file))
+       f, err := fs.Open(util.PathJoinRelX(file))
        if err != nil {
                if os.IsNotExist(err) {
-                       return false
+                       w.WriteHeader(http.StatusNotFound)
+                       return
                }
                w.WriteHeader(http.StatusInternalServerError)
                log.Error("[Static] Open %q failed: %v", file, err)
-               return true
+               return
        }
        defer f.Close()
 
@@ -88,17 +77,16 @@ func handleRequest(w http.ResponseWriter, req *http.Request, fs http.FileSystem,
        if err != nil {
                w.WriteHeader(http.StatusInternalServerError)
                log.Error("[Static] %q exists, but fails to open: %v", file, err)
-               return true
+               return
        }
 
-       // Try to serve index file
+       // need to serve index file? (no at the moment)
        if fi.IsDir() {
                w.WriteHeader(http.StatusNotFound)
-               return true
+               return
        }
 
        serveContent(w, req, fi, fi.ModTime(), f)
-       return true
 }
 
 type GzipBytesProvider interface {

diff --git a/modules/setting/server.go b/modules/setting/server.go
index 7c033bcc6ba7980aa3aafa04753b1b3a106ef28d..08eb82fb3dece4e3bc305aa214221f01feefd0a3 100644 (file)
--- a/modules/setting/server.go
+++ b/modules/setting/server.go
@@ -349,9 +349,4 @@ func loadServerFrom(rootCfg ConfigProvider) {
        default:
                LandingPageURL = LandingPage(landingPage)
        }
-
-       HasRobotsTxt, err = util.IsFile(path.Join(CustomPath, "robots.txt"))
-       if err != nil {
-               log.Error("Unable to check if %s is a file. Error: %v", path.Join(CustomPath, "robots.txt"), err)
-       }
 }

diff --git a/public/.well-known/security.txt b/public/.well-known/security.txt
new file mode 100644 (file)
index 0000000..2cae3cb
--- /dev/null
+++ b/public/.well-known/security.txt
@@ -0,0 +1,6 @@
+# This site is running a Gitea instance.
+# Gitea related security problems could be reported to Gitea community.
+# Site related security problems should be reported to this site's admin.
+Contact: https://github.com/go-gitea/gitea/blob/main/SECURITY.md
+Policy: https://github.com/go-gitea/gitea/blob/main/SECURITY.md
+Preferred-Languages: en

diff --git a/routers/install/routes.go b/routers/install/routes.go
index ce6d41b32db40880062e126ba709f54c9baad6ba..06c9d389a60f44e8a3dbba1069ec3f2801319791 100644 (file)
--- a/routers/install/routes.go
+++ b/routers/install/routes.go
@@ -20,7 +20,7 @@ import (
 func Routes() *web.Route {
        base := web.NewRoute()
        base.Use(common.ProtocolMiddlewares()...)
-       base.Methods("GET, HEAD", "/assets/*", public.AssetsHandlerFunc("/assets/"))
+       base.Methods("GET, HEAD", "/assets/*", public.FileHandlerFunc())
 
        r := web.NewRoute()
        r.Use(common.Sessioner(), Contexter())

diff --git a/routers/web/misc/misc.go b/routers/web/misc/misc.go
index 6ed3b5c3adb8d46090db0da421f55aac4c282d1a..54c93763f6a00bcbedaed0e344d475c47f82629d 100644 (file)
--- a/routers/web/misc/misc.go
+++ b/routers/web/misc/misc.go
@@ -34,9 +34,12 @@ func DummyOK(w http.ResponseWriter, req *http.Request) {
 }
 
 func RobotsTxt(w http.ResponseWriter, req *http.Request) {
-       filePath := util.FilePathJoinAbs(setting.CustomPath, "robots.txt")
+       robotsTxt := util.FilePathJoinAbs(setting.CustomPath, "public/robots.txt")
+       if ok, _ := util.IsExist(robotsTxt); !ok {
+               robotsTxt = util.FilePathJoinAbs(setting.CustomPath, "robots.txt") // the legacy "robots.txt"
+       }
        httpcache.SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
-       http.ServeFile(w, req, filePath)
+       http.ServeFile(w, req, robotsTxt)
 }
 
 func StaticRedirect(target string) func(w http.ResponseWriter, req *http.Request) {

diff --git a/routers/web/web.go b/routers/web/web.go
index f091bfefb86d4d71b37feb4d7b649505efb04247..455791ad802df8351cd43131d9d6d65c73cb087f 100644 (file)
--- a/routers/web/web.go
+++ b/routers/web/web.go
@@ -108,7 +108,7 @@ func Routes() *web.Route {
        routes := web.NewRoute()
 
        routes.Head("/", misc.DummyOK) // for health check - doesn't need to be passed through gzip handler
-       routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.AssetsHandlerFunc("/assets/"))
+       routes.Methods("GET, HEAD", "/assets/*", CorsHandler(), public.FileHandlerFunc())
        routes.Methods("GET, HEAD", "/avatars/*", storageHandler(setting.Avatar.Storage, "avatars", storage.Avatars))
        routes.Methods("GET, HEAD", "/repo-avatars/*", storageHandler(setting.RepoAvatar.Storage, "repo-avatars", storage.RepoAvatars))
        routes.Methods("GET, HEAD", "/apple-touch-icon.png", misc.StaticRedirect("/assets/img/apple-touch-icon.png"))
@@ -132,15 +132,12 @@ func Routes() *web.Route {
                routes.Methods("GET,HEAD", "/captcha/*", append(mid, captcha.Captchaer(context.GetImageCaptcha()))...)
        }
 
-       if setting.HasRobotsTxt {
-               routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
-       }
-
        if setting.Metrics.Enabled {
                prometheus.MustRegister(metrics.NewCollector())
                routes.Get("/metrics", append(mid, Metrics)...)
        }
 
+       routes.Get("/robots.txt", append(mid, misc.RobotsTxt)...)
        routes.Get("/ssh_info", misc.SSHInfo)
        routes.Get("/api/healthz", healthcheck.Check)
 
@@ -336,8 +333,7 @@ func registerRoutes(m *web.Route) {
 
        // FIXME: not all routes need go through same middleware.
        // Especially some AJAX requests, we can reduce middleware number to improve performance.
-       // Routers.
-       // for health check
+
        m.Get("/", Home)
        m.Get("/sitemap.xml", sitemapEnabled, ignExploreSignIn, HomeSitemap)
        m.Group("/.well-known", func() {
@@ -349,7 +345,8 @@ func registerRoutes(m *web.Route) {
                m.Get("/change-password", func(ctx *context.Context) {
                        ctx.Redirect(setting.AppSubURL + "/user/settings/account")
                })
-       })
+               m.Any("/*", CorsHandler(), public.FileHandlerFunc())
+       }, CorsHandler())
 
        m.Group("/explore", func() {
                m.Get("", func(ctx *context.Context) {

diff --git a/tests/integration/links_test.go b/tests/integration/links_test.go
index 9136f8f9153ae6536a47a1cd0c509e40821dda54..91655833af0412d21f77836a9958bbdaa6f15e8b 100644 (file)
--- a/tests/integration/links_test.go
+++ b/tests/integration/links_test.go
@@ -38,6 +38,7 @@ func TestLinksNoLogin(t *testing.T) {
                "/user2/repo1/projects/1",
                "/assets/img/404.png",
                "/assets/img/500.png",
+               "/.well-known/security.txt",
        }
 
        for _, link := range links {