HTTP cache rework and enable caching for storage assets (#13569)

This enabled HTTP time-based cache for storage assets, primarily
avatars. I have not observed If-Modified-Since from browsers during
tests but I guess it's good to support regardless.

It introduces a new generic httpcache module that can handle both
time-based and etag-based caching.

Additionally, manifest.json and robots.txt are now also cachable.

diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini
index a4e35d2495f525a42f4069fa7dc286298daabbc8..1311c5a65003681e62cb343fd6e4202bb8adf889 100644 (file)
--- a/custom/conf/app.example.ini
+++ b/custom/conf/app.example.ini
@@ -389,7 +389,7 @@ GRACEFUL_HAMMER_TIME = 60s
 ; Allows the setting of a startup timeout and waithint for Windows as SVC service
 ; 0 disables this.
 STARTUP_TIMEOUT = 0
-; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time, default is 6h
+; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time. Note that this cache is disabled when RUN_MODE is "dev". Default is 6h
 STATIC_CACHE_TIME = 6h
 
 ; Define allowed algorithms and their minimum key length (use -1 to disable a type)

diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
index c58e26ceb1e1e1164458742633bc53d0cd7f447d..eaf43da29a7ed7664b8316b00e20827787b56c43 100644 (file)
--- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -262,7 +262,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `KEY_FILE`: **https/key.pem**: Key file path used for HTTPS. From 1.11 paths are relative to `CUSTOM_PATH`.
 - `STATIC_ROOT_PATH`: **./**: Upper level of template and static files path.
 - `APP_DATA_PATH`: **data** (**/data/gitea** on docker): Default path for application data.
-- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
+- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. Note that this cache is disabled when `RUN_MODE` is "dev".
 - `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
 - `ENABLE_PPROF`: **false**: Application profiling (memory and cpu). For "web" command it listens on localhost:6060. For "serv" command it dumps to disk at `PPROF_DATA_PATH` as `(cpuprofile|memprofile)_<username>_<temporary id>`
 - `PPROF_DATA_PATH`: **data/tmp/pprof**: `PPROF_DATA_PATH`, use an absolute path when you start gitea as service

diff --git a/main.go b/main.go
index e4fe220e8a0187453f9a6518a2ac64929b9c76aa..8ee6ffa92ccdc82fbe56267eff04fb5ab2eafecf 100644 (file)
--- a/main.go
+++ b/main.go
@@ -11,6 +11,7 @@ import (
        "os"
        "runtime"
        "strings"
+       "time"
 
        "code.gitea.io/gitea/cmd"
        "code.gitea.io/gitea/modules/log"
@@ -40,6 +41,7 @@ var (
 func init() {
        setting.AppVer = Version
        setting.AppBuiltWith = formatBuiltWith()
+       setting.AppStartTime = time.Now().UTC()
 
        // Grab the original help templates
        originalAppHelpTemplate = cli.AppHelpTemplate

diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go
new file mode 100644 (file)
index 0000000..c4134f8
--- /dev/null
+++ b/modules/httpcache/httpcache.go
@@ -0,0 +1,59 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package httpcache
+
+import (
+       "encoding/base64"
+       "fmt"
+       "net/http"
+       "os"
+       "strconv"
+       "time"
+
+       "code.gitea.io/gitea/modules/setting"
+)
+
+// GetCacheControl returns a suitable "Cache-Control" header value
+func GetCacheControl() string {
+       if setting.RunMode == "dev" {
+               return "no-store"
+       }
+       return "private, max-age=" + strconv.FormatInt(int64(setting.StaticCacheTime.Seconds()), 10)
+}
+
+// generateETag generates an ETag based on size, filename and file modification time
+func generateETag(fi os.FileInfo) string {
+       etag := fmt.Sprint(fi.Size()) + fi.Name() + fi.ModTime().UTC().Format(http.TimeFormat)
+       return base64.StdEncoding.EncodeToString([]byte(etag))
+}
+
+// HandleTimeCache handles time-based caching for a HTTP request
+func HandleTimeCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) {
+       ifModifiedSince := req.Header.Get("If-Modified-Since")
+       if ifModifiedSince != "" {
+               t, err := time.Parse(http.TimeFormat, ifModifiedSince)
+               if err == nil && fi.ModTime().Unix() <= t.Unix() {
+                       w.WriteHeader(http.StatusNotModified)
+                       return true
+               }
+       }
+
+       w.Header().Set("Cache-Control", GetCacheControl())
+       w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
+       return false
+}
+
+// HandleEtagCache handles ETag-based caching for a HTTP request
+func HandleEtagCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) {
+       etag := generateETag(fi)
+       if req.Header.Get("If-None-Match") == etag {
+               w.WriteHeader(http.StatusNotModified)
+               return true
+       }
+
+       w.Header().Set("Cache-Control", GetCacheControl())
+       w.Header().Set("ETag", etag)
+       return false
+}

diff --git a/modules/public/public.go b/modules/public/public.go
index 3a2fa4c57c1277744d1a761087d6687d9c3a9d5c..fc933637d8f6e27175a5f090049dc68204bf33b6 100644 (file)
--- a/modules/public/public.go
+++ b/modules/public/public.go
@@ -5,15 +5,13 @@
 package public
 
 import (
-       "encoding/base64"
-       "fmt"
        "log"
        "net/http"
        "path"
        "path/filepath"
        "strings"
-       "time"
 
+       "code.gitea.io/gitea/modules/httpcache"
        "code.gitea.io/gitea/modules/setting"
 )
 
@@ -22,11 +20,8 @@ type Options struct {
        Directory   string
        IndexFile   string
        SkipLogging bool
-       // if set to true, will enable caching. Expires header will also be set to
-       // expire after the defined time.
-       ExpiresAfter time.Duration
-       FileSystem   http.FileSystem
-       Prefix       string
+       FileSystem  http.FileSystem
+       Prefix      string
 }
 
 // KnownPublicEntries list all direct children in the `public` directory
@@ -158,23 +153,10 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Optio
                log.Println("[Static] Serving " + file)
        }
 
-       // Add an Expires header to the static content
-       if opt.ExpiresAfter > 0 {
-               w.Header().Set("Expires", time.Now().Add(opt.ExpiresAfter).UTC().Format(http.TimeFormat))
-               tag := GenerateETag(fmt.Sprint(fi.Size()), fi.Name(), fi.ModTime().UTC().Format(http.TimeFormat))
-               w.Header().Set("ETag", tag)
-               if req.Header.Get("If-None-Match") == tag {
-                       w.WriteHeader(304)
-                       return true
-               }
+       if httpcache.HandleEtagCache(req, w, fi) {
+               return true
        }
 
        http.ServeContent(w, req, file, fi.ModTime(), f)
        return true
 }
-
-// GenerateETag generates an ETag based on size, filename and file modification time
-func GenerateETag(fileSize, fileName, modTime string) string {
-       etag := fileSize + fileName + modTime
-       return base64.StdEncoding.EncodeToString([]byte(etag))
-}

diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 7ae8bb352de10e02096a98b7135150c5e0e29611..708dc282337db88f23072dd96a344b6e07a3de06 100644 (file)
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -67,6 +67,7 @@ var (
        // AppVer settings
        AppVer         string
        AppBuiltWith   string
+       AppStartTime   time.Time
        AppName        string
        AppURL         string
        AppSubURL      string
@@ -362,6 +363,7 @@ var (
        PIDFile       = "/run/gitea.pid"
        WritePIDFile  bool
        ProdMode      bool
+       RunMode       string
        RunUser       string
        IsWindows     bool
        HasRobotsTxt  bool
@@ -837,6 +839,7 @@ func NewContext() {
        }
 
        RunUser = Cfg.Section("").Key("RUN_USER").MustString(user.CurrentUsername())
+       RunMode = Cfg.Section("").Key("RUN_MODE").MustString("dev")
        // Does not check run user when the install lock is off.
        if InstallLock {
                currentUser, match := IsRunUserMatchCurrentUser(RunUser)

diff --git a/routers/routes/chi.go b/routers/routes/chi.go
index 4575f1ea93be23b1eff54313ff33c0bbee13fdb8..5ff7a728ff1baf21f70d5996b22bba9806ac35f9 100644 (file)
--- a/routers/routes/chi.go
+++ b/routers/routes/chi.go
@@ -16,6 +16,7 @@ import (
        "text/template"
        "time"
 
+       "code.gitea.io/gitea/modules/httpcache"
        "code.gitea.io/gitea/modules/log"
        "code.gitea.io/gitea/modules/metrics"
        "code.gitea.io/gitea/modules/public"
@@ -162,6 +163,12 @@ func storageHandler(storageSetting setting.Storage, prefix string, objStore stor
 
                        rPath := strings.TrimPrefix(req.RequestURI, "/"+prefix)
                        rPath = strings.TrimPrefix(rPath, "/")
+
+                       fi, err := objStore.Stat(rPath)
+                       if err == nil && httpcache.HandleTimeCache(req, w, fi) {
+                               return
+                       }
+
                        //If we have matched and access to release or issue
                        fr, err := objStore.Open(rPath)
                        if err != nil {
@@ -200,21 +207,15 @@ func NewChi() chi.Router {
                setupAccessLogger(c)
        }
 
-       if setting.ProdMode {
-               log.Warn("ProdMode ignored")
-       }
-
        c.Use(public.Custom(
                &public.Options{
-                       SkipLogging:  setting.DisableRouterLog,
-                       ExpiresAfter: time.Hour * 6,
+                       SkipLogging: setting.DisableRouterLog,
                },
        ))
        c.Use(public.Static(
                &public.Options{
-                       Directory:    path.Join(setting.StaticRootPath, "public"),
-                       SkipLogging:  setting.DisableRouterLog,
-                       ExpiresAfter: time.Hour * 6,
+                       Directory:   path.Join(setting.StaticRootPath, "public"),
+                       SkipLogging: setting.DisableRouterLog,
                },
        ))
 
@@ -247,10 +248,14 @@ func NormalRoutes() http.Handler {
                w.WriteHeader(http.StatusOK)
        })
 
-       // robots.txt
        if setting.HasRobotsTxt {
                r.Get("/robots.txt", func(w http.ResponseWriter, req *http.Request) {
-                       http.ServeFile(w, req, path.Join(setting.CustomPath, "robots.txt"))
+                       filePath := path.Join(setting.CustomPath, "robots.txt")
+                       fi, err := os.Stat(filePath)
+                       if err == nil && httpcache.HandleTimeCache(req, w, fi) {
+                               return
+                       }
+                       http.ServeFile(w, req, filePath)
                })
        }
 

diff --git a/routers/routes/macaron.go b/routers/routes/macaron.go
index 1f0b21a74d36bebafb51d7210e394ed1f8a72254..170bc7d493dffa39463daabb3d9e01941642b238 100644 (file)
--- a/routers/routes/macaron.go
+++ b/routers/routes/macaron.go
@@ -6,10 +6,12 @@ package routes
 
 import (
        "encoding/gob"
+       "net/http"
 
        "code.gitea.io/gitea/models"
        "code.gitea.io/gitea/modules/auth"
        "code.gitea.io/gitea/modules/context"
+       "code.gitea.io/gitea/modules/httpcache"
        "code.gitea.io/gitea/modules/lfs"
        "code.gitea.io/gitea/modules/log"
        "code.gitea.io/gitea/modules/options"
@@ -977,6 +979,8 @@ func RegisterMacaronRoutes(m *macaron.Macaron) {
 
        // Progressive Web App
        m.Get("/manifest.json", templates.JSONRenderer(), func(ctx *context.Context) {
+               ctx.Resp.Header().Set("Cache-Control", httpcache.GetCacheControl())
+               ctx.Resp.Header().Set("Last-Modified", setting.AppStartTime.Format(http.TimeFormat))
                ctx.HTML(200, "pwa/manifest_json")
        })