HTTP cache rework and enable caching for storage assets (#13569)

This enabled HTTP time-based cache for storage assets, primarily
avatars. I have not observed If-Modified-Since from browsers during
tests but I guess it's good to support regardless.

It introduces a new generic httpcache module that can handle both
time-based and etag-based caching.

Additionally, manifest.json and robots.txt are now also cachable.

Author: silverwind

custom/conf/app.example.ini

@@ -389,7 +389,7 @@ GRACEFUL_HAMMER_TIME = 60s
 ; Allows the setting of a startup timeout and waithint for Windows as SVC service
 ; 0 disables this.
 STARTUP_TIMEOUT = 0
-; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time, default is 6h
+; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time. Note that this cache is disabled when RUN_MODE is "dev". Default is 6h
 STATIC_CACHE_TIME = 6h
 
 ; Define allowed algorithms and their minimum key length (use -1 to disable a type)

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -262,7 +262,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `KEY_FILE`: **https/key.pem**: Key file path used for HTTPS. From 1.11 paths are relative to `CUSTOM_PATH`.
 - `STATIC_ROOT_PATH`: **./**: Upper level of template and static files path.
 - `APP_DATA_PATH`: **data** (**/data/gitea** on docker): Default path for application data.
-- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
+- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. Note that this cache is disabled when `RUN_MODE` is "dev".
 - `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
 - `ENABLE_PPROF`: **false**: Application profiling (memory and cpu). For "web" command it listens on localhost:6060. For "serv" command it dumps to disk at `PPROF_DATA_PATH` as `(cpuprofile|memprofile)_<username>_<temporary id>`
 - `PPROF_DATA_PATH`: **data/tmp/pprof**: `PPROF_DATA_PATH`, use an absolute path when you start gitea as service

main.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"time"
 
 	"code.gitea.io/gitea/cmd"
 	"code.gitea.io/gitea/modules/log"
@@ -40,6 +41,7 @@ var (
 func init() {
 	setting.AppVer = Version
 	setting.AppBuiltWith = formatBuiltWith()
+	setting.AppStartTime = time.Now().UTC()
 
 	// Grab the original help templates
 	originalAppHelpTemplate = cli.AppHelpTemplate

modules/httpcache/httpcache.go

@@ -0,0 +1,59 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package httpcache
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// GetCacheControl returns a suitable "Cache-Control" header value
+func GetCacheControl() string {
+	if setting.RunMode == "dev" {
+		return "no-store"
+	}
+	return "private, max-age=" + strconv.FormatInt(int64(setting.StaticCacheTime.Seconds()), 10)
+}
+
+// generateETag generates an ETag based on size, filename and file modification time
+func generateETag(fi os.FileInfo) string {
+	etag := fmt.Sprint(fi.Size()) + fi.Name() + fi.ModTime().UTC().Format(http.TimeFormat)
+	return base64.StdEncoding.EncodeToString([]byte(etag))
+}
+
+// HandleTimeCache handles time-based caching for a HTTP request
+func HandleTimeCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) {
+	ifModifiedSince := req.Header.Get("If-Modified-Since")
+	if ifModifiedSince != "" {
+		t, err := time.Parse(http.TimeFormat, ifModifiedSince)
+		if err == nil && fi.ModTime().Unix() <= t.Unix() {
+			w.WriteHeader(http.StatusNotModified)
+			return true
+		}
+	}
+
+	w.Header().Set("Cache-Control", GetCacheControl())
+	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
+	return false
+}
+
+// HandleEtagCache handles ETag-based caching for a HTTP request
+func HandleEtagCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) {
+	etag := generateETag(fi)
+	if req.Header.Get("If-None-Match") == etag {
+		w.WriteHeader(http.StatusNotModified)
+		return true
+	}
+
+	w.Header().Set("Cache-Control", GetCacheControl())
+	w.Header().Set("ETag", etag)
+	return false
+}

modules/public/public.go

@@ -5,15 +5,13 @@
 package public
 
 import (
-	"encoding/base64"
-	"fmt"
 	"log"
 	"net/http"
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
+	"code.gitea.io/gitea/modules/httpcache"
 	"code.gitea.io/gitea/modules/setting"
 )
 
@@ -22,11 +20,8 @@ type Options struct {
 	Directory   string
 	IndexFile   string
 	SkipLogging bool
-	// if set to true, will enable caching. Expires header will also be set to
-	// expire after the defined time.
-	ExpiresAfter time.Duration
-	FileSystem   http.FileSystem
-	Prefix       string
+	FileSystem  http.FileSystem
+	Prefix      string
 }
 
 // KnownPublicEntries list all direct children in the `public` directory
@@ -158,23 +153,10 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Optio
 		log.Println("[Static] Serving " + file)
 	}
 
-	// Add an Expires header to the static content
-	if opt.ExpiresAfter > 0 {
-		w.Header().Set("Expires", time.Now().Add(opt.ExpiresAfter).UTC().Format(http.TimeFormat))
-		tag := GenerateETag(fmt.Sprint(fi.Size()), fi.Name(), fi.ModTime().UTC().Format(http.TimeFormat))
-		w.Header().Set("ETag", tag)
-		if req.Header.Get("If-None-Match") == tag {
-			w.WriteHeader(304)
-			return true
-		}
+	if httpcache.HandleEtagCache(req, w, fi) {
+		return true
 	}
 
 	http.ServeContent(w, req, file, fi.ModTime(), f)
 	return true
 }
-
-// GenerateETag generates an ETag based on size, filename and file modification time
-func GenerateETag(fileSize, fileName, modTime string) string {
-	etag := fileSize + fileName + modTime
-	return base64.StdEncoding.EncodeToString([]byte(etag))
-}

modules/setting/setting.go

@@ -67,6 +67,7 @@ var (
 	// AppVer settings
 	AppVer         string
 	AppBuiltWith   string
+	AppStartTime   time.Time
 	AppName        string
 	AppURL         string
 	AppSubURL      string
@@ -362,6 +363,7 @@ var (
 	PIDFile       = "/run/gitea.pid"
 	WritePIDFile  bool
 	ProdMode      bool
+	RunMode       string
 	RunUser       string
 	IsWindows     bool
 	HasRobotsTxt  bool
@@ -837,6 +839,7 @@ func NewContext() {
 	}
 
 	RunUser = Cfg.Section("").Key("RUN_USER").MustString(user.CurrentUsername())
+	RunMode = Cfg.Section("").Key("RUN_MODE").MustString("dev")
 	// Does not check run user when the install lock is off.
 	if InstallLock {
 		currentUser, match := IsRunUserMatchCurrentUser(RunUser)

routers/routes/chi.go

@@ -16,6 +16,7 @@ import (
 	"text/template"
 	"time"
 
+	"code.gitea.io/gitea/modules/httpcache"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/metrics"
 	"code.gitea.io/gitea/modules/public"
@@ -162,6 +163,12 @@ func storageHandler(storageSetting setting.Storage, prefix string, objStore stor
 
 			rPath := strings.TrimPrefix(req.RequestURI, "/"+prefix)
 			rPath = strings.TrimPrefix(rPath, "/")
+
+			fi, err := objStore.Stat(rPath)
+			if err == nil && httpcache.HandleTimeCache(req, w, fi) {
+				return
+			}
+
 			//If we have matched and access to release or issue
 			fr, err := objStore.Open(rPath)
 			if err != nil {
@@ -200,21 +207,15 @@ func NewChi() chi.Router {
 		setupAccessLogger(c)
 	}
 
-	if setting.ProdMode {
-		log.Warn("ProdMode ignored")
-	}
-
 	c.Use(public.Custom(
 		&public.Options{
-			SkipLogging:  setting.DisableRouterLog,
-			ExpiresAfter: time.Hour * 6,
+			SkipLogging: setting.DisableRouterLog,
 		},
 	))
 	c.Use(public.Static(
 		&public.Options{
-			Directory:    path.Join(setting.StaticRootPath, "public"),
-			SkipLogging:  setting.DisableRouterLog,
-			ExpiresAfter: time.Hour * 6,
+			Directory:   path.Join(setting.StaticRootPath, "public"),
+			SkipLogging: setting.DisableRouterLog,
 		},
 	))
 
@@ -247,10 +248,14 @@ func NormalRoutes() http.Handler {
 		w.WriteHeader(http.StatusOK)
 	})
 
-	// robots.txt
 	if setting.HasRobotsTxt {
 		r.Get("/robots.txt", func(w http.ResponseWriter, req *http.Request) {
-			http.ServeFile(w, req, path.Join(setting.CustomPath, "robots.txt"))
+			filePath := path.Join(setting.CustomPath, "robots.txt")
+			fi, err := os.Stat(filePath)
+			if err == nil && httpcache.HandleTimeCache(req, w, fi) {
+				return
+			}
+			http.ServeFile(w, req, filePath)
 		})
 	}
 

routers/routes/macaron.go

@@ -6,10 +6,12 @@ package routes
 
 import (
 	"encoding/gob"
+	"net/http"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/httpcache"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/options"
@@ -977,6 +979,8 @@ func RegisterMacaronRoutes(m *macaron.Macaron) {
 
 	// Progressive Web App
 	m.Get("/manifest.json", templates.JSONRenderer(), func(ctx *context.Context) {
+		ctx.Resp.Header().Set("Cache-Control", httpcache.GetCacheControl())
+		ctx.Resp.Header().Set("Last-Modified", setting.AppStartTime.Format(http.TimeFormat))
 		ctx.HTML(200, "pwa/manifest_json")
 	})