Add commit count caching (#2774)

* Add commit count caching

* Small refactoring

* Add different key prefix for refs and commits

* Add configuratuion option to allow to change caching time or disable it

diff --git a/conf/app.ini b/conf/app.ini
index 0ce8aae52e7d25e5df245afd04e5c2dcaade420f..3a850106c1eb1190a6ed14d6103c848338f0b421 100644 (file)
--- a/conf/app.ini
+++ b/conf/app.ini
@@ -339,6 +339,9 @@ INTERVAL = 60
 ; redis: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180
 ; memcache: `127.0.0.1:11211`
 HOST =
+; Time to keep items in cache if not used, default is 16 hours.
+; Setting it to 0 disables caching
+ITEM_TTL = 16h
 
 [session]
 ; Either "memory", "file", or "redis", default is "memory"

diff --git a/models/repo.go b/models/repo.go
index 1b1be62f875d4236c734c81161a98af902eb2af6..eca71568ee936b5b2e19e86b892cc02760b3b136 100644 (file)
--- a/models/repo.go
+++ b/models/repo.go
@@ -258,6 +258,17 @@ func (repo *Repository) APIFormat(mode AccessMode) *api.Repository {
        return repo.innerAPIFormat(mode, false)
 }
 
+// GetCommitsCountCacheKey returns cache key used for commits count caching.
+func (repo *Repository) GetCommitsCountCacheKey(contextName string, isRef bool) string {
+       var prefix string
+       if isRef {
+               prefix = "ref"
+       } else {
+               prefix = "commit"
+       }
+       return fmt.Sprintf("commits-count-%d-%s-%s", repo.ID, prefix, contextName)
+}
+
 func (repo *Repository) innerAPIFormat(mode AccessMode, isParent bool) *api.Repository {
        var parent *api.Repository
 

diff --git a/models/update.go b/models/update.go
index 62d13ce20960364f82da0cf3838e450c08b2a5f5..82369bf6367602951fe93fbb7136c6bca4db44cf 100644 (file)
--- a/models/update.go
+++ b/models/update.go
@@ -11,7 +11,7 @@ import (
        "strings"
 
        "code.gitea.io/git"
-
+       "code.gitea.io/gitea/modules/cache"
        "code.gitea.io/gitea/modules/log"
 )
 
@@ -205,19 +205,26 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
        var commits = &PushCommits{}
        if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
                // If is tag reference
+               tagName := opts.RefFullName[len(git.TagPrefix):]
                if isDelRef {
-                       err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
+                       err = pushUpdateDeleteTag(repo, gitRepo, tagName)
                        if err != nil {
                                return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err)
                        }
                } else {
-                       err = pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
+                       // Clear cache for tag commit count
+                       cache.Remove(repo.GetCommitsCountCacheKey(tagName, true))
+                       err = pushUpdateAddTag(repo, gitRepo, tagName)
                        if err != nil {
                                return nil, fmt.Errorf("pushUpdateAddTag: %v", err)
                        }
                }
        } else if !isDelRef {
                // If is branch reference
+
+               // Clear cache for branch commit count
+               cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true))
+
                newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
                if err != nil {
                        return nil, fmt.Errorf("gitRepo.GetCommit: %v", err)

diff --git a/modules/cache/cache.go b/modules/cache/cache.go
new file mode 100644 (file)
index 0000000..0a73ae8
--- /dev/null
+++ b/modules/cache/cache.go
@@ -0,0 +1,72 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+       "code.gitea.io/gitea/modules/setting"
+
+       mc "github.com/go-macaron/cache"
+)
+
+var conn mc.Cache
+
+// NewContext start cache service
+func NewContext() error {
+       if setting.CacheService == nil || conn != nil {
+               return nil
+       }
+
+       var err error
+       conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
+               Adapter:       setting.CacheService.Adapter,
+               AdapterConfig: setting.CacheService.Conn,
+               Interval:      setting.CacheService.Interval,
+       })
+       return err
+}
+
+// GetInt returns key value from cache with callback when no key exists in cache
+func GetInt(key string, getFunc func() (int, error)) (int, error) {
+       if conn == nil || setting.CacheService.TTL == 0 {
+               return getFunc()
+       }
+       if !conn.IsExist(key) {
+               var (
+                       value int
+                       err   error
+               )
+               if value, err = getFunc(); err != nil {
+                       return value, err
+               }
+               conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+       }
+       return conn.Get(key).(int), nil
+}
+
+// GetInt64 returns key value from cache with callback when no key exists in cache
+func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
+       if conn == nil || setting.CacheService.TTL == 0 {
+               return getFunc()
+       }
+       if !conn.IsExist(key) {
+               var (
+                       value int64
+                       err   error
+               )
+               if value, err = getFunc(); err != nil {
+                       return value, err
+               }
+               conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+       }
+       return conn.Get(key).(int64), nil
+}
+
+// Remove key from cache
+func Remove(key string) {
+       if conn == nil {
+               return
+       }
+       conn.Delete(key)
+}

diff --git a/modules/context/repo.go b/modules/context/repo.go
index c33396c0f926f45267425ce0d97b4b969ba13203..3aaf1ce64a8a16086e2d9dbd3459aefa77951ec5 100644 (file)
--- a/modules/context/repo.go
+++ b/modules/context/repo.go
@@ -13,7 +13,9 @@ import (
 
        "code.gitea.io/git"
        "code.gitea.io/gitea/models"
+       "code.gitea.io/gitea/modules/cache"
        "code.gitea.io/gitea/modules/setting"
+
        "github.com/Unknwon/com"
        "gopkg.in/editorconfig/editorconfig-core-go.v1"
        "gopkg.in/macaron.v1"
@@ -100,6 +102,21 @@ func (r *Repository) CanUseTimetracker(issue *models.Issue, user *models.User) b
                r.IsWriter() || issue.IsPoster(user.ID) || issue.AssigneeID == user.ID)
 }
 
+// GetCommitsCount returns cached commit count for current view
+func (r *Repository) GetCommitsCount() (int64, error) {
+       var contextName string
+       if r.IsViewBranch {
+               contextName = r.BranchName
+       } else if r.IsViewTag {
+               contextName = r.TagName
+       } else {
+               contextName = r.CommitID
+       }
+       return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, r.IsViewBranch || r.IsViewTag), func() (int64, error) {
+               return r.Commit.CommitsCount()
+       })
+}
+
 // GetEditorconfig returns the .editorconfig definition if found in the
 // HEAD of the default repo branch.
 func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
@@ -535,9 +552,9 @@ func RepoRef() macaron.Handler {
                ctx.Data["IsViewCommit"] = ctx.Repo.IsViewCommit
                ctx.Data["CanCreateBranch"] = ctx.Repo.CanCreateBranch()
 
-               ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount()
+               ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount()
                if err != nil {
-                       ctx.Handle(500, "CommitsCount", err)
+                       ctx.Handle(500, "GetCommitsCount", err)
                        return
                }
                ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount

diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 9787e0928003542917f57626be46f9b82e6f6c8d..6c89381f3b97b6c970b62474c2d1bb866064b218 100644 (file)
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -325,11 +325,6 @@ var (
        // Time settings
        TimeFormat string
 
-       // Cache settings
-       CacheAdapter  string
-       CacheInterval int
-       CacheConn     string
-
        // Session settings
        SessionConfig  session.Options
        CSRFCookieName = "_csrf"
@@ -1295,16 +1290,33 @@ func NewXORMLogService(disableConsole bool) {
        }
 }
 
+// Cache represents cache settings
+type Cache struct {
+       Adapter  string
+       Interval int
+       Conn     string
+       TTL      time.Duration
+}
+
+var (
+       // CacheService the global cache
+       CacheService *Cache
+)
+
 func newCacheService() {
-       CacheAdapter = Cfg.Section("cache").Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"})
-       switch CacheAdapter {
+       sec := Cfg.Section("cache")
+       CacheService = &Cache{
+               Adapter: sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}),
+       }
+       switch CacheService.Adapter {
        case "memory":
-               CacheInterval = Cfg.Section("cache").Key("INTERVAL").MustInt(60)
+               CacheService.Interval = sec.Key("INTERVAL").MustInt(60)
        case "redis", "memcache":
-               CacheConn = strings.Trim(Cfg.Section("cache").Key("HOST").String(), "\" ")
+               CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ")
        default:
-               log.Fatal(4, "Unknown cache adapter: %s", CacheAdapter)
+               log.Fatal(4, "Unknown cache adapter: %s", CacheService.Adapter)
        }
+       CacheService.TTL = sec.Key("ITEM_TTL").MustDuration(16 * time.Hour)
 
        log.Info("Cache Service Enabled")
 }

diff --git a/routers/admin/admin.go b/routers/admin/admin.go
index 94b88a05c3eae9e48e07fdae04e365f6354e8a49..39a8f718cab4d53826f658ea4ff51c6f21273748 100644 (file)
--- a/routers/admin/admin.go
+++ b/routers/admin/admin.go
@@ -224,9 +224,9 @@ func Config(ctx *context.Context) {
                ctx.Data["Mailer"] = setting.MailService
        }
 
-       ctx.Data["CacheAdapter"] = setting.CacheAdapter
-       ctx.Data["CacheInterval"] = setting.CacheInterval
-       ctx.Data["CacheConn"] = setting.CacheConn
+       ctx.Data["CacheAdapter"] = setting.CacheService.Adapter
+       ctx.Data["CacheInterval"] = setting.CacheService.Interval
+       ctx.Data["CacheConn"] = setting.CacheService.Conn
 
        ctx.Data["SessionConfig"] = setting.SessionConfig
 

diff --git a/routers/init.go b/routers/init.go
index 006f285266b1ce224fa5339eb3b70b7f7daf665b..8cfbe39ee5e26f807ca492c56cedcf4c2feb7639 100644 (file)
--- a/routers/init.go
+++ b/routers/init.go
@@ -11,6 +11,7 @@ import (
        "code.gitea.io/git"
        "code.gitea.io/gitea/models"
        "code.gitea.io/gitea/models/migrations"
+       "code.gitea.io/gitea/modules/cache"
        "code.gitea.io/gitea/modules/cron"
        "code.gitea.io/gitea/modules/highlight"
        "code.gitea.io/gitea/modules/log"
@@ -18,6 +19,7 @@ import (
        "code.gitea.io/gitea/modules/markup"
        "code.gitea.io/gitea/modules/setting"
        "code.gitea.io/gitea/modules/ssh"
+
        macaron "gopkg.in/macaron.v1"
 )
 
@@ -37,6 +39,7 @@ func checkRunMode() {
 func NewServices() {
        setting.NewServices()
        mailer.NewContext()
+       cache.NewContext()
 }
 
 // GlobalInit is for global configuration reload-able.

diff --git a/routers/repo/commit.go b/routers/repo/commit.go
index 21a0d9dd9f24eee9506a788b99f382ca4ac7b786..637a50543adb76af481dddbdd2a310b4741aebb6 100644 (file)
--- a/routers/repo/commit.go
+++ b/routers/repo/commit.go
@@ -55,7 +55,7 @@ func Commits(ctx *context.Context) {
        }
        ctx.Data["PageIsViewCode"] = true
 
-       commitsCount, err := ctx.Repo.Commit.CommitsCount()
+       commitsCount, err := ctx.Repo.GetCommitsCount()
        if err != nil {
                ctx.Handle(500, "GetCommitsCount", err)
                return
@@ -91,7 +91,7 @@ func Graph(ctx *context.Context) {
        ctx.Data["PageIsCommits"] = true
        ctx.Data["PageIsViewCode"] = true
 
-       commitsCount, err := ctx.Repo.Commit.CommitsCount()
+       commitsCount, err := ctx.Repo.GetCommitsCount()
        if err != nil {
                ctx.Handle(500, "GetCommitsCount", err)
                return

diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index 227b4fff9cdd926c8845a08be72ac3dad57016a8..703afbb4a73512ad913c2db69b0b62c9ed2daef1 100644 (file)
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -99,9 +99,9 @@ func NewMacaron() *macaron.Macaron {
                Redirect:    true,
        }))
        m.Use(cache.Cacher(cache.Options{
-               Adapter:       setting.CacheAdapter,
-               AdapterConfig: setting.CacheConn,
-               Interval:      setting.CacheInterval,
+               Adapter:       setting.CacheService.Adapter,
+               AdapterConfig: setting.CacheService.Conn,
+               Interval:      setting.CacheService.Interval,
        }))
        m.Use(captcha.Captchaer(captcha.Options{
                SubURL: setting.AppSubURL,
@@ -576,9 +576,9 @@ func RegisterRoutes(m *macaron.Macaron) {
                                ctx.Handle(500, "GetBranchCommit", err)
                                return
                        }
-                       ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount()
+                       ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount()
                        if err != nil {
-                               ctx.Handle(500, "CommitsCount", err)
+                               ctx.Handle(500, "GetCommitsCount", err)
                                return
                        }
                        ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount