summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--custom/conf/app.ini.sample14
-rw-r--r--docs/content/doc/advanced/config-cheat-sheet.en-us.md7
-rw-r--r--docs/content/doc/advanced/config-cheat-sheet.zh-cn.md7
-rw-r--r--integrations/repo_test.go65
-rw-r--r--modules/cache/cache.go26
-rw-r--r--modules/cache/last_commit.go64
-rw-r--r--modules/git/cache.go6
-rw-r--r--modules/git/commit_info.go45
-rw-r--r--modules/setting/cache.go54
-rw-r--r--routers/repo/view.go8
10 files changed, 273 insertions, 23 deletions
diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample
index b0aafb8b31..31e8bf0b09 100644
--- a/custom/conf/app.ini.sample
+++ b/custom/conf/app.ini.sample
@@ -43,7 +43,7 @@ DEFAULT_CLOSE_ISSUES_VIA_COMMITS_IN_ANY_BRANCH = false
ENABLE_PUSH_CREATE_USER = false
ENABLE_PUSH_CREATE_ORG = false
; Comma separated list of globally disabled repo units. Allowed values: repo.issues, repo.ext_issues, repo.pulls, repo.wiki, repo.ext_wiki
-DISABLED_REPO_UNITS =
+DISABLED_REPO_UNITS =
; Comma separated list of default repo units. Allowed values: repo.code, repo.releases, repo.issues, repo.pulls, repo.wiki.
; Note: Code and Releases can currently not be deactivated. If you specify default repo units you should still list them for future compatibility.
; External wiki and issue tracker can't be enabled by default as it requires additional settings.
@@ -632,6 +632,8 @@ SENDMAIL_PATH = sendmail
SENDMAIL_ARGS =
[cache]
+; if the cache enabled
+ENABLED = true
; Either "memory", "redis", or "memcache", default is "memory"
ADAPTER = memory
; For "memory" only, GC interval in seconds, default is 60
@@ -644,6 +646,16 @@ HOST =
; Setting it to 0 disables caching
ITEM_TTL = 16h
+; Last commit cache
+[cache.last_commit]
+; if the cache enabled
+ENABLED = true
+; Time to keep items in cache if not used, default is 8760 hours.
+; Setting it to 0 disables caching
+ITEM_TTL = 8760h
+; Only enable the cache when repository's commits count great than
+COMMITS_COUNT = 1000
+
[session]
; Either "memory", "file", or "redis", default is "memory"
PROVIDER = memory
diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
index e7b87c5272..7e5b39e480 100644
--- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -383,6 +383,7 @@ relation to port exhaustion.
## Cache (`cache`)
+- `ENABLED`: **true**: Enable the cache.
- `ADAPTER`: **memory**: Cache engine adapter, either `memory`, `redis`, or `memcache`.
- `INTERVAL`: **60**: Garbage Collection interval (sec), for memory cache only.
- `HOST`: **\<empty\>**: Connection string for `redis` and `memcache`.
@@ -390,6 +391,12 @@ relation to port exhaustion.
- Memcache: `127.0.0.1:9090;127.0.0.1:9091`
- `ITEM_TTL`: **16h**: Time to keep items in cache if not used, Setting it to 0 disables caching.
+## Cache - LastCommitCache settings (`cache.last_commit`)
+
+- `ENABLED`: **true**: Enable the cache.
+- `ITEM_TTL`: **8760h**: Time to keep items in cache if not used, Setting it to 0 disables caching.
+- `COMMITS_COUNT`: **1000**: Only enable the cache when repository's commits count great than.
+
## Session (`session`)
- `PROVIDER`: **memory**: Session engine provider \[memory, file, redis, mysql, couchbase, memcache, nodb, postgres\].
diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
index a095b3bc88..80861f457a 100644
--- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
@@ -148,6 +148,7 @@ menu:
## Cache (`cache`)
+- `ENABLED`: **true**: 是否启用。
- `ADAPTER`: **memory**: 缓存引擎,可以为 `memory`, `redis` 或 `memcache`。
- `INTERVAL`: **60**: 只对内存缓存有效,GC间隔,单位秒。
- `HOST`: **\<empty\>**: 针对redis和memcache有效,主机地址和端口。
@@ -155,6 +156,12 @@ menu:
- Memache: `127.0.0.1:9090;127.0.0.1:9091`
- `ITEM_TTL`: **16h**: 缓存项目失效时间,设置为 0 则禁用缓存。
+## Cache - LastCommitCache settings (`cache.last_commit`)
+
+- `ENABLED`: **true**: 是否启用。
+- `ITEM_TTL`: **8760h**: 缓存项目失效时间,设置为 0 则禁用缓存。
+- `COMMITS_COUNT`: **1000**: 仅当仓库的提交数大于时才启用缓存。
+
## Session (`session`)
- `PROVIDER`: Session 内容存储方式,可选 `memory`, `file`, `redis` 或 `mysql`。
diff --git a/integrations/repo_test.go b/integrations/repo_test.go
index b5ff072ea4..d2e02dd37f 100644
--- a/integrations/repo_test.go
+++ b/integrations/repo_test.go
@@ -7,8 +7,10 @@ package integrations
import (
"fmt"
"net/http"
+ "path"
"strings"
"testing"
+ "time"
"code.gitea.io/gitea/modules/setting"
@@ -29,12 +31,71 @@ func TestViewRepo(t *testing.T) {
session.MakeRequest(t, req, http.StatusNotFound)
}
-func TestViewRepo2(t *testing.T) {
+func testViewRepo(t *testing.T) {
defer prepareTestEnv(t)()
req := NewRequest(t, "GET", "/user3/repo3")
session := loginUser(t, "user2")
- session.MakeRequest(t, req, http.StatusOK)
+ resp := session.MakeRequest(t, req, http.StatusOK)
+
+ htmlDoc := NewHTMLParser(t, resp.Body)
+ files := htmlDoc.doc.Find("#repo-files-table > TBODY > TR")
+
+ type file struct {
+ fileName string
+ commitID string
+ commitMsg string
+ commitTime string
+ }
+
+ var items []file
+
+ files.Each(func(i int, s *goquery.Selection) {
+ tds := s.Find("td")
+ var f file
+ tds.Each(func(i int, s *goquery.Selection) {
+ if i == 0 {
+ f.fileName = strings.TrimSpace(s.Text())
+ } else if i == 1 {
+ a := s.Find("a")
+ f.commitMsg = strings.TrimSpace(a.Text())
+ l, _ := a.Attr("href")
+ f.commitID = path.Base(l)
+ }
+ })
+
+ f.commitTime, _ = s.Find("span.time-since").Attr("title")
+ items = append(items, f)
+ })
+
+ assert.EqualValues(t, []file{
+ {
+ fileName: "doc",
+ commitID: "2a47ca4b614a9f5a43abbd5ad851a54a616ffee6",
+ commitMsg: "init project",
+ commitTime: time.Date(2017, time.June, 14, 13, 54, 21, 0, time.UTC).Format(time.RFC1123),
+ },
+ {
+ fileName: "README.md",
+ commitID: "2a47ca4b614a9f5a43abbd5ad851a54a616ffee6",
+ commitMsg: "init project",
+ commitTime: time.Date(2017, time.June, 14, 13, 54, 21, 0, time.UTC).Format(time.RFC1123),
+ },
+ }, items)
+}
+
+func TestViewRepo2(t *testing.T) {
+ // no last commit cache
+ testViewRepo(t)
+
+ // enable last commit cache for all repositories
+ oldCommitsCount := setting.CacheService.LastCommit.CommitsCount
+ setting.CacheService.LastCommit.CommitsCount = 0
+ // first view will not hit the cache
+ testViewRepo(t)
+ // second view will hit the cache
+ testViewRepo(t)
+ setting.CacheService.LastCommit.CommitsCount = oldCommitsCount
}
func TestViewRepo3(t *testing.T) {
diff --git a/modules/cache/cache.go b/modules/cache/cache.go
index 20d23f3b5b..e3a905e3fa 100644
--- a/modules/cache/cache.go
+++ b/modules/cache/cache.go
@@ -16,20 +16,28 @@ import (
_ "gitea.com/macaron/cache/redis"
)
-var conn mc.Cache
+var (
+ conn mc.Cache
+)
+
+func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
+ return mc.NewCacher(cacheConfig.Adapter, mc.Options{
+ Adapter: cacheConfig.Adapter,
+ AdapterConfig: cacheConfig.Conn,
+ Interval: cacheConfig.Interval,
+ })
+}
// NewContext start cache service
func NewContext() error {
- if setting.CacheService == nil || conn != nil {
- return nil
+ var err error
+
+ if conn == nil && setting.CacheService.Enabled {
+ if conn, err = newCache(setting.CacheService.Cache); err != nil {
+ return err
+ }
}
- var err error
- conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
- Adapter: setting.CacheService.Adapter,
- AdapterConfig: setting.CacheService.Conn,
- Interval: setting.CacheService.Interval,
- })
return err
}
diff --git a/modules/cache/last_commit.go b/modules/cache/last_commit.go
new file mode 100644
index 0000000000..2fd9313bd1
--- /dev/null
+++ b/modules/cache/last_commit.go
@@ -0,0 +1,64 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+ "fmt"
+
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/log"
+
+ mc "gitea.com/macaron/cache"
+ "gopkg.in/src-d/go-git.v4/plumbing/object"
+)
+
+// LastCommitCache represents a cache to store last commit
+type LastCommitCache struct {
+ repoPath string
+ ttl int64
+ repo *git.Repository
+ commitCache map[string]*object.Commit
+ mc.Cache
+}
+
+// NewLastCommitCache creates a new last commit cache for repo
+func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache {
+ return &LastCommitCache{
+ repoPath: repoPath,
+ repo: gitRepo,
+ commitCache: make(map[string]*object.Commit),
+ ttl: ttl,
+ Cache: conn,
+ }
+}
+
+// Get get the last commit information by commit id and entry path
+func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
+ v := c.Cache.Get(fmt.Sprintf("last_commit:%s:%s:%s", c.repoPath, ref, entryPath))
+ if vs, ok := v.(string); ok {
+ log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
+ if commit, ok := c.commitCache[vs]; ok {
+ log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
+ return commit, nil
+ }
+ id, err := c.repo.ConvertToSHA1(vs)
+ if err != nil {
+ return nil, err
+ }
+ commit, err := c.repo.GoGitRepo().CommitObject(id)
+ if err != nil {
+ return nil, err
+ }
+ c.commitCache[vs] = commit
+ return commit, nil
+ }
+ return nil, nil
+}
+
+// Put put the last commit id with commit and entry path
+func (c LastCommitCache) Put(ref, entryPath, commitID string) error {
+ log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
+ return c.Cache.Put(fmt.Sprintf("last_commit:%s:%s:%s", c.repoPath, ref, entryPath), commitID, c.ttl)
+}
diff --git a/modules/git/cache.go b/modules/git/cache.go
index dbbbafae4c..39daf5c49d 100644
--- a/modules/git/cache.go
+++ b/modules/git/cache.go
@@ -4,8 +4,10 @@
package git
+import "gopkg.in/src-d/go-git.v4/plumbing/object"
+
// LastCommitCache cache
type LastCommitCache interface {
- Get(repoPath, ref, entryPath string) (*Commit, error)
- Put(repoPath, ref, entryPath string, commit *Commit) error
+ Get(ref, entryPath string) (*object.Commit, error)
+ Put(ref, entryPath, commitID string) error
}
diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go
index e74ddbfb05..69fd7f3563 100644
--- a/modules/git/commit_info.go
+++ b/modules/git/commit_info.go
@@ -5,6 +5,8 @@
package git
import (
+ "path"
+
"github.com/emirpasic/gods/trees/binaryheap"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
@@ -30,7 +32,29 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom
return nil, nil, err
}
- revs, err := getLastCommitForPaths(c, treePath, entryPaths)
+ var revs map[string]*object.Commit
+ if cache != nil {
+ var unHitPaths []string
+ revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
+ if err != nil {
+ return nil, nil, err
+ }
+ if len(unHitPaths) > 0 {
+ revs2, err := getLastCommitForPaths(c, treePath, unHitPaths)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ for k, v := range revs2 {
+ if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
+ return nil, nil, err
+ }
+ revs[k] = v
+ }
+ }
+ } else {
+ revs, err = getLastCommitForPaths(c, treePath, entryPaths)
+ }
if err != nil {
return nil, nil, err
}
@@ -127,6 +151,25 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[
return hashes, nil
}
+func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) {
+ var unHitEntryPaths []string
+ var results = make(map[string]*object.Commit)
+ for _, p := range paths {
+ lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
+ if err != nil {
+ return nil, nil, err
+ }
+ if lastCommit != nil {
+ results[p] = lastCommit
+ continue
+ }
+
+ unHitEntryPaths = append(unHitEntryPaths, p)
+ }
+
+ return results, unHitEntryPaths, nil
+}
+
func getLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
diff --git a/modules/setting/cache.go b/modules/setting/cache.go
index babb62baea..34a212db18 100644
--- a/modules/setting/cache.go
+++ b/modules/setting/cache.go
@@ -13,31 +13,71 @@ import (
// Cache represents cache settings
type Cache struct {
+ Enabled bool
Adapter string
Interval int
Conn string
- TTL time.Duration
+ TTL time.Duration `ini:"ITEM_TTL"`
}
var (
// CacheService the global cache
- CacheService *Cache
+ CacheService = struct {
+ Cache
+
+ LastCommit struct {
+ Enabled bool
+ TTL time.Duration `ini:"ITEM_TTL"`
+ CommitsCount int64
+ } `ini:"cache.last_commit"`
+ }{
+ Cache: Cache{
+ Enabled: true,
+ Adapter: "memory",
+ Interval: 60,
+ TTL: 16 * time.Hour,
+ },
+ LastCommit: struct {
+ Enabled bool
+ TTL time.Duration `ini:"ITEM_TTL"`
+ CommitsCount int64
+ }{
+ Enabled: true,
+ TTL: 8760 * time.Hour,
+ CommitsCount: 1000,
+ },
+ }
)
func newCacheService() {
sec := Cfg.Section("cache")
- CacheService = &Cache{
- Adapter: sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}),
+ if err := sec.MapTo(&CacheService); err != nil {
+ log.Fatal("Failed to map Cache settings: %v", err)
}
+
+ CacheService.Adapter = sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"})
switch CacheService.Adapter {
case "memory":
- CacheService.Interval = sec.Key("INTERVAL").MustInt(60)
case "redis", "memcache":
CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ")
+ case "": // disable cache
+ CacheService.Enabled = false
default:
log.Fatal("Unknown cache adapter: %s", CacheService.Adapter)
}
- CacheService.TTL = sec.Key("ITEM_TTL").MustDuration(16 * time.Hour)
- log.Info("Cache Service Enabled")
+ if CacheService.Enabled {
+ log.Info("Cache Service Enabled")
+ }
+
+ sec = Cfg.Section("cache.last_commit")
+ if !CacheService.Enabled {
+ CacheService.LastCommit.Enabled = false
+ }
+
+ CacheService.LastCommit.CommitsCount = sec.Key("COMMITS_COUNT").MustInt64(1000)
+
+ if CacheService.LastCommit.Enabled {
+ log.Info("Last Commit Cache Service Enabled")
+ }
}
diff --git a/routers/repo/view.go b/routers/repo/view.go
index 3fbff007e1..f56c524359 100644
--- a/routers/repo/view.go
+++ b/routers/repo/view.go
@@ -17,6 +17,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
+ "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
@@ -49,8 +50,13 @@ func renderDirectory(ctx *context.Context, treeLink string) {
}
entries.CustomSort(base.NaturalSortLess)
+ var c git.LastCommitCache
+ if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount {
+ c = cache.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()))
+ }
+
var latestCommit *git.Commit
- ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, nil)
+ ctx.Data["Files"], latestCommit, err = entries.GetCommitsInfo(ctx.Repo.Commit, ctx.Repo.TreePath, c)
if err != nil {
ctx.ServerError("GetCommitsInfo", err)
return