Language statistics bar for repositories (#8037)

* Implementation for calculating language statistics

Impement saving code language statistics to database

Implement rendering langauge stats

Add primary laguage to show in repository list

Implement repository stats indexer queue

Add indexer test

Refactor to use queue module

* Do not timeout for queues

7 files changed, 592 insertions, 0 deletions

diff --git a/vendor/gopkg.in/toqueteos/substring.v1/.gitignore b/vendor/gopkg.in/toqueteos/substring.v1/.gitignore
new file mode 100644
index 0000000000..daf913b1b3
--- /dev/null
+++ b/vendor/gopkg.in/toqueteos/substring.v1/.gitignore
@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof
diff --git a/vendor/gopkg.in/toqueteos/substring.v1/.travis.yml b/vendor/gopkg.in/toqueteos/substring.v1/.travis.yml
new file mode 100644
index 0000000000..34e2822397
--- /dev/null
+++ b/vendor/gopkg.in/toqueteos/substring.v1/.travis.yml
@@ -0,0 +1,11 @@
+language: go
+
+go:
+  - 1.2
+  - 1.3
+  - 1.4
+  - tip
+
+script:
+  - go get launchpad.net/gocheck
+  - go test
diff --git a/vendor/gopkg.in/toqueteos/substring.v1/LICENSE b/vendor/gopkg.in/toqueteos/substring.v1/LICENSE
new file mode 100644
index 0000000000..91ac6736d8
--- /dev/null
+++ b/vendor/gopkg.in/toqueteos/substring.v1/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Carlos Cobo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
diff --git a/vendor/gopkg.in/toqueteos/substring.v1/README.md b/vendor/gopkg.in/toqueteos/substring.v1/README.md
new file mode 100644
index 0000000000..432276387b
--- /dev/null
+++ b/vendor/gopkg.in/toqueteos/substring.v1/README.md
@@ -0,0 +1,80 @@
+# substring [![Build Status](https://travis-ci.org/toqueteos/substring.png?branch=master)](https://travis-ci.org/toqueteos/substring) [![GoDoc](http://godoc.org/github.com/toqueteos/substring?status.png)](http://godoc.org/github.com/toqueteos/substring) [![GitHub release](https://img.shields.io/github/release/toqueteos/substring.svg)](https://github.com/toqueteos/substring/releases)
+
+Simple and composable alternative to [regexp](http://golang.org/pkg/regexp/) package for fast substring searches.
+
+## Installation
+
+The recommended way to install substring
+
+```
+go get -t gopkg.in/toqueteos/substring.v1
+```
+
+The `-t` flag is for fetching [gocheck](https://gopkg.in/check.v1), required for tests and benchmarks.
+
+## Examples
+
+A basic example with two matchers:
+
+```go
+package main
+
+import (
+    "fmt"
+    "regexp"
+
+    "gopkg.in/toqueteos/substring.v1"
+)
+
+func main() {
+    m1 := substring.After("assets/", substring.Or(
+        substring.Has("jquery"),
+        substring.Has("angular"),
+        substring.Suffixes(".js", ".css", ".html"),
+    ))
+    fmt.Println(m1.Match("assets/angular/foo/bar")) //Prints: true
+    fmt.Println(m1.Match("assets/js/file.js"))      //Prints: true
+    fmt.Println(m1.Match("assets/style/bar.css"))   //Prints: true
+    fmt.Println(m1.Match("assets/foo/bar.html"))    //Prints: false
+    fmt.Println(m1.Match("assets/js/qux.json"))     //Prints: false
+    fmt.Println(m1.Match("core/file.html"))         //Prints: false
+    fmt.Println(m1.Match("foobar/that.jsx"))        //Prints: false
+
+    m2 := substring.After("vendor/", substring.Suffixes(".css", ".js", ".less"))
+
+    fmt.Println(m2.Match("foo/vendor/bar/qux.css")) //Prints: true
+    fmt.Println(m2.Match("foo/var/qux.less"))       //Prints: false
+
+    re := regexp.MustCompile(`vendor\/.*\.(css|js|less)$`)
+    fmt.Println(re.MatchString("foo/vendor/bar/qux.css")) //Prints: true
+    fmt.Println(re.MatchString("foo/var/qux.less"))       //Prints: false
+}
+```
+
+## How fast?
+
+It may vary depending on your use case but 1~2 orders of magnitude faster than `regexp` is pretty common.
+
+Test it out for yourself by running `go test -check.b`!
+
+```
+$ go test -check.b
+PASS: lib_test.go:18: LibSuite.BenchmarkExample1        10000000               221 ns/op
+PASS: lib_test.go:23: LibSuite.BenchmarkExample2        10000000               229 ns/op
+PASS: lib_test.go:28: LibSuite.BenchmarkExample3        10000000               216 ns/op
+PASS: lib_test.go:33: LibSuite.BenchmarkExample4        10000000               208 ns/op
+PASS: lib_test.go:38: LibSuite.BenchmarkExample5        20000000                82.1 ns/op
+PASS: lib_test.go:48: LibSuite.BenchmarkExampleRe1        500000              4136 ns/op
+PASS: lib_test.go:53: LibSuite.BenchmarkExampleRe2        500000              5222 ns/op
+PASS: lib_test.go:58: LibSuite.BenchmarkExampleRe3        500000              5116 ns/op
+PASS: lib_test.go:63: LibSuite.BenchmarkExampleRe4        500000              4020 ns/op
+PASS: lib_test.go:68: LibSuite.BenchmarkExampleRe5      10000000               226 ns/op
+OK: 10 passed
+PASS
+ok      gopkg.in/toqueteos/substring.v1 23.471s
+```
+
+License
+-------
+
+MIT, see [LICENSE](LICENSE)
diff --git a/vendor/gopkg.in/toqueteos/substring.v1/bytes.go b/vendor/gopkg.in/toqueteos/substring.v1/bytes.go
new file mode 100644
index 0000000000..30521d4bc9
--- /dev/null
+++ b/vendor/gopkg.in/toqueteos/substring.v1/bytes.go
@@ -0,0 +1,229 @@
+package substring
+
+import (
+	"bytes"
+	"regexp"
+
+	"github.com/toqueteos/trie"
+)
+
+type BytesMatcher interface {
+	Match(b []byte) bool
+	MatchIndex(b []byte) int
+}
+
+// regexp
+type regexpBytes struct{ re *regexp.Regexp }
+
+func BytesRegexp(pat string) *regexpBytes  { return &regexpBytes{regexp.MustCompile(pat)} }
+func (m *regexpBytes) Match(b []byte) bool { return m.re.Match(b) }
+func (m *regexpBytes) MatchIndex(b []byte) int {
+	found := m.re.FindIndex(b)
+	if found != nil {
+		return found[1]
+	}
+	return -1
+}
+
+// exact
+type exactBytes struct{ pat []byte }
+
+func BytesExact(pat string) *exactBytes { return &exactBytes{[]byte(pat)} }
+func (m *exactBytes) Match(b []byte) bool {
+	l, r := len(m.pat), len(b)
+	if l != r {
+		return false
+	}
+	for i := 0; i < l; i++ {
+		if b[i] != m.pat[i] {
+			return false
+		}
+	}
+	return true
+}
+func (m *exactBytes) MatchIndex(b []byte) int {
+	if m.Match(b) {
+		return len(b)
+	}
+	return -1
+}
+
+// any, search `s` in `.Match(pat)`
+type anyBytes struct {
+	pat []byte
+}
+
+func BytesAny(pat string) *anyBytes     { return &anyBytes{[]byte(pat)} }
+func (m *anyBytes) Match(b []byte) bool { return bytes.Index(m.pat, b) >= 0 }
+func (m *anyBytes) MatchIndex(b []byte) int {
+	if idx := bytes.Index(m.pat, b); idx >= 0 {
+		return idx + len(b)
+	}
+	return -1
+}
+
+// has, search `pat` in `.Match(s)`
+type hasBytes struct {
+	pat []byte
+}
+
+func BytesHas(pat string) *hasBytes     { return &hasBytes{[]byte(pat)} }
+func (m *hasBytes) Match(b []byte) bool { return bytes.Index(b, m.pat) >= 0 }
+func (m *hasBytes) MatchIndex(b []byte) int {
+	if idx := bytes.Index(b, m.pat); idx >= 0 {
+		return idx + len(m.pat)
+	}
+	return -1
+}
+
+// prefix
+type prefixBytes struct{ pat []byte }
+
+func BytesPrefix(pat string) *prefixBytes  { return &prefixBytes{[]byte(pat)} }
+func (m *prefixBytes) Match(b []byte) bool { return bytes.HasPrefix(b, m.pat) }
+func (m *prefixBytes) MatchIndex(b []byte) int {
+	if bytes.HasPrefix(b, m.pat) {
+		return len(m.pat)
+	}
+	return -1
+}
+
+// prefixes
+type prefixesBytes struct {
+	t *trie.Trie
+}
+
+func BytesPrefixes(pats ...string) *prefixesBytes {
+	t := trie.New()
+	for _, pat := range pats {
+		t.Insert([]byte(pat))
+	}
+	return &prefixesBytes{t}
+}
+func (m *prefixesBytes) Match(b []byte) bool { return m.t.PrefixIndex(b) >= 0 }
+func (m *prefixesBytes) MatchIndex(b []byte) int {
+	if idx := m.t.PrefixIndex(b); idx >= 0 {
+		return idx
+	}
+	return -1
+}
+
+// suffix
+type suffixBytes struct{ pat []byte }
+
+func BytesSuffix(pat string) *suffixBytes  { return &suffixBytes{[]byte(pat)} }
+func (m *suffixBytes) Match(b []byte) bool { return bytes.HasSuffix(b, m.pat) }
+func (m *suffixBytes) MatchIndex(b []byte) int {
+	if bytes.HasSuffix(b, m.pat) {
+		return len(m.pat)
+	}
+	return -1
+}
+
+// suffixes
+type suffixesBytes struct {
+	t *trie.Trie
+}
+
+func BytesSuffixes(pats ...string) *suffixesBytes {
+	t := trie.New()
+	for _, pat := range pats {
+		t.Insert(reverse([]byte(pat)))
+	}
+	return &suffixesBytes{t}
+}
+func (m *suffixesBytes) Match(b []byte) bool {
+	return m.t.PrefixIndex(reverse(b)) >= 0
+}
+func (m *suffixesBytes) MatchIndex(b []byte) int {
+	if idx := m.t.PrefixIndex(reverse(b)); idx >= 0 {
+		return idx
+	}
+	return -1
+}
+
+// after
+type afterBytes struct {
+	first   []byte
+	matcher BytesMatcher
+}
+
+func BytesAfter(first string, m BytesMatcher) *afterBytes { return &afterBytes{[]byte(first), m} }
+func (a *afterBytes) Match(b []byte) bool {
+	if idx := bytes.Index(b, a.first); idx >= 0 {
+		return a.matcher.Match(b[idx+len(a.first):])
+	}
+	return false
+}
+func (a *afterBytes) MatchIndex(b []byte) int {
+	if idx := bytes.Index(b, a.first); idx >= 0 {
+		return idx + a.matcher.MatchIndex(b[idx:])
+	}
+	return -1
+}
+
+// and, returns true iff all matchers return true
+type andBytes struct{ matchers []BytesMatcher }
+
+func BytesAnd(m ...BytesMatcher) *andBytes { return &andBytes{m} }
+func (a *andBytes) Match(b []byte) bool {
+	for _, m := range a.matchers {
+		if !m.Match(b) {
+			return false
+		}
+	}
+	return true
+}
+func (a *andBytes) MatchIndex(b []byte) int {
+	longest := 0
+	for _, m := range a.matchers {
+		if idx := m.MatchIndex(b); idx < 0 {
+			return -1
+		} else if idx > longest {
+			longest = idx
+		}
+	}
+	return longest
+}
+
+// or, returns true iff any matcher returns true
+type orBytes struct{ matchers []BytesMatcher }
+
+func BytesOr(m ...BytesMatcher) *orBytes { return &orBytes{m} }
+func (o *orBytes) Match(b []byte) bool {
+	for _, m := range o.matchers {
+		if m.Match(b) {
+			return true
+		}
+	}
+	return false
+}
+func (o *orBytes) MatchIndex(b []byte) int {
+	for _, m := range o.matchers {
+		if idx := m.MatchIndex(b); idx >= 0 {
+			return idx
+		}
+	}
+	return -1
+}
+
+type suffixGroupBytes struct {
+	suffix   BytesMatcher
+	matchers []BytesMatcher
+}
+
+func BytesSuffixGroup(s string, m ...BytesMatcher) *suffixGroupBytes {
+	return &suffixGroupBytes{BytesSuffix(s), m}
+}
+func (sg *suffixGroupBytes) Match(b []byte) bool {
+	if sg.suffix.Match(b) {
+		return BytesOr(sg.matchers...).Match(b)
+	}
+	return false
+}
+func (sg *suffixGroupBytes) MatchIndex(b []byte) int {
+	if sg.suffix.MatchIndex(b) >= 0 {
+		return BytesOr(sg.matchers...).MatchIndex(b)
+	}
+	return -1
+}
diff --git a/vendor/gopkg.in/toqueteos/substring.v1/lib.go b/vendor/gopkg.in/toqueteos/substring.v1/lib.go
new file mode 100644
index 0000000000..9cd4a30261
--- /dev/null
+++ b/vendor/gopkg.in/toqueteos/substring.v1/lib.go
@@ -0,0 +1,10 @@
+package substring
+
+// reverse is a helper fn for Suffixes
+func reverse(b []byte) []byte {
+	n := len(b)
+	for i := 0; i < n/2; i++ {
+		b[i], b[n-1-i] = b[n-1-i], b[i]
+	}
+	return b
+}
diff --git a/vendor/gopkg.in/toqueteos/substring.v1/string.go b/vendor/gopkg.in/toqueteos/substring.v1/string.go
new file mode 100644
index 0000000000..4ec4776c36
--- /dev/null
+++ b/vendor/gopkg.in/toqueteos/substring.v1/string.go
@@ -0,0 +1,216 @@
+package substring
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/toqueteos/trie"
+)
+
+type StringsMatcher interface {
+	Match(s string) bool
+	MatchIndex(s string) int
+}
+
+// regexp
+type regexpString struct{ re *regexp.Regexp }
+
+func Regexp(pat string) *regexpString       { return &regexpString{regexp.MustCompile(pat)} }
+func (m *regexpString) Match(s string) bool { return m.re.MatchString(s) }
+func (m *regexpString) MatchIndex(s string) int {
+	found := m.re.FindStringIndex(s)
+	if found != nil {
+		return found[1]
+	}
+	return -1
+}
+
+// exact
+type exactString struct{ pat string }
+
+func Exact(pat string) *exactString        { return &exactString{pat} }
+func (m *exactString) Match(s string) bool { return m.pat == s }
+func (m *exactString) MatchIndex(s string) int {
+	if m.pat == s {
+		return len(s)
+	}
+	return -1
+}
+
+// any, search `s` in `.Match(pat)`
+type anyString struct{ pat string }
+
+func Any(pat string) *anyString { return &anyString{pat} }
+func (m *anyString) Match(s string) bool {
+	return strings.Index(m.pat, s) >= 0
+}
+func (m *anyString) MatchIndex(s string) int {
+	if idx := strings.Index(m.pat, s); idx >= 0 {
+		return idx + len(s)
+	}
+	return -1
+}
+
+// has, search `pat` in `.Match(s)`
+type hasString struct{ pat string }
+
+func Has(pat string) *hasString { return &hasString{pat} }
+func (m *hasString) Match(s string) bool {
+	return strings.Index(s, m.pat) >= 0
+}
+func (m *hasString) MatchIndex(s string) int {
+	if idx := strings.Index(s, m.pat); idx >= 0 {
+		return idx + len(m.pat)
+	}
+	return -1
+}
+
+// prefix
+type prefixString struct{ pat string }
+
+func Prefix(pat string) *prefixString       { return &prefixString{pat} }
+func (m *prefixString) Match(s string) bool { return strings.HasPrefix(s, m.pat) }
+func (m *prefixString) MatchIndex(s string) int {
+	if strings.HasPrefix(s, m.pat) {
+		return len(m.pat)
+	}
+	return -1
+}
+
+// prefixes
+type prefixesString struct{ t *trie.Trie }
+
+func Prefixes(pats ...string) *prefixesString {
+	t := trie.New()
+	for _, pat := range pats {
+		t.Insert([]byte(pat))
+	}
+	return &prefixesString{t}
+}
+func (m *prefixesString) Match(s string) bool { return m.t.PrefixIndex([]byte(s)) >= 0 }
+func (m *prefixesString) MatchIndex(s string) int {
+	if idx := m.t.PrefixIndex([]byte(s)); idx >= 0 {
+		return idx
+	}
+	return -1
+}
+
+// suffix
+type suffixString struct{ pat string }
+
+func Suffix(pat string) *suffixString       { return &suffixString{pat} }
+func (m *suffixString) Match(s string) bool { return strings.HasSuffix(s, m.pat) }
+func (m *suffixString) MatchIndex(s string) int {
+	if strings.HasSuffix(s, m.pat) {
+		return len(m.pat)
+	}
+	return -1
+}
+
+// suffixes
+type suffixesString struct{ t *trie.Trie }
+
+func Suffixes(pats ...string) *suffixesString {
+	t := trie.New()
+	for _, pat := range pats {
+		t.Insert(reverse([]byte(pat)))
+	}
+	return &suffixesString{t}
+}
+func (m *suffixesString) Match(s string) bool {
+	return m.t.PrefixIndex(reverse([]byte(s))) >= 0
+}
+func (m *suffixesString) MatchIndex(s string) int {
+	if idx := m.t.PrefixIndex(reverse([]byte(s))); idx >= 0 {
+		return idx
+	}
+	return -1
+}
+
+// after
+type afterString struct {
+	first   string
+	matcher StringsMatcher
+}
+
+func After(first string, m StringsMatcher) *afterString {
+	return &afterString{first, m}
+}
+func (a *afterString) Match(s string) bool {
+	if idx := strings.Index(s, a.first); idx >= 0 {
+		return a.matcher.Match(s[idx+len(a.first):])
+	}
+	return false
+}
+func (a *afterString) MatchIndex(s string) int {
+	if idx := strings.Index(s, a.first); idx >= 0 {
+		return idx + a.matcher.MatchIndex(s[idx+len(a.first):])
+	}
+	return -1
+}
+
+// and, returns true iff all matchers return true
+type andString struct{ matchers []StringsMatcher }
+
+func And(m ...StringsMatcher) *andString { return &andString{m} }
+func (a *andString) Match(s string) bool {
+	for _, m := range a.matchers {
+		if !m.Match(s) {
+			return false
+		}
+	}
+	return true
+}
+func (a *andString) MatchIndex(s string) int {
+	longest := 0
+	for _, m := range a.matchers {
+		if idx := m.MatchIndex(s); idx < 0 {
+			return -1
+		} else if idx > longest {
+			longest = idx
+		}
+	}
+	return longest
+}
+
+// or, returns true iff any matcher returns true
+type orString struct{ matchers []StringsMatcher }
+
+func Or(m ...StringsMatcher) *orString { return &orString{m} }
+func (o *orString) Match(s string) bool {
+	for _, m := range o.matchers {
+		if m.Match(s) {
+			return true
+		}
+	}
+	return false
+}
+func (o *orString) MatchIndex(s string) int {
+	for _, m := range o.matchers {
+		if idx := m.MatchIndex(s); idx >= 0 {
+			return idx
+		}
+	}
+	return -1
+}
+
+type suffixGroupString struct {
+	suffix   StringsMatcher
+	matchers []StringsMatcher
+}
+
+func SuffixGroup(s string, m ...StringsMatcher) *suffixGroupString {
+	return &suffixGroupString{Suffix(s), m}
+}
+func (sg *suffixGroupString) Match(s string) bool {
+	if sg.suffix.Match(s) {
+		return Or(sg.matchers...).Match(s)
+	}
+	return false
+}
+func (sg *suffixGroupString) MatchIndex(s string) int {
+	if sg.suffix.MatchIndex(s) >= 0 {
+		return Or(sg.matchers...).MatchIndex(s)
+	}
+	return -1
+}