Git LFS support v2 (#122)

* Import github.com/git-lfs/lfs-test-server as lfs module base

Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198

Removed:

Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go
.dockerignore .gitignore README.md

* Remove config, add JWT support from github.com/mgit-at/lfs-test-server

Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83

* Add LFS settings

* Add LFS meta object model

* Add LFS routes and initialization

* Import github.com/dgrijalva/jwt-go into vendor/

* Adapt LFS module: handlers, routing, meta store

* Move LFS routes to /user/repo/info/lfs/*

* Add request header checks to LFS BatchHandler / PostHandler

* Implement LFS basic authentication

* Rework JWT secret generation / load

* Implement LFS SSH token authentication with JWT

Specification: https://github.com/github/git-lfs/tree/master/docs/api

* Integrate LFS settings into install process

* Remove LFS objects when repository is deleted

Only removes objects from content store when deleted repo is the only
referencing repository

* Make LFS module stateless

Fixes bug where LFS would not work after installation without
restarting Gitea

* Change 500 'Internal Server Error' to 400 'Bad Request'

* Change sql query to xorm call

* Remove unneeded type from LFS module

* Change internal imports to code.gitea.io/gitea/

* Add Gitea authors copyright

* Change basic auth realm to "gitea-lfs"

* Add unique indexes to LFS model

* Use xorm count function in LFS check on repository delete

* Return io.ReadCloser from content store and close after usage

* Add LFS info to runWeb()

* Export LFS content store base path

* LFS file download from UI

* Work around git-lfs client issue with unauthenticated requests

Returning a dummy Authorization header for unauthenticated requests
lets git-lfs client skip asking for auth credentials
See: https://github.com/github/git-lfs/issues/1088

* Fix unauthenticated UI downloads from public repositories

* Authentication check order, Finish LFS file view logic

* Ignore LFS hooks if installed for current OS user

Fixes Gitea UI actions for repositories tracking LFS files.
Checks for minimum needed git version by parsing the semantic version
string.

* Hide LFS metafile diff from commit view, marking as binary

* Show LFS notice if file in commit view is tracked

* Add notbefore/nbf JWT claim

* Correct lint suggestions - comments for structs and functions

- Add comments to LFS model
- Function comment for GetRandomBytesAsBase64
- LFS server function comments and lint variable suggestion

* Move secret generation code out of conditional

Ensures no LFS code may run with an empty secret

* Do not hand out JWT tokens if LFS server support is disabled

1 files changed, 94 insertions, 0 deletions

diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go
new file mode 100644
index 0000000000..2ca44512f1
--- /dev/null
+++ b/modules/lfs/content_store.go
@@ -0,0 +1,94 @@
+package lfs
+
+import (
+	"code.gitea.io/gitea/models"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+var (
+	errHashMismatch = errors.New("Content hash does not match OID")
+	errSizeMismatch = errors.New("Content size does not match")
+)
+
+// ContentStore provides a simple file system based storage.
+type ContentStore struct {
+	BasePath string
+}
+
+// Get takes a Meta object and retreives the content from the store, returning
+// it as an io.Reader. If fromByte > 0, the reader starts from that byte
+func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
+	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	if fromByte > 0 {
+		_, err = f.Seek(fromByte, os.SEEK_CUR)
+	}
+	return f, err
+}
+
+// Put takes a Meta object and an io.Reader and writes the content to the store.
+func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
+	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
+	tmpPath := path + ".tmp"
+
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0750); err != nil {
+		return err
+	}
+
+	file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmpPath)
+
+	hash := sha256.New()
+	hw := io.MultiWriter(hash, file)
+
+	written, err := io.Copy(hw, r)
+	if err != nil {
+		file.Close()
+		return err
+	}
+	file.Close()
+
+	if written != meta.Size {
+		return errSizeMismatch
+	}
+
+	shaStr := hex.EncodeToString(hash.Sum(nil))
+	if shaStr != meta.Oid {
+		return errHashMismatch
+	}
+
+	if err := os.Rename(tmpPath, path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Exists returns true if the object exists in the content store.
+func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
+	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func transformKey(key string) string {
+	if len(key) < 5 {
+		return key
+	}
+
+	return filepath.Join(key[0:2], key[2:4], key[4:len(key)])
+}