Git LFS support v2 (#122)

* Import github.com/git-lfs/lfs-test-server as lfs module base

Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198

Removed:

Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go
.dockerignore .gitignore README.md

* Remove config, add JWT support from github.com/mgit-at/lfs-test-server

Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83

* Add LFS settings

* Add LFS meta object model

* Add LFS routes and initialization

* Import github.com/dgrijalva/jwt-go into vendor/

* Adapt LFS module: handlers, routing, meta store

* Move LFS routes to /user/repo/info/lfs/*

* Add request header checks to LFS BatchHandler / PostHandler

* Implement LFS basic authentication

* Rework JWT secret generation / load

* Implement LFS SSH token authentication with JWT

Specification: https://github.com/github/git-lfs/tree/master/docs/api

* Integrate LFS settings into install process

* Remove LFS objects when repository is deleted

Only removes objects from content store when deleted repo is the only
referencing repository

* Make LFS module stateless

Fixes bug where LFS would not work after installation without
restarting Gitea

* Change 500 'Internal Server Error' to 400 'Bad Request'

* Change sql query to xorm call

* Remove unneeded type from LFS module

* Change internal imports to code.gitea.io/gitea/

* Add Gitea authors copyright

* Change basic auth realm to "gitea-lfs"

* Add unique indexes to LFS model

* Use xorm count function in LFS check on repository delete

* Return io.ReadCloser from content store and close after usage

* Add LFS info to runWeb()

* Export LFS content store base path

* LFS file download from UI

* Work around git-lfs client issue with unauthenticated requests

Returning a dummy Authorization header for unauthenticated requests
lets git-lfs client skip asking for auth credentials
See: https://github.com/github/git-lfs/issues/1088

* Fix unauthenticated UI downloads from public repositories

* Authentication check order, Finish LFS file view logic

* Ignore LFS hooks if installed for current OS user

Fixes Gitea UI actions for repositories tracking LFS files.
Checks for minimum needed git version by parsing the semantic version
string.

* Hide LFS metafile diff from commit view, marking as binary

* Show LFS notice if file in commit view is tracked

* Add notbefore/nbf JWT claim

* Correct lint suggestions - comments for structs and functions

- Add comments to LFS model
- Function comment for GetRandomBytesAsBase64
- LFS server function comments and lint variable suggestion

* Move secret generation code out of conditional

Ensures no LFS code may run with an empty secret

* Do not hand out JWT tokens if LFS server support is disabled

3 files changed, 663 insertions, 0 deletions

diff --git a/modules/lfs/LICENSE b/modules/lfs/LICENSE
new file mode 100644
index 0000000000..0a94a80c8c
--- /dev/null
+++ b/modules/lfs/LICENSE
@@ -0,0 +1,20 @@
+Copyright (c) 2016 The Gitea Authors
+Copyright (c) GitHub, Inc. and LFS Test Server contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go
new file mode 100644
index 0000000000..2ca44512f1
--- /dev/null
+++ b/modules/lfs/content_store.go
@@ -0,0 +1,94 @@
+package lfs
+
+import (
+	"code.gitea.io/gitea/models"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+var (
+	errHashMismatch = errors.New("Content hash does not match OID")
+	errSizeMismatch = errors.New("Content size does not match")
+)
+
+// ContentStore provides a simple file system based storage.
+type ContentStore struct {
+	BasePath string
+}
+
+// Get takes a Meta object and retreives the content from the store, returning
+// it as an io.Reader. If fromByte > 0, the reader starts from that byte
+func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
+	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	if fromByte > 0 {
+		_, err = f.Seek(fromByte, os.SEEK_CUR)
+	}
+	return f, err
+}
+
+// Put takes a Meta object and an io.Reader and writes the content to the store.
+func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
+	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
+	tmpPath := path + ".tmp"
+
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0750); err != nil {
+		return err
+	}
+
+	file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmpPath)
+
+	hash := sha256.New()
+	hw := io.MultiWriter(hash, file)
+
+	written, err := io.Copy(hw, r)
+	if err != nil {
+		file.Close()
+		return err
+	}
+	file.Close()
+
+	if written != meta.Size {
+		return errSizeMismatch
+	}
+
+	shaStr := hex.EncodeToString(hash.Sum(nil))
+	if shaStr != meta.Oid {
+		return errHashMismatch
+	}
+
+	if err := os.Rename(tmpPath, path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Exists returns true if the object exists in the content store.
+func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
+	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func transformKey(key string) string {
+	if len(key) < 5 {
+		return key
+	}
+
+	return filepath.Join(key[0:2], key[2:4], key[4:len(key)])
+}
diff --git a/modules/lfs/server.go b/modules/lfs/server.go
new file mode 100644
index 0000000000..f82cb70364
--- /dev/null
+++ b/modules/lfs/server.go
@@ -0,0 +1,549 @@
+package lfs
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"github.com/dgrijalva/jwt-go"
+	"gopkg.in/macaron.v1"
+)
+
+const (
+	contentMediaType = "application/vnd.git-lfs"
+	metaMediaType    = contentMediaType + "+json"
+)
+
+// RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and
+// some headers are stored.
+type RequestVars struct {
+	Oid           string
+	Size          int64
+	User          string
+	Password      string
+	Repo          string
+	Authorization string
+}
+
+// BatchVars contains multiple RequestVars processed in one batch operation.
+// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
+type BatchVars struct {
+	Transfers []string       `json:"transfers,omitempty"`
+	Operation string         `json:"operation"`
+	Objects   []*RequestVars `json:"objects"`
+}
+
+// BatchResponse contains multiple object metadata Representation structures
+// for use with the batch API.
+type BatchResponse struct {
+	Transfer string            `json:"transfer,omitempty"`
+	Objects  []*Representation `json:"objects"`
+}
+
+// Representation is object medata as seen by clients of the lfs server.
+type Representation struct {
+	Oid     string           `json:"oid"`
+	Size    int64            `json:"size"`
+	Actions map[string]*link `json:"actions"`
+	Error   *ObjectError     `json:"error,omitempty"`
+}
+
+// ObjectError defines the JSON structure returned to the client in case of an error
+type ObjectError struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+// ObjectLink builds a URL linking to the object.
+func (v *RequestVars) ObjectLink() string {
+	return fmt.Sprintf("%s%s/%s/info/lfs/objects/%s", setting.AppURL, v.User, v.Repo, v.Oid)
+}
+
+// link provides a structure used to build a hypermedia representation of an HTTP link.
+type link struct {
+	Href      string            `json:"href"`
+	Header    map[string]string `json:"header,omitempty"`
+	ExpiresAt time.Time         `json:"expires_at,omitempty"`
+}
+
+// ObjectOidHandler is the main request routing entry point into LFS server functions
+func ObjectOidHandler(ctx *context.Context) {
+
+	if !setting.LFS.StartServer {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" {
+		if MetaMatcher(ctx.Req) {
+			GetMetaHandler(ctx)
+			return
+		}
+		if ContentMatcher(ctx.Req) || len(ctx.Params("filename")) > 0 {
+			GetContentHandler(ctx)
+			return
+		}
+	} else if ctx.Req.Method == "PUT" && ContentMatcher(ctx.Req) {
+		PutHandler(ctx)
+		return
+	}
+
+}
+
+// GetContentHandler gets the content from the content store
+func GetContentHandler(ctx *context.Context) {
+
+	rv := unpack(ctx)
+
+	meta, err := models.GetLFSMetaObjectByOid(rv.Oid)
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	repository, err := models.GetRepositoryByID(meta.RepositoryID)
+
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !authenticate(ctx, repository, rv.Authorization, false) {
+		requireAuth(ctx)
+		return
+	}
+
+	// Support resume download using Range header
+	var fromByte int64
+	statusCode := 200
+	if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" {
+		regex := regexp.MustCompile(`bytes=(\d+)\-.*`)
+		match := regex.FindStringSubmatch(rangeHdr)
+		if match != nil && len(match) > 1 {
+			statusCode = 206
+			fromByte, _ = strconv.ParseInt(match[1], 10, 32)
+			ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, meta.Size-1, int64(meta.Size)-fromByte))
+		}
+	}
+
+	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	content, err := contentStore.Get(meta, fromByte)
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10))
+	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+
+	filename := ctx.Params("filename")
+	if len(filename) > 0 {
+		decodedFilename, err := base64.RawURLEncoding.DecodeString(filename)
+		if err == nil {
+			ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"")
+		}
+	}
+
+	ctx.Resp.WriteHeader(statusCode)
+	io.Copy(ctx.Resp, content)
+	content.Close()
+	logRequest(ctx.Req, statusCode)
+}
+
+// GetMetaHandler retrieves metadata about the object
+func GetMetaHandler(ctx *context.Context) {
+
+	rv := unpack(ctx)
+
+	meta, err := models.GetLFSMetaObjectByOid(rv.Oid)
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	repository, err := models.GetRepositoryByID(meta.RepositoryID)
+
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !authenticate(ctx, repository, rv.Authorization, false) {
+		requireAuth(ctx)
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Type", metaMediaType)
+
+	if ctx.Req.Method == "GET" {
+		enc := json.NewEncoder(ctx.Resp)
+		enc.Encode(Represent(rv, meta, true, false))
+	}
+
+	logRequest(ctx.Req, 200)
+}
+
+// PostHandler instructs the client how to upload data
+func PostHandler(ctx *context.Context) {
+
+	if !setting.LFS.StartServer {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !MetaMatcher(ctx.Req) {
+		writeStatus(ctx, 400)
+		return
+	}
+
+	rv := unpack(ctx)
+
+	repositoryString := rv.User + "/" + rv.Repo
+	repository, err := models.GetRepositoryByRef(repositoryString)
+
+	if err != nil {
+		log.Debug("Could not find repository: %s - %s", repositoryString, err)
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !authenticate(ctx, repository, rv.Authorization, true) {
+		requireAuth(ctx)
+	}
+
+	meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID})
+
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	ctx.Resp.Header().Set("Content-Type", metaMediaType)
+
+	sentStatus := 202
+	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	if meta.Existing && contentStore.Exists(meta) {
+		sentStatus = 200
+	}
+	ctx.Resp.WriteHeader(sentStatus)
+
+	enc := json.NewEncoder(ctx.Resp)
+	enc.Encode(Represent(rv, meta, meta.Existing, true))
+	logRequest(ctx.Req, sentStatus)
+}
+
+// BatchHandler provides the batch api
+func BatchHandler(ctx *context.Context) {
+
+	if !setting.LFS.StartServer {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !MetaMatcher(ctx.Req) {
+		writeStatus(ctx, 400)
+		return
+	}
+
+	bv := unpackbatch(ctx)
+
+	var responseObjects []*Representation
+
+	// Create a response object
+	for _, object := range bv.Objects {
+
+		repositoryString := object.User + "/" + object.Repo
+		repository, err := models.GetRepositoryByRef(repositoryString)
+
+		if err != nil {
+			log.Debug("Could not find repository: %s - %s", repositoryString, err)
+			writeStatus(ctx, 404)
+			return
+		}
+
+		requireWrite := false
+		if bv.Operation == "upload" {
+			requireWrite = true
+		}
+
+		if !authenticate(ctx, repository, object.Authorization, requireWrite) {
+			requireAuth(ctx)
+			return
+		}
+
+		meta, err := models.GetLFSMetaObjectByOid(object.Oid)
+
+		contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+		if err == nil && contentStore.Exists(meta) { // Object is found and exists
+			responseObjects = append(responseObjects, Represent(object, meta, true, false))
+			continue
+		}
+
+		// Object is not found
+		meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
+
+		if err == nil {
+			responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, true))
+		}
+	}
+
+	ctx.Resp.Header().Set("Content-Type", metaMediaType)
+
+	respobj := &BatchResponse{Objects: responseObjects}
+
+	enc := json.NewEncoder(ctx.Resp)
+	enc.Encode(respobj)
+	logRequest(ctx.Req, 200)
+}
+
+// PutHandler receives data from the client and puts it into the content store
+func PutHandler(ctx *context.Context) {
+	rv := unpack(ctx)
+
+	meta, err := models.GetLFSMetaObjectByOid(rv.Oid)
+
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	repository, err := models.GetRepositoryByID(meta.RepositoryID)
+
+	if err != nil {
+		writeStatus(ctx, 404)
+		return
+	}
+
+	if !authenticate(ctx, repository, rv.Authorization, true) {
+		requireAuth(ctx)
+		return
+	}
+
+	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil {
+		models.RemoveLFSMetaObjectByOid(rv.Oid)
+		ctx.Resp.WriteHeader(500)
+		fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
+		return
+	}
+
+	logRequest(ctx.Req, 200)
+}
+
+// Represent takes a RequestVars and Meta and turns it into a Representation suitable
+// for json encoding
+func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation {
+	rep := &Representation{
+		Oid:     meta.Oid,
+		Size:    meta.Size,
+		Actions: make(map[string]*link),
+	}
+
+	header := make(map[string]string)
+	header["Accept"] = contentMediaType
+
+	if rv.Authorization == "" {
+		//https://github.com/github/git-lfs/issues/1088
+		header["Authorization"] = "Authorization: Basic dummy"
+	} else {
+		header["Authorization"] = rv.Authorization
+	}
+
+	if download {
+		rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header}
+	}
+
+	if upload {
+		rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header}
+	}
+
+	return rep
+}
+
+// ContentMatcher provides a mux.MatcherFunc that only allows requests that contain
+// an Accept header with the contentMediaType
+func ContentMatcher(r macaron.Request) bool {
+	mediaParts := strings.Split(r.Header.Get("Accept"), ";")
+	mt := mediaParts[0]
+	return mt == contentMediaType
+}
+
+// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain
+// an Accept header with the metaMediaType
+func MetaMatcher(r macaron.Request) bool {
+	mediaParts := strings.Split(r.Header.Get("Accept"), ";")
+	mt := mediaParts[0]
+	return mt == metaMediaType
+}
+
+func unpack(ctx *context.Context) *RequestVars {
+	r := ctx.Req
+	rv := &RequestVars{
+		User:          ctx.Params("username"),
+		Repo:          strings.TrimSuffix(ctx.Params("reponame"), ".git"),
+		Oid:           ctx.Params("oid"),
+		Authorization: r.Header.Get("Authorization"),
+	}
+
+	if r.Method == "POST" { // Maybe also check if +json
+		var p RequestVars
+		dec := json.NewDecoder(r.Body().ReadCloser())
+		err := dec.Decode(&p)
+		if err != nil {
+			return rv
+		}
+
+		rv.Oid = p.Oid
+		rv.Size = p.Size
+	}
+
+	return rv
+}
+
+// TODO cheap hack, unify with unpack
+func unpackbatch(ctx *context.Context) *BatchVars {
+
+	r := ctx.Req
+	var bv BatchVars
+
+	dec := json.NewDecoder(r.Body().ReadCloser())
+	err := dec.Decode(&bv)
+	if err != nil {
+		return &bv
+	}
+
+	for i := 0; i < len(bv.Objects); i++ {
+		bv.Objects[i].User = ctx.Params("username")
+		bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git")
+		bv.Objects[i].Authorization = r.Header.Get("Authorization")
+	}
+
+	return &bv
+}
+
+func writeStatus(ctx *context.Context, status int) {
+	message := http.StatusText(status)
+
+	mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";")
+	mt := mediaParts[0]
+	if strings.HasSuffix(mt, "+json") {
+		message = `{"message":"` + message + `"}`
+	}
+
+	ctx.Resp.WriteHeader(status)
+	fmt.Fprint(ctx.Resp, message)
+	logRequest(ctx.Req, status)
+}
+
+func logRequest(r macaron.Request, status int) {
+	log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status)
+}
+
+// authenticate uses the authorization string to determine whether
+// or not to proceed. This server assumes an HTTP Basic auth format.
+func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool {
+
+	accessMode := models.AccessModeRead
+	if requireWrite {
+		accessMode = models.AccessModeWrite
+	}
+
+	if !repository.IsPrivate && !requireWrite {
+		return true
+	}
+
+	if ctx.IsSigned {
+		accessCheck, _ := models.HasAccess(ctx.User, repository, accessMode)
+		return accessCheck
+	}
+
+	if authorization == "" {
+		return false
+	}
+
+	if authenticateToken(repository, authorization, requireWrite) {
+		return true
+	}
+
+	if !strings.HasPrefix(authorization, "Basic ") {
+		return false
+	}
+
+	c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic "))
+	if err != nil {
+		return false
+	}
+	cs := string(c)
+	i := strings.IndexByte(cs, ':')
+	if i < 0 {
+		return false
+	}
+	user, password := cs[:i], cs[i+1:]
+
+	userModel, err := models.GetUserByName(user)
+	if err != nil {
+		return false
+	}
+
+	if !userModel.ValidatePassword(password) {
+		return false
+	}
+
+	accessCheck, _ := models.HasAccess(userModel, repository, accessMode)
+	return accessCheck
+}
+
+func authenticateToken(repository *models.Repository, authorization string, requireWrite bool) bool {
+	if !strings.HasPrefix(authorization, "Bearer ") {
+		return false
+	}
+
+	token, err := jwt.Parse(authorization[7:], func(t *jwt.Token) (interface{}, error) {
+		if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
+		}
+		return setting.LFS.JWTSecretBytes, nil
+	})
+	if err != nil {
+		return false
+	}
+	claims, claimsOk := token.Claims.(jwt.MapClaims)
+	if !token.Valid || !claimsOk {
+		return false
+	}
+
+	opStr, ok := claims["op"].(string)
+	if !ok {
+		return false
+	}
+
+	if requireWrite && opStr != "upload" {
+		return false
+	}
+
+	repoID, ok := claims["repo"].(float64)
+	if !ok {
+		return false
+	}
+
+	if repository.ID != int64(repoID) {
+		return false
+	}
+
+	return true
+}
+
+func requireAuth(ctx *context.Context) {
+	ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+	writeStatus(ctx, 401)
+}