Git LFS support v2 (#122)

* Import github.com/git-lfs/lfs-test-server as lfs module base

Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198

Removed:

Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go
.dockerignore .gitignore README.md

* Remove config, add JWT support from github.com/mgit-at/lfs-test-server

Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83

* Add LFS settings

* Add LFS meta object model

* Add LFS routes and initialization

* Import github.com/dgrijalva/jwt-go into vendor/

* Adapt LFS module: handlers, routing, meta store

* Move LFS routes to /user/repo/info/lfs/*

* Add request header checks to LFS BatchHandler / PostHandler

* Implement LFS basic authentication

* Rework JWT secret generation / load

* Implement LFS SSH token authentication with JWT

Specification: https://github.com/github/git-lfs/tree/master/docs/api

* Integrate LFS settings into install process

* Remove LFS objects when repository is deleted

Only removes objects from content store when deleted repo is the only
referencing repository

* Make LFS module stateless

Fixes bug where LFS would not work after installation without
restarting Gitea

* Change 500 'Internal Server Error' to 400 'Bad Request'

* Change sql query to xorm call

* Remove unneeded type from LFS module

* Change internal imports to code.gitea.io/gitea/

* Add Gitea authors copyright

* Change basic auth realm to "gitea-lfs"

* Add unique indexes to LFS model

* Use xorm count function in LFS check on repository delete

* Return io.ReadCloser from content store and close after usage

* Add LFS info to runWeb()

* Export LFS content store base path

* LFS file download from UI

* Work around git-lfs client issue with unauthenticated requests

Returning a dummy Authorization header for unauthenticated requests
lets git-lfs client skip asking for auth credentials
See: https://github.com/github/git-lfs/issues/1088

* Fix unauthenticated UI downloads from public repositories

* Authentication check order, Finish LFS file view logic

* Ignore LFS hooks if installed for current OS user

Fixes Gitea UI actions for repositories tracking LFS files.
Checks for minimum needed git version by parsing the semantic version
string.

* Hide LFS metafile diff from commit view, marking as binary

* Show LFS notice if file in commit view is tracked

* Add notbefore/nbf JWT claim

* Correct lint suggestions - comments for structs and functions

- Add comments to LFS model
- Function comment for GetRandomBytesAsBase64
- LFS server function comments and lint variable suggestion

* Move secret generation code out of conditional

Ensures no LFS code may run with an empty secret

* Do not hand out JWT tokens if LFS server support is disabled

2 files changed, 75 insertions, 1 deletions

diff --git a/cmd/serve.go b/cmd/serve.go
index f806db2096..8e498faba7 100644
--- a/cmd/serve.go
+++ b/cmd/serve.go
@@ -7,6 +7,7 @@ package cmd
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/exec"
@@ -21,12 +22,14 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"github.com/Unknwon/com"
+	"github.com/dgrijalva/jwt-go"
 	gouuid "github.com/satori/go.uuid"
 	"github.com/urfave/cli"
 )
 
 const (
-	accessDenied = "Repository does not exist or you do not have access"
+	accessDenied        = "Repository does not exist or you do not have access"
+	lfsAuthenticateVerb = "git-lfs-authenticate"
 )
 
 // CmdServ represents the available serv sub-command.
@@ -73,6 +76,7 @@ var (
 		"git-upload-pack":    models.AccessModeRead,
 		"git-upload-archive": models.AccessModeRead,
 		"git-receive-pack":   models.AccessModeWrite,
+		lfsAuthenticateVerb:  models.AccessModeNone,
 	}
 )
 
@@ -161,6 +165,21 @@ func runServ(c *cli.Context) error {
 	}
 
 	verb, args := parseCmd(cmd)
+
+	var lfsVerb string
+	if verb == lfsAuthenticateVerb {
+
+		if !setting.LFS.StartServer {
+			fail("Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
+		}
+
+		if strings.Contains(args, " ") {
+			argsSplit := strings.SplitN(args, " ", 2)
+			args = strings.TrimSpace(argsSplit[0])
+			lfsVerb = strings.TrimSpace(argsSplit[1])
+		}
+	}
+
 	repoPath := strings.ToLower(strings.Trim(args, "'"))
 	rr := strings.SplitN(repoPath, "/", 2)
 	if len(rr) != 2 {
@@ -196,6 +215,14 @@ func runServ(c *cli.Context) error {
 		fail("Unknown git command", "Unknown git command %s", verb)
 	}
 
+	if verb == lfsAuthenticateVerb {
+		if lfsVerb == "upload" {
+			requestedMode = models.AccessModeWrite
+		} else {
+			requestedMode = models.AccessModeRead
+		}
+	}
+
 	// Prohibit push to mirror repositories.
 	if requestedMode > models.AccessModeRead && repo.IsMirror {
 		fail("mirror repository is read-only", "")
@@ -261,6 +288,41 @@ func runServ(c *cli.Context) error {
 		}
 	}
 
+	//LFS token authentication
+
+	if verb == lfsAuthenticateVerb {
+
+		url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, repoUser.Name, repo.Name)
+
+		now := time.Now()
+		token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
+			"repo": repo.ID,
+			"op":   lfsVerb,
+			"exp":  now.Add(5 * time.Minute).Unix(),
+			"nbf":  now.Unix(),
+		})
+
+		// Sign and get the complete encoded token as a string using the secret
+		tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes)
+		if err != nil {
+			fail("Internal error", "Failed to sign JWT token: %v", err)
+		}
+
+		tokenAuthentication := &models.LFSTokenResponse{
+			Header: make(map[string]string),
+			Href:   url,
+		}
+		tokenAuthentication.Header["Authorization"] = fmt.Sprintf("Bearer %s", tokenString)
+
+		enc := json.NewEncoder(os.Stdout)
+		err = enc.Encode(tokenAuthentication)
+		if err != nil {
+			fail("Internal error", "Failed to encode LFS json response: %v", err)
+		}
+
+		return nil
+	}
+
 	uuid := gouuid.NewV4().String()
 	os.Setenv("GITEA_UUID", uuid)
 	// Keep the old env variable name for backward compability
diff --git a/cmd/web.go b/cmd/web.go
index f6baf6ad4d..e6f6820a6e 100644
--- a/cmd/web.go
+++ b/cmd/web.go
@@ -17,6 +17,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/auth"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/public"
@@ -29,6 +30,7 @@ import (
 	"code.gitea.io/gitea/routers/org"
 	"code.gitea.io/gitea/routers/repo"
 	"code.gitea.io/gitea/routers/user"
+
 	"github.com/go-macaron/binding"
 	"github.com/go-macaron/cache"
 	"github.com/go-macaron/captcha"
@@ -564,6 +566,12 @@ func runWeb(ctx *cli.Context) error {
 		}, ignSignIn, context.RepoAssignment(true), context.RepoRef())
 
 		m.Group("/:reponame", func() {
+			m.Group("/info/lfs", func() {
+				m.Post("/objects/batch", lfs.BatchHandler)
+				m.Get("/objects/:oid/:filename", lfs.ObjectOidHandler)
+				m.Any("/objects/:oid", lfs.ObjectOidHandler)
+				m.Post("/objects", lfs.PostHandler)
+			}, ignSignInAndCsrf)
 			m.Any("/*", ignSignInAndCsrf, repo.HTTP)
 			m.Head("/tasks/trigger", repo.TriggerTask)
 		})
@@ -600,6 +608,10 @@ func runWeb(ctx *cli.Context) error {
 	}
 	log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)
 
+	if setting.LFS.StartServer {
+		log.Info("LFS server enabled")
+	}
+
 	var err error
 	switch setting.Protocol {
 	case setting.HTTP: