Git LFS support v2 (#122)

* Import github.com/git-lfs/lfs-test-server as lfs module base

Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198

Removed:

Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go
.dockerignore .gitignore README.md

* Remove config, add JWT support from github.com/mgit-at/lfs-test-server

Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83

* Add LFS settings

* Add LFS meta object model

* Add LFS routes and initialization

* Import github.com/dgrijalva/jwt-go into vendor/

* Adapt LFS module: handlers, routing, meta store

* Move LFS routes to /user/repo/info/lfs/*

* Add request header checks to LFS BatchHandler / PostHandler

* Implement LFS basic authentication

* Rework JWT secret generation / load

* Implement LFS SSH token authentication with JWT

Specification: https://github.com/github/git-lfs/tree/master/docs/api

* Integrate LFS settings into install process

* Remove LFS objects when repository is deleted

Only removes objects from content store when deleted repo is the only
referencing repository

* Make LFS module stateless

Fixes bug where LFS would not work after installation without
restarting Gitea

* Change 500 'Internal Server Error' to 400 'Bad Request'

* Change sql query to xorm call

* Remove unneeded type from LFS module

* Change internal imports to code.gitea.io/gitea/

* Add Gitea authors copyright

* Change basic auth realm to "gitea-lfs"

* Add unique indexes to LFS model

* Use xorm count function in LFS check on repository delete

* Return io.ReadCloser from content store and close after usage

* Add LFS info to runWeb()

* Export LFS content store base path

* LFS file download from UI

* Work around git-lfs client issue with unauthenticated requests

Returning a dummy Authorization header for unauthenticated requests
lets git-lfs client skip asking for auth credentials
See: https://github.com/github/git-lfs/issues/1088

* Fix unauthenticated UI downloads from public repositories

* Authentication check order, Finish LFS file view logic

* Ignore LFS hooks if installed for current OS user

Fixes Gitea UI actions for repositories tracking LFS files.
Checks for minimum needed git version by parsing the semantic version
string.

* Hide LFS metafile diff from commit view, marking as binary

* Show LFS notice if file in commit view is tracked

* Add notbefore/nbf JWT claim

* Correct lint suggestions - comments for structs and functions

- Add comments to LFS model
- Function comment for GetRandomBytesAsBase64
- LFS server function comments and lint variable suggestion

* Move secret generation code out of conditional

Ensures no LFS code may run with an empty secret

* Do not hand out JWT tokens if LFS server support is disabled

1 files changed, 90 insertions, 0 deletions

diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 81fcb4b150..105d33371e 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -5,7 +5,10 @@
 package setting
 
 import (
+	"crypto/rand"
+	"encoding/base64"
 	"fmt"
+	"io"
 	"net/mail"
 	"net/url"
 	"os"
@@ -17,6 +20,7 @@ import (
 	"strings"
 	"time"
 
+	"code.gitea.io/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/user"
 	"github.com/Unknwon/com"
@@ -89,6 +93,13 @@ var (
 		MinimumKeySizes     map[string]int `ini:"-"`
 	}
 
+	LFS struct {
+		StartServer     bool   `ini:"LFS_START_SERVER"`
+		ContentPath     string `ini:"LFS_CONTENT_PATH"`
+		JWTSecretBase64 string `ini:"LFS_JWT_SECRET"`
+		JWTSecretBytes  []byte `ini:"-"`
+	}
+
 	// Security settings
 	InstallLock          bool
 	SecretKey            string
@@ -583,6 +594,85 @@ please consider changing to GITEA_CUSTOM`)
 		}
 	}
 
+	if err = Cfg.Section("server").MapTo(&LFS); err != nil {
+		log.Fatal(4, "Fail to map LFS settings: %v", err)
+	}
+
+	if LFS.StartServer {
+
+		if err := os.MkdirAll(LFS.ContentPath, 0700); err != nil {
+			log.Fatal(4, "Fail to create '%s': %v", LFS.ContentPath, err)
+		}
+
+		LFS.JWTSecretBytes = make([]byte, 32)
+		n, err := base64.RawURLEncoding.Decode(LFS.JWTSecretBytes, []byte(LFS.JWTSecretBase64))
+
+		if err != nil || n != 32 {
+			//Generate new secret and save to config
+
+			_, err := io.ReadFull(rand.Reader, LFS.JWTSecretBytes)
+
+			if err != nil {
+				log.Fatal(4, "Error reading random bytes: %s", err)
+			}
+
+			LFS.JWTSecretBase64 = base64.RawURLEncoding.EncodeToString(LFS.JWTSecretBytes)
+
+			// Save secret
+			cfg := ini.Empty()
+			if com.IsFile(CustomConf) {
+				// Keeps custom settings if there is already something.
+				if err := cfg.Append(CustomConf); err != nil {
+					log.Error(4, "Fail to load custom conf '%s': %v", CustomConf, err)
+				}
+			}
+
+			cfg.Section("server").Key("LFS_JWT_SECRET").SetValue(LFS.JWTSecretBase64)
+
+			os.MkdirAll(filepath.Dir(CustomConf), os.ModePerm)
+			if err := cfg.SaveTo(CustomConf); err != nil {
+				log.Fatal(4, "Error saving generated JWT Secret to custom config: %v", err)
+				return
+			}
+		}
+
+		//Disable LFS client hooks if installed for the current OS user
+		//Needs at least git v2.1.2
+
+		binVersion, err := git.BinVersion()
+		if err != nil {
+			log.Fatal(4, "Error retrieving git version: %s", err)
+		}
+
+		splitVersion := strings.SplitN(binVersion, ".", 3)
+
+		majorVersion, err := strconv.ParseUint(splitVersion[0], 10, 64)
+		if err != nil {
+			log.Fatal(4, "Error parsing git major version: %s", err)
+		}
+		minorVersion, err := strconv.ParseUint(splitVersion[1], 10, 64)
+		if err != nil {
+			log.Fatal(4, "Error parsing git minor version: %s", err)
+		}
+		revisionVersion, err := strconv.ParseUint(splitVersion[2], 10, 64)
+		if err != nil {
+			log.Fatal(4, "Error parsing git revision version: %s", err)
+		}
+
+		if !((majorVersion > 2) || (majorVersion == 2 && minorVersion > 1) ||
+			(majorVersion == 2 && minorVersion == 1 && revisionVersion >= 2)) {
+
+			LFS.StartServer = false
+			log.Error(4, "LFS server support needs at least Git v2.1.2")
+
+		} else {
+
+			git.GlobalCommandArgs = append(git.GlobalCommandArgs, "-c", "filter.lfs.required=",
+				"-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=")
+
+		}
+	}
+
 	sec = Cfg.Section("security")
 	InstallLock = sec.Key("INSTALL_LOCK").MustBool(false)
 	SecretKey = sec.Key("SECRET_KEY").MustString("!#@FDEWREWR&*(")