From 2e7ccecfe6f3d52b1dd5277a0eaf7a628164c8ac Mon Sep 17 00:00:00 2001 From: Fabian Zaremba Date: Mon, 26 Dec 2016 02:16:37 +0100 Subject: Git LFS support v2 (#122) * Import github.com/git-lfs/lfs-test-server as lfs module base Imported commit is 3968aac269a77b73924649b9412ae03f7ccd3198 Removed: Dockerfile CONTRIBUTING.md mgmt* script/ vendor/ kvlogger.go .dockerignore .gitignore README.md * Remove config, add JWT support from github.com/mgit-at/lfs-test-server Imported commit f0cdcc5a01599c5a955dc1bbf683bb4acecdba83 * Add LFS settings * Add LFS meta object model * Add LFS routes and initialization * Import github.com/dgrijalva/jwt-go into vendor/ * Adapt LFS module: handlers, routing, meta store * Move LFS routes to /user/repo/info/lfs/* * Add request header checks to LFS BatchHandler / PostHandler * Implement LFS basic authentication * Rework JWT secret generation / load * Implement LFS SSH token authentication with JWT Specification: https://github.com/github/git-lfs/tree/master/docs/api * Integrate LFS settings into install process * Remove LFS objects when repository is deleted Only removes objects from content store when deleted repo is the only referencing repository * Make LFS module stateless Fixes bug where LFS would not work after installation without restarting Gitea * Change 500 'Internal Server Error' to 400 'Bad Request' * Change sql query to xorm call * Remove unneeded type from LFS module * Change internal imports to code.gitea.io/gitea/ * Add Gitea authors copyright * Change basic auth realm to "gitea-lfs" * Add unique indexes to LFS model * Use xorm count function in LFS check on repository delete * Return io.ReadCloser from content store and close after usage * Add LFS info to runWeb() * Export LFS content store base path * LFS file download from UI * Work around git-lfs client issue with unauthenticated requests Returning a dummy Authorization header for unauthenticated requests lets git-lfs client skip asking for auth credentials See: https://github.com/github/git-lfs/issues/1088 * Fix unauthenticated UI downloads from public repositories * Authentication check order, Finish LFS file view logic * Ignore LFS hooks if installed for current OS user Fixes Gitea UI actions for repositories tracking LFS files. Checks for minimum needed git version by parsing the semantic version string. * Hide LFS metafile diff from commit view, marking as binary * Show LFS notice if file in commit view is tracked * Add notbefore/nbf JWT claim * Correct lint suggestions - comments for structs and functions - Add comments to LFS model - Function comment for GetRandomBytesAsBase64 - LFS server function comments and lint variable suggestion * Move secret generation code out of conditional Ensures no LFS code may run with an empty secret * Do not hand out JWT tokens if LFS server support is disabled --- modules/lfs/LICENSE | 20 ++ modules/lfs/content_store.go | 94 ++++++++ modules/lfs/server.go | 549 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 663 insertions(+) create mode 100644 modules/lfs/LICENSE create mode 100644 modules/lfs/content_store.go create mode 100644 modules/lfs/server.go (limited to 'modules/lfs') diff --git a/modules/lfs/LICENSE b/modules/lfs/LICENSE new file mode 100644 index 0000000000..0a94a80c8c --- /dev/null +++ b/modules/lfs/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2016 The Gitea Authors +Copyright (c) GitHub, Inc. and LFS Test Server contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go new file mode 100644 index 0000000000..2ca44512f1 --- /dev/null +++ b/modules/lfs/content_store.go @@ -0,0 +1,94 @@ +package lfs + +import ( + "code.gitea.io/gitea/models" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" +) + +var ( + errHashMismatch = errors.New("Content hash does not match OID") + errSizeMismatch = errors.New("Content size does not match") +) + +// ContentStore provides a simple file system based storage. +type ContentStore struct { + BasePath string +} + +// Get takes a Meta object and retreives the content from the store, returning +// it as an io.Reader. If fromByte > 0, the reader starts from that byte +func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) { + path := filepath.Join(s.BasePath, transformKey(meta.Oid)) + + f, err := os.Open(path) + if err != nil { + return nil, err + } + if fromByte > 0 { + _, err = f.Seek(fromByte, os.SEEK_CUR) + } + return f, err +} + +// Put takes a Meta object and an io.Reader and writes the content to the store. +func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error { + path := filepath.Join(s.BasePath, transformKey(meta.Oid)) + tmpPath := path + ".tmp" + + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0750); err != nil { + return err + } + + file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640) + if err != nil { + return err + } + defer os.Remove(tmpPath) + + hash := sha256.New() + hw := io.MultiWriter(hash, file) + + written, err := io.Copy(hw, r) + if err != nil { + file.Close() + return err + } + file.Close() + + if written != meta.Size { + return errSizeMismatch + } + + shaStr := hex.EncodeToString(hash.Sum(nil)) + if shaStr != meta.Oid { + return errHashMismatch + } + + if err := os.Rename(tmpPath, path); err != nil { + return err + } + return nil +} + +// Exists returns true if the object exists in the content store. +func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool { + path := filepath.Join(s.BasePath, transformKey(meta.Oid)) + if _, err := os.Stat(path); os.IsNotExist(err) { + return false + } + return true +} + +func transformKey(key string) string { + if len(key) < 5 { + return key + } + + return filepath.Join(key[0:2], key[2:4], key[4:len(key)]) +} diff --git a/modules/lfs/server.go b/modules/lfs/server.go new file mode 100644 index 0000000000..f82cb70364 --- /dev/null +++ b/modules/lfs/server.go @@ -0,0 +1,549 @@ +package lfs + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "github.com/dgrijalva/jwt-go" + "gopkg.in/macaron.v1" +) + +const ( + contentMediaType = "application/vnd.git-lfs" + metaMediaType = contentMediaType + "+json" +) + +// RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and +// some headers are stored. +type RequestVars struct { + Oid string + Size int64 + User string + Password string + Repo string + Authorization string +} + +// BatchVars contains multiple RequestVars processed in one batch operation. +// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md +type BatchVars struct { + Transfers []string `json:"transfers,omitempty"` + Operation string `json:"operation"` + Objects []*RequestVars `json:"objects"` +} + +// BatchResponse contains multiple object metadata Representation structures +// for use with the batch API. +type BatchResponse struct { + Transfer string `json:"transfer,omitempty"` + Objects []*Representation `json:"objects"` +} + +// Representation is object medata as seen by clients of the lfs server. +type Representation struct { + Oid string `json:"oid"` + Size int64 `json:"size"` + Actions map[string]*link `json:"actions"` + Error *ObjectError `json:"error,omitempty"` +} + +// ObjectError defines the JSON structure returned to the client in case of an error +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// ObjectLink builds a URL linking to the object. +func (v *RequestVars) ObjectLink() string { + return fmt.Sprintf("%s%s/%s/info/lfs/objects/%s", setting.AppURL, v.User, v.Repo, v.Oid) +} + +// link provides a structure used to build a hypermedia representation of an HTTP link. +type link struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} + +// ObjectOidHandler is the main request routing entry point into LFS server functions +func ObjectOidHandler(ctx *context.Context) { + + if !setting.LFS.StartServer { + writeStatus(ctx, 404) + return + } + + if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" { + if MetaMatcher(ctx.Req) { + GetMetaHandler(ctx) + return + } + if ContentMatcher(ctx.Req) || len(ctx.Params("filename")) > 0 { + GetContentHandler(ctx) + return + } + } else if ctx.Req.Method == "PUT" && ContentMatcher(ctx.Req) { + PutHandler(ctx) + return + } + +} + +// GetContentHandler gets the content from the content store +func GetContentHandler(ctx *context.Context) { + + rv := unpack(ctx) + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + if err != nil { + writeStatus(ctx, 404) + return + } + + repository, err := models.GetRepositoryByID(meta.RepositoryID) + + if err != nil { + writeStatus(ctx, 404) + return + } + + if !authenticate(ctx, repository, rv.Authorization, false) { + requireAuth(ctx) + return + } + + // Support resume download using Range header + var fromByte int64 + statusCode := 200 + if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" { + regex := regexp.MustCompile(`bytes=(\d+)\-.*`) + match := regex.FindStringSubmatch(rangeHdr) + if match != nil && len(match) > 1 { + statusCode = 206 + fromByte, _ = strconv.ParseInt(match[1], 10, 32) + ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, meta.Size-1, int64(meta.Size)-fromByte)) + } + } + + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + content, err := contentStore.Get(meta, fromByte) + if err != nil { + writeStatus(ctx, 404) + return + } + + ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(meta.Size, 10)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + filename := ctx.Params("filename") + if len(filename) > 0 { + decodedFilename, err := base64.RawURLEncoding.DecodeString(filename) + if err == nil { + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"") + } + } + + ctx.Resp.WriteHeader(statusCode) + io.Copy(ctx.Resp, content) + content.Close() + logRequest(ctx.Req, statusCode) +} + +// GetMetaHandler retrieves metadata about the object +func GetMetaHandler(ctx *context.Context) { + + rv := unpack(ctx) + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + if err != nil { + writeStatus(ctx, 404) + return + } + + repository, err := models.GetRepositoryByID(meta.RepositoryID) + + if err != nil { + writeStatus(ctx, 404) + return + } + + if !authenticate(ctx, repository, rv.Authorization, false) { + requireAuth(ctx) + return + } + + ctx.Resp.Header().Set("Content-Type", metaMediaType) + + if ctx.Req.Method == "GET" { + enc := json.NewEncoder(ctx.Resp) + enc.Encode(Represent(rv, meta, true, false)) + } + + logRequest(ctx.Req, 200) +} + +// PostHandler instructs the client how to upload data +func PostHandler(ctx *context.Context) { + + if !setting.LFS.StartServer { + writeStatus(ctx, 404) + return + } + + if !MetaMatcher(ctx.Req) { + writeStatus(ctx, 400) + return + } + + rv := unpack(ctx) + + repositoryString := rv.User + "/" + rv.Repo + repository, err := models.GetRepositoryByRef(repositoryString) + + if err != nil { + log.Debug("Could not find repository: %s - %s", repositoryString, err) + writeStatus(ctx, 404) + return + } + + if !authenticate(ctx, repository, rv.Authorization, true) { + requireAuth(ctx) + } + + meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID}) + + if err != nil { + writeStatus(ctx, 404) + return + } + + ctx.Resp.Header().Set("Content-Type", metaMediaType) + + sentStatus := 202 + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + if meta.Existing && contentStore.Exists(meta) { + sentStatus = 200 + } + ctx.Resp.WriteHeader(sentStatus) + + enc := json.NewEncoder(ctx.Resp) + enc.Encode(Represent(rv, meta, meta.Existing, true)) + logRequest(ctx.Req, sentStatus) +} + +// BatchHandler provides the batch api +func BatchHandler(ctx *context.Context) { + + if !setting.LFS.StartServer { + writeStatus(ctx, 404) + return + } + + if !MetaMatcher(ctx.Req) { + writeStatus(ctx, 400) + return + } + + bv := unpackbatch(ctx) + + var responseObjects []*Representation + + // Create a response object + for _, object := range bv.Objects { + + repositoryString := object.User + "/" + object.Repo + repository, err := models.GetRepositoryByRef(repositoryString) + + if err != nil { + log.Debug("Could not find repository: %s - %s", repositoryString, err) + writeStatus(ctx, 404) + return + } + + requireWrite := false + if bv.Operation == "upload" { + requireWrite = true + } + + if !authenticate(ctx, repository, object.Authorization, requireWrite) { + requireAuth(ctx) + return + } + + meta, err := models.GetLFSMetaObjectByOid(object.Oid) + + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + if err == nil && contentStore.Exists(meta) { // Object is found and exists + responseObjects = append(responseObjects, Represent(object, meta, true, false)) + continue + } + + // Object is not found + meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID}) + + if err == nil { + responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, true)) + } + } + + ctx.Resp.Header().Set("Content-Type", metaMediaType) + + respobj := &BatchResponse{Objects: responseObjects} + + enc := json.NewEncoder(ctx.Resp) + enc.Encode(respobj) + logRequest(ctx.Req, 200) +} + +// PutHandler receives data from the client and puts it into the content store +func PutHandler(ctx *context.Context) { + rv := unpack(ctx) + + meta, err := models.GetLFSMetaObjectByOid(rv.Oid) + + if err != nil { + writeStatus(ctx, 404) + return + } + + repository, err := models.GetRepositoryByID(meta.RepositoryID) + + if err != nil { + writeStatus(ctx, 404) + return + } + + if !authenticate(ctx, repository, rv.Authorization, true) { + requireAuth(ctx) + return + } + + contentStore := &ContentStore{BasePath: setting.LFS.ContentPath} + if err := contentStore.Put(meta, ctx.Req.Body().ReadCloser()); err != nil { + models.RemoveLFSMetaObjectByOid(rv.Oid) + ctx.Resp.WriteHeader(500) + fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err) + return + } + + logRequest(ctx.Req, 200) +} + +// Represent takes a RequestVars and Meta and turns it into a Representation suitable +// for json encoding +func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation { + rep := &Representation{ + Oid: meta.Oid, + Size: meta.Size, + Actions: make(map[string]*link), + } + + header := make(map[string]string) + header["Accept"] = contentMediaType + + if rv.Authorization == "" { + //https://github.com/github/git-lfs/issues/1088 + header["Authorization"] = "Authorization: Basic dummy" + } else { + header["Authorization"] = rv.Authorization + } + + if download { + rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header} + } + + if upload { + rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header} + } + + return rep +} + +// ContentMatcher provides a mux.MatcherFunc that only allows requests that contain +// an Accept header with the contentMediaType +func ContentMatcher(r macaron.Request) bool { + mediaParts := strings.Split(r.Header.Get("Accept"), ";") + mt := mediaParts[0] + return mt == contentMediaType +} + +// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain +// an Accept header with the metaMediaType +func MetaMatcher(r macaron.Request) bool { + mediaParts := strings.Split(r.Header.Get("Accept"), ";") + mt := mediaParts[0] + return mt == metaMediaType +} + +func unpack(ctx *context.Context) *RequestVars { + r := ctx.Req + rv := &RequestVars{ + User: ctx.Params("username"), + Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"), + Oid: ctx.Params("oid"), + Authorization: r.Header.Get("Authorization"), + } + + if r.Method == "POST" { // Maybe also check if +json + var p RequestVars + dec := json.NewDecoder(r.Body().ReadCloser()) + err := dec.Decode(&p) + if err != nil { + return rv + } + + rv.Oid = p.Oid + rv.Size = p.Size + } + + return rv +} + +// TODO cheap hack, unify with unpack +func unpackbatch(ctx *context.Context) *BatchVars { + + r := ctx.Req + var bv BatchVars + + dec := json.NewDecoder(r.Body().ReadCloser()) + err := dec.Decode(&bv) + if err != nil { + return &bv + } + + for i := 0; i < len(bv.Objects); i++ { + bv.Objects[i].User = ctx.Params("username") + bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git") + bv.Objects[i].Authorization = r.Header.Get("Authorization") + } + + return &bv +} + +func writeStatus(ctx *context.Context, status int) { + message := http.StatusText(status) + + mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";") + mt := mediaParts[0] + if strings.HasSuffix(mt, "+json") { + message = `{"message":"` + message + `"}` + } + + ctx.Resp.WriteHeader(status) + fmt.Fprint(ctx.Resp, message) + logRequest(ctx.Req, status) +} + +func logRequest(r macaron.Request, status int) { + log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status) +} + +// authenticate uses the authorization string to determine whether +// or not to proceed. This server assumes an HTTP Basic auth format. +func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool { + + accessMode := models.AccessModeRead + if requireWrite { + accessMode = models.AccessModeWrite + } + + if !repository.IsPrivate && !requireWrite { + return true + } + + if ctx.IsSigned { + accessCheck, _ := models.HasAccess(ctx.User, repository, accessMode) + return accessCheck + } + + if authorization == "" { + return false + } + + if authenticateToken(repository, authorization, requireWrite) { + return true + } + + if !strings.HasPrefix(authorization, "Basic ") { + return false + } + + c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic ")) + if err != nil { + return false + } + cs := string(c) + i := strings.IndexByte(cs, ':') + if i < 0 { + return false + } + user, password := cs[:i], cs[i+1:] + + userModel, err := models.GetUserByName(user) + if err != nil { + return false + } + + if !userModel.ValidatePassword(password) { + return false + } + + accessCheck, _ := models.HasAccess(userModel, repository, accessMode) + return accessCheck +} + +func authenticateToken(repository *models.Repository, authorization string, requireWrite bool) bool { + if !strings.HasPrefix(authorization, "Bearer ") { + return false + } + + token, err := jwt.Parse(authorization[7:], func(t *jwt.Token) (interface{}, error) { + if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) + } + return setting.LFS.JWTSecretBytes, nil + }) + if err != nil { + return false + } + claims, claimsOk := token.Claims.(jwt.MapClaims) + if !token.Valid || !claimsOk { + return false + } + + opStr, ok := claims["op"].(string) + if !ok { + return false + } + + if requireWrite && opStr != "upload" { + return false + } + + repoID, ok := claims["repo"].(float64) + if !ok { + return false + } + + if repository.ID != int64(repoID) { + return false + } + + return true +} + +func requireAuth(ctx *context.Context) { + ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs") + writeStatus(ctx, 401) +} -- cgit v1.2.3