From c03e488e14fdaf1c0056952f40c5fc8124719a30 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 9 Apr 2021 00:25:57 +0200 Subject: Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton Co-authored-by: zeripath --- modules/lfs/pointer.go | 123 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 modules/lfs/pointer.go (limited to 'modules/lfs/pointer.go') diff --git a/modules/lfs/pointer.go b/modules/lfs/pointer.go new file mode 100644 index 0000000000..975b5e7dc6 --- /dev/null +++ b/modules/lfs/pointer.go @@ -0,0 +1,123 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path" + "regexp" + "strconv" + "strings" +) + +const ( + blobSizeCutoff = 1024 + + // MetaFileIdentifier is the string appearing at the first line of LFS pointer files. + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" + + // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. + MetaFileOidPrefix = "oid sha256:" +) + +var ( + // ErrMissingPrefix occurs if the content lacks the LFS prefix + ErrMissingPrefix = errors.New("Content lacks the LFS prefix") + + // ErrInvalidStructure occurs if the content has an invalid structure + ErrInvalidStructure = errors.New("Content has an invalid structure") + + // ErrInvalidOIDFormat occurs if the oid has an invalid format + ErrInvalidOIDFormat = errors.New("OID has an invalid format") +) + +// ReadPointer tries to read LFS pointer data from the reader +func ReadPointer(reader io.Reader) (Pointer, error) { + buf := make([]byte, blobSizeCutoff) + n, err := io.ReadFull(reader, buf) + if err != nil && err != io.ErrUnexpectedEOF { + return Pointer{}, err + } + buf = buf[:n] + + return ReadPointerFromBuffer(buf) +} + +var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`) + +// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise. +func ReadPointerFromBuffer(buf []byte) (Pointer, error) { + var p Pointer + + headString := string(buf) + if !strings.HasPrefix(headString, MetaFileIdentifier) { + return p, ErrMissingPrefix + } + + splitLines := strings.Split(headString, "\n") + if len(splitLines) < 3 { + return p, ErrInvalidStructure + } + + oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix) + if len(oid) != 64 || !oidPattern.MatchString(oid) { + return p, ErrInvalidOIDFormat + } + size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64) + if err != nil { + return p, err + } + + p.Oid = oid + p.Size = size + + return p, nil +} + +// IsValid checks if the pointer has a valid structure. +// It doesn't check if the pointed-to-content exists. +func (p Pointer) IsValid() bool { + if len(p.Oid) != 64 { + return false + } + if !oidPattern.MatchString(p.Oid) { + return false + } + if p.Size < 0 { + return false + } + return true +} + +// StringContent returns the string representation of the pointer +// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer +func (p Pointer) StringContent() string { + return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size) +} + +// RelativePath returns the relative storage path of the pointer +func (p Pointer) RelativePath() string { + if len(p.Oid) < 5 { + return p.Oid + } + + return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:]) +} + +// GeneratePointer generates a pointer for arbitrary content +func GeneratePointer(content io.Reader) (Pointer, error) { + h := sha256.New() + c, err := io.Copy(h, content) + if err != nil { + return Pointer{}, err + } + sum := h.Sum(nil) + return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil +} -- cgit v1.2.3