From c03e488e14fdaf1c0056952f40c5fc8124719a30 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 9 Apr 2021 00:25:57 +0200 Subject: Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton Co-authored-by: zeripath --- models/lfs.go | 54 ++++++----------------------------------- models/migrations/migrations.go | 2 ++ models/migrations/v178.go | 18 ++++++++++++++ models/repo.go | 3 ++- models/repo_mirror.go | 3 +++ 5 files changed, 33 insertions(+), 47 deletions(-) create mode 100644 models/migrations/v178.go (limited to 'models') diff --git a/models/lfs.go b/models/lfs.go index 019d85545e..90f6523d4a 100644 --- a/models/lfs.go +++ b/models/lfs.go @@ -5,13 +5,9 @@ package models import ( - "crypto/sha256" - "encoding/hex" "errors" - "fmt" - "io" - "path" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" @@ -19,28 +15,13 @@ import ( // LFSMetaObject stores metadata for LFS tracked files. type LFSMetaObject struct { - ID int64 `xorm:"pk autoincr"` - Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"` - Size int64 `xorm:"NOT NULL"` + ID int64 `xorm:"pk autoincr"` + lfs.Pointer `xorm:"extends"` RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"` Existing bool `xorm:"-"` CreatedUnix timeutil.TimeStamp `xorm:"created"` } -// RelativePath returns the relative path of the lfs object -func (m *LFSMetaObject) RelativePath() string { - if len(m.Oid) < 5 { - return m.Oid - } - - return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:]) -} - -// Pointer returns the string representation of an LFS pointer file -func (m *LFSMetaObject) Pointer() string { - return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size) -} - // LFSTokenResponse defines the JSON structure in which the JWT token is stored. // This structure is fetched via SSH and passed by the Git LFS client to the server // endpoint for authorization. @@ -53,15 +34,6 @@ type LFSTokenResponse struct { // to differentiate between database and missing object errors. var ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist") -const ( - // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files. - // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1" - - // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash. - LFSMetaFileOidPrefix = "oid sha256:" -) - // NewLFSMetaObject stores a given populated LFSMetaObject structure in the database // if it is not already present. func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { @@ -90,16 +62,6 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) { return m, sess.Commit() } -// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content -func GenerateLFSOid(content io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, content); err != nil { - return "", err - } - sum := h.Sum(nil) - return hex.EncodeToString(sum), nil -} - // GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID. // It may return ErrLFSObjectNotExist or a database error. If the error is nil, // the returned pointer is a valid LFSMetaObject. @@ -108,7 +70,7 @@ func (repo *Repository) GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error return nil, ErrLFSObjectNotExist } - m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} + m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID} has, err := x.Get(m) if err != nil { return nil, err @@ -131,12 +93,12 @@ func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) (int64, error) { return -1, err } - m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID} + m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID} if _, err := sess.Delete(m); err != nil { return -1, err } - count, err := sess.Count(&LFSMetaObject{Oid: oid}) + count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) if err != nil { return count, err } @@ -168,11 +130,11 @@ func (repo *Repository) CountLFSMetaObjects() (int64, error) { // LFSObjectAccessible checks if a provided Oid is accessible to the user func LFSObjectAccessible(user *User, oid string) (bool, error) { if user.IsAdmin { - count, err := x.Count(&LFSMetaObject{Oid: oid}) + count, err := x.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) return (count > 0), err } cond := accessibleRepositoryCondition(user) - count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid}) + count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}) return (count > 0), err } diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index c8e687b382..e9d4927ae6 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -302,6 +302,8 @@ var migrations = []Migration{ NewMigration("Remove invalid labels from comments", removeInvalidLabels), // v177 -> v178 NewMigration("Delete orphaned IssueLabels", deleteOrphanedIssueLabels), + // v178 -> v179 + NewMigration("Add LFS columns to Mirror", addLFSMirrorColumns), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v178.go b/models/migrations/v178.go new file mode 100644 index 0000000000..c2a9af618e --- /dev/null +++ b/models/migrations/v178.go @@ -0,0 +1,18 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "xorm.io/xorm" +) + +func addLFSMirrorColumns(x *xorm.Engine) error { + type Mirror struct { + LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` + LFSEndpoint string `xorm:"lfs_endpoint TEXT"` + } + + return x.Sync2(new(Mirror)) +} diff --git a/models/repo.go b/models/repo.go index 7f2ec1f742..bdb84ee00d 100644 --- a/models/repo.go +++ b/models/repo.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/options" @@ -1531,7 +1532,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error { } for _, v := range lfsObjects { - count, err := sess.Count(&LFSMetaObject{Oid: v.Oid}) + count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: v.Oid}}) if err != nil { return err } diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 10b0a7b139..2c37b54aa9 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -25,6 +25,9 @@ type Mirror struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` + LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"` + LFSEndpoint string `xorm:"lfs_endpoint TEXT"` + Address string `xorm:"-"` } -- cgit v1.2.3