From c03e488e14fdaf1c0056952f40c5fc8124719a30 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 9 Apr 2021 00:25:57 +0200 Subject: Add LFS Migration and Mirror (#14726) * Implemented LFS client. * Implemented scanning for pointer files. * Implemented downloading of lfs files. * Moved model-dependent code into services. * Removed models dependency. Added TryReadPointerFromBuffer. * Migrated code from service to module. * Centralised storage creation. * Removed dependency from models. * Moved ContentStore into modules. * Share structs between server and client. * Moved method to services. * Implemented lfs download on clone. * Implemented LFS sync on clone and mirror update. * Added form fields. * Updated templates. * Fixed condition. * Use alternate endpoint. * Added missing methods. * Fixed typo and make linter happy. * Detached pointer parser from gogit dependency. * Fixed TestGetLFSRange test. * Added context to support cancellation. * Use ReadFull to probably read more data. * Removed duplicated code from models. * Moved scan implementation into pointer_scanner_nogogit. * Changed method name. * Added comments. * Added more/specific log/error messages. * Embedded lfs.Pointer into models.LFSMetaObject. * Moved code from models to module. * Moved code from models to module. * Moved code from models to module. * Reduced pointer usage. * Embedded type. * Use promoted fields. * Fixed unexpected eof. * Added unit tests. * Implemented migration of local file paths. * Show an error on invalid LFS endpoints. * Hide settings if not used. * Added LFS info to mirror struct. * Fixed comment. * Check LFS endpoint. * Manage LFS settings from mirror page. * Fixed selector. * Adjusted selector. * Added more tests. * Added local filesystem migration test. * Fixed typo. * Reset settings. * Added special windows path handling. * Added unit test for HTTPClient. * Added unit test for BasicTransferAdapter. * Moved into util package. * Test if LFS endpoint is allowed. * Added support for git:// * Just use a static placeholder as the displayed url may be invalid. * Reverted to original code. * Added "Advanced Settings". * Updated wording. * Added discovery info link. * Implemented suggestion. * Fixed missing format parameter. * Added Pointer.IsValid(). * Always remove model on error. * Added suggestions. * Use channel instead of array. * Update routers/repo/migrate.go * fmt Signed-off-by: Andrew Thornton Co-authored-by: zeripath --- modules/lfs/shared.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 modules/lfs/shared.go (limited to 'modules/lfs/shared.go') diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go new file mode 100644 index 0000000000..70b76d7512 --- /dev/null +++ b/modules/lfs/shared.go @@ -0,0 +1,69 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package lfs + +import ( + "time" +) + +const ( + // MediaType contains the media type for LFS server requests + MediaType = "application/vnd.git-lfs+json" +) + +// BatchRequest contains multiple requests processed in one batch operation. +// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#requests +type BatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Ref *Reference `json:"ref,omitempty"` + Objects []Pointer `json:"objects"` +} + +// Reference contains a git reference. +// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#ref-property +type Reference struct { + Name string `json:"name"` +} + +// Pointer contains LFS pointer data +type Pointer struct { + Oid string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"` + Size int64 `json:"size" xorm:"NOT NULL"` +} + +// BatchResponse contains multiple object metadata Representation structures +// for use with the batch API. +// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses +type BatchResponse struct { + Transfer string `json:"transfer,omitempty"` + Objects []*ObjectResponse `json:"objects"` +} + +// ObjectResponse is object metadata as seen by clients of the LFS server. +type ObjectResponse struct { + Pointer + Actions map[string]*Link `json:"actions"` + Error *ObjectError `json:"error,omitempty"` +} + +// Link provides a structure used to build a hypermedia representation of an HTTP link. +type Link struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} + +// ObjectError defines the JSON structure returned to the client in case of an error +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// PointerBlob associates a Git blob with a Pointer. +type PointerBlob struct { + Hash string + Pointer +} -- cgit v1.2.3