* Implemented LFS client.
* Implemented scanning for pointer files.
* Implemented downloading of lfs files.
* Moved model-dependent code into services.
* Removed models dependency. Added TryReadPointerFromBuffer.
* Migrated code from service to module.
* Centralised storage creation.
* Removed dependency from models.
* Moved ContentStore into modules.
* Share structs between server and client.
* Moved method to services.
* Implemented lfs download on clone.
* Implemented LFS sync on clone and mirror update.
* Added form fields.
* Updated templates.
* Fixed condition.
* Use alternate endpoint.
* Added missing methods.
* Fixed typo and make linter happy.
* Detached pointer parser from gogit dependency.
* Fixed TestGetLFSRange test.
* Added context to support cancellation.
* Use ReadFull to probably read more data.
* Removed duplicated code from models.
* Moved scan implementation into pointer_scanner_nogogit.
* Changed method name.
* Added comments.
* Added more/specific log/error messages.
* Embedded lfs.Pointer into models.LFSMetaObject.
* Moved code from models to module.
* Moved code from models to module.
* Moved code from models to module.
* Reduced pointer usage.
* Embedded type.
* Use promoted fields.
* Fixed unexpected eof.
* Added unit tests.
* Implemented migration of local file paths.
* Show an error on invalid LFS endpoints.
* Hide settings if not used.
* Added LFS info to mirror struct.
* Fixed comment.
* Check LFS endpoint.
* Manage LFS settings from mirror page.
* Fixed selector.
* Adjusted selector.
* Added more tests.
* Added local filesystem migration test.
* Fixed typo.
* Reset settings.
* Added special windows path handling.
* Added unit test for HTTPClient.
* Added unit test for BasicTransferAdapter.
* Moved into util package.
* Test if LFS endpoint is allowed.
* Added support for git://
* Just use a static placeholder as the displayed url may be invalid.
* Reverted to original code.
* Added "Advanced Settings".
* Updated wording.
* Added discovery info link.
* Implemented suggestion.
* Fixed missing format parameter.
* Added Pointer.IsValid().
* Always remove model on error.
* Added suggestions.
* Use channel instead of array.
* Update routers/repo/migrate.go
* fmt
Signed-off-by: Andrew Thornton <art27@cantab.net>
Co-authored-by: zeripath <art27@cantab.net>
"time"
"code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/pprof"
"code.gitea.io/gitea/modules/private"
"code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/services/lfs"
"github.com/dgrijalva/jwt-go"
jsoniter "github.com/json-iterator/go"
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+ "net/http"
+ "path"
+ "testing"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/lfs"
+ "code.gitea.io/gitea/modules/setting"
+ api "code.gitea.io/gitea/modules/structs"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestAPIRepoLFSMigrateLocal(t *testing.T) {
+ defer prepareTestEnv(t)()
+
+ oldImportLocalPaths := setting.ImportLocalPaths
+ oldAllowLocalNetworks := setting.Migrations.AllowLocalNetworks
+ setting.ImportLocalPaths = true
+ setting.Migrations.AllowLocalNetworks = true
+
+ user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
+ session := loginUser(t, user.Name)
+ token := getTokenForLoggedInUser(t, session)
+
+ req := NewRequestWithJSON(t, "POST", "/api/v1/repos/migrate?token="+token, &api.MigrateRepoOptions{
+ CloneAddr: path.Join(setting.RepoRootPath, "migration/lfs-test.git"),
+ RepoOwnerID: user.ID,
+ RepoName: "lfs-test-local",
+ LFS: true,
+ })
+ resp := MakeRequest(t, req, NoExpectedStatus)
+ assert.EqualValues(t, http.StatusCreated, resp.Code)
+
+ store := lfs.NewContentStore()
+ ok, _ := store.Verify(lfs.Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6})
+ assert.True(t, ok)
+ ok, _ = store.Verify(lfs.Pointer{Oid: "d6f175817f886ec6fbbc1515326465fa96c3bfd54a4ea06cfd6dbbd8340e0152", Size: 6})
+ assert.True(t, ok)
+
+ setting.ImportLocalPaths = oldImportLocalPaths
+ setting.Migrations.AllowLocalNetworks = oldAllowLocalNetworks
+}
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
assert.NotEqual(t, littleSize, resp.Body.Len())
assert.LessOrEqual(t, resp.Body.Len(), 1024)
if resp.Body.Len() != littleSize && resp.Body.Len() <= 1024 {
- assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
+ assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier)
}
}
resp := session.MakeRequest(t, req, http.StatusOK)
assert.NotEqual(t, bigSize, resp.Body.Len())
if resp.Body.Len() != bigSize && resp.Body.Len() <= 1024 {
- assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
+ assert.Contains(t, resp.Body.String(), lfs.MetaFileIdentifier)
}
}
}
--- /dev/null
+ref: refs/heads/master
--- /dev/null
+[core]
+ bare = false
+ repositoryformatversion = 0
+ filemode = false
+ symlinks = false
+ ignorecase = true
+ logallrefupdates = true
--- /dev/null
+Unnamed repository; edit this file 'description' to name the repository.
--- /dev/null
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-checkout.\n"; exit 2; }
+git lfs post-checkout "$@"
--- /dev/null
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-commit.\n"; exit 2; }
+git lfs post-commit "$@"
--- /dev/null
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/post-merge.\n"; exit 2; }
+git lfs post-merge "$@"
--- /dev/null
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting .git/hooks/pre-push.\n"; exit 2; }
+git lfs pre-push "$@"
--- /dev/null
+dummy2
\ No newline at end of file
--- /dev/null
+dummy1
\ No newline at end of file
--- /dev/null
+546244003622c64b2fc3c2cd544d7a29882c8383
import (
"archive/zip"
"bytes"
- "crypto/sha256"
- "encoding/hex"
- "io"
"io/ioutil"
"net/http"
"net/http/httptest"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/routers/routes"
gzipp "github.com/klauspost/compress/gzip"
"github.com/stretchr/testify/assert"
)
-func GenerateLFSOid(content io.Reader) (string, error) {
- h := sha256.New()
- if _, err := io.Copy(h, content); err != nil {
- return "", err
- }
- sum := h.Sum(nil)
- return hex.EncodeToString(sum), nil
-}
-
var lfsID = int64(20000)
func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string {
- oid, err := GenerateLFSOid(bytes.NewReader(*content))
+ pointer, err := lfs.GeneratePointer(bytes.NewReader(*content))
assert.NoError(t, err)
var lfsMetaObject *models.LFSMetaObject
if setting.Database.UsePostgreSQL {
- lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
+ lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Pointer: pointer, RepositoryID: repositoryID}
} else {
- lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
+ lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: repositoryID}
}
lfsID++
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
assert.NoError(t, err)
- contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
- exist, err := contentStore.Exists(lfsMetaObject)
+ contentStore := lfs.NewContentStore()
+ exist, err := contentStore.Exists(pointer)
assert.NoError(t, err)
if !exist {
- err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
+ err := contentStore.Put(pointer, bytes.NewReader(*content))
assert.NoError(t, err)
}
- return oid
+ return pointer.Oid
}
func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, expectedStatus int) *httptest.ResponseRecorder {
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/url"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "code.gitea.io/gitea/modules/lfs"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func str2url(raw string) *url.URL {
+ u, _ := url.Parse(raw)
+ return u
+}
+
+func TestDetermineLocalEndpoint(t *testing.T) {
+ defer prepareTestEnv(t)()
+
+ root, _ := ioutil.TempDir("", "lfs_test")
+ defer os.RemoveAll(root)
+
+ rootdotgit, _ := ioutil.TempDir("", "lfs_test")
+ defer os.RemoveAll(rootdotgit)
+ os.Mkdir(filepath.Join(rootdotgit, ".git"), 0700)
+
+ lfsroot, _ := ioutil.TempDir("", "lfs_test")
+ defer os.RemoveAll(lfsroot)
+
+ // Test cases
+ var cases = []struct {
+ cloneurl string
+ lfsurl string
+ expected *url.URL
+ }{
+ // case 0
+ {
+ cloneurl: root,
+ lfsurl: "",
+ expected: str2url(fmt.Sprintf("file://%s", root)),
+ },
+ // case 1
+ {
+ cloneurl: root,
+ lfsurl: lfsroot,
+ expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
+ },
+ // case 2
+ {
+ cloneurl: "https://git.com/repo.git",
+ lfsurl: lfsroot,
+ expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
+ },
+ // case 3
+ {
+ cloneurl: rootdotgit,
+ lfsurl: "",
+ expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))),
+ },
+ // case 4
+ {
+ cloneurl: "",
+ lfsurl: rootdotgit,
+ expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))),
+ },
+ // case 5
+ {
+ cloneurl: rootdotgit,
+ lfsurl: rootdotgit,
+ expected: str2url(fmt.Sprintf("file://%s", filepath.Join(rootdotgit, ".git"))),
+ },
+ // case 6
+ {
+ cloneurl: fmt.Sprintf("file://%s", root),
+ lfsurl: "",
+ expected: str2url(fmt.Sprintf("file://%s", root)),
+ },
+ // case 7
+ {
+ cloneurl: fmt.Sprintf("file://%s", root),
+ lfsurl: fmt.Sprintf("file://%s", lfsroot),
+ expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
+ },
+ // case 8
+ {
+ cloneurl: root,
+ lfsurl: fmt.Sprintf("file://%s", lfsroot),
+ expected: str2url(fmt.Sprintf("file://%s", lfsroot)),
+ },
+ // case 9
+ {
+ cloneurl: "",
+ lfsurl: "/does/not/exist",
+ expected: nil,
+ },
+ // case 10
+ {
+ cloneurl: "",
+ lfsurl: "file:///does/not/exist",
+ expected: str2url("file:///does/not/exist"),
+ },
+ }
+
+ for n, c := range cases {
+ ep := lfs.DetermineEndpoint(c.cloneurl, c.lfsurl)
+
+ assert.Equal(t, c.expected, ep, "case %d: error should match", n)
+ }
+}
package models
import (
- "crypto/sha256"
- "encoding/hex"
"errors"
- "fmt"
- "io"
- "path"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/timeutil"
"xorm.io/builder"
// LFSMetaObject stores metadata for LFS tracked files.
type LFSMetaObject struct {
- ID int64 `xorm:"pk autoincr"`
- Oid string `xorm:"UNIQUE(s) INDEX NOT NULL"`
- Size int64 `xorm:"NOT NULL"`
+ ID int64 `xorm:"pk autoincr"`
+ lfs.Pointer `xorm:"extends"`
RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
Existing bool `xorm:"-"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
}
-// RelativePath returns the relative path of the lfs object
-func (m *LFSMetaObject) RelativePath() string {
- if len(m.Oid) < 5 {
- return m.Oid
- }
-
- return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:])
-}
-
-// Pointer returns the string representation of an LFS pointer file
-func (m *LFSMetaObject) Pointer() string {
- return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
-}
-
// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
// This structure is fetched via SSH and passed by the Git LFS client to the server
// endpoint for authorization.
// to differentiate between database and missing object errors.
var ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist")
-const (
- // LFSMetaFileIdentifier is the string appearing at the first line of LFS pointer files.
- // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
- LFSMetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
-
- // LFSMetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
- LFSMetaFileOidPrefix = "oid sha256:"
-)
-
// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database
// if it is not already present.
func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
return m, sess.Commit()
}
-// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content
-func GenerateLFSOid(content io.Reader) (string, error) {
- h := sha256.New()
- if _, err := io.Copy(h, content); err != nil {
- return "", err
- }
- sum := h.Sum(nil)
- return hex.EncodeToString(sum), nil
-}
-
// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
// the returned pointer is a valid LFSMetaObject.
return nil, ErrLFSObjectNotExist
}
- m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID}
+ m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID}
has, err := x.Get(m)
if err != nil {
return nil, err
return -1, err
}
- m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID}
+ m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repo.ID}
if _, err := sess.Delete(m); err != nil {
return -1, err
}
- count, err := sess.Count(&LFSMetaObject{Oid: oid})
+ count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
if err != nil {
return count, err
}
// LFSObjectAccessible checks if a provided Oid is accessible to the user
func LFSObjectAccessible(user *User, oid string) (bool, error) {
if user.IsAdmin {
- count, err := x.Count(&LFSMetaObject{Oid: oid})
+ count, err := x.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
return (count > 0), err
}
cond := accessibleRepositoryCondition(user)
- count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid})
+ count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
return (count > 0), err
}
NewMigration("Remove invalid labels from comments", removeInvalidLabels),
// v177 -> v178
NewMigration("Delete orphaned IssueLabels", deleteOrphanedIssueLabels),
+ // v178 -> v179
+ NewMigration("Add LFS columns to Mirror", addLFSMirrorColumns),
}
// GetCurrentDBVersion returns the current db version
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "xorm.io/xorm"
+)
+
+func addLFSMirrorColumns(x *xorm.Engine) error {
+ type Mirror struct {
+ LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"`
+ LFSEndpoint string `xorm:"lfs_endpoint TEXT"`
+ }
+
+ return x.Sync2(new(Mirror))
+}
"strings"
"time"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/options"
}
for _, v := range lfsObjects {
- count, err := sess.Count(&LFSMetaObject{Oid: v.Oid})
+ count, err := sess.Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: v.Oid}})
if err != nil {
return err
}
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"`
NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"`
+ LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"`
+ LFSEndpoint string `xorm:"lfs_endpoint TEXT"`
+
Address string `xorm:"-"`
}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "context"
+ "io"
+ "net/url"
+)
+
+// Client is used to communicate with a LFS source
+type Client interface {
+ Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error)
+}
+
+// NewClient creates a LFS client
+func NewClient(endpoint *url.URL) Client {
+ if endpoint.Scheme == "file" {
+ return newFilesystemClient(endpoint)
+ }
+ return newHTTPClient(endpoint)
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "net/url"
+
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestNewClient(t *testing.T) {
+ u, _ := url.Parse("file:///test")
+ c := NewClient(u)
+ assert.IsType(t, &FilesystemClient{}, c)
+
+ u, _ = url.Parse("https://test.com/lfs")
+ c = NewClient(u)
+ assert.IsType(t, &HTTPClient{}, c)
+}
"io"
"os"
- "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage"
)
var (
- errHashMismatch = errors.New("Content hash does not match OID")
- errSizeMismatch = errors.New("Content size does not match")
+ // ErrHashMismatch occurs if the content has does not match OID
+ ErrHashMismatch = errors.New("Content hash does not match OID")
+ // ErrSizeMismatch occurs if the content size does not match
+ ErrSizeMismatch = errors.New("Content size does not match")
)
// ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
FromByte int64
}
-func (err ErrRangeNotSatisfiable) Error() string {
- return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
-}
-
// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
func IsErrRangeNotSatisfiable(err error) bool {
_, ok := err.(ErrRangeNotSatisfiable)
return ok
}
+func (err ErrRangeNotSatisfiable) Error() string {
+ return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
+}
+
// ContentStore provides a simple file system based storage.
type ContentStore struct {
storage.ObjectStorage
}
+// NewContentStore creates the default ContentStore
+func NewContentStore() *ContentStore {
+ contentStore := &ContentStore{ObjectStorage: storage.LFS}
+ return contentStore
+}
+
// Get takes a Meta object and retrieves the content from the store, returning
// it as an io.ReadSeekCloser.
-func (s *ContentStore) Get(meta *models.LFSMetaObject) (storage.Object, error) {
- f, err := s.Open(meta.RelativePath())
+func (s *ContentStore) Get(pointer Pointer) (storage.Object, error) {
+ f, err := s.Open(pointer.RelativePath())
if err != nil {
- log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
+ log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", pointer.Oid, err)
return nil, err
}
return f, err
}
// Put takes a Meta object and an io.Reader and writes the content to the store.
-func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
- p := meta.RelativePath()
+func (s *ContentStore) Put(pointer Pointer, r io.Reader) error {
+ p := pointer.RelativePath()
// Wrap the provided reader with an inline hashing and size checker
- wrappedRd := newHashingReader(meta.Size, meta.Oid, r)
+ wrappedRd := newHashingReader(pointer.Size, pointer.Oid, r)
// now pass the wrapped reader to Save - if there is a size mismatch or hash mismatch then
// the errors returned by the newHashingReader should percolate up to here
- written, err := s.Save(p, wrappedRd, meta.Size)
+ written, err := s.Save(p, wrappedRd, pointer.Size)
if err != nil {
- log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
+ log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", pointer.Oid, p, err)
return err
}
// This shouldn't happen but it is sensible to test
- if written != meta.Size {
+ if written != pointer.Size {
if err := s.Delete(p); err != nil {
- log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+ log.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err)
}
- return errSizeMismatch
+ return ErrSizeMismatch
}
return nil
}
// Exists returns true if the object exists in the content store.
-func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
- _, err := s.ObjectStorage.Stat(meta.RelativePath())
+func (s *ContentStore) Exists(pointer Pointer) (bool, error) {
+ _, err := s.ObjectStorage.Stat(pointer.RelativePath())
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
// Verify returns true if the object exists in the content store and size is correct.
-func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
- p := meta.RelativePath()
+func (s *ContentStore) Verify(pointer Pointer) (bool, error) {
+ p := pointer.RelativePath()
fi, err := s.ObjectStorage.Stat(p)
- if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
+ if os.IsNotExist(err) || (err == nil && fi.Size() != pointer.Size) {
return false, nil
} else if err != nil {
- log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
+ log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, pointer.Oid, err)
return false, err
}
return true, nil
}
+// ReadMetaObject will read a models.LFSMetaObject and return a reader
+func ReadMetaObject(pointer Pointer) (io.ReadCloser, error) {
+ contentStore := NewContentStore()
+ return contentStore.Get(pointer)
+}
+
type hashingReader struct {
internal io.Reader
currentSize int64
if err != nil && err == io.EOF {
if r.currentSize != r.expectedSize {
- return n, errSizeMismatch
+ return n, ErrSizeMismatch
}
shaStr := hex.EncodeToString(r.hash.Sum(nil))
if shaStr != r.expectedHash {
- return n, errHashMismatch
+ return n, ErrHashMismatch
}
}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "fmt"
+ "net/url"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+
+ "code.gitea.io/gitea/modules/log"
+)
+
+// DetermineEndpoint determines an endpoint from the clone url or uses the specified LFS url.
+func DetermineEndpoint(cloneurl, lfsurl string) *url.URL {
+ if len(lfsurl) > 0 {
+ return endpointFromURL(lfsurl)
+ }
+ return endpointFromCloneURL(cloneurl)
+}
+
+func endpointFromCloneURL(rawurl string) *url.URL {
+ ep := endpointFromURL(rawurl)
+ if ep == nil {
+ return ep
+ }
+
+ if strings.HasSuffix(ep.Path, "/") {
+ ep.Path = ep.Path[:len(ep.Path)-1]
+ }
+
+ if ep.Scheme == "file" {
+ return ep
+ }
+
+ if path.Ext(ep.Path) == ".git" {
+ ep.Path += "/info/lfs"
+ } else {
+ ep.Path += ".git/info/lfs"
+ }
+
+ return ep
+}
+
+func endpointFromURL(rawurl string) *url.URL {
+ if strings.HasPrefix(rawurl, "/") {
+ return endpointFromLocalPath(rawurl)
+ }
+
+ u, err := url.Parse(rawurl)
+ if err != nil {
+ log.Error("lfs.endpointFromUrl: %v", err)
+ return nil
+ }
+
+ switch u.Scheme {
+ case "http", "https":
+ return u
+ case "git":
+ u.Scheme = "https"
+ return u
+ case "file":
+ return u
+ default:
+ if _, err := os.Stat(rawurl); err == nil {
+ return endpointFromLocalPath(rawurl)
+ }
+
+ log.Error("lfs.endpointFromUrl: unknown url")
+ return nil
+ }
+}
+
+func endpointFromLocalPath(path string) *url.URL {
+ var slash string
+ if abs, err := filepath.Abs(path); err == nil {
+ if !strings.HasPrefix(abs, "/") {
+ slash = "/"
+ }
+ path = abs
+ }
+
+ var gitpath string
+ if filepath.Base(path) == ".git" {
+ gitpath = path
+ path = filepath.Dir(path)
+ } else {
+ gitpath = filepath.Join(path, ".git")
+ }
+
+ if _, err := os.Stat(gitpath); err == nil {
+ path = gitpath
+ } else if _, err := os.Stat(path); err != nil {
+ return nil
+ }
+
+ path = fmt.Sprintf("file://%s%s", slash, filepath.ToSlash(path))
+
+ u, _ := url.Parse(path)
+
+ return u
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "net/url"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func str2url(raw string) *url.URL {
+ u, _ := url.Parse(raw)
+ return u
+}
+
+func TestDetermineEndpoint(t *testing.T) {
+ // Test cases
+ var cases = []struct {
+ cloneurl string
+ lfsurl string
+ expected *url.URL
+ }{
+ // case 0
+ {
+ cloneurl: "",
+ lfsurl: "",
+ expected: nil,
+ },
+ // case 1
+ {
+ cloneurl: "https://git.com/repo",
+ lfsurl: "",
+ expected: str2url("https://git.com/repo.git/info/lfs"),
+ },
+ // case 2
+ {
+ cloneurl: "https://git.com/repo.git",
+ lfsurl: "",
+ expected: str2url("https://git.com/repo.git/info/lfs"),
+ },
+ // case 3
+ {
+ cloneurl: "",
+ lfsurl: "https://gitlfs.com/repo",
+ expected: str2url("https://gitlfs.com/repo"),
+ },
+ // case 4
+ {
+ cloneurl: "https://git.com/repo.git",
+ lfsurl: "https://gitlfs.com/repo",
+ expected: str2url("https://gitlfs.com/repo"),
+ },
+ // case 5
+ {
+ cloneurl: "git://git.com/repo.git",
+ lfsurl: "",
+ expected: str2url("https://git.com/repo.git/info/lfs"),
+ },
+ // case 6
+ {
+ cloneurl: "",
+ lfsurl: "git://gitlfs.com/repo",
+ expected: str2url("https://gitlfs.com/repo"),
+ },
+ }
+
+ for n, c := range cases {
+ ep := DetermineEndpoint(c.cloneurl, c.lfsurl)
+
+ assert.Equal(t, c.expected, ep, "case %d: error should match", n)
+ }
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "context"
+ "io"
+ "net/url"
+ "os"
+ "path/filepath"
+
+ "code.gitea.io/gitea/modules/util"
+)
+
+// FilesystemClient is used to read LFS data from a filesystem path
+type FilesystemClient struct {
+ lfsdir string
+}
+
+func newFilesystemClient(endpoint *url.URL) *FilesystemClient {
+ path, _ := util.FileURLToPath(endpoint)
+
+ lfsdir := filepath.Join(path, "lfs", "objects")
+
+ client := &FilesystemClient{lfsdir}
+
+ return client
+}
+
+func (c *FilesystemClient) objectPath(oid string) string {
+ return filepath.Join(c.lfsdir, oid[0:2], oid[2:4], oid)
+}
+
+// Download reads the specific LFS object from the target repository
+func (c *FilesystemClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) {
+ objectPath := c.objectPath(oid)
+
+ if _, err := os.Stat(objectPath); os.IsNotExist(err) {
+ return nil, err
+ }
+
+ file, err := os.Open(objectPath)
+ if err != nil {
+ return nil, err
+ }
+
+ return file, nil
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+
+ "code.gitea.io/gitea/modules/log"
+)
+
+// HTTPClient is used to communicate with the LFS server
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
+type HTTPClient struct {
+ client *http.Client
+ endpoint string
+ transfers map[string]TransferAdapter
+}
+
+func newHTTPClient(endpoint *url.URL) *HTTPClient {
+ hc := &http.Client{}
+
+ client := &HTTPClient{
+ client: hc,
+ endpoint: strings.TrimSuffix(endpoint.String(), "/"),
+ transfers: make(map[string]TransferAdapter),
+ }
+
+ basic := &BasicTransferAdapter{hc}
+
+ client.transfers[basic.Name()] = basic
+
+ return client
+}
+
+func (c *HTTPClient) transferNames() []string {
+ keys := make([]string, len(c.transfers))
+
+ i := 0
+ for k := range c.transfers {
+ keys[i] = k
+ i++
+ }
+
+ return keys
+}
+
+func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Pointer) (*BatchResponse, error) {
+ url := fmt.Sprintf("%s/objects/batch", c.endpoint)
+
+ request := &BatchRequest{operation, c.transferNames(), nil, objects}
+
+ payload := new(bytes.Buffer)
+ err := json.NewEncoder(payload).Encode(request)
+ if err != nil {
+ return nil, fmt.Errorf("lfs.HTTPClient.batch json.Encode: %w", err)
+ }
+
+ log.Trace("lfs.HTTPClient.batch NewRequestWithContext: %s", url)
+
+ req, err := http.NewRequestWithContext(ctx, "POST", url, payload)
+ if err != nil {
+ return nil, fmt.Errorf("lfs.HTTPClient.batch http.NewRequestWithContext: %w", err)
+ }
+ req.Header.Set("Content-type", MediaType)
+ req.Header.Set("Accept", MediaType)
+
+ res, err := c.client.Do(req)
+ if err != nil {
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ default:
+ }
+ return nil, fmt.Errorf("lfs.HTTPClient.batch http.Do: %w", err)
+ }
+ defer res.Body.Close()
+
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("lfs.HTTPClient.batch: Unexpected servers response: %s", res.Status)
+ }
+
+ var response BatchResponse
+ err = json.NewDecoder(res.Body).Decode(&response)
+ if err != nil {
+ return nil, fmt.Errorf("lfs.HTTPClient.batch json.Decode: %w", err)
+ }
+
+ if len(response.Transfer) == 0 {
+ response.Transfer = "basic"
+ }
+
+ return &response, nil
+}
+
+// Download reads the specific LFS object from the LFS server
+func (c *HTTPClient) Download(ctx context.Context, oid string, size int64) (io.ReadCloser, error) {
+ var objects []Pointer
+ objects = append(objects, Pointer{oid, size})
+
+ result, err := c.batch(ctx, "download", objects)
+ if err != nil {
+ return nil, err
+ }
+
+ transferAdapter, ok := c.transfers[result.Transfer]
+ if !ok {
+ return nil, fmt.Errorf("lfs.HTTPClient.Download Transferadapter not found: %s", result.Transfer)
+ }
+
+ if len(result.Objects) == 0 {
+ return nil, errors.New("lfs.HTTPClient.Download: No objects in result")
+ }
+
+ content, err := transferAdapter.Download(ctx, result.Objects[0])
+ if err != nil {
+ return nil, err
+ }
+ return content, nil
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+type RoundTripFunc func(req *http.Request) *http.Response
+
+func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+ return f(req), nil
+}
+
+type DummyTransferAdapter struct {
+}
+
+func (a *DummyTransferAdapter) Name() string {
+ return "dummy"
+}
+
+func (a *DummyTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) {
+ return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil
+}
+
+func TestHTTPClientDownload(t *testing.T) {
+ oid := "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041"
+ size := int64(6)
+
+ roundTripHandler := func(req *http.Request) *http.Response {
+ url := req.URL.String()
+ if strings.Contains(url, "status-not-ok") {
+ return &http.Response{StatusCode: http.StatusBadRequest}
+ }
+ if strings.Contains(url, "invalid-json-response") {
+ return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))}
+ }
+ if strings.Contains(url, "valid-batch-request-download") {
+ assert.Equal(t, "POST", req.Method)
+ assert.Equal(t, MediaType, req.Header.Get("Content-type"), "case %s: error should match", url)
+ assert.Equal(t, MediaType, req.Header.Get("Accept"), "case %s: error should match", url)
+
+ var batchRequest BatchRequest
+ err := json.NewDecoder(req.Body).Decode(&batchRequest)
+ assert.NoError(t, err)
+
+ assert.Equal(t, "download", batchRequest.Operation)
+ assert.Equal(t, 1, len(batchRequest.Objects))
+ assert.Equal(t, oid, batchRequest.Objects[0].Oid)
+ assert.Equal(t, size, batchRequest.Objects[0].Size)
+
+ batchResponse := &BatchResponse{
+ Transfer: "dummy",
+ Objects: make([]*ObjectResponse, 1),
+ }
+
+ payload := new(bytes.Buffer)
+ json.NewEncoder(payload).Encode(batchResponse)
+
+ return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+ }
+ if strings.Contains(url, "invalid-response-no-objects") {
+ batchResponse := &BatchResponse{Transfer: "dummy"}
+
+ payload := new(bytes.Buffer)
+ json.NewEncoder(payload).Encode(batchResponse)
+
+ return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+ }
+ if strings.Contains(url, "unknown-transfer-adapter") {
+ batchResponse := &BatchResponse{Transfer: "unknown_adapter"}
+
+ payload := new(bytes.Buffer)
+ json.NewEncoder(payload).Encode(batchResponse)
+
+ return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
+ }
+
+ t.Errorf("Unknown test case: %s", url)
+
+ return nil
+ }
+
+ hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)}
+ dummy := &DummyTransferAdapter{}
+
+ var cases = []struct {
+ endpoint string
+ expectederror string
+ }{
+ // case 0
+ {
+ endpoint: "https://status-not-ok.io",
+ expectederror: "Unexpected servers response: ",
+ },
+ // case 1
+ {
+ endpoint: "https://invalid-json-response.io",
+ expectederror: "json.Decode: ",
+ },
+ // case 2
+ {
+ endpoint: "https://valid-batch-request-download.io",
+ expectederror: "",
+ },
+ // case 3
+ {
+ endpoint: "https://invalid-response-no-objects.io",
+ expectederror: "No objects in result",
+ },
+ // case 4
+ {
+ endpoint: "https://unknown-transfer-adapter.io",
+ expectederror: "Transferadapter not found: ",
+ },
+ }
+
+ for n, c := range cases {
+ client := &HTTPClient{
+ client: hc,
+ endpoint: c.endpoint,
+ transfers: make(map[string]TransferAdapter),
+ }
+ client.transfers["dummy"] = dummy
+
+ _, err := client.Download(context.Background(), oid, size)
+ if len(c.expectederror) > 0 {
+ assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+ } else {
+ assert.NoError(t, err, "case %d", n)
+ }
+ }
+}
+++ /dev/null
-// Copyright 2017 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package lfs
-
-import (
- "net/http"
- "strconv"
- "strings"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/context"
- "code.gitea.io/gitea/modules/convert"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/setting"
- api "code.gitea.io/gitea/modules/structs"
- jsoniter "github.com/json-iterator/go"
-)
-
-//checkIsValidRequest check if it a valid request in case of bad request it write the response to ctx.
-func checkIsValidRequest(ctx *context.Context) bool {
- if !setting.LFS.StartServer {
- log.Debug("Attempt to access LFS server but LFS server is disabled")
- writeStatus(ctx, http.StatusNotFound)
- return false
- }
- if !MetaMatcher(ctx.Req) {
- log.Info("Attempt access LOCKs without accepting the correct media type: %s", metaMediaType)
- writeStatus(ctx, http.StatusBadRequest)
- return false
- }
- if !ctx.IsSigned {
- user, _, _, err := parseToken(ctx.Req.Header.Get("Authorization"))
- if err != nil {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- writeStatus(ctx, http.StatusUnauthorized)
- return false
- }
- ctx.User = user
- }
- return true
-}
-
-func handleLockListOut(ctx *context.Context, repo *models.Repository, lock *models.LFSLock, err error) {
- if err != nil {
- if models.IsErrLFSLockNotExist(err) {
- ctx.JSON(http.StatusOK, api.LFSLockList{
- Locks: []*api.LFSLock{},
- })
- return
- }
- ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
- Message: "unable to list locks : Internal Server Error",
- })
- return
- }
- if repo.ID != lock.RepoID {
- ctx.JSON(http.StatusOK, api.LFSLockList{
- Locks: []*api.LFSLock{},
- })
- return
- }
- ctx.JSON(http.StatusOK, api.LFSLockList{
- Locks: []*api.LFSLock{convert.ToLFSLock(lock)},
- })
-}
-
-// GetListLockHandler list locks
-func GetListLockHandler(ctx *context.Context) {
- if !checkIsValidRequest(ctx) {
- // Status is written in checkIsValidRequest
- return
- }
- ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
- rv := unpack(ctx)
-
- repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
- if err != nil {
- log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
- writeStatus(ctx, 404)
- return
- }
- repository.MustOwner()
-
- authenticated := authenticate(ctx, repository, rv.Authorization, false)
- if !authenticated {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
- Message: "You must have pull access to list locks",
- })
- return
- }
-
- cursor := ctx.QueryInt("cursor")
- if cursor < 0 {
- cursor = 0
- }
- limit := ctx.QueryInt("limit")
- if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
- limit = setting.LFS.LocksPagingNum
- } else if limit < 0 {
- limit = 0
- }
- id := ctx.Query("id")
- if id != "" { //Case where we request a specific id
- v, err := strconv.ParseInt(id, 10, 64)
- if err != nil {
- ctx.JSON(http.StatusBadRequest, api.LFSLockError{
- Message: "bad request : " + err.Error(),
- })
- return
- }
- lock, err := models.GetLFSLockByID(v)
- if err != nil && !models.IsErrLFSLockNotExist(err) {
- log.Error("Unable to get lock with ID[%s]: Error: %v", v, err)
- }
- handleLockListOut(ctx, repository, lock, err)
- return
- }
-
- path := ctx.Query("path")
- if path != "" { //Case where we request a specific id
- lock, err := models.GetLFSLock(repository, path)
- if err != nil && !models.IsErrLFSLockNotExist(err) {
- log.Error("Unable to get lock for repository %-v with path %s: Error: %v", repository, path, err)
- }
- handleLockListOut(ctx, repository, lock, err)
- return
- }
-
- //If no query params path or id
- lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
- if err != nil {
- log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
- ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
- Message: "unable to list locks : Internal Server Error",
- })
- return
- }
- lockListAPI := make([]*api.LFSLock, len(lockList))
- next := ""
- for i, l := range lockList {
- lockListAPI[i] = convert.ToLFSLock(l)
- }
- if limit > 0 && len(lockList) == limit {
- next = strconv.Itoa(cursor + 1)
- }
- ctx.JSON(http.StatusOK, api.LFSLockList{
- Locks: lockListAPI,
- Next: next,
- })
-}
-
-// PostLockHandler create lock
-func PostLockHandler(ctx *context.Context) {
- if !checkIsValidRequest(ctx) {
- // Status is written in checkIsValidRequest
- return
- }
- ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
- userName := ctx.Params("username")
- repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
- authorization := ctx.Req.Header.Get("Authorization")
-
- repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
- if err != nil {
- log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
- writeStatus(ctx, 404)
- return
- }
- repository.MustOwner()
-
- authenticated := authenticate(ctx, repository, authorization, true)
- if !authenticated {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
- Message: "You must have push access to create locks",
- })
- return
- }
-
- var req api.LFSLockRequest
- bodyReader := ctx.Req.Body
- defer bodyReader.Close()
- json := jsoniter.ConfigCompatibleWithStandardLibrary
- dec := json.NewDecoder(bodyReader)
- if err := dec.Decode(&req); err != nil {
- log.Warn("Failed to decode lock request as json. Error: %v", err)
- writeStatus(ctx, 400)
- return
- }
-
- lock, err := models.CreateLFSLock(&models.LFSLock{
- Repo: repository,
- Path: req.Path,
- Owner: ctx.User,
- })
- if err != nil {
- if models.IsErrLFSLockAlreadyExist(err) {
- ctx.JSON(http.StatusConflict, api.LFSLockError{
- Lock: convert.ToLFSLock(lock),
- Message: "already created lock",
- })
- return
- }
- if models.IsErrLFSUnauthorizedAction(err) {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
- Message: "You must have push access to create locks : " + err.Error(),
- })
- return
- }
- log.Error("Unable to CreateLFSLock in repository %-v at %s for user %-v: Error: %v", repository, req.Path, ctx.User, err)
- ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
- Message: "internal server error : Internal Server Error",
- })
- return
- }
- ctx.JSON(http.StatusCreated, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
-}
-
-// VerifyLockHandler list locks for verification
-func VerifyLockHandler(ctx *context.Context) {
- if !checkIsValidRequest(ctx) {
- // Status is written in checkIsValidRequest
- return
- }
- ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
- userName := ctx.Params("username")
- repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
- authorization := ctx.Req.Header.Get("Authorization")
-
- repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
- if err != nil {
- log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
- writeStatus(ctx, 404)
- return
- }
- repository.MustOwner()
-
- authenticated := authenticate(ctx, repository, authorization, true)
- if !authenticated {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
- Message: "You must have push access to verify locks",
- })
- return
- }
-
- cursor := ctx.QueryInt("cursor")
- if cursor < 0 {
- cursor = 0
- }
- limit := ctx.QueryInt("limit")
- if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
- limit = setting.LFS.LocksPagingNum
- } else if limit < 0 {
- limit = 0
- }
- lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
- if err != nil {
- log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
- ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
- Message: "unable to list locks : Internal Server Error",
- })
- return
- }
- next := ""
- if limit > 0 && len(lockList) == limit {
- next = strconv.Itoa(cursor + 1)
- }
- lockOursListAPI := make([]*api.LFSLock, 0, len(lockList))
- lockTheirsListAPI := make([]*api.LFSLock, 0, len(lockList))
- for _, l := range lockList {
- if l.Owner.ID == ctx.User.ID {
- lockOursListAPI = append(lockOursListAPI, convert.ToLFSLock(l))
- } else {
- lockTheirsListAPI = append(lockTheirsListAPI, convert.ToLFSLock(l))
- }
- }
- ctx.JSON(http.StatusOK, api.LFSLockListVerify{
- Ours: lockOursListAPI,
- Theirs: lockTheirsListAPI,
- Next: next,
- })
-}
-
-// UnLockHandler delete locks
-func UnLockHandler(ctx *context.Context) {
- if !checkIsValidRequest(ctx) {
- // Status is written in checkIsValidRequest
- return
- }
- ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
- userName := ctx.Params("username")
- repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
- authorization := ctx.Req.Header.Get("Authorization")
-
- repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
- if err != nil {
- log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
- writeStatus(ctx, 404)
- return
- }
- repository.MustOwner()
-
- authenticated := authenticate(ctx, repository, authorization, true)
- if !authenticated {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
- Message: "You must have push access to delete locks",
- })
- return
- }
-
- var req api.LFSLockDeleteRequest
- bodyReader := ctx.Req.Body
- defer bodyReader.Close()
- json := jsoniter.ConfigCompatibleWithStandardLibrary
- dec := json.NewDecoder(bodyReader)
- if err := dec.Decode(&req); err != nil {
- log.Warn("Failed to decode lock request as json. Error: %v", err)
- writeStatus(ctx, 400)
- return
- }
-
- lock, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, req.Force)
- if err != nil {
- if models.IsErrLFSUnauthorizedAction(err) {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
- Message: "You must have push access to delete locks : " + err.Error(),
- })
- return
- }
- log.Error("Unable to DeleteLFSLockByID[%d] by user %-v with force %t: Error: %v", ctx.ParamsInt64("lid"), ctx.User, req.Force, err)
- ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
- Message: "unable to delete lock : Internal Server Error",
- })
- return
- }
- ctx.JSON(http.StatusOK, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
-}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+const (
+ blobSizeCutoff = 1024
+
+ // MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
+ // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
+ MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+
+ // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
+ MetaFileOidPrefix = "oid sha256:"
+)
+
+var (
+ // ErrMissingPrefix occurs if the content lacks the LFS prefix
+ ErrMissingPrefix = errors.New("Content lacks the LFS prefix")
+
+ // ErrInvalidStructure occurs if the content has an invalid structure
+ ErrInvalidStructure = errors.New("Content has an invalid structure")
+
+ // ErrInvalidOIDFormat occurs if the oid has an invalid format
+ ErrInvalidOIDFormat = errors.New("OID has an invalid format")
+)
+
+// ReadPointer tries to read LFS pointer data from the reader
+func ReadPointer(reader io.Reader) (Pointer, error) {
+ buf := make([]byte, blobSizeCutoff)
+ n, err := io.ReadFull(reader, buf)
+ if err != nil && err != io.ErrUnexpectedEOF {
+ return Pointer{}, err
+ }
+ buf = buf[:n]
+
+ return ReadPointerFromBuffer(buf)
+}
+
+var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`)
+
+// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise.
+func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
+ var p Pointer
+
+ headString := string(buf)
+ if !strings.HasPrefix(headString, MetaFileIdentifier) {
+ return p, ErrMissingPrefix
+ }
+
+ splitLines := strings.Split(headString, "\n")
+ if len(splitLines) < 3 {
+ return p, ErrInvalidStructure
+ }
+
+ oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
+ if len(oid) != 64 || !oidPattern.MatchString(oid) {
+ return p, ErrInvalidOIDFormat
+ }
+ size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+ if err != nil {
+ return p, err
+ }
+
+ p.Oid = oid
+ p.Size = size
+
+ return p, nil
+}
+
+// IsValid checks if the pointer has a valid structure.
+// It doesn't check if the pointed-to-content exists.
+func (p Pointer) IsValid() bool {
+ if len(p.Oid) != 64 {
+ return false
+ }
+ if !oidPattern.MatchString(p.Oid) {
+ return false
+ }
+ if p.Size < 0 {
+ return false
+ }
+ return true
+}
+
+// StringContent returns the string representation of the pointer
+// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer
+func (p Pointer) StringContent() string {
+ return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size)
+}
+
+// RelativePath returns the relative storage path of the pointer
+func (p Pointer) RelativePath() string {
+ if len(p.Oid) < 5 {
+ return p.Oid
+ }
+
+ return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
+}
+
+// GeneratePointer generates a pointer for arbitrary content
+func GeneratePointer(content io.Reader) (Pointer, error) {
+ h := sha256.New()
+ c, err := io.Copy(h, content)
+ if err != nil {
+ return Pointer{}, err
+ }
+ sum := h.Sum(nil)
+ return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build gogit
+
+package lfs
+
+import (
+ "context"
+ "fmt"
+
+ "code.gitea.io/gitea/modules/git"
+
+ "github.com/go-git/go-git/v5/plumbing/object"
+)
+
+// SearchPointerBlobs scans the whole repository for LFS pointer files
+func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) {
+ gitRepo := repo.GoGitRepo()
+
+ err := func() error {
+ blobs, err := gitRepo.BlobObjects()
+ if err != nil {
+ return fmt.Errorf("lfs.SearchPointerBlobs BlobObjects: %w", err)
+ }
+
+ return blobs.ForEach(func(blob *object.Blob) error {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ }
+
+ if blob.Size > blobSizeCutoff {
+ return nil
+ }
+
+ reader, err := blob.Reader()
+ if err != nil {
+ return fmt.Errorf("lfs.SearchPointerBlobs blob.Reader: %w", err)
+ }
+ defer reader.Close()
+
+ pointer, _ := ReadPointer(reader)
+ if pointer.IsValid() {
+ pointerChan <- PointerBlob{Hash: blob.Hash.String(), Pointer: pointer}
+ }
+
+ return nil
+ })
+ }()
+
+ if err != nil {
+ select {
+ case <-ctx.Done():
+ default:
+ errChan <- err
+ }
+ }
+
+ close(pointerChan)
+ close(errChan)
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !gogit
+
+package lfs
+
+import (
+ "bufio"
+ "context"
+ "io"
+ "strconv"
+ "sync"
+
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/git/pipeline"
+)
+
+// SearchPointerBlobs scans the whole repository for LFS pointer files
+func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) {
+ basePath := repo.Path
+
+ catFileCheckReader, catFileCheckWriter := io.Pipe()
+ shasToBatchReader, shasToBatchWriter := io.Pipe()
+ catFileBatchReader, catFileBatchWriter := io.Pipe()
+
+ wg := sync.WaitGroup{}
+ wg.Add(4)
+
+ // Create the go-routines in reverse order.
+
+ // 4. Take the output of cat-file --batch and check if each file in turn
+ // to see if they're pointers to files in the LFS store
+ go createPointerResultsFromCatFileBatch(ctx, catFileBatchReader, &wg, pointerChan)
+
+ // 3. Take the shas of the blobs and batch read them
+ go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
+
+ // 2. From the provided objects restrict to blobs <=1k
+ go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
+
+ // 1. Run batch-check on all objects in the repository
+ if git.CheckGitVersionAtLeast("2.6.0") != nil {
+ revListReader, revListWriter := io.Pipe()
+ shasToCheckReader, shasToCheckWriter := io.Pipe()
+ wg.Add(2)
+ go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
+ go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
+ go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
+ } else {
+ go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
+ }
+ wg.Wait()
+
+ close(pointerChan)
+ close(errChan)
+}
+
+func createPointerResultsFromCatFileBatch(ctx context.Context, catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) {
+ defer wg.Done()
+ defer catFileBatchReader.Close()
+
+ bufferedReader := bufio.NewReader(catFileBatchReader)
+ buf := make([]byte, 1025)
+
+loop:
+ for {
+ select {
+ case <-ctx.Done():
+ break loop
+ default:
+ }
+
+ // File descriptor line: sha
+ sha, err := bufferedReader.ReadString(' ')
+ if err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ // Throw away the blob
+ if _, err := bufferedReader.ReadString(' '); err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ sizeStr, err := bufferedReader.ReadString('\n')
+ if err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
+ if err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ pointerBuf := buf[:size+1]
+ if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
+ _ = catFileBatchReader.CloseWithError(err)
+ break
+ }
+ pointerBuf = pointerBuf[:size]
+ // Now we need to check if the pointerBuf is an LFS pointer
+ pointer, _ := ReadPointerFromBuffer(pointerBuf)
+ if !pointer.IsValid() {
+ continue
+ }
+
+ pointerChan <- PointerBlob{Hash: sha, Pointer: pointer}
+ }
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "path"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestStringContent(t *testing.T) {
+ p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393", Size: 1234}
+ expected := "version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"
+ assert.Equal(t, p.StringContent(), expected)
+}
+
+func TestRelativePath(t *testing.T) {
+ p := Pointer{Oid: "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393"}
+ expected := path.Join("4d", "7a", "214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+ assert.Equal(t, p.RelativePath(), expected)
+
+ p2 := Pointer{Oid: "4d7a"}
+ assert.Equal(t, p2.RelativePath(), "4d7a")
+}
+
+func TestIsValid(t *testing.T) {
+ p := Pointer{}
+ assert.False(t, p.IsValid())
+
+ p = Pointer{Oid: "123"}
+ assert.False(t, p.IsValid())
+
+ p = Pointer{Oid: "z4cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"}
+ assert.False(t, p.IsValid())
+
+ p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc"}
+ assert.True(t, p.IsValid())
+
+ p = Pointer{Oid: "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc", Size: -1}
+ assert.False(t, p.IsValid())
+}
+
+func TestGeneratePointer(t *testing.T) {
+ p, err := GeneratePointer(strings.NewReader("Gitea"))
+ assert.NoError(t, err)
+ assert.True(t, p.IsValid())
+ assert.Equal(t, p.Oid, "94cb57646c54a297c9807697e80a30946f79a4b82cb079d2606847825b1812cc")
+ assert.Equal(t, p.Size, int64(5))
+}
+
+func TestReadPointerFromBuffer(t *testing.T) {
+ p, err := ReadPointerFromBuffer([]byte{})
+ assert.ErrorIs(t, err, ErrMissingPrefix)
+ assert.False(t, p.IsValid())
+
+ p, err = ReadPointerFromBuffer([]byte("test"))
+ assert.ErrorIs(t, err, ErrMissingPrefix)
+ assert.False(t, p.IsValid())
+
+ p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\n"))
+ assert.ErrorIs(t, err, ErrInvalidStructure)
+ assert.False(t, p.IsValid())
+
+ p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a\nsize 1234\n"))
+ assert.ErrorIs(t, err, ErrInvalidOIDFormat)
+ assert.False(t, p.IsValid())
+
+ p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a2146z4ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"))
+ assert.ErrorIs(t, err, ErrInvalidOIDFormat)
+ assert.False(t, p.IsValid())
+
+ p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\ntest 1234\n"))
+ assert.Error(t, err)
+ assert.False(t, p.IsValid())
+
+ p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize test\n"))
+ assert.Error(t, err)
+ assert.False(t, p.IsValid())
+
+ p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"))
+ assert.NoError(t, err)
+ assert.True(t, p.IsValid())
+ assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+ assert.Equal(t, p.Size, int64(1234))
+
+ p, err = ReadPointerFromBuffer([]byte("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\ntest"))
+ assert.NoError(t, err)
+ assert.True(t, p.IsValid())
+ assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+ assert.Equal(t, p.Size, int64(1234))
+}
+
+func TestReadPointer(t *testing.T) {
+ p, err := ReadPointer(strings.NewReader("version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 1234\n"))
+ assert.NoError(t, err)
+ assert.True(t, p.IsValid())
+ assert.Equal(t, p.Oid, "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393")
+ assert.Equal(t, p.Size, int64(1234))
+}
+++ /dev/null
-// Copyright 2019 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package lfs
-
-import (
- "io"
- "strconv"
- "strings"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/base"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
-)
-
-// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
-func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) {
- if !setting.LFS.StartServer {
- return nil, nil
- }
-
- buf := make([]byte, 1024)
- n, _ := reader.Read(buf)
- buf = buf[:n]
-
- if isTextFile := base.IsTextFile(buf); !isTextFile {
- return nil, nil
- }
-
- return IsPointerFile(&buf), &buf
-}
-
-// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file
-func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
- if !setting.LFS.StartServer {
- return nil
- }
-
- headString := string(*buf)
- if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) {
- return nil
- }
-
- splitLines := strings.Split(headString, "\n")
- if len(splitLines) < 3 {
- return nil
- }
-
- oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix)
- size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
- if len(oid) != 64 || err != nil {
- return nil
- }
-
- contentStore := &ContentStore{ObjectStorage: storage.LFS}
- meta := &models.LFSMetaObject{Oid: oid, Size: size}
- exist, err := contentStore.Exists(meta)
- if err != nil || !exist {
- return nil
- }
-
- return meta
-}
-
-// ReadMetaObject will read a models.LFSMetaObject and return a reader
-func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
- contentStore := &ContentStore{ObjectStorage: storage.LFS}
- return contentStore.Get(meta)
-}
+++ /dev/null
-// Copyright 2020 The Gitea Authors. All rights reserved.
-// Use of this source code is governed by a MIT-style
-// license that can be found in the LICENSE file.
-
-package lfs
-
-import (
- "encoding/base64"
- "fmt"
- "io"
- "net/http"
- "path"
- "regexp"
- "strconv"
- "strings"
- "time"
-
- "code.gitea.io/gitea/models"
- "code.gitea.io/gitea/modules/context"
- "code.gitea.io/gitea/modules/log"
- "code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
-
- "github.com/dgrijalva/jwt-go"
- jsoniter "github.com/json-iterator/go"
-)
-
-const (
- metaMediaType = "application/vnd.git-lfs+json"
-)
-
-// RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and
-// some headers are stored.
-type RequestVars struct {
- Oid string
- Size int64
- User string
- Password string
- Repo string
- Authorization string
-}
-
-// BatchVars contains multiple RequestVars processed in one batch operation.
-// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
-type BatchVars struct {
- Transfers []string `json:"transfers,omitempty"`
- Operation string `json:"operation"`
- Objects []*RequestVars `json:"objects"`
-}
-
-// BatchResponse contains multiple object metadata Representation structures
-// for use with the batch API.
-type BatchResponse struct {
- Transfer string `json:"transfer,omitempty"`
- Objects []*Representation `json:"objects"`
-}
-
-// Representation is object metadata as seen by clients of the lfs server.
-type Representation struct {
- Oid string `json:"oid"`
- Size int64 `json:"size"`
- Actions map[string]*link `json:"actions"`
- Error *ObjectError `json:"error,omitempty"`
-}
-
-// ObjectError defines the JSON structure returned to the client in case of an error
-type ObjectError struct {
- Code int `json:"code"`
- Message string `json:"message"`
-}
-
-// Claims is a JWT Token Claims
-type Claims struct {
- RepoID int64
- Op string
- UserID int64
- jwt.StandardClaims
-}
-
-// ObjectLink builds a URL linking to the object.
-func (v *RequestVars) ObjectLink() string {
- return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/objects", v.Oid)
-}
-
-// VerifyLink builds a URL for verifying the object.
-func (v *RequestVars) VerifyLink() string {
- return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify")
-}
-
-// link provides a structure used to build a hypermedia representation of an HTTP link.
-type link struct {
- Href string `json:"href"`
- Header map[string]string `json:"header,omitempty"`
- ExpiresAt time.Time `json:"expires_at,omitempty"`
-}
-
-var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`)
-
-func isOidValid(oid string) bool {
- return oidRegExp.MatchString(oid)
-}
-
-// ObjectOidHandler is the main request routing entry point into LFS server functions
-func ObjectOidHandler(ctx *context.Context) {
- if !setting.LFS.StartServer {
- log.Debug("Attempt to access LFS server but LFS server is disabled")
- writeStatus(ctx, 404)
- return
- }
-
- if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" {
- if MetaMatcher(ctx.Req) {
- getMetaHandler(ctx)
- return
- }
-
- getContentHandler(ctx)
- return
- } else if ctx.Req.Method == "PUT" {
- PutHandler(ctx)
- return
- }
-
- log.Warn("Unhandled LFS method: %s for %s/%s OID[%s]", ctx.Req.Method, ctx.Params("username"), ctx.Params("reponame"), ctx.Params("oid"))
- writeStatus(ctx, 404)
-}
-
-func getAuthenticatedRepoAndMeta(ctx *context.Context, rv *RequestVars, requireWrite bool) (*models.LFSMetaObject, *models.Repository) {
- if !isOidValid(rv.Oid) {
- log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", rv.Oid, rv.User, rv.Repo)
- writeStatus(ctx, 404)
- return nil, nil
- }
-
- repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
- if err != nil {
- log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err)
- writeStatus(ctx, 404)
- return nil, nil
- }
-
- if !authenticate(ctx, repository, rv.Authorization, requireWrite) {
- requireAuth(ctx)
- return nil, nil
- }
-
- meta, err := repository.GetLFSMetaObjectByOid(rv.Oid)
- if err != nil {
- log.Error("Unable to get LFS OID[%s] Error: %v", rv.Oid, err)
- writeStatus(ctx, 404)
- return nil, nil
- }
-
- return meta, repository
-}
-
-// getContentHandler gets the content from the content store
-func getContentHandler(ctx *context.Context) {
- rv := unpack(ctx)
-
- meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
- if meta == nil {
- // Status already written in getAuthenticatedRepoAndMeta
- return
- }
-
- // Support resume download using Range header
- var fromByte, toByte int64
- toByte = meta.Size - 1
- statusCode := 200
- if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" {
- regex := regexp.MustCompile(`bytes=(\d+)\-(\d*).*`)
- match := regex.FindStringSubmatch(rangeHdr)
- if len(match) > 1 {
- statusCode = 206
- fromByte, _ = strconv.ParseInt(match[1], 10, 32)
-
- if fromByte >= meta.Size {
- writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
- return
- }
-
- if match[2] != "" {
- _toByte, _ := strconv.ParseInt(match[2], 10, 32)
- if _toByte >= fromByte && _toByte < toByte {
- toByte = _toByte
- }
- }
-
- ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, toByte, meta.Size-fromByte))
- ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Range")
- }
- }
-
- contentStore := &ContentStore{ObjectStorage: storage.LFS}
- content, err := contentStore.Get(meta)
- if err != nil {
- // Errors are logged in contentStore.Get
- writeStatus(ctx, http.StatusNotFound)
- return
- }
- defer content.Close()
-
- if fromByte > 0 {
- _, err = content.Seek(fromByte, io.SeekStart)
- if err != nil {
- log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
-
- writeStatus(ctx, http.StatusInternalServerError)
- return
- }
- }
-
- contentLength := toByte + 1 - fromByte
- ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
- ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
-
- filename := ctx.Params("filename")
- if len(filename) > 0 {
- decodedFilename, err := base64.RawURLEncoding.DecodeString(filename)
- if err == nil {
- ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"")
- ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
- }
- }
-
- ctx.Resp.WriteHeader(statusCode)
- if written, err := io.CopyN(ctx.Resp, content, contentLength); err != nil {
- log.Error("Error whilst copying LFS OID[%s] to the response after %d bytes. Error: %v", meta.Oid, written, err)
- }
- logRequest(ctx.Req, statusCode)
-}
-
-// getMetaHandler retrieves metadata about the object
-func getMetaHandler(ctx *context.Context) {
- rv := unpack(ctx)
-
- meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
- if meta == nil {
- // Status already written in getAuthenticatedRepoAndMeta
- return
- }
-
- ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
- if ctx.Req.Method == "GET" {
- json := jsoniter.ConfigCompatibleWithStandardLibrary
- enc := json.NewEncoder(ctx.Resp)
- if err := enc.Encode(Represent(rv, meta, true, false)); err != nil {
- log.Error("Failed to encode representation as json. Error: %v", err)
- }
- }
-
- logRequest(ctx.Req, 200)
-}
-
-// PostHandler instructs the client how to upload data
-func PostHandler(ctx *context.Context) {
- if !setting.LFS.StartServer {
- log.Debug("Attempt to access LFS server but LFS server is disabled")
- writeStatus(ctx, 404)
- return
- }
-
- if !MetaMatcher(ctx.Req) {
- log.Info("Attempt to POST without accepting the correct media type: %s", metaMediaType)
- writeStatus(ctx, 400)
- return
- }
-
- rv := unpack(ctx)
-
- repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
- if err != nil {
- log.Error("Unable to get repository: %s/%s Error: %v", rv.User, rv.Repo, err)
- writeStatus(ctx, 404)
- return
- }
-
- if !authenticate(ctx, repository, rv.Authorization, true) {
- requireAuth(ctx)
- return
- }
-
- if !isOidValid(rv.Oid) {
- log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", rv.Oid, rv.User, rv.Repo)
- writeStatus(ctx, 404)
- return
- }
-
- if setting.LFS.MaxFileSize > 0 && rv.Size > setting.LFS.MaxFileSize {
- log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", rv.Oid, rv.Size, rv.User, rv.Repo, setting.LFS.MaxFileSize)
- writeStatus(ctx, 413)
- return
- }
-
- meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID})
- if err != nil {
- log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", rv.Oid, rv.Size, rv.User, rv.Repo, err)
- writeStatus(ctx, 404)
- return
- }
-
- ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
- sentStatus := 202
- contentStore := &ContentStore{ObjectStorage: storage.LFS}
- exist, err := contentStore.Exists(meta)
- if err != nil {
- log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", rv.Oid, rv.User, rv.Repo, err)
- writeStatus(ctx, 500)
- return
- }
- if meta.Existing && exist {
- sentStatus = 200
- }
- ctx.Resp.WriteHeader(sentStatus)
-
- json := jsoniter.ConfigCompatibleWithStandardLibrary
- enc := json.NewEncoder(ctx.Resp)
- if err := enc.Encode(Represent(rv, meta, meta.Existing, true)); err != nil {
- log.Error("Failed to encode representation as json. Error: %v", err)
- }
- logRequest(ctx.Req, sentStatus)
-}
-
-// BatchHandler provides the batch api
-func BatchHandler(ctx *context.Context) {
- if !setting.LFS.StartServer {
- log.Debug("Attempt to access LFS server but LFS server is disabled")
- writeStatus(ctx, 404)
- return
- }
-
- if !MetaMatcher(ctx.Req) {
- log.Info("Attempt to BATCH without accepting the correct media type: %s", metaMediaType)
- writeStatus(ctx, 400)
- return
- }
-
- bv := unpackbatch(ctx)
-
- var responseObjects []*Representation
-
- // Create a response object
- for _, object := range bv.Objects {
- if !isOidValid(object.Oid) {
- log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, object.User, object.Repo)
- continue
- }
-
- repository, err := models.GetRepositoryByOwnerAndName(object.User, object.Repo)
- if err != nil {
- log.Error("Unable to get repository: %s/%s Error: %v", object.User, object.Repo, err)
- writeStatus(ctx, 404)
- return
- }
-
- requireWrite := false
- if bv.Operation == "upload" {
- requireWrite = true
- }
-
- if !authenticate(ctx, repository, object.Authorization, requireWrite) {
- requireAuth(ctx)
- return
- }
-
- contentStore := &ContentStore{ObjectStorage: storage.LFS}
-
- meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
- if err == nil { // Object is found and exists
- exist, err := contentStore.Exists(meta)
- if err != nil {
- log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
- writeStatus(ctx, 500)
- return
- }
- if exist {
- responseObjects = append(responseObjects, Represent(object, meta, true, false))
- continue
- }
- }
-
- if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
- log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, object.User, object.Repo, setting.LFS.MaxFileSize)
- writeStatus(ctx, 413)
- return
- }
-
- // Object is not found
- meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
- if err == nil {
- exist, err := contentStore.Exists(meta)
- if err != nil {
- log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, object.User, object.Repo, err)
- writeStatus(ctx, 500)
- return
- }
- responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !exist))
- } else {
- log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, object.User, object.Repo, err)
- }
- }
-
- ctx.Resp.Header().Set("Content-Type", metaMediaType)
-
- respobj := &BatchResponse{Objects: responseObjects}
-
- json := jsoniter.ConfigCompatibleWithStandardLibrary
- enc := json.NewEncoder(ctx.Resp)
- if err := enc.Encode(respobj); err != nil {
- log.Error("Failed to encode representation as json. Error: %v", err)
- }
- logRequest(ctx.Req, 200)
-}
-
-// PutHandler receives data from the client and puts it into the content store
-func PutHandler(ctx *context.Context) {
- rv := unpack(ctx)
-
- meta, repository := getAuthenticatedRepoAndMeta(ctx, rv, true)
- if meta == nil {
- // Status already written in getAuthenticatedRepoAndMeta
- return
- }
-
- contentStore := &ContentStore{ObjectStorage: storage.LFS}
- defer ctx.Req.Body.Close()
- if err := contentStore.Put(meta, ctx.Req.Body); err != nil {
- // Put will log the error itself
- ctx.Resp.WriteHeader(500)
- if err == errSizeMismatch || err == errHashMismatch {
- fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
- } else {
- fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
- }
- if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
- log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", rv.Oid, err)
- }
- return
- }
-
- logRequest(ctx.Req, 200)
-}
-
-// VerifyHandler verify oid and its size from the content store
-func VerifyHandler(ctx *context.Context) {
- if !setting.LFS.StartServer {
- log.Debug("Attempt to access LFS server but LFS server is disabled")
- writeStatus(ctx, 404)
- return
- }
-
- if !MetaMatcher(ctx.Req) {
- log.Info("Attempt to VERIFY without accepting the correct media type: %s", metaMediaType)
- writeStatus(ctx, 400)
- return
- }
-
- rv := unpack(ctx)
-
- meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, true)
- if meta == nil {
- // Status already written in getAuthenticatedRepoAndMeta
- return
- }
-
- contentStore := &ContentStore{ObjectStorage: storage.LFS}
- ok, err := contentStore.Verify(meta)
- if err != nil {
- // Error will be logged in Verify
- ctx.Resp.WriteHeader(500)
- fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
- return
- }
- if !ok {
- writeStatus(ctx, 422)
- return
- }
-
- logRequest(ctx.Req, 200)
-}
-
-// Represent takes a RequestVars and Meta and turns it into a Representation suitable
-// for json encoding
-func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation {
- rep := &Representation{
- Oid: meta.Oid,
- Size: meta.Size,
- Actions: make(map[string]*link),
- }
-
- header := make(map[string]string)
-
- if rv.Authorization == "" {
- //https://github.com/github/git-lfs/issues/1088
- header["Authorization"] = "Authorization: Basic dummy"
- } else {
- header["Authorization"] = rv.Authorization
- }
-
- if download {
- rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header}
- }
-
- if upload {
- rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header}
- }
-
- if upload && !download {
- // Force client side verify action while gitea lacks proper server side verification
- verifyHeader := make(map[string]string)
- for k, v := range header {
- verifyHeader[k] = v
- }
-
- // This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662
- verifyHeader["Accept"] = metaMediaType
-
- rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: verifyHeader}
- }
-
- return rep
-}
-
-// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain
-// an Accept header with the metaMediaType
-func MetaMatcher(r *http.Request) bool {
- mediaParts := strings.Split(r.Header.Get("Accept"), ";")
- mt := mediaParts[0]
- return mt == metaMediaType
-}
-
-func unpack(ctx *context.Context) *RequestVars {
- r := ctx.Req
- rv := &RequestVars{
- User: ctx.Params("username"),
- Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"),
- Oid: ctx.Params("oid"),
- Authorization: r.Header.Get("Authorization"),
- }
-
- if r.Method == "POST" { // Maybe also check if +json
- var p RequestVars
- bodyReader := r.Body
- defer bodyReader.Close()
- json := jsoniter.ConfigCompatibleWithStandardLibrary
- dec := json.NewDecoder(bodyReader)
- err := dec.Decode(&p)
- if err != nil {
- // The error is logged as a WARN here because this may represent misbehaviour rather than a true error
- log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", rv.Oid, rv.User, rv.Repo, err)
- return rv
- }
-
- rv.Oid = p.Oid
- rv.Size = p.Size
- }
-
- return rv
-}
-
-// TODO cheap hack, unify with unpack
-func unpackbatch(ctx *context.Context) *BatchVars {
-
- r := ctx.Req
- var bv BatchVars
-
- bodyReader := r.Body
- defer bodyReader.Close()
- json := jsoniter.ConfigCompatibleWithStandardLibrary
- dec := json.NewDecoder(bodyReader)
- err := dec.Decode(&bv)
- if err != nil {
- // The error is logged as a WARN here because this may represent misbehaviour rather than a true error
- log.Warn("Unable to decode BATCH request vars in %s/%s: Error: %v", ctx.Params("username"), strings.TrimSuffix(ctx.Params("reponame"), ".git"), err)
- return &bv
- }
-
- for i := 0; i < len(bv.Objects); i++ {
- bv.Objects[i].User = ctx.Params("username")
- bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git")
- bv.Objects[i].Authorization = r.Header.Get("Authorization")
- }
-
- return &bv
-}
-
-func writeStatus(ctx *context.Context, status int) {
- message := http.StatusText(status)
-
- mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";")
- mt := mediaParts[0]
- if strings.HasSuffix(mt, "+json") {
- message = `{"message":"` + message + `"}`
- }
-
- ctx.Resp.WriteHeader(status)
- fmt.Fprint(ctx.Resp, message)
- logRequest(ctx.Req, status)
-}
-
-func logRequest(r *http.Request, status int) {
- log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status)
-}
-
-// authenticate uses the authorization string to determine whether
-// or not to proceed. This server assumes an HTTP Basic auth format.
-func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool {
- accessMode := models.AccessModeRead
- if requireWrite {
- accessMode = models.AccessModeWrite
- }
-
- // ctx.IsSigned is unnecessary here, this will be checked in perm.CanAccess
- perm, err := models.GetUserRepoPermission(repository, ctx.User)
- if err != nil {
- log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
- return false
- }
-
- canRead := perm.CanAccess(accessMode, models.UnitTypeCode)
- if canRead {
- return true
- }
-
- user, repo, opStr, err := parseToken(authorization)
- if err != nil {
- // Most of these are Warn level - the true internal server errors are logged in parseToken already
- log.Warn("Authentication failure for provided token with Error: %v", err)
- return false
- }
- ctx.User = user
- if opStr == "basic" {
- perm, err = models.GetUserRepoPermission(repository, ctx.User)
- if err != nil {
- log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
- return false
- }
- return perm.CanAccess(accessMode, models.UnitTypeCode)
- }
- if repository.ID == repo.ID {
- if requireWrite && opStr != "upload" {
- return false
- }
- return true
- }
- return false
-}
-
-func parseToken(authorization string) (*models.User, *models.Repository, string, error) {
- if authorization == "" {
- return nil, nil, "unknown", fmt.Errorf("No token")
- }
- if strings.HasPrefix(authorization, "Bearer ") {
- token, err := jwt.ParseWithClaims(authorization[7:], &Claims{}, func(t *jwt.Token) (interface{}, error) {
- if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
- return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
- }
- return setting.LFS.JWTSecretBytes, nil
- })
- if err != nil {
- // The error here is WARN level because it is caused by bad authorization rather than an internal server error
- return nil, nil, "unknown", err
- }
- claims, claimsOk := token.Claims.(*Claims)
- if !token.Valid || !claimsOk {
- return nil, nil, "unknown", fmt.Errorf("Token claim invalid")
- }
- r, err := models.GetRepositoryByID(claims.RepoID)
- if err != nil {
- log.Error("Unable to GetRepositoryById[%d]: Error: %v", claims.RepoID, err)
- return nil, nil, claims.Op, err
- }
- u, err := models.GetUserByID(claims.UserID)
- if err != nil {
- log.Error("Unable to GetUserById[%d]: Error: %v", claims.UserID, err)
- return nil, r, claims.Op, err
- }
- return u, r, claims.Op, nil
- }
-
- if strings.HasPrefix(authorization, "Basic ") {
- c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic "))
- if err != nil {
- return nil, nil, "basic", err
- }
- cs := string(c)
- i := strings.IndexByte(cs, ':')
- if i < 0 {
- return nil, nil, "basic", fmt.Errorf("Basic auth invalid")
- }
- user, password := cs[:i], cs[i+1:]
- u, err := models.GetUserByName(user)
- if err != nil {
- log.Error("Unable to GetUserByName[%d]: Error: %v", user, err)
- return nil, nil, "basic", err
- }
- if !u.IsPasswordSet() || !u.ValidatePassword(password) {
- return nil, nil, "basic", fmt.Errorf("Basic auth failed")
- }
- return u, nil, "basic", nil
- }
-
- return nil, nil, "unknown", fmt.Errorf("Token not found")
-}
-
-func requireAuth(ctx *context.Context) {
- ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
- writeStatus(ctx, 401)
-}
--- /dev/null
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "time"
+)
+
+const (
+ // MediaType contains the media type for LFS server requests
+ MediaType = "application/vnd.git-lfs+json"
+)
+
+// BatchRequest contains multiple requests processed in one batch operation.
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#requests
+type BatchRequest struct {
+ Operation string `json:"operation"`
+ Transfers []string `json:"transfers,omitempty"`
+ Ref *Reference `json:"ref,omitempty"`
+ Objects []Pointer `json:"objects"`
+}
+
+// Reference contains a git reference.
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#ref-property
+type Reference struct {
+ Name string `json:"name"`
+}
+
+// Pointer contains LFS pointer data
+type Pointer struct {
+ Oid string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"`
+ Size int64 `json:"size" xorm:"NOT NULL"`
+}
+
+// BatchResponse contains multiple object metadata Representation structures
+// for use with the batch API.
+// https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses
+type BatchResponse struct {
+ Transfer string `json:"transfer,omitempty"`
+ Objects []*ObjectResponse `json:"objects"`
+}
+
+// ObjectResponse is object metadata as seen by clients of the LFS server.
+type ObjectResponse struct {
+ Pointer
+ Actions map[string]*Link `json:"actions"`
+ Error *ObjectError `json:"error,omitempty"`
+}
+
+// Link provides a structure used to build a hypermedia representation of an HTTP link.
+type Link struct {
+ Href string `json:"href"`
+ Header map[string]string `json:"header,omitempty"`
+ ExpiresAt time.Time `json:"expires_at,omitempty"`
+}
+
+// ObjectError defines the JSON structure returned to the client in case of an error
+type ObjectError struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+// PointerBlob associates a Git blob with a Pointer.
+type PointerBlob struct {
+ Hash string
+ Pointer
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+)
+
+// TransferAdapter represents an adapter for downloading/uploading LFS objects
+type TransferAdapter interface {
+ Name() string
+ Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error)
+ //Upload(ctx context.Context, reader io.Reader) error
+}
+
+// BasicTransferAdapter implements the "basic" adapter
+type BasicTransferAdapter struct {
+ client *http.Client
+}
+
+// Name returns the name of the adapter
+func (a *BasicTransferAdapter) Name() string {
+ return "basic"
+}
+
+// Download reads the download location and downloads the data
+func (a *BasicTransferAdapter) Download(ctx context.Context, r *ObjectResponse) (io.ReadCloser, error) {
+ download, ok := r.Actions["download"]
+ if !ok {
+ return nil, errors.New("lfs.BasicTransferAdapter.Download: Action 'download' not found")
+ }
+
+ req, err := http.NewRequestWithContext(ctx, "GET", download.Href, nil)
+ if err != nil {
+ return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.NewRequestWithContext: %w", err)
+ }
+ for key, value := range download.Header {
+ req.Header.Set(key, value)
+ }
+
+ res, err := a.client.Do(req)
+ if err != nil {
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ default:
+ }
+ return nil, fmt.Errorf("lfs.BasicTransferAdapter.Download http.Do: %w", err)
+ }
+
+ return res.Body, nil
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "bytes"
+ "context"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestBasicTransferAdapterName(t *testing.T) {
+ a := &BasicTransferAdapter{}
+
+ assert.Equal(t, "basic", a.Name())
+}
+
+func TestBasicTransferAdapterDownload(t *testing.T) {
+ roundTripHandler := func(req *http.Request) *http.Response {
+ url := req.URL.String()
+ if strings.Contains(url, "valid-download-request") {
+ assert.Equal(t, "GET", req.Method)
+ assert.Equal(t, "test-value", req.Header.Get("test-header"))
+
+ return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("dummy"))}
+ }
+
+ t.Errorf("Unknown test case: %s", url)
+
+ return nil
+ }
+
+ hc := &http.Client{Transport: RoundTripFunc(roundTripHandler)}
+ a := &BasicTransferAdapter{hc}
+
+ var cases = []struct {
+ response *ObjectResponse
+ expectederror string
+ }{
+ // case 0
+ {
+ response: &ObjectResponse{},
+ expectederror: "Action 'download' not found",
+ },
+ // case 1
+ {
+ response: &ObjectResponse{
+ Actions: map[string]*Link{"upload": nil},
+ },
+ expectederror: "Action 'download' not found",
+ },
+ // case 2
+ {
+ response: &ObjectResponse{
+ Actions: map[string]*Link{"download": {
+ Href: "https://valid-download-request.io",
+ Header: map[string]string{"test-header": "test-value"},
+ }},
+ },
+ expectederror: "",
+ },
+ }
+
+ for n, c := range cases {
+ _, err := a.Download(context.Background(), c.response)
+ if len(c.expectederror) > 0 {
+ assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
+ } else {
+ assert.NoError(t, err, "case %d", n)
+ }
+ }
+}
// required: true
RepoName string `json:"repo_name" binding:"Required"`
Mirror bool `json:"mirror"`
+ LFS bool `json:"lfs"`
+ LFSEndpoint string `json:"lfs_endpoint"`
Private bool `json:"private"`
Description string `json:"description"`
OriginalURL string
OriginalURL: repo.OriginalURL,
GitServiceType: opts.GitServiceType,
Mirror: repo.IsMirror,
+ LFS: opts.LFS,
+ LFSEndpoint: opts.LFSEndpoint,
CloneAddr: repo.CloneURL,
Private: repo.IsPrivate,
Wiki: opts.Wiki,
if err != nil {
return nil, err
}
+ if opts.LFS && len(opts.LFSEndpoint) > 0 {
+ err := IsMigrateURLAllowed(opts.LFSEndpoint, doer)
+ if err != nil {
+ return nil, err
+ }
+ }
downloader, err := newDownloader(ctx, ownerName, opts)
if err != nil {
return nil, err
"code.gitea.io/gitea/modules/log"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
stdcharset "golang.org/x/net/html/charset"
buf = buf[:n]
if setting.LFS.StartServer {
- meta := lfs.IsPointerFile(&buf)
- if meta != nil {
- meta, err = repo.GetLFSMetaObjectByOid(meta.Oid)
+ pointer, _ := lfs.ReadPointerFromBuffer(buf)
+ if pointer.IsValid() {
+ meta, err := repo.GetLFSMetaObjectByOid(pointer.Oid)
if err != nil && err != models.ErrLFSObjectNotExist {
// return default
return "UTF-8", false
}
- }
- if meta != nil {
- dataRc, err := lfs.ReadMetaObject(meta)
- if err != nil {
- // return default
- return "UTF-8", false
- }
- defer dataRc.Close()
- buf = make([]byte, 1024)
- n, err = dataRc.Read(buf)
- if err != nil {
- // return default
- return "UTF-8", false
+ if meta != nil {
+ dataRc, err := lfs.ReadMetaObject(pointer)
+ if err != nil {
+ // return default
+ return "UTF-8", false
+ }
+ defer dataRc.Close()
+ buf = make([]byte, 1024)
+ n, err = dataRc.Read(buf)
+ if err != nil {
+ // return default
+ return "UTF-8", false
+ }
+ buf = buf[:n]
}
- buf = buf[:n]
}
-
}
encoding, err := charset.DetectEncoding(buf)
if filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" {
// OK so we are supposed to LFS this data!
- oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content))
+ pointer, err := lfs.GeneratePointer(strings.NewReader(opts.Content))
if err != nil {
return nil, err
}
- lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID}
- content = lfsMetaObject.Pointer()
+ lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: repo.ID}
+ content = pointer.StringContent()
}
}
// Add the object to the database
if err != nil {
return nil, err
}
- contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
- exist, err := contentStore.Exists(lfsMetaObject)
+ contentStore := lfs.NewContentStore()
+ exist, err := contentStore.Exists(lfsMetaObject.Pointer)
if err != nil {
return nil, err
}
if !exist {
- if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
+ if err := contentStore.Put(lfsMetaObject.Pointer, strings.NewReader(opts.Content)); err != nil {
if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
}
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
- "code.gitea.io/gitea/modules/storage"
)
// UploadRepoFileOptions contains the uploaded repository file options
// OK now we can insert the data into the store - there's no way to clean up the store
// once it's in there, it's in there.
- contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
+ contentStore := lfs.NewContentStore()
for _, info := range infos {
if err := uploadToLFSContentStore(info, contentStore); err != nil {
return cleanUpAfterFailure(&infos, t, err)
if setting.LFS.StartServer && filename2attribute2info[info.upload.Name] != nil && filename2attribute2info[info.upload.Name]["filter"] == "lfs" {
// Handle LFS
// FIXME: Inefficient! this should probably happen in models.Upload
- oid, err := models.GenerateLFSOid(file)
- if err != nil {
- return err
- }
- fileInfo, err := file.Stat()
+ pointer, err := lfs.GeneratePointer(file)
if err != nil {
return err
}
- info.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID}
+ info.lfsMetaObject = &models.LFSMetaObject{Pointer: pointer, RepositoryID: t.repo.ID}
- if objectHash, err = t.HashObject(strings.NewReader(info.lfsMetaObject.Pointer())); err != nil {
+ if objectHash, err = t.HashObject(strings.NewReader(pointer.StringContent())); err != nil {
return err
}
} else if objectHash, err = t.HashObject(file); err != nil {
if info.lfsMetaObject == nil {
return nil
}
- exist, err := contentStore.Exists(info.lfsMetaObject)
+ exist, err := contentStore.Exists(info.lfsMetaObject.Pointer)
if err != nil {
return err
}
defer file.Close()
// FIXME: Put regenerates the hash and copies the file over.
// I guess this strictly ensures the soundness of the store but this is inefficient.
- if err := contentStore.Put(info.lfsMetaObject, file); err != nil {
+ if err := contentStore.Put(info.lfsMetaObject.Pointer, file); err != nil {
// OK Now we need to cleanup
// Can't clean up the store, once uploaded there they're there.
return err
import (
"context"
"fmt"
+ "net/url"
"path"
"strings"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
migration "code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting"
log.Error("Failed to synchronize tags to releases for repository: %v", err)
}
}
+
+ if opts.LFS {
+ ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
+ if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil {
+ log.Error("Failed to store missing LFS objects for repository: %v", err)
+ }
+ }
}
if err = repo.UpdateSize(models.DefaultDBContext()); err != nil {
Interval: setting.Mirror.DefaultInterval,
EnablePrune: true,
NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
+ LFS: opts.LFS,
+ }
+ if opts.LFS {
+ mirrorModel.LFSEndpoint = opts.LFSEndpoint
}
if opts.MirrorInterval != "" {
return models.SaveOrUpdateTag(repo, &rel)
}
+
+// StoreMissingLfsObjectsInRepository downloads missing LFS objects
+func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error {
+ client := lfs.NewClient(endpoint)
+ contentStore := lfs.NewContentStore()
+
+ pointerChan := make(chan lfs.PointerBlob)
+ errChan := make(chan error, 1)
+ go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan)
+
+ err := func() error {
+ for pointerBlob := range pointerChan {
+ meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: pointerBlob.Pointer, RepositoryID: repo.ID})
+ if err != nil {
+ return fmt.Errorf("StoreMissingLfsObjectsInRepository models.NewLFSMetaObject: %w", err)
+ }
+ if meta.Existing {
+ continue
+ }
+
+ log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] not present in repository %s", pointerBlob.Oid, repo.FullName())
+
+ err = func() error {
+ exist, err := contentStore.Exists(pointerBlob.Pointer)
+ if err != nil {
+ return fmt.Errorf("StoreMissingLfsObjectsInRepository contentStore.Exists: %w", err)
+ }
+ if !exist {
+ if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize {
+ log.Info("LFS OID[%s] download denied because of LFS_MAX_FILE_SIZE=%d < size %d", pointerBlob.Oid, setting.LFS.MaxFileSize, pointerBlob.Size)
+ return nil
+ }
+
+ stream, err := client.Download(ctx, pointerBlob.Oid, pointerBlob.Size)
+ if err != nil {
+ return fmt.Errorf("StoreMissingLfsObjectsInRepository: LFS OID[%s] failed to download: %w", pointerBlob.Oid, err)
+ }
+ defer stream.Close()
+
+ if err := contentStore.Put(pointerBlob.Pointer, stream); err != nil {
+ return fmt.Errorf("StoreMissingLfsObjectsInRepository LFS OID[%s] contentStore.Put: %w", pointerBlob.Oid, err)
+ }
+ } else {
+ log.Trace("StoreMissingLfsObjectsInRepository: LFS OID[%s] already present in content store", pointerBlob.Oid)
+ }
+ return nil
+ }()
+ if err != nil {
+ if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil {
+ log.Error("StoreMissingLfsObjectsInRepository RemoveLFSMetaObjectByOid[Oid: %s]: %w", meta.Oid, err2)
+ }
+
+ select {
+ case <-ctx.Done():
+ return nil
+ default:
+ }
+ return err
+ }
+ }
+ return nil
+ }()
+ if err != nil {
+ return err
+ }
+
+ err, has := <-errChan
+ if has {
+ return err
+ }
+
+ return nil
+}
AuthToken string `json:"auth_token"`
Mirror bool `json:"mirror"`
+ LFS bool `json:"lfs"`
+ LFSEndpoint string `json:"lfs_endpoint"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
import (
"errors"
+ "net/url"
"os"
"path"
"path/filepath"
+ "regexp"
+ "runtime"
"strings"
)
}
return statDir(rootPath, "", isIncludeDir, false, false)
}
+
+// FileURLToPath extracts the path informations from a file://... url.
+func FileURLToPath(u *url.URL) (string, error) {
+ if u.Scheme != "file" {
+ return "", errors.New("URL scheme is not 'file': " + u.String())
+ }
+
+ path := u.Path
+
+ if runtime.GOOS != "windows" {
+ return path, nil
+ }
+
+ // If it looks like there's a Windows drive letter at the beginning, strip off the leading slash.
+ re := regexp.MustCompile("/[A-Za-z]:/")
+ if re.MatchString(path) {
+ return path[1:], nil
+ }
+ return path, nil
+}
--- /dev/null
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package util
+
+import (
+ "net/url"
+ "runtime"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestFileURLToPath(t *testing.T) {
+ var cases = []struct {
+ url string
+ expected string
+ haserror bool
+ windows bool
+ }{
+ // case 0
+ {
+ url: "",
+ haserror: true,
+ },
+ // case 1
+ {
+ url: "http://test.io",
+ haserror: true,
+ },
+ // case 2
+ {
+ url: "file:///path",
+ expected: "/path",
+ },
+ // case 3
+ {
+ url: "file:///C:/path",
+ expected: "C:/path",
+ windows: true,
+ },
+ }
+
+ for n, c := range cases {
+ if c.windows && runtime.GOOS != "windows" {
+ continue
+ }
+ u, _ := url.Parse(c.url)
+ p, err := FileURLToPath(u)
+ if c.haserror {
+ assert.Error(t, err, "case %d: should return error", n)
+ } else {
+ assert.NoError(t, err, "case %d: should not return error", n)
+ assert.Equal(t, c.expected, p, "case %d: should be equal", n)
+ }
+ }
+}
mirror_address_desc = Put any required credentials in the Clone Authorization section.
mirror_address_url_invalid = The provided url is invalid. You must escape all components of the url correctly.
mirror_address_protocol_invalid = The provided url is invalid. Only http(s):// or git:// locations can be mirrored from.
+mirror_lfs = Large File System (LFS)
+mirror_lfs_desc = Activate mirroring of LFS data.
+mirror_lfs_endpoint = LFS Endpoint
+mirror_lfs_endpoint_desc = Sync will attempt to use the clone url to <a target="_blank" rel="noopener noreferrer" href="%s">determine the LFS server</a>. You can also specify a custom endpoint if the repository LFS data is stored somewhere else.
mirror_last_synced = Last Synchronized
watchers = Watchers
stargazers = Stargazers
migrate_service = Migration Service
migrate_options_mirror_helper = This repository will be a <span class="text blue">mirror</span>
migrate_options_mirror_disabled = Your site administrator has disabled new mirrors.
+migrate_options_lfs = Migrate LFS files
+migrate_options_lfs_endpoint.label = LFS Endpoint
+migrate_options_lfs_endpoint.description = Migration will attempt to use your Git remote to <a target="_blank" rel="noopener noreferrer" href="%s">determine the LFS server</a>. You can also specify a custom endpoint if the repository LFS data is stored somewhere else.
+migrate_options_lfs_endpoint.description.local = A local server path is supported too.
+migrate_options_lfs_endpoint.placeholder = Leave blank to derive from clone URL
migrate_items = Migration Items
migrate_items_wiki = Wiki
migrate_items_milestones = Milestones
migrate.permission_denied_blocked = You are not allowed to import from blocked hosts.
migrate.permission_denied_private_ip = You are not allowed to import from private IPs.
migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory."
+migrate.invalid_lfs_endpoint = The LFS endpoint is not valid.
migrate.failed = Migration failed: %v
-migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead.
migrate.migrate_items_options = Access Token is required to migrate additional items
migrated_from = Migrated from <a href="%[1]s">%[2]s</a>
migrated_from_fake = Migrated From %[1]s
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/convert"
"code.gitea.io/gitea/modules/graceful"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/migrations/base"
err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User)
}
if err != nil {
- if models.IsErrInvalidCloneAddr(err) {
- addrErr := err.(*models.ErrInvalidCloneAddr)
- switch {
- case addrErr.IsURLError:
- ctx.Error(http.StatusUnprocessableEntity, "", err)
- case addrErr.IsPermissionDenied:
- if addrErr.LocalPath {
- ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.")
- } else if len(addrErr.PrivateNet) == 0 {
- ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.")
- } else {
- ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.")
- }
- case addrErr.IsInvalidPath:
- ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.")
- default:
- ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", "Unknown error type (ErrInvalidCloneAddr): "+err.Error())
- }
- } else {
- ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", err)
- }
+ handleRemoteAddrError(ctx, err)
return
}
return
}
+ form.LFS = form.LFS && setting.LFS.StartServer
+
+ if form.LFS && len(form.LFSEndpoint) > 0 {
+ ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
+ if ep == nil {
+ ctx.Error(http.StatusInternalServerError, "", ctx.Tr("repo.migrate.invalid_lfs_endpoint"))
+ return
+ }
+ err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
+ if err != nil {
+ handleRemoteAddrError(ctx, err)
+ return
+ }
+ }
+
var opts = migrations.MigrateOptions{
CloneAddr: remoteAddr,
RepoName: form.RepoName,
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror,
+ LFS: form.LFS,
+ LFSEndpoint: form.LFSEndpoint,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
}
}
}
+
+func handleRemoteAddrError(ctx *context.APIContext, err error) {
+ if models.IsErrInvalidCloneAddr(err) {
+ addrErr := err.(*models.ErrInvalidCloneAddr)
+ switch {
+ case addrErr.IsURLError:
+ ctx.Error(http.StatusUnprocessableEntity, "", err)
+ case addrErr.IsPermissionDenied:
+ if addrErr.LocalPath {
+ ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import local repositories.")
+ } else if len(addrErr.PrivateNet) == 0 {
+ ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from blocked hosts.")
+ } else {
+ ctx.Error(http.StatusUnprocessableEntity, "", "You are not allowed to import from private IPs.")
+ }
+ case addrErr.IsInvalidPath:
+ ctx.Error(http.StatusUnprocessableEntity, "", "Invalid local path, it does not exist or not a directory.")
+ default:
+ ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", "Unknown error type (ErrInvalidCloneAddr): "+err.Error())
+ }
+ } else {
+ ctx.Error(http.StatusInternalServerError, "ParseRemoteAddr", err)
+ }
+}
}
}()
- if meta, _ := lfs.ReadPointerFile(dataRc); meta != nil {
- meta, _ = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
+ pointer, _ := lfs.ReadPointer(dataRc)
+ if pointer.IsValid() {
+ meta, _ := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
if meta == nil {
return ServeBlob(ctx, blob)
}
- lfsDataRc, err := lfs.ReadMetaObject(meta)
+ lfsDataRc, err := lfs.ReadMetaObject(meta.Pointer)
if err != nil {
return err
}
package repo
import (
- "bufio"
"bytes"
"fmt"
gotemplate "html/template"
"path"
"strconv"
"strings"
- "sync"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
return
}
ctx.Data["LFSFile"] = meta
- dataRc, err := lfs.ReadMetaObject(meta)
+ dataRc, err := lfs.ReadMetaObject(meta.Pointer)
if err != nil {
ctx.ServerError("LFSFileGet", err)
return
ctx.Data["PageIsSettingsLFS"] = true
var hash git.SHA1
if len(sha) == 0 {
- meta := models.LFSMetaObject{Oid: oid, Size: size}
- pointer := meta.Pointer()
- hash = git.ComputeBlobHash([]byte(pointer))
+ pointer := lfs.Pointer{Oid: oid, Size: size}
+ hash = git.ComputeBlobHash([]byte(pointer.StringContent()))
sha = hash.String()
} else {
hash = git.MustIDFromString(sha)
}
ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs"
- basePath := ctx.Repo.Repository.RepoPath()
+ err = func() error {
+ pointerChan := make(chan lfs.PointerBlob)
+ errChan := make(chan error, 1)
+ go lfs.SearchPointerBlobs(ctx.Req.Context(), ctx.Repo.GitRepo, pointerChan, errChan)
+
+ numPointers := 0
+ var numAssociated, numNoExist, numAssociatable int
+
+ type pointerResult struct {
+ SHA string
+ Oid string
+ Size int64
+ InRepo bool
+ Exists bool
+ Accessible bool
+ }
+
+ results := []pointerResult{}
- pointerChan := make(chan pointerResult)
+ contentStore := lfs.NewContentStore()
+ repo := ctx.Repo.Repository
- catFileCheckReader, catFileCheckWriter := io.Pipe()
- shasToBatchReader, shasToBatchWriter := io.Pipe()
- catFileBatchReader, catFileBatchWriter := io.Pipe()
- errChan := make(chan error, 1)
- wg := sync.WaitGroup{}
- wg.Add(5)
+ for pointerBlob := range pointerChan {
+ numPointers++
+
+ result := pointerResult{
+ SHA: pointerBlob.Hash,
+ Oid: pointerBlob.Oid,
+ Size: pointerBlob.Size,
+ }
- var numPointers, numAssociated, numNoExist, numAssociatable int
+ if _, err := repo.GetLFSMetaObjectByOid(pointerBlob.Oid); err != nil {
+ if err != models.ErrLFSObjectNotExist {
+ return err
+ }
+ } else {
+ result.InRepo = true
+ }
- go func() {
- defer wg.Done()
- pointers := make([]pointerResult, 0, 50)
- for pointer := range pointerChan {
- pointers = append(pointers, pointer)
- if pointer.InRepo {
+ result.Exists, err = contentStore.Exists(pointerBlob.Pointer)
+ if err != nil {
+ return err
+ }
+
+ if result.Exists {
+ if !result.InRepo {
+ // Can we fix?
+ // OK well that's "simple"
+ // - we need to check whether current user has access to a repo that has access to the file
+ result.Accessible, err = models.LFSObjectAccessible(ctx.User, pointerBlob.Oid)
+ if err != nil {
+ return err
+ }
+ } else {
+ result.Accessible = true
+ }
+ }
+
+ if result.InRepo {
numAssociated++
}
- if !pointer.Exists {
+ if !result.Exists {
numNoExist++
}
- if !pointer.InRepo && pointer.Accessible {
+ if !result.InRepo && result.Accessible {
numAssociatable++
}
+
+ results = append(results, result)
+ }
+
+ err, has := <-errChan
+ if has {
+ return err
}
- numPointers = len(pointers)
- ctx.Data["Pointers"] = pointers
+
+ ctx.Data["Pointers"] = results
ctx.Data["NumPointers"] = numPointers
ctx.Data["NumAssociated"] = numAssociated
ctx.Data["NumAssociatable"] = numAssociatable
ctx.Data["NumNoExist"] = numNoExist
ctx.Data["NumNotAssociated"] = numPointers - numAssociated
+
+ return nil
}()
- go createPointerResultsFromCatFileBatch(catFileBatchReader, &wg, pointerChan, ctx.Repo.Repository, ctx.User)
- go pipeline.CatFileBatch(shasToBatchReader, catFileBatchWriter, &wg, basePath)
- go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
- if git.CheckGitVersionAtLeast("2.6.0") != nil {
- revListReader, revListWriter := io.Pipe()
- shasToCheckReader, shasToCheckWriter := io.Pipe()
- wg.Add(2)
- go pipeline.CatFileBatchCheck(shasToCheckReader, catFileCheckWriter, &wg, basePath)
- go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
- go pipeline.RevListAllObjects(revListWriter, &wg, basePath, errChan)
- } else {
- go pipeline.CatFileBatchCheckAllObjects(catFileCheckWriter, &wg, basePath, errChan)
+ if err != nil {
+ ctx.ServerError("LFSPointerFiles", err)
+ return
}
- wg.Wait()
- select {
- case err, has := <-errChan:
- if has {
- ctx.ServerError("LFSPointerFiles", err)
- }
- default:
- }
ctx.HTML(http.StatusOK, tplSettingsLFSPointers)
}
-type pointerResult struct {
- SHA string
- Oid string
- Size int64
- InRepo bool
- Exists bool
- Accessible bool
-}
-
-func createPointerResultsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- pointerResult, repo *models.Repository, user *models.User) {
- defer wg.Done()
- defer catFileBatchReader.Close()
- contentStore := lfs.ContentStore{ObjectStorage: storage.LFS}
-
- bufferedReader := bufio.NewReader(catFileBatchReader)
- buf := make([]byte, 1025)
- for {
- // File descriptor line: sha
- sha, err := bufferedReader.ReadString(' ')
- if err != nil {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
- // Throw away the blob
- if _, err := bufferedReader.ReadString(' '); err != nil {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
- sizeStr, err := bufferedReader.ReadString('\n')
- if err != nil {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
- size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
- if err != nil {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
- pointerBuf := buf[:size+1]
- if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
- pointerBuf = pointerBuf[:size]
- // Now we need to check if the pointerBuf is an LFS pointer
- pointer := lfs.IsPointerFile(&pointerBuf)
- if pointer == nil {
- continue
- }
-
- result := pointerResult{
- SHA: strings.TrimSpace(sha),
- Oid: pointer.Oid,
- Size: pointer.Size,
- }
-
- // Then we need to check that this pointer is in the db
- if _, err := repo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
- if err != models.ErrLFSObjectNotExist {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
- } else {
- result.InRepo = true
- }
-
- result.Exists, err = contentStore.Exists(pointer)
- if err != nil {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
-
- if result.Exists {
- if !result.InRepo {
- // Can we fix?
- // OK well that's "simple"
- // - we need to check whether current user has access to a repo that has access to the file
- result.Accessible, err = models.LFSObjectAccessible(user, result.Oid)
- if err != nil {
- _ = catFileBatchReader.CloseWithError(err)
- break
- }
- } else {
- result.Accessible = true
- }
- }
- pointerChan <- result
- }
- close(pointerChan)
-}
-
// LFSAutoAssociate auto associates accessible lfs files
func LFSAutoAssociate(ctx *context.Context) {
if !setting.LFS.StartServer {
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/setting"
ctx.Data["private"] = getRepoPrivate(ctx)
ctx.Data["mirror"] = ctx.Query("mirror") == "1"
+ ctx.Data["lfs"] = ctx.Query("lfs") == "1"
ctx.Data["wiki"] = ctx.Query("wiki") == "1"
ctx.Data["milestones"] = ctx.Query("milestones") == "1"
ctx.Data["labels"] = ctx.Query("labels") == "1"
}
}
+func handleMigrateRemoteAddrError(ctx *context.Context, err error, tpl base.TplName, form *forms.MigrateRepoForm) {
+ if models.IsErrInvalidCloneAddr(err) {
+ addrErr := err.(*models.ErrInvalidCloneAddr)
+ switch {
+ case addrErr.IsProtocolInvalid:
+ ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, form)
+ case addrErr.IsURLError:
+ ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
+ case addrErr.IsPermissionDenied:
+ if addrErr.LocalPath {
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, form)
+ } else if len(addrErr.PrivateNet) == 0 {
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, form)
+ } else {
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, form)
+ }
+ case addrErr.IsInvalidPath:
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, form)
+ default:
+ log.Error("Error whilst updating url: %v", err)
+ ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
+ }
+ } else {
+ log.Error("Error whilst updating url: %v", err)
+ ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, form)
+ }
+}
+
// MigratePost response for migrating from external git repository
func MigratePost(ctx *context.Context) {
form := web.GetForm(ctx).(*forms.MigrateRepoForm)
err = migrations.IsMigrateURLAllowed(remoteAddr, ctx.User)
}
if err != nil {
- if models.IsErrInvalidCloneAddr(err) {
- ctx.Data["Err_CloneAddr"] = true
- addrErr := err.(*models.ErrInvalidCloneAddr)
- switch {
- case addrErr.IsProtocolInvalid:
- ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tpl, &form)
- case addrErr.IsURLError:
- ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form)
- case addrErr.IsPermissionDenied:
- if addrErr.LocalPath {
- ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tpl, &form)
- } else if len(addrErr.PrivateNet) == 0 {
- ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tpl, &form)
- } else {
- ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tpl, &form)
- }
- case addrErr.IsInvalidPath:
- ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tpl, &form)
- default:
- log.Error("Error whilst updating url: %v", err)
- ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form)
- }
- } else {
- log.Error("Error whilst updating url: %v", err)
- ctx.RenderWithErr(ctx.Tr("form.url_error"), tpl, &form)
- }
+ ctx.Data["Err_CloneAddr"] = true
+ handleMigrateRemoteAddrError(ctx, err, tpl, form)
return
}
+ form.LFS = form.LFS && setting.LFS.StartServer
+
+ if form.LFS && len(form.LFSEndpoint) > 0 {
+ ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
+ if ep == nil {
+ ctx.Data["Err_LFSEndpoint"] = true
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tpl, &form)
+ return
+ }
+ err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
+ if err != nil {
+ ctx.Data["Err_LFSEndpoint"] = true
+ handleMigrateRemoteAddrError(ctx, err, tpl, form)
+ return
+ }
+ }
+
var opts = migrations.MigrateOptions{
OriginalURL: form.CloneAddr,
GitServiceType: serviceType,
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror && !setting.Repository.DisableMirrors,
+ LFS: form.LFS,
+ LFSEndpoint: form.LFSEndpoint,
AuthUsername: form.AuthUsername,
AuthPassword: form.AuthPassword,
AuthToken: form.AuthToken,
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/repository"
err = migrations.IsMigrateURLAllowed(address, ctx.User)
}
if err != nil {
- if models.IsErrInvalidCloneAddr(err) {
- ctx.Data["Err_MirrorAddress"] = true
- addrErr := err.(*models.ErrInvalidCloneAddr)
- switch {
- case addrErr.IsProtocolInvalid:
- ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, &form)
- case addrErr.IsURLError:
- ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, &form)
- case addrErr.IsPermissionDenied:
- if addrErr.LocalPath {
- ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, &form)
- } else if len(addrErr.PrivateNet) == 0 {
- ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, &form)
- } else {
- ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, &form)
- }
- case addrErr.IsInvalidPath:
- ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, &form)
- default:
- ctx.ServerError("Unknown error", err)
- }
- }
ctx.Data["Err_MirrorAddress"] = true
- ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, &form)
+ handleSettingRemoteAddrError(ctx, err, form)
return
}
return
}
+ form.LFS = form.LFS && setting.LFS.StartServer
+
+ if len(form.LFSEndpoint) > 0 {
+ ep := lfs.DetermineEndpoint("", form.LFSEndpoint)
+ if ep == nil {
+ ctx.Data["Err_LFSEndpoint"] = true
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_lfs_endpoint"), tplSettingsOptions, &form)
+ return
+ }
+ err = migrations.IsMigrateURLAllowed(ep.String(), ctx.User)
+ if err != nil {
+ ctx.Data["Err_LFSEndpoint"] = true
+ handleSettingRemoteAddrError(ctx, err, form)
+ return
+ }
+ }
+
+ ctx.Repo.Mirror.LFS = form.LFS
+ ctx.Repo.Mirror.LFSEndpoint = form.LFSEndpoint
+ if err := models.UpdateMirror(ctx.Repo.Mirror); err != nil {
+ ctx.ServerError("UpdateMirror", err)
+ return
+ }
+
ctx.Flash.Success(ctx.Tr("repo.settings.update_settings_success"))
ctx.Redirect(repo.Link() + "/settings")
}
}
+func handleSettingRemoteAddrError(ctx *context.Context, err error, form *forms.RepoSettingForm) {
+ if models.IsErrInvalidCloneAddr(err) {
+ addrErr := err.(*models.ErrInvalidCloneAddr)
+ switch {
+ case addrErr.IsProtocolInvalid:
+ ctx.RenderWithErr(ctx.Tr("repo.mirror_address_protocol_invalid"), tplSettingsOptions, form)
+ case addrErr.IsURLError:
+ ctx.RenderWithErr(ctx.Tr("form.url_error"), tplSettingsOptions, form)
+ case addrErr.IsPermissionDenied:
+ if addrErr.LocalPath {
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied"), tplSettingsOptions, form)
+ } else if len(addrErr.PrivateNet) == 0 {
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_blocked"), tplSettingsOptions, form)
+ } else {
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.permission_denied_private_ip"), tplSettingsOptions, form)
+ }
+ case addrErr.IsInvalidPath:
+ ctx.RenderWithErr(ctx.Tr("repo.migrate.invalid_local_path"), tplSettingsOptions, form)
+ default:
+ ctx.ServerError("Unknown error", err)
+ }
+ }
+ ctx.RenderWithErr(ctx.Tr("repo.mirror_address_url_invalid"), tplSettingsOptions, form)
+}
+
// Collaboration render a repository's collaboration page
func Collaboration(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("repo.settings")
// FIXME: what happens when README file is an image?
if isTextFile && setting.LFS.StartServer {
- meta := lfs.IsPointerFile(&buf)
- if meta != nil {
- meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
+ pointer, _ := lfs.ReadPointerFromBuffer(buf)
+ if pointer.IsValid() {
+ meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
if err != nil && err != models.ErrLFSObjectNotExist {
ctx.ServerError("GetLFSMetaObject", err)
return
}
- }
+ if meta != nil {
+ ctx.Data["IsLFSFile"] = true
+ isLFSFile = true
+
+ // OK read the lfs object
+ var err error
+ dataRc, err = lfs.ReadMetaObject(pointer)
+ if err != nil {
+ ctx.ServerError("ReadMetaObject", err)
+ return
+ }
+ defer dataRc.Close()
- if meta != nil {
- ctx.Data["IsLFSFile"] = true
- isLFSFile = true
+ buf = make([]byte, 1024)
+ n, err = dataRc.Read(buf)
+ if err != nil {
+ ctx.ServerError("Data", err)
+ return
+ }
+ buf = buf[:n]
- // OK read the lfs object
- var err error
- dataRc, err = lfs.ReadMetaObject(meta)
- if err != nil {
- ctx.ServerError("ReadMetaObject", err)
- return
- }
- defer dataRc.Close()
+ isTextFile = base.IsTextFile(buf)
+ ctx.Data["IsTextFile"] = isTextFile
- buf = make([]byte, 1024)
- n, err = dataRc.Read(buf)
- if err != nil {
- ctx.ServerError("Data", err)
- return
+ fileSize = meta.Size
+ ctx.Data["FileSize"] = meta.Size
+ filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
+ ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64)
}
- buf = buf[:n]
-
- isTextFile = base.IsTextFile(buf)
- ctx.Data["IsTextFile"] = isTextFile
-
- fileSize = meta.Size
- ctx.Data["FileSize"] = meta.Size
- filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.name))
- ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s.git/info/lfs/objects/%s/%s", setting.AppURL, ctx.Repo.Repository.FullName(), meta.Oid, filenameBase64)
}
}
//Check for LFS meta file
if isTextFile && setting.LFS.StartServer {
- meta := lfs.IsPointerFile(&buf)
- if meta != nil {
- meta, err = ctx.Repo.Repository.GetLFSMetaObjectByOid(meta.Oid)
+ pointer, _ := lfs.ReadPointerFromBuffer(buf)
+ if pointer.IsValid() {
+ meta, err := ctx.Repo.Repository.GetLFSMetaObjectByOid(pointer.Oid)
if err != nil && err != models.ErrLFSObjectNotExist {
ctx.ServerError("GetLFSMetaObject", err)
return
}
- }
- if meta != nil {
- isLFSFile = true
+ if meta != nil {
+ isLFSFile = true
- // OK read the lfs object
- var err error
- dataRc, err = lfs.ReadMetaObject(meta)
- if err != nil {
- ctx.ServerError("ReadMetaObject", err)
- return
- }
- defer dataRc.Close()
-
- buf = make([]byte, 1024)
- n, err = dataRc.Read(buf)
- // Error EOF don't mean there is an error, it just means we read to
- // the end
- if err != nil && err != io.EOF {
- ctx.ServerError("Data", err)
- return
- }
- buf = buf[:n]
+ // OK read the lfs object
+ var err error
+ dataRc, err = lfs.ReadMetaObject(pointer)
+ if err != nil {
+ ctx.ServerError("ReadMetaObject", err)
+ return
+ }
+ defer dataRc.Close()
+
+ buf = make([]byte, 1024)
+ n, err = dataRc.Read(buf)
+ // Error EOF don't mean there is an error, it just means we read to
+ // the end
+ if err != nil && err != io.EOF {
+ ctx.ServerError("Data", err)
+ return
+ }
+ buf = buf[:n]
- isTextFile = base.IsTextFile(buf)
- fileSize = meta.Size
- ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath)
+ isTextFile = base.IsTextFile(buf)
+ fileSize = meta.Size
+ ctx.Data["RawFileLink"] = fmt.Sprintf("%s/media/%s/%s", ctx.Repo.RepoLink, ctx.Repo.BranchNameSubURL(), ctx.Repo.TreePath)
+ }
}
}
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/httpcache"
- "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/metrics"
"code.gitea.io/gitea/modules/public"
"code.gitea.io/gitea/routers/user"
userSetting "code.gitea.io/gitea/routers/user/setting"
"code.gitea.io/gitea/services/forms"
+ "code.gitea.io/gitea/services/lfs"
"code.gitea.io/gitea/services/mailer"
// to registers all internal adapters
// required: true
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
Mirror bool `json:"mirror"`
+ LFS bool `json:"lfs"`
+ LFSEndpoint string `json:"lfs_endpoint"`
Private bool `json:"private"`
Description string `json:"description" binding:"MaxSize(255)"`
Wiki bool `json:"wiki"`
MirrorAddress string
MirrorUsername string
MirrorPassword string
+ LFS bool `form:"mirror_lfs"`
+ LFSEndpoint string `form:"mirror_lfs_endpoint"`
Private bool
Template bool
EnablePrune bool
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/highlight"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
curSection.Lines[len(curSection.Lines)-1].Content = line
// handle LFS
- if line[1:] == models.LFSMetaFileIdentifier {
+ if line[1:] == lfs.MetaFileIdentifier {
curFileLFSPrefix = true
- } else if curFileLFSPrefix && strings.HasPrefix(line[1:], models.LFSMetaFileOidPrefix) {
- oid := strings.TrimPrefix(line[1:], models.LFSMetaFileOidPrefix)
+ } else if curFileLFSPrefix && strings.HasPrefix(line[1:], lfs.MetaFileOidPrefix) {
+ oid := strings.TrimPrefix(line[1:], lfs.MetaFileOidPrefix)
if len(oid) == 64 {
- m := &models.LFSMetaObject{Oid: oid}
+ m := &models.LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}}
count, err := models.Count(m)
if err == nil && count > 0 {
--- /dev/null
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "net/http"
+ "strconv"
+ "strings"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/convert"
+ lfs_module "code.gitea.io/gitea/modules/lfs"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+ api "code.gitea.io/gitea/modules/structs"
+ jsoniter "github.com/json-iterator/go"
+)
+
+//checkIsValidRequest check if it a valid request in case of bad request it write the response to ctx.
+func checkIsValidRequest(ctx *context.Context) bool {
+ if !setting.LFS.StartServer {
+ log.Debug("Attempt to access LFS server but LFS server is disabled")
+ writeStatus(ctx, http.StatusNotFound)
+ return false
+ }
+ if !MetaMatcher(ctx.Req) {
+ log.Info("Attempt access LOCKs without accepting the correct media type: %s", lfs_module.MediaType)
+ writeStatus(ctx, http.StatusBadRequest)
+ return false
+ }
+ if !ctx.IsSigned {
+ user, _, _, err := parseToken(ctx.Req.Header.Get("Authorization"))
+ if err != nil {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ writeStatus(ctx, http.StatusUnauthorized)
+ return false
+ }
+ ctx.User = user
+ }
+ return true
+}
+
+func handleLockListOut(ctx *context.Context, repo *models.Repository, lock *models.LFSLock, err error) {
+ if err != nil {
+ if models.IsErrLFSLockNotExist(err) {
+ ctx.JSON(http.StatusOK, api.LFSLockList{
+ Locks: []*api.LFSLock{},
+ })
+ return
+ }
+ ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+ Message: "unable to list locks : Internal Server Error",
+ })
+ return
+ }
+ if repo.ID != lock.RepoID {
+ ctx.JSON(http.StatusOK, api.LFSLockList{
+ Locks: []*api.LFSLock{},
+ })
+ return
+ }
+ ctx.JSON(http.StatusOK, api.LFSLockList{
+ Locks: []*api.LFSLock{convert.ToLFSLock(lock)},
+ })
+}
+
+// GetListLockHandler list locks
+func GetListLockHandler(ctx *context.Context) {
+ if !checkIsValidRequest(ctx) {
+ // Status is written in checkIsValidRequest
+ return
+ }
+ ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+ rv, _ := unpack(ctx)
+
+ repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
+ if err != nil {
+ log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
+ writeStatus(ctx, 404)
+ return
+ }
+ repository.MustOwner()
+
+ authenticated := authenticate(ctx, repository, rv.Authorization, false)
+ if !authenticated {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+ Message: "You must have pull access to list locks",
+ })
+ return
+ }
+
+ cursor := ctx.QueryInt("cursor")
+ if cursor < 0 {
+ cursor = 0
+ }
+ limit := ctx.QueryInt("limit")
+ if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
+ limit = setting.LFS.LocksPagingNum
+ } else if limit < 0 {
+ limit = 0
+ }
+ id := ctx.Query("id")
+ if id != "" { //Case where we request a specific id
+ v, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ ctx.JSON(http.StatusBadRequest, api.LFSLockError{
+ Message: "bad request : " + err.Error(),
+ })
+ return
+ }
+ lock, err := models.GetLFSLockByID(v)
+ if err != nil && !models.IsErrLFSLockNotExist(err) {
+ log.Error("Unable to get lock with ID[%s]: Error: %v", v, err)
+ }
+ handleLockListOut(ctx, repository, lock, err)
+ return
+ }
+
+ path := ctx.Query("path")
+ if path != "" { //Case where we request a specific id
+ lock, err := models.GetLFSLock(repository, path)
+ if err != nil && !models.IsErrLFSLockNotExist(err) {
+ log.Error("Unable to get lock for repository %-v with path %s: Error: %v", repository, path, err)
+ }
+ handleLockListOut(ctx, repository, lock, err)
+ return
+ }
+
+ //If no query params path or id
+ lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
+ if err != nil {
+ log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
+ ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+ Message: "unable to list locks : Internal Server Error",
+ })
+ return
+ }
+ lockListAPI := make([]*api.LFSLock, len(lockList))
+ next := ""
+ for i, l := range lockList {
+ lockListAPI[i] = convert.ToLFSLock(l)
+ }
+ if limit > 0 && len(lockList) == limit {
+ next = strconv.Itoa(cursor + 1)
+ }
+ ctx.JSON(http.StatusOK, api.LFSLockList{
+ Locks: lockListAPI,
+ Next: next,
+ })
+}
+
+// PostLockHandler create lock
+func PostLockHandler(ctx *context.Context) {
+ if !checkIsValidRequest(ctx) {
+ // Status is written in checkIsValidRequest
+ return
+ }
+ ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+ userName := ctx.Params("username")
+ repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
+ authorization := ctx.Req.Header.Get("Authorization")
+
+ repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
+ if err != nil {
+ log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
+ writeStatus(ctx, 404)
+ return
+ }
+ repository.MustOwner()
+
+ authenticated := authenticate(ctx, repository, authorization, true)
+ if !authenticated {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+ Message: "You must have push access to create locks",
+ })
+ return
+ }
+
+ var req api.LFSLockRequest
+ bodyReader := ctx.Req.Body
+ defer bodyReader.Close()
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ dec := json.NewDecoder(bodyReader)
+ if err := dec.Decode(&req); err != nil {
+ log.Warn("Failed to decode lock request as json. Error: %v", err)
+ writeStatus(ctx, 400)
+ return
+ }
+
+ lock, err := models.CreateLFSLock(&models.LFSLock{
+ Repo: repository,
+ Path: req.Path,
+ Owner: ctx.User,
+ })
+ if err != nil {
+ if models.IsErrLFSLockAlreadyExist(err) {
+ ctx.JSON(http.StatusConflict, api.LFSLockError{
+ Lock: convert.ToLFSLock(lock),
+ Message: "already created lock",
+ })
+ return
+ }
+ if models.IsErrLFSUnauthorizedAction(err) {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+ Message: "You must have push access to create locks : " + err.Error(),
+ })
+ return
+ }
+ log.Error("Unable to CreateLFSLock in repository %-v at %s for user %-v: Error: %v", repository, req.Path, ctx.User, err)
+ ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+ Message: "internal server error : Internal Server Error",
+ })
+ return
+ }
+ ctx.JSON(http.StatusCreated, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
+}
+
+// VerifyLockHandler list locks for verification
+func VerifyLockHandler(ctx *context.Context) {
+ if !checkIsValidRequest(ctx) {
+ // Status is written in checkIsValidRequest
+ return
+ }
+ ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+ userName := ctx.Params("username")
+ repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
+ authorization := ctx.Req.Header.Get("Authorization")
+
+ repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
+ if err != nil {
+ log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
+ writeStatus(ctx, 404)
+ return
+ }
+ repository.MustOwner()
+
+ authenticated := authenticate(ctx, repository, authorization, true)
+ if !authenticated {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+ Message: "You must have push access to verify locks",
+ })
+ return
+ }
+
+ cursor := ctx.QueryInt("cursor")
+ if cursor < 0 {
+ cursor = 0
+ }
+ limit := ctx.QueryInt("limit")
+ if limit > setting.LFS.LocksPagingNum && setting.LFS.LocksPagingNum > 0 {
+ limit = setting.LFS.LocksPagingNum
+ } else if limit < 0 {
+ limit = 0
+ }
+ lockList, err := models.GetLFSLockByRepoID(repository.ID, cursor, limit)
+ if err != nil {
+ log.Error("Unable to list locks for repository ID[%d]: Error: %v", repository.ID, err)
+ ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+ Message: "unable to list locks : Internal Server Error",
+ })
+ return
+ }
+ next := ""
+ if limit > 0 && len(lockList) == limit {
+ next = strconv.Itoa(cursor + 1)
+ }
+ lockOursListAPI := make([]*api.LFSLock, 0, len(lockList))
+ lockTheirsListAPI := make([]*api.LFSLock, 0, len(lockList))
+ for _, l := range lockList {
+ if l.Owner.ID == ctx.User.ID {
+ lockOursListAPI = append(lockOursListAPI, convert.ToLFSLock(l))
+ } else {
+ lockTheirsListAPI = append(lockTheirsListAPI, convert.ToLFSLock(l))
+ }
+ }
+ ctx.JSON(http.StatusOK, api.LFSLockListVerify{
+ Ours: lockOursListAPI,
+ Theirs: lockTheirsListAPI,
+ Next: next,
+ })
+}
+
+// UnLockHandler delete locks
+func UnLockHandler(ctx *context.Context) {
+ if !checkIsValidRequest(ctx) {
+ // Status is written in checkIsValidRequest
+ return
+ }
+ ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+ userName := ctx.Params("username")
+ repoName := strings.TrimSuffix(ctx.Params("reponame"), ".git")
+ authorization := ctx.Req.Header.Get("Authorization")
+
+ repository, err := models.GetRepositoryByOwnerAndName(userName, repoName)
+ if err != nil {
+ log.Error("Unable to get repository: %s/%s Error: %v", userName, repoName, err)
+ writeStatus(ctx, 404)
+ return
+ }
+ repository.MustOwner()
+
+ authenticated := authenticate(ctx, repository, authorization, true)
+ if !authenticated {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+ Message: "You must have push access to delete locks",
+ })
+ return
+ }
+
+ var req api.LFSLockDeleteRequest
+ bodyReader := ctx.Req.Body
+ defer bodyReader.Close()
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ dec := json.NewDecoder(bodyReader)
+ if err := dec.Decode(&req); err != nil {
+ log.Warn("Failed to decode lock request as json. Error: %v", err)
+ writeStatus(ctx, 400)
+ return
+ }
+
+ lock, err := models.DeleteLFSLockByID(ctx.ParamsInt64("lid"), ctx.User, req.Force)
+ if err != nil {
+ if models.IsErrLFSUnauthorizedAction(err) {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ ctx.JSON(http.StatusUnauthorized, api.LFSLockError{
+ Message: "You must have push access to delete locks : " + err.Error(),
+ })
+ return
+ }
+ log.Error("Unable to DeleteLFSLockByID[%d] by user %-v with force %t: Error: %v", ctx.ParamsInt64("lid"), ctx.User, req.Force, err)
+ ctx.JSON(http.StatusInternalServerError, api.LFSLockError{
+ Message: "unable to delete lock : Internal Server Error",
+ })
+ return
+ }
+ ctx.JSON(http.StatusOK, api.LFSLockResponse{Lock: convert.ToLFSLock(lock)})
+}
--- /dev/null
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+ "encoding/base64"
+ "fmt"
+ "io"
+ "net/http"
+ "path"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/context"
+ lfs_module "code.gitea.io/gitea/modules/lfs"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+
+ "github.com/dgrijalva/jwt-go"
+ jsoniter "github.com/json-iterator/go"
+)
+
+// requestContext contain variables from the HTTP request.
+type requestContext struct {
+ User string
+ Repo string
+ Authorization string
+}
+
+// Claims is a JWT Token Claims
+type Claims struct {
+ RepoID int64
+ Op string
+ UserID int64
+ jwt.StandardClaims
+}
+
+// ObjectLink builds a URL linking to the object.
+func (rc *requestContext) ObjectLink(oid string) string {
+ return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/objects", oid)
+}
+
+// VerifyLink builds a URL for verifying the object.
+func (rc *requestContext) VerifyLink() string {
+ return setting.AppURL + path.Join(rc.User, rc.Repo+".git", "info/lfs/verify")
+}
+
+var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`)
+
+func isOidValid(oid string) bool {
+ return oidRegExp.MatchString(oid)
+}
+
+// ObjectOidHandler is the main request routing entry point into LFS server functions
+func ObjectOidHandler(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ log.Debug("Attempt to access LFS server but LFS server is disabled")
+ writeStatus(ctx, 404)
+ return
+ }
+
+ if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" {
+ if MetaMatcher(ctx.Req) {
+ getMetaHandler(ctx)
+ return
+ }
+
+ getContentHandler(ctx)
+ return
+ } else if ctx.Req.Method == "PUT" {
+ PutHandler(ctx)
+ return
+ }
+
+ log.Warn("Unhandled LFS method: %s for %s/%s OID[%s]", ctx.Req.Method, ctx.Params("username"), ctx.Params("reponame"), ctx.Params("oid"))
+ writeStatus(ctx, 404)
+}
+
+func getAuthenticatedRepoAndMeta(ctx *context.Context, rc *requestContext, p lfs_module.Pointer, requireWrite bool) (*models.LFSMetaObject, *models.Repository) {
+ if !isOidValid(p.Oid) {
+ log.Info("Attempt to access invalid LFS OID[%s] in %s/%s", p.Oid, rc.User, rc.Repo)
+ writeStatus(ctx, 404)
+ return nil, nil
+ }
+
+ repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo)
+ if err != nil {
+ log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err)
+ writeStatus(ctx, 404)
+ return nil, nil
+ }
+
+ if !authenticate(ctx, repository, rc.Authorization, requireWrite) {
+ requireAuth(ctx)
+ return nil, nil
+ }
+
+ meta, err := repository.GetLFSMetaObjectByOid(p.Oid)
+ if err != nil {
+ log.Error("Unable to get LFS OID[%s] Error: %v", p.Oid, err)
+ writeStatus(ctx, 404)
+ return nil, nil
+ }
+
+ return meta, repository
+}
+
+// getContentHandler gets the content from the content store
+func getContentHandler(ctx *context.Context) {
+ rc, p := unpack(ctx)
+
+ meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false)
+ if meta == nil {
+ // Status already written in getAuthenticatedRepoAndMeta
+ return
+ }
+
+ // Support resume download using Range header
+ var fromByte, toByte int64
+ toByte = meta.Size - 1
+ statusCode := 200
+ if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" {
+ regex := regexp.MustCompile(`bytes=(\d+)\-(\d*).*`)
+ match := regex.FindStringSubmatch(rangeHdr)
+ if len(match) > 1 {
+ statusCode = 206
+ fromByte, _ = strconv.ParseInt(match[1], 10, 32)
+
+ if fromByte >= meta.Size {
+ writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
+ return
+ }
+
+ if match[2] != "" {
+ _toByte, _ := strconv.ParseInt(match[2], 10, 32)
+ if _toByte >= fromByte && _toByte < toByte {
+ toByte = _toByte
+ }
+ }
+
+ ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, toByte, meta.Size-fromByte))
+ ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Range")
+ }
+ }
+
+ contentStore := lfs_module.NewContentStore()
+ content, err := contentStore.Get(meta.Pointer)
+ if err != nil {
+ // Errors are logged in contentStore.Get
+ writeStatus(ctx, http.StatusNotFound)
+ return
+ }
+ defer content.Close()
+
+ if fromByte > 0 {
+ _, err = content.Seek(fromByte, io.SeekStart)
+ if err != nil {
+ log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
+
+ writeStatus(ctx, http.StatusInternalServerError)
+ return
+ }
+ }
+
+ contentLength := toByte + 1 - fromByte
+ ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
+ ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+
+ filename := ctx.Params("filename")
+ if len(filename) > 0 {
+ decodedFilename, err := base64.RawURLEncoding.DecodeString(filename)
+ if err == nil {
+ ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"")
+ ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition")
+ }
+ }
+
+ ctx.Resp.WriteHeader(statusCode)
+ if written, err := io.CopyN(ctx.Resp, content, contentLength); err != nil {
+ log.Error("Error whilst copying LFS OID[%s] to the response after %d bytes. Error: %v", meta.Oid, written, err)
+ }
+ logRequest(ctx.Req, statusCode)
+}
+
+// getMetaHandler retrieves metadata about the object
+func getMetaHandler(ctx *context.Context) {
+ rc, p := unpack(ctx)
+
+ meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, false)
+ if meta == nil {
+ // Status already written in getAuthenticatedRepoAndMeta
+ return
+ }
+
+ ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+ if ctx.Req.Method == "GET" {
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ enc := json.NewEncoder(ctx.Resp)
+ if err := enc.Encode(represent(rc, meta.Pointer, true, false)); err != nil {
+ log.Error("Failed to encode representation as json. Error: %v", err)
+ }
+ }
+
+ logRequest(ctx.Req, 200)
+}
+
+// PostHandler instructs the client how to upload data
+func PostHandler(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ log.Debug("Attempt to access LFS server but LFS server is disabled")
+ writeStatus(ctx, 404)
+ return
+ }
+
+ if !MetaMatcher(ctx.Req) {
+ log.Info("Attempt to POST without accepting the correct media type: %s", lfs_module.MediaType)
+ writeStatus(ctx, 400)
+ return
+ }
+
+ rc, p := unpack(ctx)
+
+ repository, err := models.GetRepositoryByOwnerAndName(rc.User, rc.Repo)
+ if err != nil {
+ log.Error("Unable to get repository: %s/%s Error: %v", rc.User, rc.Repo, err)
+ writeStatus(ctx, 404)
+ return
+ }
+
+ if !authenticate(ctx, repository, rc.Authorization, true) {
+ requireAuth(ctx)
+ return
+ }
+
+ if !isOidValid(p.Oid) {
+ log.Info("Invalid LFS OID[%s] attempt to POST in %s/%s", p.Oid, rc.User, rc.Repo)
+ writeStatus(ctx, 404)
+ return
+ }
+
+ if setting.LFS.MaxFileSize > 0 && p.Size > setting.LFS.MaxFileSize {
+ log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", p.Oid, p.Size, rc.User, rc.Repo, setting.LFS.MaxFileSize)
+ writeStatus(ctx, 413)
+ return
+ }
+
+ meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: p, RepositoryID: repository.ID})
+ if err != nil {
+ log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", p.Oid, p.Size, rc.User, rc.Repo, err)
+ writeStatus(ctx, 404)
+ return
+ }
+
+ ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+ sentStatus := 202
+ contentStore := lfs_module.NewContentStore()
+ exist, err := contentStore.Exists(p)
+ if err != nil {
+ log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", p.Oid, rc.User, rc.Repo, err)
+ writeStatus(ctx, 500)
+ return
+ }
+ if meta.Existing && exist {
+ sentStatus = 200
+ }
+ ctx.Resp.WriteHeader(sentStatus)
+
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ enc := json.NewEncoder(ctx.Resp)
+ if err := enc.Encode(represent(rc, meta.Pointer, meta.Existing, true)); err != nil {
+ log.Error("Failed to encode representation as json. Error: %v", err)
+ }
+ logRequest(ctx.Req, sentStatus)
+}
+
+// BatchHandler provides the batch api
+func BatchHandler(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ log.Debug("Attempt to access LFS server but LFS server is disabled")
+ writeStatus(ctx, 404)
+ return
+ }
+
+ if !MetaMatcher(ctx.Req) {
+ log.Info("Attempt to BATCH without accepting the correct media type: %s", lfs_module.MediaType)
+ writeStatus(ctx, 400)
+ return
+ }
+
+ bv := unpackbatch(ctx)
+
+ reqCtx := &requestContext{
+ User: ctx.Params("username"),
+ Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"),
+ Authorization: ctx.Req.Header.Get("Authorization"),
+ }
+
+ var responseObjects []*lfs_module.ObjectResponse
+
+ // Create a response object
+ for _, object := range bv.Objects {
+ if !isOidValid(object.Oid) {
+ log.Info("Invalid LFS OID[%s] attempt to BATCH in %s/%s", object.Oid, reqCtx.User, reqCtx.Repo)
+ continue
+ }
+
+ repository, err := models.GetRepositoryByOwnerAndName(reqCtx.User, reqCtx.Repo)
+ if err != nil {
+ log.Error("Unable to get repository: %s/%s Error: %v", reqCtx.User, reqCtx.Repo, err)
+ writeStatus(ctx, 404)
+ return
+ }
+
+ requireWrite := false
+ if bv.Operation == "upload" {
+ requireWrite = true
+ }
+
+ if !authenticate(ctx, repository, reqCtx.Authorization, requireWrite) {
+ requireAuth(ctx)
+ return
+ }
+
+ contentStore := lfs_module.NewContentStore()
+
+ meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
+ if err == nil { // Object is found and exists
+ exist, err := contentStore.Exists(meta.Pointer)
+ if err != nil {
+ log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err)
+ writeStatus(ctx, 500)
+ return
+ }
+ if exist {
+ responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, true, false))
+ continue
+ }
+ }
+
+ if requireWrite && setting.LFS.MaxFileSize > 0 && object.Size > setting.LFS.MaxFileSize {
+ log.Info("Denied LFS OID[%s] upload of size %d to %s/%s because of LFS_MAX_FILE_SIZE=%d", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, setting.LFS.MaxFileSize)
+ writeStatus(ctx, 413)
+ return
+ }
+
+ // Object is not found
+ meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Pointer: object, RepositoryID: repository.ID})
+ if err == nil {
+ exist, err := contentStore.Exists(meta.Pointer)
+ if err != nil {
+ log.Error("Unable to check if LFS OID[%s] exist on %s / %s. Error: %v", object.Oid, reqCtx.User, reqCtx.Repo, err)
+ writeStatus(ctx, 500)
+ return
+ }
+ responseObjects = append(responseObjects, represent(reqCtx, meta.Pointer, meta.Existing, !exist))
+ } else {
+ log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", object.Oid, object.Size, reqCtx.User, reqCtx.Repo, err)
+ }
+ }
+
+ ctx.Resp.Header().Set("Content-Type", lfs_module.MediaType)
+
+ respobj := &lfs_module.BatchResponse{Objects: responseObjects}
+
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ enc := json.NewEncoder(ctx.Resp)
+ if err := enc.Encode(respobj); err != nil {
+ log.Error("Failed to encode representation as json. Error: %v", err)
+ }
+ logRequest(ctx.Req, 200)
+}
+
+// PutHandler receives data from the client and puts it into the content store
+func PutHandler(ctx *context.Context) {
+ rc, p := unpack(ctx)
+
+ meta, repository := getAuthenticatedRepoAndMeta(ctx, rc, p, true)
+ if meta == nil {
+ // Status already written in getAuthenticatedRepoAndMeta
+ return
+ }
+
+ contentStore := lfs_module.NewContentStore()
+ defer ctx.Req.Body.Close()
+ if err := contentStore.Put(meta.Pointer, ctx.Req.Body); err != nil {
+ // Put will log the error itself
+ ctx.Resp.WriteHeader(500)
+ if err == lfs_module.ErrSizeMismatch || err == lfs_module.ErrHashMismatch {
+ fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
+ } else {
+ fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
+ }
+ if _, err = repository.RemoveLFSMetaObjectByOid(p.Oid); err != nil {
+ log.Error("Whilst removing metaobject for LFS OID[%s] due to preceding error there was another Error: %v", p.Oid, err)
+ }
+ return
+ }
+
+ logRequest(ctx.Req, 200)
+}
+
+// VerifyHandler verify oid and its size from the content store
+func VerifyHandler(ctx *context.Context) {
+ if !setting.LFS.StartServer {
+ log.Debug("Attempt to access LFS server but LFS server is disabled")
+ writeStatus(ctx, 404)
+ return
+ }
+
+ if !MetaMatcher(ctx.Req) {
+ log.Info("Attempt to VERIFY without accepting the correct media type: %s", lfs_module.MediaType)
+ writeStatus(ctx, 400)
+ return
+ }
+
+ rc, p := unpack(ctx)
+
+ meta, _ := getAuthenticatedRepoAndMeta(ctx, rc, p, true)
+ if meta == nil {
+ // Status already written in getAuthenticatedRepoAndMeta
+ return
+ }
+
+ contentStore := lfs_module.NewContentStore()
+ ok, err := contentStore.Verify(meta.Pointer)
+ if err != nil {
+ // Error will be logged in Verify
+ ctx.Resp.WriteHeader(500)
+ fmt.Fprintf(ctx.Resp, `{"message":"Internal Server Error"}`)
+ return
+ }
+ if !ok {
+ writeStatus(ctx, 422)
+ return
+ }
+
+ logRequest(ctx.Req, 200)
+}
+
+// represent takes a requestContext and Meta and turns it into a ObjectResponse suitable
+// for json encoding
+func represent(rc *requestContext, pointer lfs_module.Pointer, download, upload bool) *lfs_module.ObjectResponse {
+ rep := &lfs_module.ObjectResponse{
+ Pointer: pointer,
+ Actions: make(map[string]*lfs_module.Link),
+ }
+
+ header := make(map[string]string)
+
+ if rc.Authorization == "" {
+ //https://github.com/github/git-lfs/issues/1088
+ header["Authorization"] = "Authorization: Basic dummy"
+ } else {
+ header["Authorization"] = rc.Authorization
+ }
+
+ if download {
+ rep.Actions["download"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header}
+ }
+
+ if upload {
+ rep.Actions["upload"] = &lfs_module.Link{Href: rc.ObjectLink(pointer.Oid), Header: header}
+ }
+
+ if upload && !download {
+ // Force client side verify action while gitea lacks proper server side verification
+ verifyHeader := make(map[string]string)
+ for k, v := range header {
+ verifyHeader[k] = v
+ }
+
+ // This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662
+ verifyHeader["Accept"] = lfs_module.MediaType
+
+ rep.Actions["verify"] = &lfs_module.Link{Href: rc.VerifyLink(), Header: verifyHeader}
+ }
+
+ return rep
+}
+
+// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain
+// an Accept header with the lfs_module.MediaType
+func MetaMatcher(r *http.Request) bool {
+ mediaParts := strings.Split(r.Header.Get("Accept"), ";")
+ mt := mediaParts[0]
+ return mt == lfs_module.MediaType
+}
+
+func unpack(ctx *context.Context) (*requestContext, lfs_module.Pointer) {
+ r := ctx.Req
+ rc := &requestContext{
+ User: ctx.Params("username"),
+ Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"),
+ Authorization: r.Header.Get("Authorization"),
+ }
+ p := lfs_module.Pointer{Oid: ctx.Params("oid")}
+
+ if r.Method == "POST" { // Maybe also check if +json
+ var p2 lfs_module.Pointer
+ bodyReader := r.Body
+ defer bodyReader.Close()
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ dec := json.NewDecoder(bodyReader)
+ err := dec.Decode(&p2)
+ if err != nil {
+ // The error is logged as a WARN here because this may represent misbehaviour rather than a true error
+ log.Warn("Unable to decode POST request vars for LFS OID[%s] in %s/%s: Error: %v", p.Oid, rc.User, rc.Repo, err)
+ return rc, p
+ }
+
+ p.Oid = p2.Oid
+ p.Size = p2.Size
+ }
+
+ return rc, p
+}
+
+// TODO cheap hack, unify with unpack
+func unpackbatch(ctx *context.Context) *lfs_module.BatchRequest {
+
+ r := ctx.Req
+ var bv lfs_module.BatchRequest
+
+ bodyReader := r.Body
+ defer bodyReader.Close()
+ json := jsoniter.ConfigCompatibleWithStandardLibrary
+ dec := json.NewDecoder(bodyReader)
+ err := dec.Decode(&bv)
+ if err != nil {
+ // The error is logged as a WARN here because this may represent misbehaviour rather than a true error
+ log.Warn("Unable to decode BATCH request vars in %s/%s: Error: %v", ctx.Params("username"), strings.TrimSuffix(ctx.Params("reponame"), ".git"), err)
+ return &bv
+ }
+
+ return &bv
+}
+
+func writeStatus(ctx *context.Context, status int) {
+ message := http.StatusText(status)
+
+ mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";")
+ mt := mediaParts[0]
+ if strings.HasSuffix(mt, "+json") {
+ message = `{"message":"` + message + `"}`
+ }
+
+ ctx.Resp.WriteHeader(status)
+ fmt.Fprint(ctx.Resp, message)
+ logRequest(ctx.Req, status)
+}
+
+func logRequest(r *http.Request, status int) {
+ log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status)
+}
+
+// authenticate uses the authorization string to determine whether
+// or not to proceed. This server assumes an HTTP Basic auth format.
+func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool {
+ accessMode := models.AccessModeRead
+ if requireWrite {
+ accessMode = models.AccessModeWrite
+ }
+
+ // ctx.IsSigned is unnecessary here, this will be checked in perm.CanAccess
+ perm, err := models.GetUserRepoPermission(repository, ctx.User)
+ if err != nil {
+ log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
+ return false
+ }
+
+ canRead := perm.CanAccess(accessMode, models.UnitTypeCode)
+ if canRead {
+ return true
+ }
+
+ user, repo, opStr, err := parseToken(authorization)
+ if err != nil {
+ // Most of these are Warn level - the true internal server errors are logged in parseToken already
+ log.Warn("Authentication failure for provided token with Error: %v", err)
+ return false
+ }
+ ctx.User = user
+ if opStr == "basic" {
+ perm, err = models.GetUserRepoPermission(repository, ctx.User)
+ if err != nil {
+ log.Error("Unable to GetUserRepoPermission for user %-v in repo %-v Error: %v", ctx.User, repository)
+ return false
+ }
+ return perm.CanAccess(accessMode, models.UnitTypeCode)
+ }
+ if repository.ID == repo.ID {
+ if requireWrite && opStr != "upload" {
+ return false
+ }
+ return true
+ }
+ return false
+}
+
+func parseToken(authorization string) (*models.User, *models.Repository, string, error) {
+ if authorization == "" {
+ return nil, nil, "unknown", fmt.Errorf("No token")
+ }
+ if strings.HasPrefix(authorization, "Bearer ") {
+ token, err := jwt.ParseWithClaims(authorization[7:], &Claims{}, func(t *jwt.Token) (interface{}, error) {
+ if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
+ return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
+ }
+ return setting.LFS.JWTSecretBytes, nil
+ })
+ if err != nil {
+ // The error here is WARN level because it is caused by bad authorization rather than an internal server error
+ return nil, nil, "unknown", err
+ }
+ claims, claimsOk := token.Claims.(*Claims)
+ if !token.Valid || !claimsOk {
+ return nil, nil, "unknown", fmt.Errorf("Token claim invalid")
+ }
+ r, err := models.GetRepositoryByID(claims.RepoID)
+ if err != nil {
+ log.Error("Unable to GetRepositoryById[%d]: Error: %v", claims.RepoID, err)
+ return nil, nil, claims.Op, err
+ }
+ u, err := models.GetUserByID(claims.UserID)
+ if err != nil {
+ log.Error("Unable to GetUserById[%d]: Error: %v", claims.UserID, err)
+ return nil, r, claims.Op, err
+ }
+ return u, r, claims.Op, nil
+ }
+
+ if strings.HasPrefix(authorization, "Basic ") {
+ c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic "))
+ if err != nil {
+ return nil, nil, "basic", err
+ }
+ cs := string(c)
+ i := strings.IndexByte(cs, ':')
+ if i < 0 {
+ return nil, nil, "basic", fmt.Errorf("Basic auth invalid")
+ }
+ user, password := cs[:i], cs[i+1:]
+ u, err := models.GetUserByName(user)
+ if err != nil {
+ log.Error("Unable to GetUserByName[%d]: Error: %v", user, err)
+ return nil, nil, "basic", err
+ }
+ if !u.IsPasswordSet() || !u.ValidatePassword(password) {
+ return nil, nil, "basic", fmt.Errorf("Basic auth failed")
+ }
+ return u, nil, "basic", nil
+ }
+
+ return nil, nil, "unknown", fmt.Errorf("Token not found")
+}
+
+func requireAuth(ctx *context.Context) {
+ ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
+ writeStatus(ctx, 401)
+}
"code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/graceful"
+ "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/notification"
repo_module "code.gitea.io/gitea/modules/repository"
}
// runSync returns true if sync finished without error.
-func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
+func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool) {
repoPath := m.Repo.RepoPath()
wikiPath := m.Repo.WikiPath()
timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
log.Error("OpenRepository: %v", err)
return nil, false
}
+ defer gitRepo.Close()
log.Trace("SyncMirrors [repo: %-v]: syncing releases with tags...", m.Repo)
if err = repo_module.SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
- gitRepo.Close()
log.Error("Failed to synchronize tags to releases for repository: %v", err)
}
- gitRepo.Close()
+
+ if m.LFS && setting.LFS.StartServer {
+ log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
+ readAddress(m)
+ ep := lfs.DetermineEndpoint(m.Address, m.LFSEndpoint)
+ if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil {
+ log.Error("Failed to synchronize LFS objects for repository: %v", err)
+ }
+ }
log.Trace("SyncMirrors [repo: %-v]: updating size of repository", m.Repo)
if err := m.Repo.UpdateSize(models.DefaultDBContext()); err != nil {
mirrorQueue.Close()
return
case repoID := <-mirrorQueue.Queue():
- syncMirror(repoID)
+ syncMirror(ctx, repoID)
}
}
}
-func syncMirror(repoID string) {
+func syncMirror(ctx context.Context, repoID string) {
log.Trace("SyncMirrors [repo_id: %v]", repoID)
defer func() {
err := recover()
}
log.Trace("SyncMirrors [repo: %-v]: Running Sync", m.Repo)
- results, ok := runSync(m)
+ results, ok := runSync(ctx, m)
if !ok {
return
}
})
assert.NoError(t, err)
- mirror, err := repository.MigrateRepositoryGitData(context.Background(), user, mirrorRepo, opts)
+ ctx := context.Background()
+
+ mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts)
assert.NoError(t, err)
gitRepo, err := git.OpenRepository(repoPath)
err = mirror.GetMirror()
assert.NoError(t, err)
- _, ok := runSync(mirror.Mirror)
+ _, ok := runSync(ctx, mirror.Mirror)
assert.True(t, ok)
count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions)
assert.NoError(t, err)
assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true))
- _, ok = runSync(mirror.Mirror)
+ _, ok = runSync(ctx, mirror.Mirror)
assert.True(t, ok)
count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions)
defer wg.Done()
defer catFileBatchReader.Close()
+ contentStore := lfs.NewContentStore()
+
bufferedReader := bufio.NewReader(catFileBatchReader)
buf := make([]byte, 1025)
for {
}
pointerBuf = pointerBuf[:size]
// Now we need to check if the pointerBuf is an LFS pointer
- pointer := lfs.IsPointerFile(&pointerBuf)
- if pointer == nil {
+ pointer, _ := lfs.ReadPointerFromBuffer(pointerBuf)
+ if !pointer.IsValid() {
continue
}
+
+ exist, _ := contentStore.Exists(pointer)
+ if !exist {
+ continue
+ }
+
// Then we need to check that this pointer is in the db
if _, err := pr.HeadRepo.GetLFSMetaObjectByOid(pointer.Oid); err != nil {
if err == models.ErrLFSObjectNotExist {
// OK we have a pointer that is associated with the head repo
// and is actually a file in the LFS
// Therefore it should be associated with the base repo
- pointer.RepositoryID = pr.BaseRepoID
- if _, err := models.NewLFSMetaObject(pointer); err != nil {
+ meta := &models.LFSMetaObject{Pointer: pointer}
+ meta.RepositoryID = pr.BaseRepoID
+ if _, err := models.NewLFSMetaObject(meta); err != nil {
_ = catFileBatchReader.CloseWithError(err)
break
}
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
- {{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>
<div class="inline field {{if .Err_Auth}}error{{end}}">
<input id="auth_password" name="auth_password" type="password" value="{{.auth_password}}">
</div>
- <div class="inline field">
- <label>{{.i18n.Tr "repo.migrate_options"}}</label>
- <div class="ui checkbox">
- {{if .DisableMirrors}}
- <input id="mirror" name="mirror" type="checkbox" readonly>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
- {{else}}
- <input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
- {{end}}
- </div>
- </div>
+ {{template "repo/migrate/options" .}}
<div class="ui divider"></div>
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
- {{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>
<div class="inline field {{if .Err_Auth}}error{{end}}">
<label for="auth_token">{{.i18n.Tr "access_token"}}</label>
<input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}} data-need-clear="true" {{end}}>
- <a target=”_blank” href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a>
+ <a target="_blank" href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a>
</div>
- <div class="inline field">
- <label>{{.i18n.Tr "repo.migrate_options"}}</label>
- <div class="ui checkbox">
- {{if .DisableMirrors}}
- <input id="mirror" name="mirror" type="checkbox" readonly>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
- {{else}}
- <input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
- {{end}}
- </div>
- </div>
+ {{template "repo/migrate/options" .}}
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
<div id="migrate_items">
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
- {{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>
<div class="inline field {{if .Err_Auth}}error{{end}}">
<label for="auth_token">{{.i18n.Tr "access_token"}}</label>
<input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}}data-need-clear="true"{{end}}>
- <a target=”_blank” href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token">{{svg "octicon-question"}}</a>
+ <a target="_blank" href="https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token">{{svg "octicon-question"}}</a>
</div>
- <div class="inline field">
- <label>{{.i18n.Tr "repo.migrate_options"}}</label>
- <div class="ui checkbox">
- {{if .DisableMirrors}}
- <input id="mirror" name="mirror" type="checkbox" readonly>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
- {{else}}
- <input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
- {{end}}
- </div>
- </div>
+ {{template "repo/migrate/options" .}}
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
<div id="migrate_items">
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
- {{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>
<div class="inline field {{if .Err_Auth}}error{{end}}">
<label for="auth_token">{{.i18n.Tr "access_token"}}</label>
<input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}}data-need-clear="true"{{end}}>
- <a target=”_blank” href="https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html">{{svg "octicon-question"}}</a>
+ <a target="_blank" href="https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html">{{svg "octicon-question"}}</a>
</div>
- <div class="inline field">
- <label>{{.i18n.Tr "repo.migrate_options"}}</label>
- <div class="ui checkbox">
- {{if .DisableMirrors}}
- <input id="mirror" name="mirror" type="checkbox" readonly>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
- {{else}}
- <input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
- {{end}}
- </div>
- </div>
+ {{template "repo/migrate/options" .}}
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
<div id="migrate_items">
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
<span class="help">
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
- {{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
</span>
</div>
<div class="inline field {{if .Err_Auth}}error{{end}}">
<label for="auth_token">{{.i18n.Tr "access_token"}}</label>
<input id="auth_token" name="auth_token" value="{{.auth_token}}" {{if not .auth_token}} data-need-clear="true" {{end}}>
- <!-- <a target=”_blank” href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a> -->
+ <!-- <a target="_blank" href="https://docs.gitea.io/en-us/api-usage">{{svg "octicon-question"}}</a> -->
</div>
- <div class="inline field">
- <label>{{.i18n.Tr "repo.migrate_options"}}</label>
- <div class="ui checkbox">
- {{if .DisableMirrors}}
- <input id="mirror" name="mirror" type="checkbox" readonly>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
- {{else}}
- <input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
- <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
- {{end}}
- </div>
- </div>
+ {{template "repo/migrate/options" .}}
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>
<div id="migrate_items">
--- /dev/null
+<div class="inline field">
+ <label>{{.i18n.Tr "repo.migrate_options"}}</label>
+ <div class="ui checkbox">
+ {{if .DisableMirrors}}
+ <input id="mirror" name="mirror" type="checkbox" readonly>
+ <label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
+ {{else}}
+ <input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
+ <label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
+ {{end}}
+ </div>
+</div>
+{{if .LFSActive}}
+<div class="inline field">
+ <label></label>
+ <div class="ui checkbox">
+ <input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
+ <label>{{.i18n.Tr "repo.migrate_options_lfs"}}</label>
+ </div>
+ <span id="lfs_settings" style="display:none">(<a id="lfs_settings_show" href="#">{{.i18n.Tr "repo.settings.advanced_settings"}}</a>)</span>
+</div>
+<div id="lfs_endpoint" style="display:none">
+ <span class="help">{{.i18n.Tr "repo.migrate_options_lfs_endpoint.description" "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md#server-discovery" | Str2html}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate_options_lfs_endpoint.description.local"}}{{end}}</span>
+ <div class="inline field {{if .Err_LFSEndpoint}}error{{end}}">
+ <label>{{.i18n.Tr "repo.migrate_options_lfs_endpoint.label"}}</label>
+ <input name="lfs_endpoint" value="{{.lfs_endpoint}}" placeholder="{{.i18n.Tr "repo.migrate_options_lfs_endpoint.placeholder"}}">
+ </div>
+</div>
+{{end}}
<div class="inline field {{if .Err_EnablePrune}}error{{end}}">
<label>{{.i18n.Tr "repo.mirror_prune"}}</label>
<div class="ui checkbox">
- <input id="enable_prune" name="enable_prune" type="checkbox" {{if .MirrorEnablePrune}}checked{{end}}>
- <label>{{.i18n.Tr "repo.mirror_prune_desc"}}</label>
+ <input id="enable_prune" name="enable_prune" type="checkbox" {{if .MirrorEnablePrune}}checked{{end}}>
+ <label>{{.i18n.Tr "repo.mirror_prune_desc"}}</label>
</div>
</div>
<div class="inline field {{if .Err_Interval}}error{{end}}">
</div>
</div>
+ {{if .LFSStartServer}}
+ <div class="inline field">
+ <label>{{.i18n.Tr "repo.mirror_lfs"}}</label>
+ <div class="ui checkbox">
+ <input id="mirror_lfs" name="mirror_lfs" type="checkbox" {{if .Mirror.LFS}}checked{{end}}>
+ <label>{{.i18n.Tr "repo.mirror_lfs_desc"}}</label>
+ </div>
+ </div>
+ <div class="field {{if .Err_LFSEndpoint}}error{{end}}">
+ <label for="mirror_lfs_endpoint">{{.i18n.Tr "repo.mirror_lfs_endpoint"}}</label>
+ <input id="mirror_lfs_endpoint" name="mirror_lfs_endpoint" value="{{.Mirror.LFSEndpoint}}" placeholder="{{.i18n.Tr "repo.migrate_options_lfs_endpoint.placeholder"}}">
+ <p class="help">{{.i18n.Tr "repo.mirror_lfs_endpoint_desc" "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md#server-discovery" | Str2html}}</p>
+ </div>
+ {{end}}
+
<div class="field">
<button class="ui green button">{{$.i18n.Tr "repo.settings.update_settings"}}</button>
</div>
"type": "boolean",
"x-go-name": "Labels"
},
+ "lfs": {
+ "type": "boolean",
+ "x-go-name": "LFS"
+ },
+ "lfs_endpoint": {
+ "type": "string",
+ "x-go-name": "LFSEndpoint"
+ },
"milestones": {
"type": "boolean",
"x-go-name": "Milestones"
"type": "boolean",
"x-go-name": "Labels"
},
+ "lfs": {
+ "type": "boolean",
+ "x-go-name": "LFS"
+ },
+ "lfs_endpoint": {
+ "type": "string",
+ "x-go-name": "LFSEndpoint"
+ },
"milestones": {
"type": "boolean",
"x-go-name": "Milestones"
const $pass = $('#auth_password');
const $token = $('#auth_token');
const $mirror = $('#mirror');
+const $lfs = $('#lfs');
+const $lfsSettings = $('#lfs_settings');
+const $lfsEndpoint = $('#lfs_endpoint');
const $items = $('#migrate_items').find('input[type=checkbox]');
export default function initMigration() {
checkAuth();
+ setLFSSettingsVisibility();
$user.on('keyup', () => {checkItems(false)});
$pass.on('keyup', () => {checkItems(false)});
$token.on('keyup', () => {checkItems(true)});
$mirror.on('change', () => {checkItems(true)});
+ $('#lfs_settings_show').on('click', () => { $lfsEndpoint.show(); return false });
+ $lfs.on('change', setLFSSettingsVisibility);
const $cloneAddr = $('#clone_addr');
$cloneAddr.on('change', () => {
$items.attr('disabled', true);
}
}
+
+function setLFSSettingsVisibility() {
+ const visible = $lfs.is(':checked');
+ $lfsSettings.toggle(visible);
+ $lfsEndpoint.hide();
+}
projects / gitea.git / commitdiff