]> source.dussan.org Git - jgit.git/commit
Support LFS protocol and a file system based LFS storage 77/52177/16
authorMatthias Sohn <matthias.sohn@sap.com>
Tue, 22 Dec 2015 15:11:43 +0000 (16:11 +0100)
committerSaša Živkov <sasa.zivkov@sap.com>
Thu, 4 Feb 2016 16:49:43 +0000 (17:49 +0100)
commit3bae524f6f80f9cd9b4a29f456005b51fdaba1ee
treec0c145c9d026a5a04010dd80c90ab7fe3f6c4352
parent377616aca702144f6f79e1b376f3342ec81ec42b
Support LFS protocol and a file system based LFS storage

Implement LfsProtocolServlet handling the "Git LFS v1 Batch API"
protocol [1]. Add a simple file system based LFS content store and the
debug-lfs-store command to simplify testing.

Introduce a LargeFileRepository interface to enable additional storage
implementation while reusing the same protocol implementation.

At the client side we have to configure the lfs.url, specify that
we use the batch API and we don't use authentication:

  [lfs]
  url = http://host:port/lfs
  batch = true
  [lfs "http://host:port/lfs"]
  access = none

the git-lfs client appends the "objects/batch" to the lfs.url.

Hard code an Authorization header in the FileLfsRepository.getAction
because then git-lfs client will skip asking for credentials. It will
just forward the Authorization header from the response to the
download/upload request.

The FileLfsServlet supports file content storage for "Large File
Storage" (LFS) server as defined by the Github LFS API [2].

- upload and download of large files is probably network bound hence use
  an asynchronous servlet for good scalability
- simple object storage in file system with 2 level fan-out
- use LockFile to protect writing large objects against multiple
  concurrent uploads of the same object
- to prevent corrupt uploads the uploaded file is rejected if its hash
  doesn't match id given in URL

The debug-lfs-store command is used to run the LfsProtocolServlet and,
optionally, the FileLfsServlet which makes it easier to setup a
local test server.

[1]
https://github.com/github/git-lfs/blob/master/docs/api/http-v1-batch.md
[2] https://github.com/github/git-lfs/tree/master/docs/api

Bug: 472961
Change-Id: I7378da5575159d2195138d799704880c5c82d5f3
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
Signed-off-by: Sasa Zivkov <sasa.zivkov@sap.com>
66 files changed:
BUCK
lib/BUCK
org.eclipse.jgit.junit.http/src/org/eclipse/jgit/junit/http/AppServer.java
org.eclipse.jgit.lfs.server.test/.classpath [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.gitignore [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.project [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.core.resources.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.core.runtime.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.jdt.core.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.jdt.ui.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.mylyn.tasks.ui.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.mylyn.team.ui.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.pde.api.tools.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/.settings/org.eclipse.pde.core.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/BUCK [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/META-INF/MANIFEST.MF [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/build.properties [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/plugin.properties [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/pom.xml [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/DownloadTest.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/UploadTest.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.classpath [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.fbprefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.gitignore [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.project [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.core.resources.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.core.runtime.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.jdt.core.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.jdt.ui.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.mylyn.tasks.ui.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.mylyn.team.ui.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.pde.api.tools.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/.settings/org.eclipse.pde.core.prefs [new file with mode: 0644]
org.eclipse.jgit.lfs.server/BUCK [new file with mode: 0644]
org.eclipse.jgit.lfs.server/META-INF/MANIFEST.MF [new file with mode: 0644]
org.eclipse.jgit.lfs.server/about.html [new file with mode: 0644]
org.eclipse.jgit.lfs.server/build.properties [new file with mode: 0644]
org.eclipse.jgit.lfs.server/plugin.properties [new file with mode: 0644]
org.eclipse.jgit.lfs.server/pom.xml [new file with mode: 0644]
org.eclipse.jgit.lfs.server/resources/org/eclipse/jgit/lfs/server/internal/LfsServerText.properties [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LargeFileRepository.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsObject.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsProtocolServlet.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/Response.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/TransferHandler.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/AtomicObjectOutputStream.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsRepository.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsServlet.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectDownloadListener.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java [new file with mode: 0644]
org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/internal/LfsServerText.java [new file with mode: 0644]
org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF
org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptLongObjectException.java [new file with mode: 0644]
org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java
org.eclipse.jgit.packaging/org.eclipse.jgit.lfs.feature/feature.xml
org.eclipse.jgit.packaging/org.eclipse.jgit.lfs.feature/pom.xml
org.eclipse.jgit.packaging/org.eclipse.jgit.repository/pom.xml
org.eclipse.jgit.pgm/BUCK
org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
org.eclipse.jgit.pgm/pom.xml
org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties
org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java [new file with mode: 0644]
org.eclipse.jgit/META-INF/MANIFEST.MF
pom.xml