diff options
author | Matthias Sohn <matthias.sohn@sap.com> | 2015-12-22 16:11:43 +0100 |
---|---|---|
committer | Saša Živkov <sasa.zivkov@sap.com> | 2016-02-04 17:49:43 +0100 |
commit | 3bae524f6f80f9cd9b4a29f456005b51fdaba1ee (patch) | |
tree | c0c145c9d026a5a04010dd80c90ab7fe3f6c4352 /org.eclipse.jgit.pgm | |
parent | 377616aca702144f6f79e1b376f3342ec81ec42b (diff) | |
download | jgit-3bae524f6f80f9cd9b4a29f456005b51fdaba1ee.tar.gz jgit-3bae524f6f80f9cd9b4a29f456005b51fdaba1ee.zip |
Support LFS protocol and a file system based LFS storage
Implement LfsProtocolServlet handling the "Git LFS v1 Batch API"
protocol [1]. Add a simple file system based LFS content store and the
debug-lfs-store command to simplify testing.
Introduce a LargeFileRepository interface to enable additional storage
implementation while reusing the same protocol implementation.
At the client side we have to configure the lfs.url, specify that
we use the batch API and we don't use authentication:
[lfs]
url = http://host:port/lfs
batch = true
[lfs "http://host:port/lfs"]
access = none
the git-lfs client appends the "objects/batch" to the lfs.url.
Hard code an Authorization header in the FileLfsRepository.getAction
because then git-lfs client will skip asking for credentials. It will
just forward the Authorization header from the response to the
download/upload request.
The FileLfsServlet supports file content storage for "Large File
Storage" (LFS) server as defined by the Github LFS API [2].
- upload and download of large files is probably network bound hence use
an asynchronous servlet for good scalability
- simple object storage in file system with 2 level fan-out
- use LockFile to protect writing large objects against multiple
concurrent uploads of the same object
- to prevent corrupt uploads the uploaded file is rejected if its hash
doesn't match id given in URL
The debug-lfs-store command is used to run the LfsProtocolServlet and,
optionally, the FileLfsServlet which makes it easier to setup a
local test server.
[1]
https://github.com/github/git-lfs/blob/master/docs/api/http-v1-batch.md
[2] https://github.com/github/git-lfs/tree/master/docs/api
Bug: 472961
Change-Id: I7378da5575159d2195138d799704880c5c82d5f3
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
Signed-off-by: Sasa Zivkov <sasa.zivkov@sap.com>
Diffstat (limited to 'org.eclipse.jgit.pgm')
6 files changed, 315 insertions, 2 deletions
diff --git a/org.eclipse.jgit.pgm/BUCK b/org.eclipse.jgit.pgm/BUCK index edcf2fc28f..47711b5c6b 100644 --- a/org.eclipse.jgit.pgm/BUCK +++ b/org.eclipse.jgit.pgm/BUCK @@ -9,8 +9,18 @@ java_library( '//org.eclipse.jgit:jgit', '//org.eclipse.jgit.archive:jgit-archive', '//org.eclipse.jgit.http.apache:http-apache', + '//org.eclipse.jgit.lfs.server:jgit-lfs-server', '//org.eclipse.jgit.ui:ui', '//lib:args4j', + '//lib:httpcomponents', + '//lib:httpcore', + '//lib/jetty:http', + '//lib/jetty:io', + '//lib/jetty:server', + '//lib/jetty:servlet', + '//lib/jetty:security', + '//lib/jetty:util', + '//lib:servlet-api' ], visibility = ['PUBLIC'], ) diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF index bf1fc5dd86..b654fc935b 100644 --- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF @@ -7,9 +7,25 @@ Bundle-Vendor: %provider_name Bundle-ActivationPolicy: lazy Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.7 -Import-Package: org.apache.commons.compress.archivers;version="[1.3,2.0)", +Import-Package: javax.servlet;version="[3.1.0,4.0.0)", + javax.servlet.http;version="[3.1.0,4.0.0)", + org.apache.commons.compress.archivers;version="[1.3,2.0)", org.apache.commons.compress.archivers.tar;version="[1.3,2.0)", org.apache.commons.compress.archivers.zip;version="[1.3,2.0)", + org.eclipse.jetty.continuation;version="[9.0.0,10.0.0)", + org.eclipse.jetty.http;version="[9.0.0,10.0.0)", + org.eclipse.jetty.io;version="[9.0.0,10.0.0)", + org.eclipse.jetty.security;version="[9.0.0,10.0.0)", + org.eclipse.jetty.security.authentication;version="[9.0.0,10.0.0)", + org.eclipse.jetty.server;version="[9.0.0,10.0.0)", + org.eclipse.jetty.server.handler;version="[9.0.0,10.0.0)", + org.eclipse.jetty.server.nio;version="[9.0.0,10.0.0)", + org.eclipse.jetty.servlet;version="[9.0.0,10.0.0)", + org.eclipse.jetty.util;version="[9.0.0,10.0.0)", + org.eclipse.jetty.util.component;version="[9.0.0,10.0.0)", + org.eclipse.jetty.util.log;version="[9.0.0,10.0.0)", + org.eclipse.jetty.util.security;version="[9.0.0,10.0.0)", + org.eclipse.jetty.util.thread;version="[9.0.0,10.0.0)", org.eclipse.jgit.api;version="[4.3.0,4.4.0)", org.eclipse.jgit.api.errors;version="[4.3.0,4.4.0)", org.eclipse.jgit.archive;version="[4.3.0,4.4.0)", @@ -23,8 +39,11 @@ Import-Package: org.apache.commons.compress.archivers;version="[1.3,2.0)", org.eclipse.jgit.internal.storage.file;version="[4.3.0,4.4.0)", org.eclipse.jgit.internal.storage.pack;version="[4.3.0,4.4.0)", org.eclipse.jgit.internal.storage.reftree;version="[4.3.0,4.4.0)", + org.eclipse.jgit.lfs.lib;version="[4.3.0,4.4.0)", + org.eclipse.jgit.lfs.server;version="[4.3.0,4.4.0)", + org.eclipse.jgit.lfs.server.fs;version="[4.3.0,4.4.0)", org.eclipse.jgit.lib;version="[4.3.0,4.4.0)", - org.eclipse.jgit.merge;version="4.3.0", + org.eclipse.jgit.merge;version="[4.3.0,4.4.0)", org.eclipse.jgit.nls;version="[4.3.0,4.4.0)", org.eclipse.jgit.notes;version="[4.3.0,4.4.0)", org.eclipse.jgit.revplot;version="[4.3.0,4.4.0)", diff --git a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin index 6aa20041b5..c159f7d609 100644 --- a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin +++ b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin @@ -38,6 +38,7 @@ org.eclipse.jgit.pgm.UploadPack org.eclipse.jgit.pgm.Version org.eclipse.jgit.pgm.debug.DiffAlgorithms +org.eclipse.jgit.pgm.debug.LfsStore org.eclipse.jgit.pgm.debug.MakeCacheTree org.eclipse.jgit.pgm.debug.ReadDirCache org.eclipse.jgit.pgm.debug.RebuildCommitGraph diff --git a/org.eclipse.jgit.pgm/pom.xml b/org.eclipse.jgit.pgm/pom.xml index 0531e44400..8d0bcf9be6 100644 --- a/org.eclipse.jgit.pgm/pom.xml +++ b/org.eclipse.jgit.pgm/pom.xml @@ -122,6 +122,34 @@ <artifactId>log4j</artifactId> <version>${log4j-version}</version> </dependency> + + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-servlet</artifactId> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + + <dependency> + <groupId>org.eclipse.jgit</groupId> + <artifactId>org.eclipse.jgit.junit.http</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.eclipse.jgit</groupId> + <artifactId>org.eclipse.jgit.lfs</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>org.eclipse.jgit</groupId> + <artifactId>org.eclipse.jgit.lfs.server</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> <build> diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index b4b1261b37..544e234e3a 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -125,6 +125,7 @@ metaVar_remoteName=name metaVar_revision=REVISION metaVar_seconds=SECONDS metaVar_service=SERVICE +metaVar_lfsStorage=STORAGE metaVar_treeish=tree-ish metaVar_uriish=uri-ish metaVar_url=URL @@ -218,6 +219,10 @@ usage_DisplayTheVersionOfJgit=Display the version of jgit usage_Gc=Cleanup unnecessary files and optimize the local repository usage_Glog=View commit history as a graph usage_IndexPack=Build pack index file for an existing packed archive +usage_LFSDirectory=Directory to store large objects +usage_LFSPort=Server http port +usage_LFSRunStore=fs: store lfs objects in file system +usage_LFSStoreUrl=URL of the LFS store usage_LongFormat=Always output the long format usage_LsRemote=List references in a remote repository usage_lsRemoteHeads=Show only refs starting with refs/heads @@ -347,6 +352,7 @@ usage_resetReference=Reset to given reference name usage_resetHard=Resets the index and working tree usage_resetSoft=Resets without touching the index file nor the working tree usage_resetMixed=Resets the index but not the working tree +usage_runLfsStore=Run LFS Store in a given directory usage_setTheGitRepositoryToOperateOn=set the git repository to operate on usage_show=display one commit usage_showRefNamesMatchingCommits=Show ref names matching commits diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java new file mode 100644 index 0000000000..18398f664e --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/LfsStore.java @@ -0,0 +1,249 @@ +/* + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.pgm.debug; + +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jgit.lfs.server.LargeFileRepository; +import org.eclipse.jgit.lfs.server.LfsProtocolServlet; +import org.eclipse.jgit.lfs.server.fs.FileLfsServlet; +import org.eclipse.jgit.lfs.server.fs.FileLfsRepository; +import org.eclipse.jgit.pgm.Command; +import org.eclipse.jgit.pgm.TextBuiltin; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.Option; + +@Command(common = true, usage = "usage_runLfsStore") +class LfsStore extends TextBuiltin { + + /** + * Tiny web application server for testing + */ + class AppServer { + + private final Server server; + + private final ServerConnector connector; + + private final ContextHandlerCollection contexts; + + private URI uri; + + AppServer(int port) { + server = new Server(); + + HttpConfiguration http_config = new HttpConfiguration(); + http_config.setOutputBufferSize(32768); + + connector = new ServerConnector(server, + new HttpConnectionFactory(http_config)); + connector.setPort(port); + try { + String host = InetAddress.getByName("localhost") //$NON-NLS-1$ + .getHostAddress(); + connector.setHost(host); + if (host.contains(":") && !host.startsWith("[")) //$NON-NLS-1$ //$NON-NLS-2$ + host = "[" + host + "]"; //$NON-NLS-1$//$NON-NLS-2$ + uri = new URI("http://" + host + ":" + port); //$NON-NLS-1$ //$NON-NLS-2$ + } catch (UnknownHostException e) { + throw new RuntimeException("Cannot find localhost", e); //$NON-NLS-1$ + } catch (URISyntaxException e) { + throw new RuntimeException("Unexpected URI error on " + uri, e); //$NON-NLS-1$ + } + + contexts = new ContextHandlerCollection(); + server.setHandler(contexts); + server.setConnectors(new Connector[] { connector }); + } + + /** + * Create a new servlet context within the server. + * <p> + * This method should be invoked before the server is started, once for + * each context the caller wants to register. + * + * @param path + * path of the context; use "/" for the root context if + * binding to the root is desired. + * @return the context to add servlets into. + */ + ServletContextHandler addContext(String path) { + assertNotRunning(); + if ("".equals(path)) //$NON-NLS-1$ + path = "/"; //$NON-NLS-1$ + + ServletContextHandler ctx = new ServletContextHandler(); + ctx.setContextPath(path); + contexts.addHandler(ctx); + + return ctx; + } + + void start() throws Exception { + server.start(); + } + + void stop() throws Exception { + server.stop(); + } + + URI getURI() { + return uri; + } + + private void assertNotRunning() { + if (server.isRunning()) { + throw new IllegalStateException("server is running"); //$NON-NLS-1$ + } + } + } + + private static enum StoreType { + FS; + } + + private static final String OBJECTS = "objects/"; //$NON-NLS-1$ + + private static final String STORE_PATH = "/" + OBJECTS + "*"; //$NON-NLS-1$//$NON-NLS-2$ + + private static final String PROTOCOL_PATH = "/lfs/objects/batch"; //$NON-NLS-1$ + + @Option(name = "--port", aliases = {"-p" }, metaVar = "metaVar_port", + usage = "usage_LFSPort") + int port; + + @Option(name = "--store", metaVar = "metaVar_lfsStorage", usage = "usage_LFSRunStore") + StoreType storeType; + + @Option(name = "--store-url", aliases = {"-u" }, metaVar = "metaVar_url", + usage = "usage_LFSStoreUrl") + String storeUrl; + + @Argument(required = false, metaVar = "metaVar_directory", usage = "usage_LFSDirectory") + String directory; + + String protocolUrl; + + String accessKey; + + String secretKey; + + @Override + protected boolean requiresRepository() { + return false; + } + + protected void run() throws Exception { + AppServer server = new AppServer(port); + URI baseURI = server.getURI(); + ServletContextHandler app = server.addContext("/"); //$NON-NLS-1$ + + final LargeFileRepository repository; + switch (storeType) { + case FS: + Path dir = Paths.get(directory); + FileLfsRepository fsRepo = new FileLfsRepository( + getStoreUrl(baseURI), dir); + FileLfsServlet content = new FileLfsServlet(fsRepo, 30000); + app.addServlet(new ServletHolder(content), STORE_PATH); + repository = fsRepo; + break; + + default: + throw new IllegalArgumentException( + "Unknown store type: " + storeType); //$NON-NLS-1$ + } + + LfsProtocolServlet protocol = new LfsProtocolServlet() { + + private static final long serialVersionUID = 1L; + + @Override + protected LargeFileRepository getLargeFileRepository() { + return repository; + } + + }; + app.addServlet(new ServletHolder(protocol), PROTOCOL_PATH); + + server.start(); + + outw.println("LFS protocol URL: " + getProtocolUrl(baseURI)); //$NON-NLS-1$ + if (storeType == StoreType.FS) { + outw.println("LFS objects located in: " + directory); //$NON-NLS-1$ + outw.println("LFS store URL: " + getStoreUrl(baseURI)); //$NON-NLS-1$ + } + } + + private String getStoreUrl(URI baseURI) { + if (storeUrl == null) { + if (storeType == StoreType.FS) { + storeUrl = baseURI + "/" + OBJECTS; //$NON-NLS-1$ + } else { + die("Local store not running and no --store-url specified"); //$NON-NLS-1$ + } + } + return storeUrl; + } + + private String getProtocolUrl(URI baseURI) { + if (protocolUrl == null) { + protocolUrl = baseURI + PROTOCOL_PATH; + } + return protocolUrl; + } +} |