org.eclipse.jetty.util.log;version="[9.4.5,10.0.0)",
org.eclipse.jetty.util.security;version="[9.4.5,10.0.0)",
org.eclipse.jetty.util.thread;version="[9.4.5,10.0.0)",
+ org.eclipse.jgit.api;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.internal.storage.file;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.junit;version="[4.11.0,4.12.0)",
org.eclipse.jgit.junit.http;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.lfs;version="[4.11.0,4.12.0)",
org.eclipse.jgit.lfs.lib;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.lfs.errors;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.lfs.server;version="[4.11.0,4.12.0)",
org.eclipse.jgit.lfs.server.fs;version="[4.11.0,4.12.0)",
org.eclipse.jgit.lfs.test;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.lib;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.storage.file;version="[4.11.0,4.12.0)",
org.eclipse.jgit.util;version="[4.11.0,4.12.0)",
org.hamcrest.core;version="[1.1.0,2.0.0)",
org.junit;version="[4.12,5.0.0)",
--- /dev/null
+/*
+ * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs.server.fs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.junit.JGitTestUtil;
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.lfs.CleanFilter;
+import org.eclipse.jgit.lfs.SmudgeFilter;
+import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
+import org.eclipse.jgit.util.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CheckoutTest extends LfsServerTest {
+
+ Git git;
+ private TestRepository tdb;
+
+ @Override
+ @Before
+ public void setup() throws Exception {
+ super.setup();
+
+ SmudgeFilter.register();
+ CleanFilter.register();
+
+ Path tmp = Files.createTempDirectory("jgit_test_");
+ Repository db = FileRepositoryBuilder
+ .create(tmp.resolve(".git").toFile());
+ db.create();
+ StoredConfig cfg = db.getConfig();
+ cfg.setString("filter", "lfs", "usejgitbuiltin", "true");
+ cfg.setString("lfs", null, "url", server.getURI().toString() + "/lfs");
+ cfg.save();
+
+ tdb = new TestRepository<>(db);
+ tdb.branch("test").commit()
+ .add(".gitattributes",
+ "*.bin filter=lfs diff=lfs merge=lfs -text ")
+ .add("a.bin",
+ "version https://git-lfs.github.com/spec/v1\noid sha256:8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414\nsize 7\n")
+ .create();
+ git = Git.wrap(db);
+ tdb.branch("test2").commit().add(".gitattributes",
+ "*.bin filter=lfs diff=lfs merge=lfs -text ").create();
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ tdb.getRepository().close();
+ FileUtils.delete(tdb.getRepository().getWorkTree(),
+ FileUtils.RECURSIVE);
+ }
+
+ @Test
+ public void testUnknownContent() throws Exception {
+ git.checkout().setName("test").call();
+ // unknown content. We will see the pointer file
+ assertEquals(
+ "version https://git-lfs.github.com/spec/v1\noid sha256:8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414\nsize 7\n",
+ JGitTestUtil.read(git.getRepository(), "a.bin"));
+ assertEquals("[POST /lfs/objects/batch 200]",
+ server.getRequests().toString());
+ }
+
+ @Test
+ public void testKnownContent() throws Exception {
+ putContent(
+ LongObjectId.fromString(
+ "8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414"),
+ "1234567");
+ git.checkout().setName("test").call();
+ // known content. we will see the actual content of the LFS blob.
+ assertEquals(
+ "1234567",
+ JGitTestUtil.read(git.getRepository(), "a.bin"));
+ assertEquals(
+ "[PUT /lfs/objects/8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414 200"
+ + ", POST /lfs/objects/batch 200"
+ + ", GET /lfs/objects/8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414 200]",
+ server.getRequests().toString());
+
+ git.checkout().setName("test2").call();
+ assertFalse(JGitTestUtil.check(git.getRepository(), "a.bin"));
+ git.checkout().setName("test").call();
+ // unknown content. We will see the pointer file
+ assertEquals("1234567",
+ JGitTestUtil.read(git.getRepository(), "a.bin"));
+ assertEquals(3, server.getRequests().size());
+ }
+
+}
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jgit.junit.http.AppServer;
+import org.eclipse.jgit.lfs.errors.LfsException;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.lib.Constants;
import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.eclipse.jgit.lfs.server.LargeFileRepository;
+import org.eclipse.jgit.lfs.server.LfsProtocolServlet;
import org.eclipse.jgit.lfs.test.LongObjectIdTestUtils;
import org.eclipse.jgit.util.FileUtils;
import org.eclipse.jgit.util.IO;
this.repository = new FileLfsRepository(null, dir);
servlet = new FileLfsServlet(repository, timeout);
app.addServlet(new ServletHolder(servlet), "/objects/*");
+
+ LfsProtocolServlet protocol = new LfsProtocolServlet() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected LargeFileRepository getLargeFileRepository(
+ LfsRequest request, String path) {
+ return repository;
+ }
+
+ @Override
+ protected LargeFileRepository getLargeFileRepository(
+ LfsRequest request, String path, String auth)
+ throws LfsException {
+ return repository;
+ }
+ };
+ app.addServlet(new ServletHolder(protocol), "/objects/batch");
+
server.setUp();
+ this.repository.setUrl(server.getURI() + "/lfs/objects/");
}
@After
*/
public class FileLfsRepository implements LargeFileRepository {
- private final String url;
+ private String url;
private final Path dir;
/**
while (o >= p)
dst[o--] = '0';
}
+
+ /**
+ * @return the url of the content server
+ * @since 4.11
+ */
+ public String getUrl() {
+ return url;
+ }
+
+ /**
+ * @param url
+ * the url of the content server
+ * @since 4.11
+ */
+ public void setUrl(String url) {
+ this.url = url;
+ }
}
org.eclipse.jgit.lfs.internal;version="4.11.0";x-friends:="org.eclipse.jgit.lfs.test,org.eclipse.jgit.lfs.server.fs,org.eclipse.jgit.lfs.server",
org.eclipse.jgit.lfs.lib;version="4.11.0"
Bundle-RequiredExecutionEnvironment: JavaSE-1.8
-Import-Package: org.eclipse.jgit.annotations;version="[4.11.0,4.12.0)";resolution:=optional,
+Import-Package: com.google.gson;version="[2.8.2,3.0.0)",
+ com.google.gson.stream;version="[2.8.2,3.0.0)",
+ org.apache.http.impl.client;version="[4.2.6,5.0.0)",
+ org.apache.http.impl.conn;version="[4.2.6,5.0.0)",
+ org.eclipse.jgit.annotations;version="[4.11.0,4.12.0)";resolution:=optional,
org.eclipse.jgit.attributes;version="[4.11.0,4.12.0)",
org.eclipse.jgit.errors;version="[4.11.0,4.12.0)",
org.eclipse.jgit.internal.storage.file;version="[4.11.0,4.12.0)",
org.eclipse.jgit.lib;version="[4.11.0,4.12.0)",
org.eclipse.jgit.nls;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.storage.file;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.transport;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.transport.http;version="[4.11.0,4.12.0)",
org.eclipse.jgit.treewalk;version="[4.11.0,4.12.0)",
org.eclipse.jgit.treewalk.filter;version="[4.11.0,4.12.0)",
- org.eclipse.jgit.util;version="[4.11.0,4.12.0)"
+ org.eclipse.jgit.util;version="[4.11.0,4.12.0)",
+ org.eclipse.jgit.util.io;version="[4.11.0,4.12.0)"
<artifactId>org.eclipse.jgit</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/</sourceDirectory>
corruptLongObject=The content hash ''{0}'' of the long object ''{1}'' doesn''t match its id, the corrupt object will be deleted.
incorrectLONG_OBJECT_ID_LENGTH=Incorrect LONG_OBJECT_ID_LENGTH.
-inconsistentMediafileLength=mediafile {0} has unexpected length; expected {1} but found {2}.
+inconsistentMediafileLength=Mediafile {0} has unexpected length; expected {1} but found {2}.
+inconsistentContentLength=Unexpected content length reported by LFS server ({0}), expected {1} but reported was {2}
invalidLongId=Invalid id: {0}
invalidLongIdLength=Invalid id length {0}; should be {1}
+lfsUnavailable=LFS is not available for repository {0}
requiredHashFunctionNotAvailable=Required hash function {0} not available.
repositoryNotFound=Repository {0} not found
repositoryReadOnly=Repository {0} is read-only
lfsUnavailable=LFS is not available for repository {0}
lfsUnathorized=Not authorized to perform operation {0} on repository {1}
lfsFailedToGetRepository=failed to get repository {0}
+lfsNoDownloadUrl="Need to download object from LFS server but couldn't determine LFS server URL"
+serverFailure=When trying to open a connection to {0} the server responded with an error code. rc={1}
+wrongAmoutOfDataReceived=While downloading data from the content server {0} {1} bytes have been received while {2} have been expected
+userConfigInvalid="User config file {0} invalid {1}"
\ No newline at end of file
import org.eclipse.jgit.lfs.errors.CorruptMediaFile;
import org.eclipse.jgit.lfs.internal.AtomicObjectOutputStream;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.Constants;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.util.FileUtils;
* {@link FilterCommandRegistry#register(String, FilterCommandFactory)}
*/
public final static void register() {
- FilterCommandRegistry.register(
- org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
- + "lfs/clean", //$NON-NLS-1$
- FACTORY);
+ FilterCommandRegistry
+ .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+ + Constants.ATTR_FILTER_DRIVER_PREFIX
+ + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_CLEAN,
+ FACTORY);
}
// Used to compute the hash for the original content
public CleanFilter(Repository db, InputStream in, OutputStream out)
throws IOException {
super(in, out);
- lfsUtil = new Lfs(FileUtils.toPath(db.getDirectory()).resolve("lfs")); //$NON-NLS-1$
+ lfsUtil = new Lfs(db);
Files.createDirectories(lfsUtil.getLfsTmpDir());
tmpFile = lfsUtil.createTmpFile();
this.aOut = new AtomicObjectOutputStream(tmpFile.toAbsolutePath());
--- /dev/null
+/*
+ * Copyright (C) 2018, Markus Duft <markus.duft@ssi-schaefer.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.concurrent.Callable;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lfs.internal.LfsText;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.SystemReader;
+
+/**
+ * Installs all required LFS properties for the current user, analogous to 'git
+ * lfs install', but defaulting to using JGit builtin hooks.
+ *
+ * @since 4.11
+ */
+public class InstallLfsCommand implements Callable<Void>{
+
+ private static final String[] ARGS_USER = new String[] { "lfs", "install" }; //$NON-NLS-1$//$NON-NLS-2$
+
+ private static final String[] ARGS_LOCAL = new String[] { "lfs", "install", //$NON-NLS-1$//$NON-NLS-2$
+ "--local" }; //$NON-NLS-1$
+
+ private Repository repository;
+
+ /** {@inheritDoc} */
+ @Override
+ public Void call() throws Exception {
+ StoredConfig cfg = null;
+ if (repository == null) {
+ cfg = loadUserConfig();
+ } else {
+ cfg = repository.getConfig();
+ }
+
+ cfg.setBoolean(ConfigConstants.CONFIG_FILTER_SECTION, Constants.LFS,
+ ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, true);
+ cfg.setBoolean(ConfigConstants.CONFIG_FILTER_SECTION, Constants.LFS,
+ ConfigConstants.CONFIG_KEY_REQUIRED, true);
+
+ cfg.save();
+
+ // try to run git lfs install, we really don't care if it is present
+ // and/or works here (yet).
+ ProcessBuilder builder = FS.DETECTED.runInShell("git", //$NON-NLS-1$
+ repository == null ? ARGS_USER : ARGS_LOCAL);
+ if (repository != null) {
+ builder.directory(repository.isBare() ? repository.getDirectory()
+ : repository.getWorkTree());
+ }
+ FS.DETECTED.runProcess(builder, null, null, (String) null);
+
+ return null;
+ }
+
+ /**
+ * @param repo
+ * the repository to install LFS into locally instead of the user
+ * configuration
+ */
+ public void setRepository(Repository repo) {
+ this.repository = repo;
+ }
+
+ private StoredConfig loadUserConfig() throws IOException {
+ FileBasedConfig c = SystemReader.getInstance().openUserConfig(null,
+ FS.DETECTED);
+ try {
+ c.load();
+ } catch (ConfigInvalidException e1) {
+ throw new IOException(MessageFormat
+ .format(LfsText.get().userConfigInvalid, c.getFile()
+ .getAbsolutePath(), e1),
+ e1);
+ }
+
+ return c;
+ }
+
+}
import java.nio.file.Path;
import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.Repository;
/**
* Class which represents the lfs folder hierarchy inside a {@code .git} folder
* @param root
* the path to the LFS media directory. Will be
* {@code "<repo>/.git/lfs"}
+ * @deprecated use {@link #Lfs(Repository)} instead.
*/
+ @Deprecated
public Lfs(Path root) {
this.root = root;
}
+ /**
+ * Constructor for Lfs.
+ *
+ * @param db
+ * the associated repo
+ *
+ * @since 4.11
+ */
+ public Lfs(Repository db) {
+ this.root = db.getDirectory().toPath().resolve(Constants.LFS);
+ }
+
/**
* Get the LFS root directory
*
public Path getMediaFile(AnyLongObjectId id) {
String idStr = id.name();
return getLfsObjDir().resolve(idStr.substring(0, 2))
- .resolve(idStr.substring(2));
+ .resolve(idStr.substring(2, 4)).resolve(idStr);
}
/**
--- /dev/null
+/*
+ * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com>
+ * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This interface describes the network protocol used between lfs client and lfs
+ * server
+ *
+ * @since 4.11
+ */
+public interface Protocol {
+ /** A request sent to an LFS server */
+ class Request {
+ /** The operation of this request */
+ public String operation;
+
+ /** The objects of this request */
+ public List<ObjectSpec> objects;
+ }
+
+ /** A response received from an LFS server */
+ class Response {
+ public List<ObjectInfo> objects;
+ }
+
+ /**
+ * MetaData of an LFS object. Needs to be specified when requesting objects
+ * from the LFS server and is also returned in the response
+ */
+ class ObjectSpec {
+ public String oid; // the objectid
+
+ public long size; // the size of the object
+ }
+
+ /**
+ * Describes in a response all actions the LFS server offers for a single
+ * object
+ */
+ class ObjectInfo extends ObjectSpec {
+ public Map<String, Action> actions; // Maps operation to action
+
+ public Error error;
+ }
+
+ /**
+ * Describes in a Response a single action the client can execute on a
+ * single object
+ */
+ class Action {
+ public String href;
+
+ public Map<String, String> header;
+ }
+
+ /** Describes an error to be returned by the LFS batch API */
+ class Error {
+ public int code;
+
+ public String message;
+ }
+
+ /**
+ * The "download" operation
+ */
+ String OPERATION_DOWNLOAD = "download"; //$NON-NLS-1$
+
+ /**
+ * The "upload" operation
+ */
+ String OPERATION_UPLOAD = "upload"; //$NON-NLS-1$
+
+ /**
+ * The contenttype used in LFS requests
+ */
+ String CONTENTTYPE_VND_GIT_LFS_JSON = "application/vnd.git-lfs+json; charset=utf-8"; //$NON-NLS-1$
+
+ /**
+ * Authorization header when auto-discovering via SSH.
+ */
+ String HDR_AUTH = "Authorization"; //$NON-NLS-1$
+
+ /**
+ * Prefix of authentication token obtained through SSH.
+ */
+ String HDR_AUTH_SSH_PREFIX = "Ssh: "; //$NON-NLS-1$
+
+ /**
+ * Path to the LFS info servlet.
+ */
+ String INFO_LFS_ENDPOINT = "/info/lfs"; //$NON-NLS-1$
+
+ /**
+ * Path to the LFS objects servlet.
+ */
+ String OBJECTS_LFS_ENDPOINT = "/objects/batch"; //$NON-NLS-1$
+}
*/
package org.eclipse.jgit.lfs;
+import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
+import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
+import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
+import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
+
+import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.net.ProxySelector;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.TreeMap;
import org.eclipse.jgit.attributes.FilterCommand;
import org.eclipse.jgit.attributes.FilterCommandFactory;
import org.eclipse.jgit.attributes.FilterCommandRegistry;
+import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException;
+import org.eclipse.jgit.lfs.internal.LfsText;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.util.FileUtils;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.transport.HttpConfig;
+import org.eclipse.jgit.transport.HttpTransport;
+import org.eclipse.jgit.transport.RemoteSession;
+import org.eclipse.jgit.transport.SshSessionFactory;
+import org.eclipse.jgit.transport.URIish;
+import org.eclipse.jgit.transport.http.HttpConnection;
+import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.HttpSupport;
+import org.eclipse.jgit.util.io.MessageWriter;
+import org.eclipse.jgit.util.io.StreamCopyThread;
+
+import com.google.gson.Gson;
+import com.google.gson.stream.JsonReader;
/**
* Built-in LFS smudge filter
* and this filter is configured for that content, then this filter will replace
* the content of LFS pointer files with the original content. This happens e.g.
* when a checkout needs to update a working tree file which is under LFS
- * control. This implementation expects that the origin content is already
- * available in the .git/lfs/objects folder. This implementation will not
- * contact any LFS servers in order to get the missing content.
+ * control.
*
* @since 4.6
*/
public class SmudgeFilter extends FilterCommand {
+
+ /**
+ * Max number of bytes to copy in a single {@link #run()} call.
+ */
+ private static final int MAX_COPY_BYTES = 1024 * 1024 * 256;
+
/**
* The factory is responsible for creating instances of
* {@link org.eclipse.jgit.lfs.SmudgeFilter}
* Registers this filter in JGit by calling
*/
public final static void register() {
- FilterCommandRegistry.register(
- org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
- + "lfs/smudge", //$NON-NLS-1$
- FACTORY);
+ FilterCommandRegistry
+ .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+ + Constants.ATTR_FILTER_DRIVER_PREFIX
+ + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE,
+ FACTORY);
}
private Lfs lfs;
public SmudgeFilter(Repository db, InputStream in, OutputStream out)
throws IOException {
super(in, out);
- lfs = new Lfs(FileUtils.toPath(db.getDirectory()).resolve(Constants.LFS));
+ lfs = new Lfs(db);
LfsPointer res = LfsPointer.parseLfsPointer(in);
if (res != null) {
- Path mediaFile = lfs.getMediaFile(res.getOid());
- if (Files.exists(mediaFile)) {
- this.in = Files.newInputStream(mediaFile);
+ AnyLongObjectId oid = res.getOid();
+ Path mediaFile = lfs.getMediaFile(oid);
+ if (!Files.exists(mediaFile)) {
+ downloadLfsResource(db, res);
+ }
+ this.in = Files.newInputStream(mediaFile);
+ }
+ }
+
+ /**
+ * Download content which is hosted on a LFS server
+ *
+ * @param db
+ * the repository to work with
+ * @param res
+ * the objects to download
+ * @return the paths of all mediafiles which have been downloaded
+ * @throws IOException
+ */
+ private Collection<Path> downloadLfsResource(Repository db,
+ LfsPointer... res)
+ throws IOException {
+ Collection<Path> downloadedPaths = new ArrayList<>();
+ Map<String, LfsPointer> oidStr2ptr = new HashMap<>();
+ for (LfsPointer p : res) {
+ oidStr2ptr.put(p.getOid().name(), p);
+ }
+ HttpConnection lfsServerConn = getLfsConnection(db,
+ HttpSupport.METHOD_POST);
+ Gson gson = new Gson();
+ lfsServerConn.getOutputStream()
+ .write(gson.toJson(body(res)).getBytes(StandardCharsets.UTF_8));
+ int responseCode = lfsServerConn.getResponseCode();
+ if (responseCode != HttpConnection.HTTP_OK) {
+ throw new IOException(
+ MessageFormat.format(LfsText.get().serverFailure,
+ lfsServerConn.getURL(),
+ Integer.valueOf(responseCode)));
+ }
+ try (JsonReader reader = new JsonReader(
+ new InputStreamReader(lfsServerConn.getInputStream()))) {
+ Protocol.Response resp = gson.fromJson(reader,
+ Protocol.Response.class);
+ for (Protocol.ObjectInfo o : resp.objects) {
+ if (o.actions == null) {
+ continue;
+ }
+ LfsPointer ptr = oidStr2ptr.get(o.oid);
+ if (ptr == null) {
+ // received an object we didn't request
+ continue;
+ }
+ if (ptr.getSize() != o.size) {
+ throw new IOException(MessageFormat.format(
+ LfsText.get().inconsistentContentLength,
+ lfsServerConn.getURL(), Long.valueOf(ptr.getSize()),
+ Long.valueOf(o.size)));
+ }
+ Protocol.Action downloadAction = o.actions
+ .get(Protocol.OPERATION_DOWNLOAD);
+ if (downloadAction == null || downloadAction.href == null) {
+ continue;
+ }
+ URL contentUrl = new URL(downloadAction.href);
+ HttpConnection contentServerConn = HttpTransport
+ .getConnectionFactory().create(contentUrl,
+ HttpSupport.proxyFor(ProxySelector.getDefault(),
+ contentUrl));
+ contentServerConn.setRequestMethod(HttpSupport.METHOD_GET);
+ downloadAction.header.forEach(
+ (k, v) -> contentServerConn.setRequestProperty(k, v));
+ if (contentUrl.getProtocol().equals("https") && !db.getConfig() //$NON-NLS-1$
+ .getBoolean(HttpConfig.HTTP, HttpConfig.SSL_VERIFY_KEY,
+ true)) {
+ HttpSupport.disableSslVerify(contentServerConn);
+ }
+ contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING,
+ ENCODING_GZIP);
+ responseCode = contentServerConn.getResponseCode();
+ if (responseCode != HttpConnection.HTTP_OK) {
+ throw new IOException(
+ MessageFormat.format(LfsText.get().serverFailure,
+ contentServerConn.getURL(),
+ Integer.valueOf(responseCode)));
+ }
+ Path path = lfs.getMediaFile(ptr.getOid());
+ path.getParent().toFile().mkdirs();
+ try (InputStream contentIn = contentServerConn
+ .getInputStream()) {
+ long bytesCopied = Files.copy(contentIn, path);
+ if (bytesCopied != o.size) {
+ throw new IOException(MessageFormat.format(
+ LfsText.get().wrongAmoutOfDataReceived,
+ contentServerConn.getURL(),
+ Long.valueOf(bytesCopied),
+ Long.valueOf(o.size)));
+ }
+ downloadedPaths.add(path);
+ }
+ }
+ }
+ return downloadedPaths;
+ }
+
+ private Protocol.Request body(LfsPointer... resources) {
+ Protocol.Request req = new Protocol.Request();
+ req.operation = Protocol.OPERATION_DOWNLOAD;
+ if (resources != null) {
+ req.objects = new LinkedList<>();
+ for (LfsPointer res : resources) {
+ Protocol.ObjectSpec o = new Protocol.ObjectSpec();
+ o.oid = res.getOid().getName();
+ o.size = res.getSize();
+ req.objects.add(o);
+ }
+ }
+ return req;
+ }
+
+ /**
+ * Determine URL of LFS server by looking into config parameters lfs.url,
+ * lfs.<remote>.url or remote.<remote>.url. The LFS server URL is computed
+ * from remote.<remote>.url by appending "/info/lfs"
+ *
+ * @param db
+ * the repository to work with
+ * @param method
+ * the method (GET,PUT,...) of the request this connection will
+ * be used for
+ * @return the url for the lfs server. e.g.
+ * "https://github.com/github/git-lfs.git/info/lfs"
+ * @throws IOException
+ */
+ private HttpConnection getLfsConnection(Repository db, String method)
+ throws IOException {
+ StoredConfig config = db.getConfig();
+ String lfsEndpoint = config.getString(Constants.LFS, null,
+ ConfigConstants.CONFIG_KEY_URL);
+ Map<String, String> additionalHeaders = new TreeMap<>();
+ if (lfsEndpoint == null) {
+ String remoteUrl = null;
+ for (String remote : db.getRemoteNames()) {
+ lfsEndpoint = config.getString(Constants.LFS, remote,
+ ConfigConstants.CONFIG_KEY_URL);
+ if (lfsEndpoint == null
+ && (remote.equals(
+ org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) {
+ remoteUrl = config.getString(
+ ConfigConstants.CONFIG_KEY_REMOTE, remote,
+ ConfigConstants.CONFIG_KEY_URL);
+ }
+ break;
+ }
+ if (lfsEndpoint == null && remoteUrl != null) {
+ try {
+ URIish u = new URIish(remoteUrl);
+
+ if ("ssh".equals(u.getScheme())) { //$NON-NLS-1$
+ // discover and authenticate; git-lfs does "ssh -p
+ // <port> -- <host> git-lfs-authenticate <project>
+ // <upload/download>"
+ String json = runSshCommand(u.setPath(""), db.getFS(), //$NON-NLS-1$
+ "git-lfs-authenticate " + extractProjectName(u) //$NON-NLS-1$
+ + " " + Protocol.OPERATION_DOWNLOAD); //$NON-NLS-1$
+
+ Protocol.Action action = new Gson().fromJson(json,
+ Protocol.Action.class);
+ additionalHeaders.putAll(action.header);
+ lfsEndpoint = action.href;
+ } else {
+ lfsEndpoint = remoteUrl + Protocol.INFO_LFS_ENDPOINT;
+ }
+ } catch (Exception e) {
+ lfsEndpoint = null; // could not discover
+ }
+ } else {
+ lfsEndpoint = lfsEndpoint + Protocol.INFO_LFS_ENDPOINT;
+ }
+ }
+ if (lfsEndpoint == null) {
+ throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl);
+ }
+ URL url = new URL(lfsEndpoint + Protocol.OBJECTS_LFS_ENDPOINT);
+ HttpConnection connection = HttpTransport.getConnectionFactory().create(
+ url, HttpSupport.proxyFor(ProxySelector.getDefault(), url));
+ connection.setDoOutput(true);
+ if (url.getProtocol().equals("https") //$NON-NLS-1$
+ && !config.getBoolean(HttpConfig.HTTP,
+ HttpConfig.SSL_VERIFY_KEY, true)) {
+ HttpSupport.disableSslVerify(connection);
+ }
+ connection.setRequestMethod(method);
+ connection.setRequestProperty(HDR_ACCEPT,
+ Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
+ connection.setRequestProperty(HDR_CONTENT_TYPE,
+ Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
+ additionalHeaders
+ .forEach((k, v) -> connection.setRequestProperty(k, v));
+ return connection;
+ }
+
+ private String extractProjectName(URIish u) {
+ String path = u.getPath().substring(1);
+ if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) {
+ return path.substring(0, path.length() - 4);
+ } else {
+ return path;
+ }
+ }
+
+ private String runSshCommand(URIish sshUri, FS fs, String command)
+ throws IOException {
+ RemoteSession session = null;
+ Process process = null;
+ StreamCopyThread errorThread = null;
+ try (MessageWriter stderr = new MessageWriter()) {
+ session = SshSessionFactory.getInstance().getSession(sshUri,
+ null, fs, 5_000);
+ process = session.exec(command, 0);
+ errorThread = new StreamCopyThread(process.getErrorStream(),
+ stderr.getRawStream());
+ errorThread.start();
+ try (BufferedReader reader = new BufferedReader(
+ new InputStreamReader(process.getInputStream(),
+ org.eclipse.jgit.lib.Constants.CHARSET))) {
+ return reader.readLine();
+ }
+ } finally {
+ if (process != null) {
+ process.destroy();
+ }
+ if (errorThread != null) {
+ try {
+ errorThread.halt();
+ } catch (InterruptedException e) {
+ // Stop waiting and return anyway.
+ } finally {
+ errorThread = null;
+ }
+ }
+ if (session != null) {
+ SshSessionFactory.getInstance().releaseSession(session);
}
}
}
/** {@inheritDoc} */
@Override
public int run() throws IOException {
- int b;
+ int totalRead = 0;
+ int length = 0;
if (in != null) {
- while ((b = in.read()) != -1) {
- out.write(b);
+ byte[] buf = new byte[8192];
+ while ((length = in.read(buf)) != -1) {
+ out.write(buf, 0, length);
+ totalRead += length;
+
+ // when threshold reached, loop back to the caller. otherwise we
+ // could only support files up to 2GB (int return type)
+ // properly. we will be called again as long as we don't return
+ // -1 here.
+ if (totalRead >= MAX_COPY_BYTES) {
+ // leave streams open - we need them in the next call.
+ return totalRead;
+ }
}
+ }
+
+ if (totalRead == 0 && length == -1) {
+ // we're totally done :)
in.close();
+ out.close();
+ return length;
}
- out.close();
- return -1;
+
+ return totalRead;
}
+
}
--- /dev/null
+/*
+ * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs.errors;
+
+import java.io.IOException;
+
+/**
+ * Thrown when a LFS configuration problem has been detected (i.e. unable to
+ * find the remote LFS repository URL).
+ *
+ * @since 4.11
+ */
+public class LfsConfigInvalidException extends IOException {
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Constructor for LfsConfigInvalidException.
+ *
+ * @param msg
+ * the error description
+ */
+ public LfsConfigInvalidException(String msg) {
+ super(msg);
+ }
+
+}
// @formatter:off
/***/ public String corruptLongObject;
/***/ public String inconsistentMediafileLength;
+ /***/ public String inconsistentContentLength;
/***/ public String incorrectLONG_OBJECT_ID_LENGTH;
/***/ public String invalidLongId;
/***/ public String invalidLongIdLength;
+ /***/ public String lfsUnavailable;
/***/ public String requiredHashFunctionNotAvailable;
/***/ public String repositoryNotFound;
/***/ public String repositoryReadOnly;
- /***/ public String lfsUnavailable;
/***/ public String lfsUnathorized;
/***/ public String lfsFailedToGetRepository;
+ /***/ public String lfsNoDownloadUrl;
+ /***/ public String serverFailure;
+ /***/ public String wrongAmoutOfDataReceived;
+ /***/ public String userConfigInvalid;
}
*/
public static final String VERIFY = "verify";
+ /**
+ * Prefix for all LFS related filters.
+ *
+ * @since 4.11
+ */
+ public static final String ATTR_FILTER_DRIVER_PREFIX = "lfs/";
+
/**
* Create a new digest function for objects.
*
com.jcraft.jsch;version="[0.1.37,0.2.0)",
javax.crypto,
javax.net.ssl,
+ javax.servlet.http;version="[2.5.0,3.2.0)",
org.slf4j;version="[1.7.0,2.0.0)",
org.xml.sax,
org.xml.sax.helpers
* @since 4.7
*/
public static final String CONFIG_KEY_RECURSE_SUBMODULES = "recurseSubmodules";
+
+ /**
+ * The "required" key
+ * @since 4.11
+ */
+ public static final String CONFIG_KEY_REQUIRED = "required";
}
/**
* Inspect config and attributes to return a filtercommand applicable for
- * the current path
+ * the current path, but without expanding %f occurences
*
* @param filterCommandType
* which type of filterCommand should be executed. E.g. "clean",
projects / jgit.git / commitdiff