LFS: Add remote download to SmudgeFilter

Transfer data in chunks of 8k Transferring data byte per byte is slow,
running checkout with CleanFilter on a 2.9MB file takes 20 seconds.
Using a buffer of 8k shrinks this time to 70ms.

Also register the filter commands in a way that the native GIT LFS can
be used alongside with JGit.

Implements auto-discovery of LFS server URL when cloning from a Gerrit
LFS server.

Change-Id: I452a5aa177dcb346d92af08b27c2e35200f246fd
Also-by: Christian Halstrick <christian.halstrick@sap.com>
Signed-off-by: Markus Duft <markus.duft@ssi-schaefer.com>

8 files changed, 686 insertions, 24 deletions

diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java
index 4a98286ddb..3e6f9961a8 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java
@@ -55,6 +55,7 @@ import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.lfs.errors.CorruptMediaFile;
 import org.eclipse.jgit.lfs.internal.AtomicObjectOutputStream;
 import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.Constants;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.util.FileUtils;
 
@@ -91,10 +92,11 @@ public class CleanFilter extends FilterCommand {
 	 * {@link FilterCommandRegistry#register(String, FilterCommandFactory)}
 	 */
 	public final static void register() {
-		FilterCommandRegistry.register(
-				org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
-						+ "lfs/clean", //$NON-NLS-1$
-				FACTORY);
+		FilterCommandRegistry
+				.register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+						+ Constants.ATTR_FILTER_DRIVER_PREFIX
+						+ org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_CLEAN,
+						FACTORY);
 	}
 
 	// Used to compute the hash for the original content
@@ -127,7 +129,7 @@ public class CleanFilter extends FilterCommand {
 	public CleanFilter(Repository db, InputStream in, OutputStream out)
 			throws IOException {
 		super(in, out);
-		lfsUtil = new Lfs(FileUtils.toPath(db.getDirectory()).resolve("lfs")); //$NON-NLS-1$
+		lfsUtil = new Lfs(db);
 		Files.createDirectories(lfsUtil.getLfsTmpDir());
 		tmpFile = lfsUtil.createTmpFile();
 		this.aOut = new AtomicObjectOutputStream(tmpFile.toAbsolutePath());
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/InstallLfsCommand.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/InstallLfsCommand.java
new file mode 100644
index 0000000000..19c7fe430a
--- /dev/null
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/InstallLfsCommand.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2018, Markus Duft <markus.duft@ssi-schaefer.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.concurrent.Callable;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+import org.eclipse.jgit.lfs.internal.LfsText;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.SystemReader;
+
+/**
+ * Installs all required LFS properties for the current user, analogous to 'git
+ * lfs install', but defaulting to using JGit builtin hooks.
+ *
+ * @since 4.11
+ */
+public class InstallLfsCommand implements Callable<Void>{
+
+	private static final String[] ARGS_USER = new String[] { "lfs", "install" }; //$NON-NLS-1$//$NON-NLS-2$
+
+	private static final String[] ARGS_LOCAL = new String[] { "lfs", "install", //$NON-NLS-1$//$NON-NLS-2$
+			"--local" }; //$NON-NLS-1$
+
+	private Repository repository;
+
+	/** {@inheritDoc} */
+	@Override
+	public Void call() throws Exception {
+		StoredConfig cfg = null;
+		if (repository == null) {
+			cfg = loadUserConfig();
+		} else {
+			cfg = repository.getConfig();
+		}
+
+		cfg.setBoolean(ConfigConstants.CONFIG_FILTER_SECTION, Constants.LFS,
+				ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, true);
+		cfg.setBoolean(ConfigConstants.CONFIG_FILTER_SECTION, Constants.LFS,
+				ConfigConstants.CONFIG_KEY_REQUIRED, true);
+
+		cfg.save();
+
+		// try to run git lfs install, we really don't care if it is present
+		// and/or works here (yet).
+		ProcessBuilder builder = FS.DETECTED.runInShell("git", //$NON-NLS-1$
+				repository == null ? ARGS_USER : ARGS_LOCAL);
+		if (repository != null) {
+			builder.directory(repository.isBare() ? repository.getDirectory()
+					: repository.getWorkTree());
+		}
+		FS.DETECTED.runProcess(builder, null, null, (String) null);
+
+		return null;
+	}
+
+	/**
+	 * @param repo
+	 *            the repository to install LFS into locally instead of the user
+	 *            configuration
+	 */
+	public void setRepository(Repository repo) {
+		this.repository = repo;
+	}
+
+	private StoredConfig loadUserConfig() throws IOException {
+		FileBasedConfig c = SystemReader.getInstance().openUserConfig(null,
+				FS.DETECTED);
+		try {
+			c.load();
+		} catch (ConfigInvalidException e1) {
+			throw new IOException(MessageFormat
+					.format(LfsText.get().userConfigInvalid, c.getFile()
+							.getAbsolutePath(), e1),
+					e1);
+		}
+
+		return c;
+	}
+
+}
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java
index 138996d82f..40d83420ec 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java
@@ -47,6 +47,8 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 
 import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.Repository;
 
 /**
  * Class which represents the lfs folder hierarchy inside a {@code .git} folder
@@ -66,12 +68,26 @@ public class Lfs {
 	 * @param root
 	 *            the path to the LFS media directory. Will be
 	 *            {@code "<repo>/.git/lfs"}
+	 * @deprecated use {@link #Lfs(Repository)} instead.
 	 */
+	@Deprecated
 	public Lfs(Path root) {
 		this.root = root;
 	}
 
 	/**
+	 * Constructor for Lfs.
+	 *
+	 * @param db
+	 *            the associated repo
+	 *
+	 * @since 4.11
+	 */
+	public Lfs(Repository db) {
+		this.root = db.getDirectory().toPath().resolve(Constants.LFS);
+	}
+
+	/**
 	 * Get the LFS root directory
 	 *
 	 * @return the path to the LFS directory
@@ -118,7 +134,7 @@ public class Lfs {
 	public Path getMediaFile(AnyLongObjectId id) {
 		String idStr = id.name();
 		return getLfsObjDir().resolve(idStr.substring(0, 2))
-				.resolve(idStr.substring(2));
+				.resolve(idStr.substring(2, 4)).resolve(idStr);
 	}
 
 	/**
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java
new file mode 100644
index 0000000000..81b1810208
--- /dev/null
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com>
+ * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This interface describes the network protocol used between lfs client and lfs
+ * server
+ *
+ * @since 4.11
+ */
+public interface Protocol {
+	/** A request sent to an LFS server */
+	class Request {
+		/** The operation of this request */
+		public String operation;
+
+		/** The objects of this request */
+		public List<ObjectSpec> objects;
+	}
+
+	/** A response received from an LFS server */
+	class Response {
+		public List<ObjectInfo> objects;
+	}
+
+	/**
+	 * MetaData of an LFS object. Needs to be specified when requesting objects
+	 * from the LFS server and is also returned in the response
+	 */
+	class ObjectSpec {
+		public String oid; // the objectid
+
+		public long size; // the size of the object
+	}
+
+	/**
+	 * Describes in a response all actions the LFS server offers for a single
+	 * object
+	 */
+	class ObjectInfo extends ObjectSpec {
+		public Map<String, Action> actions; // Maps operation to action
+
+		public Error error;
+	}
+
+	/**
+	 * Describes in a Response a single action the client can execute on a
+	 * single object
+	 */
+	class Action {
+		public String href;
+
+		public Map<String, String> header;
+	}
+
+	/** Describes an error to be returned by the LFS batch API */
+	class Error {
+		public int code;
+
+		public String message;
+	}
+
+	/**
+	 * The "download" operation
+	 */
+	String OPERATION_DOWNLOAD = "download"; //$NON-NLS-1$
+
+	/**
+	 * The "upload" operation
+	 */
+	String OPERATION_UPLOAD = "upload"; //$NON-NLS-1$
+
+	/**
+	 * The contenttype used in LFS requests
+	 */
+	String CONTENTTYPE_VND_GIT_LFS_JSON = "application/vnd.git-lfs+json; charset=utf-8"; //$NON-NLS-1$
+
+	/**
+	 * Authorization header when auto-discovering via SSH.
+	 */
+	String HDR_AUTH = "Authorization"; //$NON-NLS-1$
+
+	/**
+	 * Prefix of authentication token obtained through SSH.
+	 */
+	String HDR_AUTH_SSH_PREFIX = "Ssh: "; //$NON-NLS-1$
+
+	/**
+	 * Path to the LFS info servlet.
+	 */
+	String INFO_LFS_ENDPOINT = "/info/lfs"; //$NON-NLS-1$
+
+	/**
+	 * Path to the LFS objects servlet.
+	 */
+	String OBJECTS_LFS_ENDPOINT = "/objects/batch"; //$NON-NLS-1$
+}
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
index 941edec1d8..841c30a335 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java
@@ -42,18 +42,52 @@
  */
 package org.eclipse.jgit.lfs;
 
+import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
+import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
+import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
+import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
+
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.net.ProxySelector;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.TreeMap;
 
 import org.eclipse.jgit.attributes.FilterCommand;
 import org.eclipse.jgit.attributes.FilterCommandFactory;
 import org.eclipse.jgit.attributes.FilterCommandRegistry;
+import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException;
+import org.eclipse.jgit.lfs.internal.LfsText;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
 import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.util.FileUtils;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.transport.HttpConfig;
+import org.eclipse.jgit.transport.HttpTransport;
+import org.eclipse.jgit.transport.RemoteSession;
+import org.eclipse.jgit.transport.SshSessionFactory;
+import org.eclipse.jgit.transport.URIish;
+import org.eclipse.jgit.transport.http.HttpConnection;
+import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.HttpSupport;
+import org.eclipse.jgit.util.io.MessageWriter;
+import org.eclipse.jgit.util.io.StreamCopyThread;
+
+import com.google.gson.Gson;
+import com.google.gson.stream.JsonReader;
 
 /**
  * Built-in LFS smudge filter
@@ -62,13 +96,17 @@ import org.eclipse.jgit.util.FileUtils;
  * and this filter is configured for that content, then this filter will replace
  * the content of LFS pointer files with the original content. This happens e.g.
  * when a checkout needs to update a working tree file which is under LFS
- * control. This implementation expects that the origin content is already
- * available in the .git/lfs/objects folder. This implementation will not
- * contact any LFS servers in order to get the missing content.
+ * control.
  *
  * @since 4.6
  */
 public class SmudgeFilter extends FilterCommand {
+
+	/**
+	 * Max number of bytes to copy in a single {@link #run()} call.
+	 */
+	private static final int MAX_COPY_BYTES = 1024 * 1024 * 256;
+
 	/**
 	 * The factory is responsible for creating instances of
 	 * {@link org.eclipse.jgit.lfs.SmudgeFilter}
@@ -85,10 +123,11 @@ public class SmudgeFilter extends FilterCommand {
 	 * Registers this filter in JGit by calling
 	 */
 	public final static void register() {
-		FilterCommandRegistry.register(
-				org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
-						+ "lfs/smudge", //$NON-NLS-1$
-				FACTORY);
+		FilterCommandRegistry
+				.register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+						+ Constants.ATTR_FILTER_DRIVER_PREFIX
+						+ org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE,
+						FACTORY);
 	}
 
 	private Lfs lfs;
@@ -107,12 +146,251 @@ public class SmudgeFilter extends FilterCommand {
 	public SmudgeFilter(Repository db, InputStream in, OutputStream out)
 			throws IOException {
 		super(in, out);
-		lfs = new Lfs(FileUtils.toPath(db.getDirectory()).resolve(Constants.LFS));
+		lfs = new Lfs(db);
 		LfsPointer res = LfsPointer.parseLfsPointer(in);
 		if (res != null) {
-			Path mediaFile = lfs.getMediaFile(res.getOid());
-			if (Files.exists(mediaFile)) {
-				this.in = Files.newInputStream(mediaFile);
+			AnyLongObjectId oid = res.getOid();
+			Path mediaFile = lfs.getMediaFile(oid);
+			if (!Files.exists(mediaFile)) {
+				downloadLfsResource(db, res);
+			}
+			this.in = Files.newInputStream(mediaFile);
+		}
+	}
+
+	/**
+	 * Download content which is hosted on a LFS server
+	 *
+	 * @param db
+	 *            the repository to work with
+	 * @param res
+	 *            the objects to download
+	 * @return the paths of all mediafiles which have been downloaded
+	 * @throws IOException
+	 */
+	private Collection<Path> downloadLfsResource(Repository db,
+			LfsPointer... res)
+			throws IOException {
+		Collection<Path> downloadedPaths = new ArrayList<>();
+		Map<String, LfsPointer> oidStr2ptr = new HashMap<>();
+		for (LfsPointer p : res) {
+			oidStr2ptr.put(p.getOid().name(), p);
+		}
+		HttpConnection lfsServerConn = getLfsConnection(db,
+				HttpSupport.METHOD_POST);
+		Gson gson = new Gson();
+		lfsServerConn.getOutputStream()
+				.write(gson.toJson(body(res)).getBytes(StandardCharsets.UTF_8));
+		int responseCode = lfsServerConn.getResponseCode();
+		if (responseCode != HttpConnection.HTTP_OK) {
+			throw new IOException(
+					MessageFormat.format(LfsText.get().serverFailure,
+							lfsServerConn.getURL(),
+							Integer.valueOf(responseCode)));
+		}
+		try (JsonReader reader = new JsonReader(
+				new InputStreamReader(lfsServerConn.getInputStream()))) {
+			Protocol.Response resp = gson.fromJson(reader,
+					Protocol.Response.class);
+			for (Protocol.ObjectInfo o : resp.objects) {
+				if (o.actions == null) {
+					continue;
+				}
+				LfsPointer ptr = oidStr2ptr.get(o.oid);
+				if (ptr == null) {
+					// received an object we didn't request
+					continue;
+				}
+				if (ptr.getSize() != o.size) {
+					throw new IOException(MessageFormat.format(
+							LfsText.get().inconsistentContentLength,
+							lfsServerConn.getURL(), Long.valueOf(ptr.getSize()),
+							Long.valueOf(o.size)));
+				}
+				Protocol.Action downloadAction = o.actions
+						.get(Protocol.OPERATION_DOWNLOAD);
+				if (downloadAction == null || downloadAction.href == null) {
+					continue;
+				}
+				URL contentUrl = new URL(downloadAction.href);
+				HttpConnection contentServerConn = HttpTransport
+						.getConnectionFactory().create(contentUrl,
+								HttpSupport.proxyFor(ProxySelector.getDefault(),
+										contentUrl));
+				contentServerConn.setRequestMethod(HttpSupport.METHOD_GET);
+				downloadAction.header.forEach(
+						(k, v) -> contentServerConn.setRequestProperty(k, v));
+				if (contentUrl.getProtocol().equals("https") && !db.getConfig() //$NON-NLS-1$
+						.getBoolean(HttpConfig.HTTP, HttpConfig.SSL_VERIFY_KEY,
+								true)) {
+					HttpSupport.disableSslVerify(contentServerConn);
+				}
+				contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING,
+						ENCODING_GZIP);
+				responseCode = contentServerConn.getResponseCode();
+				if (responseCode != HttpConnection.HTTP_OK) {
+					throw new IOException(
+							MessageFormat.format(LfsText.get().serverFailure,
+									contentServerConn.getURL(),
+									Integer.valueOf(responseCode)));
+				}
+				Path path = lfs.getMediaFile(ptr.getOid());
+				path.getParent().toFile().mkdirs();
+				try (InputStream contentIn = contentServerConn
+						.getInputStream()) {
+					long bytesCopied = Files.copy(contentIn, path);
+					if (bytesCopied != o.size) {
+						throw new IOException(MessageFormat.format(
+								LfsText.get().wrongAmoutOfDataReceived,
+								contentServerConn.getURL(),
+								Long.valueOf(bytesCopied),
+								Long.valueOf(o.size)));
+					}
+					downloadedPaths.add(path);
+				}
+			}
+		}
+		return downloadedPaths;
+	}
+
+	private Protocol.Request body(LfsPointer... resources) {
+		Protocol.Request req = new Protocol.Request();
+		req.operation = Protocol.OPERATION_DOWNLOAD;
+		if (resources != null) {
+			req.objects = new LinkedList<>();
+			for (LfsPointer res : resources) {
+				Protocol.ObjectSpec o = new Protocol.ObjectSpec();
+				o.oid = res.getOid().getName();
+				o.size = res.getSize();
+				req.objects.add(o);
+			}
+		}
+		return req;
+	}
+
+	/**
+	 * Determine URL of LFS server by looking into config parameters lfs.url,
+	 * lfs.<remote>.url or remote.<remote>.url. The LFS server URL is computed
+	 * from remote.<remote>.url by appending "/info/lfs"
+	 *
+	 * @param db
+	 *            the repository to work with
+	 * @param method
+	 *            the method (GET,PUT,...) of the request this connection will
+	 *            be used for
+	 * @return the url for the lfs server. e.g.
+	 *         "https://github.com/github/git-lfs.git/info/lfs"
+	 * @throws IOException
+	 */
+	private HttpConnection getLfsConnection(Repository db, String method)
+			throws IOException {
+		StoredConfig config = db.getConfig();
+		String lfsEndpoint = config.getString(Constants.LFS, null,
+				ConfigConstants.CONFIG_KEY_URL);
+		Map<String, String> additionalHeaders = new TreeMap<>();
+		if (lfsEndpoint == null) {
+			String remoteUrl = null;
+			for (String remote : db.getRemoteNames()) {
+				lfsEndpoint = config.getString(Constants.LFS, remote,
+						ConfigConstants.CONFIG_KEY_URL);
+				if (lfsEndpoint == null
+						&& (remote.equals(
+								org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) {
+					remoteUrl = config.getString(
+							ConfigConstants.CONFIG_KEY_REMOTE, remote,
+							ConfigConstants.CONFIG_KEY_URL);
+				}
+				break;
+			}
+			if (lfsEndpoint == null && remoteUrl != null) {
+				try {
+					URIish u = new URIish(remoteUrl);
+
+					if ("ssh".equals(u.getScheme())) { //$NON-NLS-1$
+						// discover and authenticate; git-lfs does "ssh -p
+						// <port> -- <host> git-lfs-authenticate <project>
+						// <upload/download>"
+						String json = runSshCommand(u.setPath(""), db.getFS(), //$NON-NLS-1$
+								"git-lfs-authenticate " + extractProjectName(u) //$NON-NLS-1$
+										+ " " + Protocol.OPERATION_DOWNLOAD); //$NON-NLS-1$
+
+						Protocol.Action action = new Gson().fromJson(json,
+								Protocol.Action.class);
+						additionalHeaders.putAll(action.header);
+						lfsEndpoint = action.href;
+					} else {
+						lfsEndpoint = remoteUrl + Protocol.INFO_LFS_ENDPOINT;
+					}
+				} catch (Exception e) {
+					lfsEndpoint = null; // could not discover
+				}
+			} else {
+				lfsEndpoint = lfsEndpoint + Protocol.INFO_LFS_ENDPOINT;
+			}
+		}
+		if (lfsEndpoint == null) {
+			throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl);
+		}
+		URL url = new URL(lfsEndpoint + Protocol.OBJECTS_LFS_ENDPOINT);
+		HttpConnection connection = HttpTransport.getConnectionFactory().create(
+				url, HttpSupport.proxyFor(ProxySelector.getDefault(), url));
+		connection.setDoOutput(true);
+		if (url.getProtocol().equals("https") //$NON-NLS-1$
+				&& !config.getBoolean(HttpConfig.HTTP,
+						HttpConfig.SSL_VERIFY_KEY, true)) {
+			HttpSupport.disableSslVerify(connection);
+		}
+		connection.setRequestMethod(method);
+		connection.setRequestProperty(HDR_ACCEPT,
+				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
+		connection.setRequestProperty(HDR_CONTENT_TYPE,
+				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
+		additionalHeaders
+				.forEach((k, v) -> connection.setRequestProperty(k, v));
+		return connection;
+	}
+
+	private String extractProjectName(URIish u) {
+		String path = u.getPath().substring(1);
+		if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) {
+			return path.substring(0, path.length() - 4);
+		} else {
+			return path;
+		}
+	}
+
+	private String runSshCommand(URIish sshUri, FS fs, String command)
+			throws IOException {
+		RemoteSession session = null;
+		Process process = null;
+		StreamCopyThread errorThread = null;
+		try (MessageWriter stderr = new MessageWriter()) {
+			session = SshSessionFactory.getInstance().getSession(sshUri,
+					null, fs, 5_000);
+			process = session.exec(command, 0);
+			errorThread = new StreamCopyThread(process.getErrorStream(),
+					stderr.getRawStream());
+			errorThread.start();
+			try (BufferedReader reader = new BufferedReader(
+					new InputStreamReader(process.getInputStream(),
+							org.eclipse.jgit.lib.Constants.CHARSET))) {
+				return reader.readLine();
+			}
+		} finally {
+			if (process != null) {
+				process.destroy();
+			}
+			if (errorThread != null) {
+				try {
+					errorThread.halt();
+				} catch (InterruptedException e) {
+					// Stop waiting and return anyway.
+				} finally {
+					errorThread = null;
+				}
+			}
+			if (session != null) {
+				SshSessionFactory.getInstance().releaseSession(session);
 			}
 		}
 	}
@@ -120,14 +398,33 @@ public class SmudgeFilter extends FilterCommand {
 	/** {@inheritDoc} */
 	@Override
 	public int run() throws IOException {
-		int b;
+		int totalRead = 0;
+		int length = 0;
 		if (in != null) {
-			while ((b = in.read()) != -1) {
-				out.write(b);
+			byte[] buf = new byte[8192];
+			while ((length = in.read(buf)) != -1) {
+				out.write(buf, 0, length);
+				totalRead += length;
+
+				// when threshold reached, loop back to the caller. otherwise we
+				// could only support files up to 2GB (int return type)
+				// properly. we will be called again as long as we don't return
+				// -1 here.
+				if (totalRead >= MAX_COPY_BYTES) {
+					// leave streams open - we need them in the next call.
+					return totalRead;
+				}
 			}
+		}
+
+		if (totalRead == 0 && length == -1) {
+			// we're totally done :)
 			in.close();
+			out.close();
+			return length;
 		}
-		out.close();
-		return -1;
+
+		return totalRead;
 	}
+
 }
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsConfigInvalidException.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsConfigInvalidException.java
new file mode 100644
index 0000000000..5320af0b78
--- /dev/null
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsConfigInvalidException.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs.errors;
+
+import java.io.IOException;
+
+/**
+ * Thrown when a LFS configuration problem has been detected (i.e. unable to
+ * find the remote LFS repository URL).
+ *
+ * @since 4.11
+ */
+public class LfsConfigInvalidException extends IOException {
+	private static final long serialVersionUID = 1L;
+
+	/**
+	 * Constructor for LfsConfigInvalidException.
+	 *
+	 * @param msg
+	 *            the error description
+	 */
+	public LfsConfigInvalidException(String msg) {
+		super(msg);
+	}
+
+}
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java
index ae5548c85c..8fc2b1bef5 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java
@@ -62,13 +62,18 @@ public class LfsText extends TranslationBundle {
 	// @formatter:off
 	/***/ public String corruptLongObject;
 	/***/ public String inconsistentMediafileLength;
+	/***/ public String inconsistentContentLength;
 	/***/ public String incorrectLONG_OBJECT_ID_LENGTH;
 	/***/ public String invalidLongId;
 	/***/ public String invalidLongIdLength;
+	/***/ public String lfsUnavailable;
 	/***/ public String requiredHashFunctionNotAvailable;
 	/***/ public String repositoryNotFound;
 	/***/ public String repositoryReadOnly;
-	/***/ public String lfsUnavailable;
 	/***/ public String lfsUnathorized;
 	/***/ public String lfsFailedToGetRepository;
+	/***/ public String lfsNoDownloadUrl;
+	/***/ public String serverFailure;
+	/***/ public String wrongAmoutOfDataReceived;
+	/***/ public String userConfigInvalid;
 }
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java
index d5b96ab0fd..fbfbf377bd 100644
--- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java
+++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java
@@ -108,6 +108,13 @@ public final class Constants {
 	public static final String VERIFY = "verify";
 
 	/**
+	 * Prefix for all LFS related filters.
+	 *
+	 * @since 4.11
+	 */
+	public static final String ATTR_FILTER_DRIVER_PREFIX = "lfs/";
+
+	/**
 	 * Create a new digest function for objects.
 	 *
 	 * @return a new digest object.