Add built-in LFS smudge filter for local case

Adds a JGit built-in implementation of the "git lfs smudge" filter. This
filter should do the same as the one described in [1] besides that it
only supports the local case when the lfs objects are already present in
the media directory. Remote cases where download of LFS objects from an
LFS server is needed will be done in a later commit.

[1] https://github.com/github/git-lfs/blob/master/docs/man/git-lfs-smudge.1.ronn

Change-Id: I8ff661d4edd3667ef7f86f3b4fa33e568eb4c8f4

org.eclipse.jgit.lfs/META-INF/MANIFEST.MF

@@ -10,7 +10,8 @@ Export-Package: org.eclipse.jgit.lfs;version="4.6.0",
  org.eclipse.jgit.lfs.internal;version="4.6.0";x-friends:="org.eclipse.jgit.lfs.test",
  org.eclipse.jgit.lfs.lib;version="4.6.0"
 Bundle-RequiredExecutionEnvironment: JavaSE-1.7
-Import-Package:  org.eclipse.jgit.attributes;version="[4.6.0,4.7.0)",
+Import-Package: org.eclipse.jgit.annotations;version="[4.6.0,4.7.0)";resolution:=optional,
+ org.eclipse.jgit.attributes;version="[4.6.0,4.7.0)",
  org.eclipse.jgit.internal.storage.file;version="[4.6.0,4.7.0)",
  org.eclipse.jgit.lib;version="[4.6.0,4.7.0)",
  org.eclipse.jgit.nls;version="[4.6.0,4.7.0)",

org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java

@@ -42,9 +42,14 @@
  */
 package org.eclipse.jgit.lfs;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.PrintStream;
 
+import org.eclipse.jgit.annotations.Nullable;
 import org.eclipse.jgit.lfs.lib.Constants;
 import org.eclipse.jgit.lfs.lib.LongObjectId;
 
@@ -113,9 +118,50 @@ public class LfsPointer {
 		}
 	}
 
+	/**
+	 * Try to parse the data provided by an InputStream to the format defined by
+	 * {@link #VERSION}
+	 *
+	 * @param in
+	 *            the {@link InputStream} from where to read the data
+	 * @return an {@link LfsPointer} or <code>null</code> if the stream was not
+	 *         parseable as LfsPointer
+	 * @throws IOException
+	 */
+	@Nullable
+	public static LfsPointer parseLfsPointer(InputStream in)
+			throws IOException {
+		boolean versionLine = false;
+		LongObjectId id = null;
+		long sz = -1;
+
+		try (BufferedReader br = new BufferedReader(
+				new InputStreamReader(in))) {
+			for (String s = br.readLine(); s != null; s = br.readLine()) {
+				if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$
+					continue;
+				} else if (s.startsWith("version") && s.length() > 8 //$NON-NLS-1$
+						&& s.substring(8).trim().equals(VERSION)) {
+					versionLine = true;
+				} else if (s.startsWith("oid sha256:")) { //$NON-NLS-1$
+					id = LongObjectId.fromString(s.substring(11).trim());
+				} else if (s.startsWith("size") && s.length() > 5) { //$NON-NLS-1$
+					sz = Long.parseLong(s.substring(5).trim());
+				} else {
+					return null;
+				}
+			}
+			if (versionLine && id != null && sz > -1) {
+				return new LfsPointer(id, sz);
+			}
+		}
+		return null;
+	}
+
 	@Override
 	public String toString() {
 		return "LfsPointer: oid=" + LongObjectId.toString(oid) + ", size=" //$NON-NLS-1$ //$NON-NLS-2$
 				+ size;
 	}
-}
+}
+

org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.eclipse.jgit.attributes.FilterCommand;
+import org.eclipse.jgit.attributes.FilterCommandFactory;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lib.Repository;
+
+/**
+ * Built-in LFS smudge filter
+ *
+ * When content is read from git's object-database and written to the filesystem
+ * and this filter is configured for that content, then this filter will replace
+ * the content of LFS pointer files with the original content. This happens e.g.
+ * when a checkout needs to update a working tree file which is under LFS
+ * control. This implementation expects that the origin content is already
+ * available in the .git/lfs/objects folder. This implementation will not
+ * contact any LFS servers in order to get the missing content.
+ *
+ * @since 4.6
+ */
+public class SmudgeFilter extends FilterCommand {
+	/**
+	 * The factory is responsible for creating instances of {@link SmudgeFilter}
+	 */
+	public final static FilterCommandFactory FACTORY = new FilterCommandFactory() {
+		@Override
+		public FilterCommand create(Repository db, InputStream in,
+				OutputStream out) throws IOException {
+			return new SmudgeFilter(db, in, out);
+		}
+	};
+
+	/**
+	 * Registers this filter in JGit by calling
+	 */
+	public final static void register() {
+		FilterCommandRegistry.register(
+				org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+						+ "lfs/smudge", //$NON-NLS-1$
+				FACTORY);
+	}
+
+	private Lfs lfs;
+
+	/**
+	 * @param db
+	 * @param in
+	 * @param out
+	 * @throws IOException
+	 */
+	public SmudgeFilter(Repository db, InputStream in, OutputStream out)
+			throws IOException {
+		super(in, out);
+		lfs = new Lfs(db.getDirectory().toPath().resolve(Constants.LFS));
+		LfsPointer res = LfsPointer.parseLfsPointer(in);
+		if (res != null) {
+			Path mediaFile = lfs.getMediaFile(res.getOid());
+			if (Files.exists(mediaFile)) {
+				this.in = Files.newInputStream(mediaFile);
+			}
+		}
+	}
+
+	@Override
+	public int run() throws IOException {
+		int b;
+		if (in != null) {
+			while ((b = in.read()) != -1) {
+				out.write(b);
+			}
+			in.close();
+		}
+		out.close();
+		return -1;
+	}
+}

org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java

@@ -55,6 +55,13 @@ import org.eclipse.jgit.lfs.internal.LfsText;
  **/
 @SuppressWarnings("nls")
 public final class Constants {
+	/**
+	 * lfs folder
+	 *
+	 * @since 4.6
+	 */
+	public static final String LFS = "lfs";
+
 	/**
 	 * Hash function used natively by Git LFS extension for large objects.
 	 *

org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Main.java

@@ -58,6 +58,7 @@ import org.eclipse.jgit.awtui.AwtAuthenticator;
 import org.eclipse.jgit.awtui.AwtCredentialsProvider;
 import org.eclipse.jgit.errors.TransportException;
 import org.eclipse.jgit.lfs.CleanFilter;
+import org.eclipse.jgit.lfs.SmudgeFilter;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.RepositoryBuilder;
 import org.eclipse.jgit.pgm.internal.CLIText;
@@ -99,6 +100,7 @@ public class Main {
 	public Main() {
 		HttpTransport.setConnectionFactory(new HttpClientConnectionFactory());
 		CleanFilter.register();
+		SmudgeFilter.register();
 	}
 
 	/**

org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java

@@ -57,13 +57,21 @@ import java.util.Set;
 import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.NoFilepatternException;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheBuilder;
 import org.eclipse.jgit.dircache.DirCacheEntry;
 import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.RepositoryTestCase;
 import org.eclipse.jgit.lfs.CleanFilter;
-import org.eclipse.jgit.lib.*;
+import org.eclipse.jgit.lfs.SmudgeFilter;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.FileMode;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectInserter;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
 import org.eclipse.jgit.treewalk.TreeWalk;
@@ -79,12 +87,13 @@ import org.junit.runner.RunWith;
 @RunWith(Theories.class)
 public class AddCommandTest extends RepositoryTestCase {
 	@DataPoints
-	public static boolean[] smudge = { true, false };
+	public static boolean[] sleepBeforeAddOptions = { true, false };
 
 
 	@Override
 	public void setUp() throws Exception {
 		CleanFilter.register();
+		SmudgeFilter.register();
 		super.setUp();
 	}
 
@@ -149,7 +158,7 @@ public class AddCommandTest extends RepositoryTestCase {
 	}
 
 	@Theory
-	public void testBuiltinFilter(boolean doSmudge)
+	public void testBuiltinFilters(boolean sleepBeforeAdd)
 			throws IOException,
 			GitAPIException, InterruptedException {
 		writeTrashFile(".gitattributes", "*.txt filter=lfs");
@@ -160,7 +169,7 @@ public class AddCommandTest extends RepositoryTestCase {
 		File f = writeTrashFile("src/a.txt", "foo\n");
 
 		try (Git git = new Git(db)) {
-			if (!doSmudge) {
+			if (!sleepBeforeAdd) {
 				fsTick(f);
 			}
 			git.add().addFilepattern(".gitattributes").call();
@@ -172,7 +181,7 @@ public class AddCommandTest extends RepositoryTestCase {
 			config.setBoolean("filter", "lfs", "useJGitBuiltin", true);
 			config.save();
 
-			if (!doSmudge) {
+			if (!sleepBeforeAdd) {
 				fsTick(f);
 			}
 			git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
@@ -185,7 +194,67 @@ public class AddCommandTest extends RepositoryTestCase {
 			RevCommit c1 = git.commit().setMessage("c1").call();
 			assertTrue(git.status().call().isClean());
 			f = writeTrashFile("src/a.txt", "foobar\n");
-			if (!doSmudge) {
+			if (!sleepBeforeAdd) {
+				fsTick(f);
+			}
+			git.add().addFilepattern("src/a.txt").call();
+			git.commit().setMessage("c2").call();
+			assertTrue(git.status().call().isClean());
+			assertEquals(
+					"[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f\nsize 7\n]",
+					indexState(CONTENT));
+			assertEquals("foobar\n", read("src/a.txt"));
+			git.checkout().setName(c1.getName()).call();
+			assertEquals(
+					"[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]",
+					indexState(CONTENT));
+			assertEquals(
+					"foo\n", read("src/a.txt"));
+		}
+	}
+
+	@Theory
+	public void testBuiltinCleanFilter(boolean sleepBeforeAdd)
+			throws IOException, GitAPIException, InterruptedException {
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
+		writeTrashFile("src/a.tmp", "foo");
+		// Caution: we need a trailing '\n' since sed on mac always appends
+		// linefeeds if missing
+		File script = writeTempFile("sed s/o/e/g");
+		File f = writeTrashFile("src/a.txt", "foo\n");
+
+		// unregister the smudge filter. Only clean filter should be builtin
+		FilterCommandRegistry.unregister(
+				org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+						+ "lfs/smudge");
+
+		try (Git git = new Git(db)) {
+			if (!sleepBeforeAdd) {
+				fsTick(f);
+			}
+			git.add().addFilepattern(".gitattributes").call();
+			StoredConfig config = git.getRepository().getConfig();
+			config.setString("filter", "lfs", "clean",
+					"sh " + slashify(script.getPath()));
+			config.setString("filter", "lfs", "smudge",
+					"sh " + slashify(script.getPath()));
+			config.setBoolean("filter", "lfs", "useJGitBuiltin", true);
+			config.save();
+
+			if (!sleepBeforeAdd) {
+				fsTick(f);
+			}
+			git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
+					.addFilepattern(".gitattributes").call();
+
+			assertEquals(
+					"[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]",
+					indexState(CONTENT));
+
+			RevCommit c1 = git.commit().setMessage("c1").call();
+			assertTrue(git.status().call().isClean());
+			f = writeTrashFile("src/a.txt", "foobar\n");
+			if (!sleepBeforeAdd) {
 				fsTick(f);
 			}
 			git.add().addFilepattern("src/a.txt").call();

org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java

@@ -74,6 +74,8 @@ import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheEntry;
 import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.eclipse.jgit.lfs.CleanFilter;
+import org.eclipse.jgit.lfs.SmudgeFilter;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.Ref;
@@ -100,6 +102,8 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 	@Override
 	@Before
 	public void setUp() throws Exception {
+		CleanFilter.register();
+		SmudgeFilter.register();
 		super.setUp();
 		git = new Git(db);
 		// commit something
@@ -563,11 +567,11 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 	public void testSmudgeFilter_modifyExisting() throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -589,7 +593,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.checkout().setName(content2.getName()).call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals(Sets.of("src/a.txt"), git.status().call().getModified());
 		assertEquals("foo", read("src/a.tmp"));
@@ -601,7 +605,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -609,7 +613,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		RevCommit initial = git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -625,7 +629,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.checkout().setName(content.getName()).call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -636,7 +640,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -644,7 +648,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -661,7 +665,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 				.call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -672,7 +676,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -680,7 +684,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -696,7 +700,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.checkout().addPath("src/a.txt").call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -707,7 +711,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -715,7 +719,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -732,7 +736,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 				.setStartPoint(content).addPath("src/a.txt").call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -745,12 +749,13 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 
 		try (Git git2 = new Git(db)) {
 			StoredConfig config = git.getRepository().getConfig();
-			config.setString("filter", "tstFilter", "smudge",
+			config.setString("filter", "lfs", "smudge",
 					"sh " + slashify(smudge_filter.getPath()));
-			config.setString("filter", "tstFilter", "clean",
+			config.setString("filter", "lfs", "clean",
 					"sh " + slashify(clean_filter.getPath()));
+			config.setBoolean("filter", "lfs", "useJGitBuiltin", true);
 			config.save();
-			writeTrashFile(".gitattributes", "filterTest.txt filter=tstFilter");
+			writeTrashFile(".gitattributes", "filterTest.txt filter=lfs");
 			git2.add().addFilepattern(".gitattributes").call();
 			git2.commit().setMessage("add attributes").call();
 
@@ -758,7 +763,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			git2.add().addFilepattern("filterTest.txt").call();
 			RevCommit one = git2.commit().setMessage("add filterText.txt").call();
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:hello world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:7bd5d32e5c494354aa4c2473a1306d0ce7b52cc3bffeb342c03cd517ef8cf8da\nsize 16\n]",
 					indexState(CONTENT));
 
 			fsTick(writeTrashFile("filterTest.txt", "bon giorno world, V1\n"));
@@ -767,20 +772,20 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 
 			assertTrue(git2.status().call().isClean());
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:bon giorno world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:087148cccf53b0049c56475c1595113c9da4b638997c3489af8ac7108d51ef13\nsize 21\n]",
 					indexState(CONTENT));
 
 			git2.checkout().setName(one.getName()).call();
 			assertTrue(git2.status().call().isClean());
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:hello world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:7bd5d32e5c494354aa4c2473a1306d0ce7b52cc3bffeb342c03cd517ef8cf8da\nsize 16\n]",
 					indexState(CONTENT));
 			assertEquals("hello world, V1\n", read("filterTest.txt"));
 
 			git2.checkout().setName(two.getName()).call();
 			assertTrue(git2.status().call().isClean());
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:bon giorno world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:087148cccf53b0049c56475c1595113c9da4b638997c3489af8ac7108d51ef13\nsize 21\n]",
 					indexState(CONTENT));
 			assertEquals("bon giorno world, V1\n", read("filterTest.txt"));
 		}

org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java

@@ -224,6 +224,11 @@ public class AddCommand extends GitCommand<DirCache> {
 					entry.setLength(f.getEntryLength());
 					entry.setLastModified(f.getEntryLastModified());
 					long len = f.getEntryContentLength();
+					// We read and filter the content multiple times.
+					// f.getEntryContentLength() reads and filters the input and
+					// inserter.insert(...) does it again. That's because an
+					// ObjectInserter needs to know the length before it starts
+					// inserting. TODO: Fix this by using Buffers.
 					try (InputStream in = f.openEntryStream()) {
 						ObjectId id = inserter.insert(OBJ_BLOB, len, in);
 						entry.setObjectId(id);

org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java

@@ -267,7 +267,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 			// If there is a matching DirCacheIterator, we can reuse
 			// its idBuffer, but only if we appear to be clean against
 			// the cached index information for the path.
-			//
 			DirCacheIterator i = state.walk.getTree(state.dirCacheTree,
 							DirCacheIterator.class);
 			if (i != null) {
@@ -397,15 +396,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 
 		if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) {
 			ByteBuffer rawbuf = IO.readWholeStream(is, (int) len);
-			byte[] raw = rawbuf.array();
-			int n = rawbuf.limit();
-			if (!isBinary(raw, n)) {
-				rawbuf = filterClean(raw, n, opType);
-				raw = rawbuf.array();
-				n = rawbuf.limit();
-			}
-			canonLen = n;
-			return new ByteArrayInputStream(raw, 0, n);
+			rawbuf = filterClean(rawbuf.array(), rawbuf.limit(), opType);
+			canonLen = rawbuf.limit();
+			return new ByteArrayInputStream(rawbuf.array(), 0, (int) canonLen);
 		}
 
 		if (getCleanFilterCommand() == null && isBinary(e)) {
@@ -433,10 +426,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		}
 	}
 
-	private static boolean isBinary(byte[] content, int sz) {
-		return RawText.isBinary(content, sz);
-	}
-
 	private static boolean isBinary(Entry entry) throws IOException {
 		InputStream in = entry.openInputStream();
 		try {