Add built-in LFS smudge filter for local case

Adds a JGit built-in implementation of the "git lfs smudge" filter. This
filter should do the same as the one described in [1] besides that it
only supports the local case when the lfs objects are already present in
the media directory. Remote cases where download of LFS objects from an
LFS server is needed will be done in a later commit.

[1] https://github.com/github/git-lfs/blob/master/docs/man/git-lfs-smudge.1.ronn

Change-Id: I8ff661d4edd3667ef7f86f3b4fa33e568eb4c8f4

2 files changed, 102 insertions, 28 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java
index a3cae4b056..5ad73f17b0 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java
@@ -57,13 +57,21 @@ import java.util.Set;
 import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.NoFilepatternException;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheBuilder;
 import org.eclipse.jgit.dircache.DirCacheEntry;
 import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.RepositoryTestCase;
 import org.eclipse.jgit.lfs.CleanFilter;
-import org.eclipse.jgit.lib.*;
+import org.eclipse.jgit.lfs.SmudgeFilter;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.FileMode;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectInserter;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
 import org.eclipse.jgit.treewalk.TreeWalk;
@@ -79,12 +87,13 @@ import org.junit.runner.RunWith;
 @RunWith(Theories.class)
 public class AddCommandTest extends RepositoryTestCase {
 	@DataPoints
-	public static boolean[] smudge = { true, false };
+	public static boolean[] sleepBeforeAddOptions = { true, false };
 
 
 	@Override
 	public void setUp() throws Exception {
 		CleanFilter.register();
+		SmudgeFilter.register();
 		super.setUp();
 	}
 
@@ -149,7 +158,7 @@ public class AddCommandTest extends RepositoryTestCase {
 	}
 
 	@Theory
-	public void testBuiltinFilter(boolean doSmudge)
+	public void testBuiltinFilters(boolean sleepBeforeAdd)
 			throws IOException,
 			GitAPIException, InterruptedException {
 		writeTrashFile(".gitattributes", "*.txt filter=lfs");
@@ -160,7 +169,7 @@ public class AddCommandTest extends RepositoryTestCase {
 		File f = writeTrashFile("src/a.txt", "foo\n");
 
 		try (Git git = new Git(db)) {
-			if (!doSmudge) {
+			if (!sleepBeforeAdd) {
 				fsTick(f);
 			}
 			git.add().addFilepattern(".gitattributes").call();
@@ -172,7 +181,7 @@ public class AddCommandTest extends RepositoryTestCase {
 			config.setBoolean("filter", "lfs", "useJGitBuiltin", true);
 			config.save();
 
-			if (!doSmudge) {
+			if (!sleepBeforeAdd) {
 				fsTick(f);
 			}
 			git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
@@ -185,7 +194,67 @@ public class AddCommandTest extends RepositoryTestCase {
 			RevCommit c1 = git.commit().setMessage("c1").call();
 			assertTrue(git.status().call().isClean());
 			f = writeTrashFile("src/a.txt", "foobar\n");
-			if (!doSmudge) {
+			if (!sleepBeforeAdd) {
+				fsTick(f);
+			}
+			git.add().addFilepattern("src/a.txt").call();
+			git.commit().setMessage("c2").call();
+			assertTrue(git.status().call().isClean());
+			assertEquals(
+					"[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f\nsize 7\n]",
+					indexState(CONTENT));
+			assertEquals("foobar\n", read("src/a.txt"));
+			git.checkout().setName(c1.getName()).call();
+			assertEquals(
+					"[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]",
+					indexState(CONTENT));
+			assertEquals(
+					"foo\n", read("src/a.txt"));
+		}
+	}
+
+	@Theory
+	public void testBuiltinCleanFilter(boolean sleepBeforeAdd)
+			throws IOException, GitAPIException, InterruptedException {
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
+		writeTrashFile("src/a.tmp", "foo");
+		// Caution: we need a trailing '\n' since sed on mac always appends
+		// linefeeds if missing
+		File script = writeTempFile("sed s/o/e/g");
+		File f = writeTrashFile("src/a.txt", "foo\n");
+
+		// unregister the smudge filter. Only clean filter should be builtin
+		FilterCommandRegistry.unregister(
+				org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
+						+ "lfs/smudge");
+
+		try (Git git = new Git(db)) {
+			if (!sleepBeforeAdd) {
+				fsTick(f);
+			}
+			git.add().addFilepattern(".gitattributes").call();
+			StoredConfig config = git.getRepository().getConfig();
+			config.setString("filter", "lfs", "clean",
+					"sh " + slashify(script.getPath()));
+			config.setString("filter", "lfs", "smudge",
+					"sh " + slashify(script.getPath()));
+			config.setBoolean("filter", "lfs", "useJGitBuiltin", true);
+			config.save();
+
+			if (!sleepBeforeAdd) {
+				fsTick(f);
+			}
+			git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
+					.addFilepattern(".gitattributes").call();
+
+			assertEquals(
+					"[.gitattributes, mode:100644, content:*.txt filter=lfs][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c\nsize 4\n]",
+					indexState(CONTENT));
+
+			RevCommit c1 = git.commit().setMessage("c1").call();
+			assertTrue(git.status().call().isClean());
+			f = writeTrashFile("src/a.txt", "foobar\n");
+			if (!sleepBeforeAdd) {
 				fsTick(f);
 			}
 			git.add().addFilepattern("src/a.txt").call();
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java
index 0bb6610a24..998b5fbfc2 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CheckoutCommandTest.java
@@ -74,6 +74,8 @@ import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheEntry;
 import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.eclipse.jgit.lfs.CleanFilter;
+import org.eclipse.jgit.lfs.SmudgeFilter;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.Ref;
@@ -100,6 +102,8 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 	@Override
 	@Before
 	public void setUp() throws Exception {
+		CleanFilter.register();
+		SmudgeFilter.register();
 		super.setUp();
 		git = new Git(db);
 		// commit something
@@ -563,11 +567,11 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 	public void testSmudgeFilter_modifyExisting() throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -589,7 +593,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.checkout().setName(content2.getName()).call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals(Sets.of("src/a.txt"), git.status().call().getModified());
 		assertEquals("foo", read("src/a.tmp"));
@@ -601,7 +605,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -609,7 +613,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		RevCommit initial = git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -625,7 +629,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.checkout().setName(content.getName()).call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -636,7 +640,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -644,7 +648,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -661,7 +665,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 				.call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -672,7 +676,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -680,7 +684,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -696,7 +700,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.checkout().addPath("src/a.txt").call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -707,7 +711,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			throws IOException, GitAPIException {
 		File script = writeTempFile("sed s/o/e/g");
 		StoredConfig config = git.getRepository().getConfig();
-		config.setString("filter", "tstFilter", "smudge",
+		config.setString("filter", "lfs", "smudge",
 				"sh " + slashify(script.getPath()));
 		config.save();
 
@@ -715,7 +719,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 		git.add().addFilepattern("foo").call();
 		git.commit().setMessage("initial").call();
 
-		writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+		writeTrashFile(".gitattributes", "*.txt filter=lfs");
 		git.add().addFilepattern(".gitattributes").call();
 		git.commit().setMessage("add filter").call();
 
@@ -732,7 +736,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 				.setStartPoint(content).addPath("src/a.txt").call();
 
 		assertEquals(
-				"[.gitattributes, mode:100644, content:*.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
+				"[.gitattributes, mode:100644, content:*.txt filter=lfs][Test.txt, mode:100644, content:Some change][foo, mode:100644, content:foo][src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:foo\n]",
 				indexState(CONTENT));
 		assertEquals("foo", read("src/a.tmp"));
 		assertEquals("fee\n", read("src/a.txt"));
@@ -745,12 +749,13 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 
 		try (Git git2 = new Git(db)) {
 			StoredConfig config = git.getRepository().getConfig();
-			config.setString("filter", "tstFilter", "smudge",
+			config.setString("filter", "lfs", "smudge",
 					"sh " + slashify(smudge_filter.getPath()));
-			config.setString("filter", "tstFilter", "clean",
+			config.setString("filter", "lfs", "clean",
 					"sh " + slashify(clean_filter.getPath()));
+			config.setBoolean("filter", "lfs", "useJGitBuiltin", true);
 			config.save();
-			writeTrashFile(".gitattributes", "filterTest.txt filter=tstFilter");
+			writeTrashFile(".gitattributes", "filterTest.txt filter=lfs");
 			git2.add().addFilepattern(".gitattributes").call();
 			git2.commit().setMessage("add attributes").call();
 
@@ -758,7 +763,7 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 			git2.add().addFilepattern("filterTest.txt").call();
 			RevCommit one = git2.commit().setMessage("add filterText.txt").call();
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:hello world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:7bd5d32e5c494354aa4c2473a1306d0ce7b52cc3bffeb342c03cd517ef8cf8da\nsize 16\n]",
 					indexState(CONTENT));
 
 			fsTick(writeTrashFile("filterTest.txt", "bon giorno world, V1\n"));
@@ -767,20 +772,20 @@ public class CheckoutCommandTest extends RepositoryTestCase {
 
 			assertTrue(git2.status().call().isClean());
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:bon giorno world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:087148cccf53b0049c56475c1595113c9da4b638997c3489af8ac7108d51ef13\nsize 21\n]",
 					indexState(CONTENT));
 
 			git2.checkout().setName(one.getName()).call();
 			assertTrue(git2.status().call().isClean());
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:hello world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:7bd5d32e5c494354aa4c2473a1306d0ce7b52cc3bffeb342c03cd517ef8cf8da\nsize 16\n]",
 					indexState(CONTENT));
 			assertEquals("hello world, V1\n", read("filterTest.txt"));
 
 			git2.checkout().setName(two.getName()).call();
 			assertTrue(git2.status().call().isClean());
 			assertEquals(
-					"[.gitattributes, mode:100644, content:filterTest.txt filter=tstFilter][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:bon giorno world, @version\n]",
+					"[.gitattributes, mode:100644, content:filterTest.txt filter=lfs][Test.txt, mode:100644, content:Some change][filterTest.txt, mode:100644, content:version https://git-lfs.github.com/spec/v1\noid sha256:087148cccf53b0049c56475c1595113c9da4b638997c3489af8ac7108d51ef13\nsize 21\n]",
 					indexState(CONTENT));
 			assertEquals("bon giorno world, V1\n", read("filterTest.txt"));
 		}