From d97248467a95d26934ddeb8617c3bbf3f673f69d Mon Sep 17 00:00:00 2001 From: Christian Halstrick Date: Tue, 3 May 2016 14:59:57 +0200 Subject: [PATCH] Add support for built-in smudge filters JGit supports smudge filters defined in repository configuration. The filters are implemented as external programs filtering content by accepting the original content (as seen in git's object database) on stdin and which emit the filtered content on stdout. This content is then written to the file in the working tree. To run such a filter JGit has to start an external process and pump data into/from this process. This commit adds support for built-in smudge filters which are implemented in Java and which are executed by jgit's main thread. When a filter is defined in the configuration as "jgit://builtin//smudge" then JGit will lookup in a static map whether a builtin filter is registered under this name. If found such a filter is called to do the filtering. The functionality in this commit requires that a program using JGit explicitly calls the JGit API to register built-in implementations for specific smudge filters. In follow-up commits configuration parameters will be added which trigger such registrations. Change-Id: Ia743aa0dbed795e71e5792f35ae55660e0eb3c24 --- .../eclipse/jgit/util/FilterCommandsTest.java | 94 +++++++++++++++ .../eclipse/jgit/internal/JGitText.properties | 1 + .../jgit/dircache/DirCacheCheckout.java | 111 ++++++++++++------ .../org/eclipse/jgit/internal/JGitText.java | 1 + 4 files changed, 170 insertions(+), 37 deletions(-) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FilterCommandsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FilterCommandsTest.java index 5fb894e911..0d31811257 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FilterCommandsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FilterCommandsTest.java @@ -154,4 +154,98 @@ public class FilterCommandsTest extends RepositoryTestCase { config.setString("filter", "test", "clean", null); config.save(); } + + @Test + public void testBuiltinSmudgeFilter() throws IOException, GitAPIException { + String builtinCommandName = "jgit://builtin/test/smudge"; + FilterCommandRegistry.register(builtinCommandName, + new TestCommandFactory('s')); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "test", "smudge", builtinCommandName); + config.save(); + + writeTrashFile(".gitattributes", "*.txt filter=test"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("add filter").call(); + + writeTrashFile("Test.txt", "Hello again"); + git.add().addFilepattern("Test.txt").call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=test][Test.txt, mode:100644, content:Hello again]", + indexState(CONTENT)); + assertEquals("Hello again", read("Test.txt")); + deleteTrashFile("Test.txt"); + git.checkout().addPath("Test.txt").call(); + assertEquals("sHseslslsos sasgsasisn", read("Test.txt")); + + writeTrashFile("Test.bin", "Hello again"); + git.add().addFilepattern("Test.bin").call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:Hello again]", + indexState(CONTENT)); + deleteTrashFile("Test.bin"); + git.checkout().addPath("Test.bin").call(); + assertEquals("Hello again", read("Test.bin")); + + config.setString("filter", "test", "clean", null); + config.save(); + + git.add().addFilepattern("Test.txt").call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:sHseslslsos sasgsasisn]", + indexState(CONTENT)); + + config.setString("filter", "test", "clean", null); + config.save(); + } + + @Test + public void testBuiltinCleanAndSmudgeFilter() throws IOException, GitAPIException { + String builtinCommandPrefix = "jgit://builtin/test/"; + FilterCommandRegistry.register(builtinCommandPrefix + "smudge", + new TestCommandFactory('s')); + FilterCommandRegistry.register(builtinCommandPrefix + "clean", + new TestCommandFactory('c')); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "test", "smudge", builtinCommandPrefix+"smudge"); + config.setString("filter", "test", "clean", + builtinCommandPrefix + "clean"); + config.save(); + + writeTrashFile(".gitattributes", "*.txt filter=test"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("add filter").call(); + + writeTrashFile("Test.txt", "Hello again"); + git.add().addFilepattern("Test.txt").call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=test][Test.txt, mode:100644, content:cHceclclcoc cacgcacicn]", + indexState(CONTENT)); + assertEquals("Hello again", read("Test.txt")); + deleteTrashFile("Test.txt"); + git.checkout().addPath("Test.txt").call(); + assertEquals("scsHscsescslscslscsoscs scsascsgscsascsiscsn", + read("Test.txt")); + + writeTrashFile("Test.bin", "Hello again"); + git.add().addFilepattern("Test.bin").call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:cHceclclcoc cacgcacicn]", + indexState(CONTENT)); + deleteTrashFile("Test.bin"); + git.checkout().addPath("Test.bin").call(); + assertEquals("Hello again", read("Test.bin")); + + config.setString("filter", "test", "clean", null); + config.save(); + + git.add().addFilepattern("Test.txt").call(); + assertEquals( + "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:scsHscsescslscslscsoscs scsascsgscsascsiscsn]", + indexState(CONTENT)); + + config.setString("filter", "test", "clean", null); + config.save(); + } + } diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 327ca0a10b..2c721eabb6 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -279,6 +279,7 @@ expectedLessThanGot=expected less than ''{0}'', got ''{1}'' expectedPktLineWithService=expected pkt-line with ''# service=-'', got ''{0}'' expectedReceivedContentType=expected Content-Type {0}; received Content-Type {1} expectedReportForRefNotReceived={0}: expected report for ref {1} not received +failedToDetermineFilterDefinition=An exception occured while determining filter definitions failedUpdatingRefs=failed updating refs failureDueToOneOfTheFollowing=Failure due to one of the following: failureUpdatingFETCH_HEAD=Failure updating FETCH_HEAD: {0} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 8af7e46a07..c3184437b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -54,6 +54,8 @@ import java.util.List; import java.util.Map; import org.eclipse.jgit.api.errors.FilterFailedException; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.errors.CheckoutConflictException; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -86,11 +88,15 @@ import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.SystemReader; import org.eclipse.jgit.util.io.EolStreamTypeUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class handles checking out one or two trees merging with the index. */ public class DirCacheCheckout { + private static Logger LOG = LoggerFactory.getLogger(DirCacheCheckout.class); + private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024; /** @@ -1303,45 +1309,19 @@ public class DirCacheCheckout { } else { nonNullEolStreamType = EolStreamType.DIRECT; } - OutputStream channel = EolStreamTypeUtil.wrapOutputStream( - new FileOutputStream(tmpFile), nonNullEolStreamType); - if (checkoutMetadata.smudgeFilterCommand != null) { - ProcessBuilder filterProcessBuilder = fs.runInShell( - checkoutMetadata.smudgeFilterCommand, new String[0]); - filterProcessBuilder.directory(repo.getWorkTree()); - filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, - repo.getDirectory().getAbsolutePath()); - ExecutionResult result; - int rc; - try { - // TODO: wire correctly with AUTOCRLF - result = fs.execute(filterProcessBuilder, ol.openStream()); - rc = result.getRc(); - if (rc == 0) { - result.getStdout().writeTo(channel, - NullProgressMonitor.INSTANCE); + try (OutputStream channel = EolStreamTypeUtil.wrapOutputStream( + new FileOutputStream(tmpFile), nonNullEolStreamType)) { + if (checkoutMetadata.smudgeFilterCommand != null) { + if (FilterCommandRegistry + .isRegistered(checkoutMetadata.smudgeFilterCommand)) { + runBuiltinFilterCommand(repo, checkoutMetadata, ol, + channel); + } else { + runExternalFilterCommand(repo, entry, checkoutMetadata, ol, + fs, channel); } - } catch (IOException | InterruptedException e) { - throw new IOException(new FilterFailedException(e, - checkoutMetadata.smudgeFilterCommand, - entry.getPathString())); - - } finally { - channel.close(); - } - if (rc != 0) { - throw new IOException(new FilterFailedException(rc, - checkoutMetadata.smudgeFilterCommand, - entry.getPathString(), - result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE), - RawParseUtils.decode(result.getStderr() - .toByteArray(MAX_EXCEPTION_TEXT_SIZE)))); - } - } else { - try { + } else { ol.copyTo(channel); - } finally { - channel.close(); } } // The entry needs to correspond to the on-disk filesize. If the content @@ -1382,6 +1362,63 @@ public class DirCacheCheckout { entry.setLastModified(fs.lastModified(f)); } + // Run an external filter command + private static void runExternalFilterCommand(Repository repo, + DirCacheEntry entry, + CheckoutMetadata checkoutMetadata, ObjectLoader ol, FS fs, + OutputStream channel) throws IOException { + ProcessBuilder filterProcessBuilder = fs.runInShell( + checkoutMetadata.smudgeFilterCommand, new String[0]); + filterProcessBuilder.directory(repo.getWorkTree()); + filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, + repo.getDirectory().getAbsolutePath()); + ExecutionResult result; + int rc; + try { + // TODO: wire correctly with AUTOCRLF + result = fs.execute(filterProcessBuilder, ol.openStream()); + rc = result.getRc(); + if (rc == 0) { + result.getStdout().writeTo(channel, + NullProgressMonitor.INSTANCE); + } + } catch (IOException | InterruptedException e) { + throw new IOException(new FilterFailedException(e, + checkoutMetadata.smudgeFilterCommand, + entry.getPathString())); + } + if (rc != 0) { + throw new IOException(new FilterFailedException(rc, + checkoutMetadata.smudgeFilterCommand, + entry.getPathString(), + result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE), + RawParseUtils.decode(result.getStderr() + .toByteArray(MAX_EXCEPTION_TEXT_SIZE)))); + } + } + + // Run a builtin filter command + private static void runBuiltinFilterCommand(Repository repo, + CheckoutMetadata checkoutMetadata, ObjectLoader ol, + OutputStream channel) throws MissingObjectException, IOException { + FilterCommand command = null; + try { + command = FilterCommandRegistry.createFilterCommand( + checkoutMetadata.smudgeFilterCommand, repo, ol.openStream(), + channel); + } catch (IOException e) { + LOG.error(JGitText.get().failedToDetermineFilterDefinition, e); + // In case an IOException occurred during creating of the command + // then proceed as if there would not have been a builtin filter. + ol.copyTo(channel); + } + if (command != null) { + while (command.run() != -1) { + // loop as long as command.run() tells there is work to do + } + } + } + @SuppressWarnings("deprecation") private static void checkValidPath(CanonicalTreeParser t) throws InvalidPathException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 758f71d27a..956171b127 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -338,6 +338,7 @@ public class JGitText extends TranslationBundle { /***/ public String expectedPktLineWithService; /***/ public String expectedReceivedContentType; /***/ public String expectedReportForRefNotReceived; + /***/ public String failedToDetermineFilterDefinition; /***/ public String failedUpdatingRefs; /***/ public String failureDueToOneOfTheFollowing; /***/ public String failureUpdatingFETCH_HEAD; -- 2.39.5