]> source.dussan.org Git - jgit.git/commitdiff
Add support for built-in smudge filters 83/71983/12
authorChristian Halstrick <christian.halstrick@sap.com>
Tue, 3 May 2016 12:59:57 +0000 (14:59 +0200)
committerMatthias Sohn <matthias.sohn@sap.com>
Tue, 20 Sep 2016 08:02:20 +0000 (10:02 +0200)
JGit supports smudge filters defined in repository configuration. The
filters are implemented as external programs filtering content by
accepting the original content (as seen in git's object database) on
stdin and which emit the filtered content on stdout. This content is
then written to the file in the working tree. To run such a filter JGit
has to start an external process and pump data into/from this process.

This commit adds support for built-in smudge filters which are
implemented in Java and which are executed by jgit's main thread. When a
filter is defined in the configuration as
"jgit://builtin/<filterDriverName>/smudge" then JGit will lookup in a
static map whether a builtin filter is registered under this name. If
found such a filter is called to do the filtering.

The functionality in this commit requires that a program using JGit
explicitly calls the JGit API to register built-in implementations for
specific smudge filters. In follow-up commits configuration parameters
will be added which trigger such registrations.

Change-Id: Ia743aa0dbed795e71e5792f35ae55660e0eb3c24

org.eclipse.jgit.test/tst/org/eclipse/jgit/util/FilterCommandsTest.java
org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java
org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java

index 5fb894e9110fe299df9bd118e7e1899939a20714..0d31811257ef8811a6917d322688a83f6f60ba5e 100644 (file)
@@ -154,4 +154,98 @@ public class FilterCommandsTest extends RepositoryTestCase {
                config.setString("filter", "test", "clean", null);
                config.save();
        }
+
+       @Test
+       public void testBuiltinSmudgeFilter() throws IOException, GitAPIException {
+               String builtinCommandName = "jgit://builtin/test/smudge";
+               FilterCommandRegistry.register(builtinCommandName,
+                               new TestCommandFactory('s'));
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "test", "smudge", builtinCommandName);
+               config.save();
+
+               writeTrashFile(".gitattributes", "*.txt filter=test");
+               git.add().addFilepattern(".gitattributes").call();
+               git.commit().setMessage("add filter").call();
+
+               writeTrashFile("Test.txt", "Hello again");
+               git.add().addFilepattern("Test.txt").call();
+               assertEquals(
+                               "[.gitattributes, mode:100644, content:*.txt filter=test][Test.txt, mode:100644, content:Hello again]",
+                               indexState(CONTENT));
+               assertEquals("Hello again", read("Test.txt"));
+               deleteTrashFile("Test.txt");
+               git.checkout().addPath("Test.txt").call();
+               assertEquals("sHseslslsos sasgsasisn", read("Test.txt"));
+
+               writeTrashFile("Test.bin", "Hello again");
+               git.add().addFilepattern("Test.bin").call();
+               assertEquals(
+                               "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:Hello again]",
+                               indexState(CONTENT));
+               deleteTrashFile("Test.bin");
+               git.checkout().addPath("Test.bin").call();
+               assertEquals("Hello again", read("Test.bin"));
+
+               config.setString("filter", "test", "clean", null);
+               config.save();
+
+               git.add().addFilepattern("Test.txt").call();
+               assertEquals(
+                               "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:sHseslslsos sasgsasisn]",
+                               indexState(CONTENT));
+
+               config.setString("filter", "test", "clean", null);
+               config.save();
+       }
+
+       @Test
+       public void testBuiltinCleanAndSmudgeFilter() throws IOException, GitAPIException {
+               String builtinCommandPrefix = "jgit://builtin/test/";
+               FilterCommandRegistry.register(builtinCommandPrefix + "smudge",
+                               new TestCommandFactory('s'));
+               FilterCommandRegistry.register(builtinCommandPrefix + "clean",
+                               new TestCommandFactory('c'));
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "test", "smudge", builtinCommandPrefix+"smudge");
+               config.setString("filter", "test", "clean",
+                               builtinCommandPrefix + "clean");
+               config.save();
+
+               writeTrashFile(".gitattributes", "*.txt filter=test");
+               git.add().addFilepattern(".gitattributes").call();
+               git.commit().setMessage("add filter").call();
+
+               writeTrashFile("Test.txt", "Hello again");
+               git.add().addFilepattern("Test.txt").call();
+               assertEquals(
+                               "[.gitattributes, mode:100644, content:*.txt filter=test][Test.txt, mode:100644, content:cHceclclcoc cacgcacicn]",
+                               indexState(CONTENT));
+               assertEquals("Hello again", read("Test.txt"));
+               deleteTrashFile("Test.txt");
+               git.checkout().addPath("Test.txt").call();
+               assertEquals("scsHscsescslscslscsoscs scsascsgscsascsiscsn",
+                               read("Test.txt"));
+
+               writeTrashFile("Test.bin", "Hello again");
+               git.add().addFilepattern("Test.bin").call();
+               assertEquals(
+                               "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:cHceclclcoc cacgcacicn]",
+                               indexState(CONTENT));
+               deleteTrashFile("Test.bin");
+               git.checkout().addPath("Test.bin").call();
+               assertEquals("Hello again", read("Test.bin"));
+
+               config.setString("filter", "test", "clean", null);
+               config.save();
+
+               git.add().addFilepattern("Test.txt").call();
+               assertEquals(
+                               "[.gitattributes, mode:100644, content:*.txt filter=test][Test.bin, mode:100644, content:Hello again][Test.txt, mode:100644, content:scsHscsescslscslscsoscs scsascsgscsascsiscsn]",
+                               indexState(CONTENT));
+
+               config.setString("filter", "test", "clean", null);
+               config.save();
+       }
+
 }
index 327ca0a10bdfdf6f05dbc809ecbc882039aa1216..2c721eabb6ec3237c9ee4b35db7de611f14e6398 100644 (file)
@@ -279,6 +279,7 @@ expectedLessThanGot=expected less than ''{0}'', got ''{1}''
 expectedPktLineWithService=expected pkt-line with ''# service=-'', got ''{0}''
 expectedReceivedContentType=expected Content-Type {0}; received Content-Type {1}
 expectedReportForRefNotReceived={0}: expected report for ref {1} not received
+failedToDetermineFilterDefinition=An exception occured while determining filter definitions
 failedUpdatingRefs=failed updating refs
 failureDueToOneOfTheFollowing=Failure due to one of the following:
 failureUpdatingFETCH_HEAD=Failure updating FETCH_HEAD: {0}
index 8af7e46a074fa6bd3e819bb75c4d4f8cd58d182b..c3184437b4af79f966ea57d7d287cfbaa3142b65 100644 (file)
@@ -54,6 +54,8 @@ import java.util.List;
 import java.util.Map;
 
 import org.eclipse.jgit.api.errors.FilterFailedException;
+import org.eclipse.jgit.attributes.FilterCommand;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.errors.CheckoutConflictException;
 import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
@@ -86,11 +88,15 @@ import org.eclipse.jgit.util.FileUtils;
 import org.eclipse.jgit.util.RawParseUtils;
 import org.eclipse.jgit.util.SystemReader;
 import org.eclipse.jgit.util.io.EolStreamTypeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class handles checking out one or two trees merging with the index.
  */
 public class DirCacheCheckout {
+       private static Logger LOG = LoggerFactory.getLogger(DirCacheCheckout.class);
+
        private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024;
 
        /**
@@ -1303,45 +1309,19 @@ public class DirCacheCheckout {
                } else {
                        nonNullEolStreamType = EolStreamType.DIRECT;
                }
-               OutputStream channel = EolStreamTypeUtil.wrapOutputStream(
-                               new FileOutputStream(tmpFile), nonNullEolStreamType);
-               if (checkoutMetadata.smudgeFilterCommand != null) {
-                       ProcessBuilder filterProcessBuilder = fs.runInShell(
-                                       checkoutMetadata.smudgeFilterCommand, new String[0]);
-                       filterProcessBuilder.directory(repo.getWorkTree());
-                       filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY,
-                                       repo.getDirectory().getAbsolutePath());
-                       ExecutionResult result;
-                       int rc;
-                       try {
-                               // TODO: wire correctly with AUTOCRLF
-                               result = fs.execute(filterProcessBuilder, ol.openStream());
-                               rc = result.getRc();
-                               if (rc == 0) {
-                                       result.getStdout().writeTo(channel,
-                                                       NullProgressMonitor.INSTANCE);
+               try (OutputStream channel = EolStreamTypeUtil.wrapOutputStream(
+                               new FileOutputStream(tmpFile), nonNullEolStreamType)) {
+                       if (checkoutMetadata.smudgeFilterCommand != null) {
+                               if (FilterCommandRegistry
+                                               .isRegistered(checkoutMetadata.smudgeFilterCommand)) {
+                                       runBuiltinFilterCommand(repo, checkoutMetadata, ol,
+                                                       channel);
+                               } else {
+                                       runExternalFilterCommand(repo, entry, checkoutMetadata, ol,
+                                                       fs, channel);
                                }
-                       } catch (IOException | InterruptedException e) {
-                               throw new IOException(new FilterFailedException(e,
-                                               checkoutMetadata.smudgeFilterCommand,
-                                               entry.getPathString()));
-
-                       } finally {
-                               channel.close();
-                       }
-                       if (rc != 0) {
-                               throw new IOException(new FilterFailedException(rc,
-                                               checkoutMetadata.smudgeFilterCommand,
-                                               entry.getPathString(),
-                                               result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE),
-                                               RawParseUtils.decode(result.getStderr()
-                                                               .toByteArray(MAX_EXCEPTION_TEXT_SIZE))));
-                       }
-               } else {
-                       try {
+                       } else {
                                ol.copyTo(channel);
-                       } finally {
-                               channel.close();
                        }
                }
                // The entry needs to correspond to the on-disk filesize. If the content
@@ -1382,6 +1362,63 @@ public class DirCacheCheckout {
                entry.setLastModified(fs.lastModified(f));
        }
 
+       // Run an external filter command
+       private static void runExternalFilterCommand(Repository repo,
+                       DirCacheEntry entry,
+                       CheckoutMetadata checkoutMetadata, ObjectLoader ol, FS fs,
+                       OutputStream channel) throws IOException {
+               ProcessBuilder filterProcessBuilder = fs.runInShell(
+                               checkoutMetadata.smudgeFilterCommand, new String[0]);
+               filterProcessBuilder.directory(repo.getWorkTree());
+               filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY,
+                               repo.getDirectory().getAbsolutePath());
+               ExecutionResult result;
+               int rc;
+               try {
+                       // TODO: wire correctly with AUTOCRLF
+                       result = fs.execute(filterProcessBuilder, ol.openStream());
+                       rc = result.getRc();
+                       if (rc == 0) {
+                               result.getStdout().writeTo(channel,
+                                               NullProgressMonitor.INSTANCE);
+                       }
+               } catch (IOException | InterruptedException e) {
+                       throw new IOException(new FilterFailedException(e,
+                                       checkoutMetadata.smudgeFilterCommand,
+                                       entry.getPathString()));
+               }
+               if (rc != 0) {
+                       throw new IOException(new FilterFailedException(rc,
+                                       checkoutMetadata.smudgeFilterCommand,
+                                       entry.getPathString(),
+                                       result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE),
+                                       RawParseUtils.decode(result.getStderr()
+                                                       .toByteArray(MAX_EXCEPTION_TEXT_SIZE))));
+               }
+       }
+
+       // Run a builtin filter command
+       private static void runBuiltinFilterCommand(Repository repo,
+                       CheckoutMetadata checkoutMetadata, ObjectLoader ol,
+                       OutputStream channel) throws MissingObjectException, IOException {
+               FilterCommand command = null;
+               try {
+                       command = FilterCommandRegistry.createFilterCommand(
+                                       checkoutMetadata.smudgeFilterCommand, repo, ol.openStream(),
+                                       channel);
+               } catch (IOException e) {
+                       LOG.error(JGitText.get().failedToDetermineFilterDefinition, e);
+                       // In case an IOException occurred during creating of the command
+                       // then proceed as if there would not have been a builtin filter.
+                       ol.copyTo(channel);
+               }
+               if (command != null) {
+                       while (command.run() != -1) {
+                               // loop as long as command.run() tells there is work to do
+                       }
+               }
+       }
+
        @SuppressWarnings("deprecation")
        private static void checkValidPath(CanonicalTreeParser t)
                        throws InvalidPathException {
index 758f71d27ac944f59e1f6201947b51cc485dcf65..956171b127bc4355ee9ac6c4afaad1846fe21b0d 100644 (file)
@@ -338,6 +338,7 @@ public class JGitText extends TranslationBundle {
        /***/ public String expectedPktLineWithService;
        /***/ public String expectedReceivedContentType;
        /***/ public String expectedReportForRefNotReceived;
+       /***/ public String failedToDetermineFilterDefinition;
        /***/ public String failedUpdatingRefs;
        /***/ public String failureDueToOneOfTheFollowing;
        /***/ public String failureUpdatingFETCH_HEAD;