]> source.dussan.org Git - jgit.git/commitdiff
Add support for clean filters 72/50372/25
authorChristian Halstrick <christian.halstrick@sap.com>
Wed, 28 Oct 2015 12:25:09 +0000 (13:25 +0100)
committerMatthias Sohn <matthias.sohn@sap.com>
Fri, 27 Nov 2015 22:23:09 +0000 (23:23 +0100)
When filters are defined for certain paths in gitattributes make
sure that clean filters are processed when adding new content to the
object database.

Change-Id: Iffd72914cec5b434ba4d0de232e285b7492db868
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
13 files changed:
org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java
org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java
org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java
org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java
org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java [new file with mode: 0644]
org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java
org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java
org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java
org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java
org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java

index 56962e869f0efcaef643a835a71547580fb7b563..c649eb9086cb06ede4ed221b928d82fb6213399d 100644 (file)
@@ -282,6 +282,19 @@ public abstract class RepositoryTestCase extends LocalDiskRepositoryTestCase {
                return name;
        }
 
+       /**
+        * Replaces '\' by '/'
+        *
+        * @param str
+        *            the string in which backslashes should be replaced
+        * @return the resulting string with slashes
+         * @since 4.2
+        */
+       public static String slashify(String str) {
+               str = str.replace('\\', '/');
+               return str;
+       }
+
        /**
         * Waits until it is guaranteed that a subsequent file modification has a
         * younger modification timestamp than the modification timestamp of the
index 2abed3adc8b405904d3c51f1ad25eff47e4f11cd..a5ad18d10231a16e39286c1b1da74e9a227bbd51 100644 (file)
@@ -45,6 +45,7 @@ package org.eclipse.jgit.api;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.io.File;
@@ -52,11 +53,13 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
 
+import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.NoFilepatternException;
 import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheBuilder;
 import org.eclipse.jgit.dircache.DirCacheEntry;
+import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.RepositoryTestCase;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
@@ -111,6 +114,191 @@ public class AddCommandTest extends RepositoryTestCase {
                                indexState(CONTENT));
        }
 
+       @Test
+       public void testCleanFilter() throws IOException,
+                       GitAPIException {
+               writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+               writeTrashFile("src/a.tmp", "foo");
+               // Caution: we need a trailing '\n' since sed on mac always appends
+               // linefeeds if missing
+               writeTrashFile("src/a.txt", "foo\n");
+               File script = writeTempFile("sed s/o/e/g");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "clean",
+                               "sh " + slashify(script.getPath()));
+               config.save();
+
+               git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
+                               .call();
+
+               assertEquals(
+                               "[src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:fee\n]",
+                               indexState(CONTENT));
+       }
+
+       @Test
+       public void testCleanFilterEnvironment()
+                       throws IOException, GitAPIException {
+               writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+               writeTrashFile("src/a.txt", "foo");
+               File script = writeTempFile("echo $GIT_DIR; echo 1 >xyz");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "clean",
+                               "sh " + slashify(script.getPath()));
+               config.save();
+               git.add().addFilepattern("src/a.txt").call();
+
+               String gitDir = db.getDirectory().getAbsolutePath();
+               assertEquals("[src/a.txt, mode:100644, content:" + gitDir
+                               + "\n]", indexState(CONTENT));
+               assertTrue(new File(db.getWorkTree(), "xyz").exists());
+       }
+
+       @Test
+       public void testMultipleCleanFilter() throws IOException, GitAPIException {
+               writeTrashFile(".gitattributes",
+                               "*.txt filter=tstFilter\n*.tmp filter=tstFilter2");
+               // Caution: we need a trailing '\n' since sed on mac always appends
+               // linefeeds if missing
+               writeTrashFile("src/a.tmp", "foo\n");
+               writeTrashFile("src/a.txt", "foo\n");
+               File script = writeTempFile("sed s/o/e/g");
+               File script2 = writeTempFile("sed s/f/x/g");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "clean",
+                               "sh " + slashify(script.getPath()));
+               config.setString("filter", "tstFilter2", "clean",
+                               "sh " + slashify(script2.getPath()));
+               config.save();
+
+               git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp")
+                               .call();
+
+               assertEquals(
+                               "[src/a.tmp, mode:100644, content:xoo\n][src/a.txt, mode:100644, content:fee\n]",
+                               indexState(CONTENT));
+
+               // TODO: multiple clean filters for one file???
+       }
+
+       /**
+        * The path of an added file name contains ';' and afterwards malicious
+        * commands. Make sure when calling filter commands to properly escape the
+        * filenames
+        *
+        * @throws IOException
+        * @throws GitAPIException
+        */
+       @Test
+       public void testCommandInjection() throws IOException, GitAPIException {
+               // Caution: we need a trailing '\n' since sed on mac always appends
+               // linefeeds if missing
+               writeTrashFile("; echo virus", "foo\n");
+               File script = writeTempFile("sed s/o/e/g");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "clean",
+                               "sh " + slashify(script.getPath()) + " %f");
+               writeTrashFile(".gitattributes", "* filter=tstFilter");
+
+               git.add().addFilepattern("; echo virus").call();
+               // Without proper escaping the content would be "feovirus". The sed
+               // command and the "echo virus" would contribute to the content
+               assertEquals("[; echo virus, mode:100644, content:fee\n]",
+                               indexState(CONTENT));
+       }
+
+       @Test
+       public void testBadCleanFilter() throws IOException, GitAPIException {
+               writeTrashFile("a.txt", "foo");
+               File script = writeTempFile("sedfoo s/o/e/g");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "clean",
+                               "sh " + script.getPath());
+               config.save();
+               writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+               try {
+                       git.add().addFilepattern("a.txt").call();
+                       fail("Didn't received the expected exception");
+               } catch (FilterFailedException e) {
+                       assertEquals(127, e.getReturnCode());
+               }
+       }
+
+       @Test
+       public void testBadCleanFilter2() throws IOException, GitAPIException {
+               writeTrashFile("a.txt", "foo");
+               File script = writeTempFile("sed s/o/e/g");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "clean",
+                               "shfoo " + script.getPath());
+               config.save();
+               writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+               try {
+                       git.add().addFilepattern("a.txt").call();
+                       fail("Didn't received the expected exception");
+               } catch (FilterFailedException e) {
+                       assertEquals(127, e.getReturnCode());
+               }
+       }
+
+       @Test
+       public void testCleanFilterReturning12() throws IOException,
+                       GitAPIException {
+               writeTrashFile("a.txt", "foo");
+               File script = writeTempFile("exit 12");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "clean",
+                               "sh " + slashify(script.getPath()));
+               config.save();
+               writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+               try {
+                       git.add().addFilepattern("a.txt").call();
+                       fail("Didn't received the expected exception");
+               } catch (FilterFailedException e) {
+                       assertEquals(12, e.getReturnCode());
+               }
+       }
+
+       @Test
+       public void testNotApplicableFilter() throws IOException, GitAPIException {
+               writeTrashFile("a.txt", "foo");
+               File script = writeTempFile("sed s/o/e/g");
+
+               Git git = new Git(db);
+               StoredConfig config = git.getRepository().getConfig();
+               config.setString("filter", "tstFilter", "something",
+                               "sh " + script.getPath());
+               config.save();
+               writeTrashFile(".gitattributes", "*.txt filter=tstFilter");
+
+               git.add().addFilepattern("a.txt").call();
+
+               assertEquals("[a.txt, mode:100644, content:foo]", indexState(CONTENT));
+       }
+
+       private File writeTempFile(String body) throws IOException {
+               File f = File.createTempFile("AddCommandTest_", "");
+               JGitTestUtil.write(f, body);
+               return f;
+       }
+
        @Test
        public void testAddExistingSingleSmallFileWithNewLine() throws IOException,
                        GitAPIException {
index 5f80b8103be828b831980a7aa2c6b416e7ad5250..d0e1c779e4630a74e34532bdc5120d886ed25533 100644 (file)
@@ -281,6 +281,8 @@ fileCannotBeDeleted=File cannot be deleted: {0}
 fileIsTooBigForThisConvenienceMethod=File is too big for this convenience method ({0} bytes).
 fileIsTooLarge=File is too large: {0}
 fileModeNotSetForPath=FileMode not set for path {0}
+filterExecutionFailed=Execution of filter command ''{0}'' on file ''{1}'' failed
+filterExecutionFailedRc=Execution of filter command ''{0}'' on file ''{1}'' failed with return code ''{2}'', message on stderr: ''{3}''
 findingGarbage=Finding garbage
 flagIsDisposed={0} is disposed.
 flagNotFromThis={0} not from this.
index ae297a6438ef22ec1a3f39c854c9023955a9485e..67fb342fe22f1c228be34beb8f40fc2b20cc354e 100644 (file)
@@ -48,6 +48,7 @@ import java.io.InputStream;
 import java.util.Collection;
 import java.util.LinkedList;
 
+import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.api.errors.NoFilepatternException;
@@ -211,6 +212,9 @@ public class AddCommand extends GitCommand<DirCache> {
                        builder.commit();
                        setCallable(false);
                } catch (IOException e) {
+                       Throwable cause = e.getCause();
+                       if (cause != null && cause instanceof FilterFailedException)
+                               throw (FilterFailedException) cause;
                        throw new JGitInternalException(
                                        JGitText.get().exceptionCaughtDuringExecutionOfAddCommand, e);
                } finally {
index 53e18df479df97b699d4fe11c37c652b006497cf..9466dab74eae7fd9b5171776d62d04facd923b3b 100644 (file)
@@ -332,7 +332,9 @@ public class CommitCommand extends GitCommand<RevCommit> {
                        treeWalk.setOperationType(OperationType.CHECKIN_OP);
                        int dcIdx = treeWalk
                                        .addTree(new DirCacheBuildIterator(existingBuilder));
-                       int fIdx = treeWalk.addTree(new FileTreeIterator(repo));
+                       FileTreeIterator fti = new FileTreeIterator(repo);
+                       fti.setDirCacheIterator(treeWalk, 0);
+                       int fIdx = treeWalk.addTree(fti);
                        int hIdx = -1;
                        if (headId != null)
                                hIdx = treeWalk.addTree(rw.parseTree(headId));
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java
new file mode 100644 (file)
index 0000000..fbc30ef
--- /dev/null
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2015, Christian Halstrick <christian.halstrick@sap.com> and
+ * other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Distribution License v1.0 which accompanies this
+ * distribution, is reproduced below, and is available at
+ * http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.api.errors;
+
+import java.text.MessageFormat;
+
+import org.eclipse.jgit.internal.JGitText;
+
+/**
+ * Exception thrown when the execution of a filter command failed
+ *
+ * @since 4.2
+ */
+public class FilterFailedException extends GitAPIException {
+       private static final long serialVersionUID = 1L;
+
+       private String filterCommand;
+
+       private String path;
+
+       private byte[] stdout;
+
+       private String stderr;
+
+       private int rc;
+
+       /**
+        * Thrown if during execution of filter command an exception occurred
+        *
+        * @param cause
+        *            the exception
+        * @param filterCommand
+        *            the command which failed
+        * @param path
+        *            the path processed by the filter
+        */
+       public FilterFailedException(Exception cause, String filterCommand,
+                       String path) {
+               super(MessageFormat.format(JGitText.get().filterExecutionFailed,
+                               filterCommand, path), cause);
+               this.filterCommand = filterCommand;
+               this.path = path;
+       }
+
+       /**
+        * Thrown if a filter command returns a non-zero return code
+        *
+        * @param rc
+        *            the return code
+        * @param filterCommand
+        *            the command which failed
+        * @param path
+        *            the path processed by the filter
+        * @param stdout
+        *            the output the filter generated so far. This should be limited
+        *            to reasonable size.
+        * @param stderr
+        *            the stderr output of the filter
+        */
+       @SuppressWarnings("boxing")
+       public FilterFailedException(int rc, String filterCommand, String path,
+                       byte[] stdout, String stderr) {
+               super(MessageFormat.format(JGitText.get().filterExecutionFailedRc,
+                               filterCommand, path, rc, stderr));
+               this.rc = rc;
+               this.filterCommand = filterCommand;
+               this.path = path;
+               this.stdout = stdout;
+               this.stderr = stderr;
+       }
+
+       /**
+        * @return the filterCommand
+        */
+       public String getFilterCommand() {
+               return filterCommand;
+       }
+
+       /**
+        * @return the path of the file processed by the filter command
+        */
+       public String getPath() {
+               return path;
+       }
+
+       /**
+        * @return the output generated by the filter command. Might be truncated to
+        *         limit memory consumption.
+        */
+       public byte[] getOutput() {
+               return stdout;
+       }
+
+       /**
+        * @return the error output returned by the filter command
+        */
+       public String getError() {
+               return stderr;
+       }
+
+       /**
+        * @return the return code returned by the filter command
+        */
+       public int getReturnCode() {
+               return rc;
+       }
+
+}
index 387d8ce7390a25000761432fe745d729fec7eba6..a3980d2126020bb78389ceb2818fc9624c59f566 100644 (file)
@@ -983,6 +983,7 @@ public class DirCache {
                        FileTreeIterator fIter = new FileTreeIterator(repository);
                        walk.addTree(iIter);
                        walk.addTree(fIter);
+                       fIter.setDirCacheIterator(walk, 0);
                        walk.setRecursive(true);
                        while (walk.next()) {
                                iIter = walk.getTree(0, DirCacheIterator.class);
index 6680564ade2067566973d9dc36c52a9ab1fde910..f6fd8a396a7707f9cc47200761e4edb636b1f2c7 100644 (file)
@@ -340,6 +340,8 @@ public class JGitText extends TranslationBundle {
        /***/ public String fileIsTooBigForThisConvenienceMethod;
        /***/ public String fileIsTooLarge;
        /***/ public String fileModeNotSetForPath;
+       /***/ public String filterExecutionFailed;
+       /***/ public String filterExecutionFailedRc;
        /***/ public String findingGarbage;
        /***/ public String flagIsDisposed;
        /***/ public String flagNotFromThis;
index 613df37a7596d64e47198444919c252e71eddc8e..1a3111ab499b4ab190765cc0b2a38793374f3a51 100644 (file)
@@ -370,6 +370,20 @@ public final class Constants {
         */
        public static final String DOT_GIT_ATTRIBUTES = ".gitattributes";
 
+       /**
+        * Key for filters in .gitattributes
+        *
+        * @since 4.2
+        */
+       public static final String ATTR_FILTER = "filter";
+
+       /**
+        * clean command name, used to call filter driver
+        *
+        * @since 4.2
+        */
+       public static final String ATTR_FILTER_TYPE_CLEAN = "clean";
+
        /** Name of the ignore file */
        public static final String DOT_GIT_IGNORE = ".gitignore";
 
index 281cde8750764965ea60994444ba5174b6f34d2b..9e474f86a8ea449d9d892c67145fd3a7c775b869 100644 (file)
@@ -413,6 +413,7 @@ public class IndexDiff {
                                treeWalk.addTree(new EmptyTreeIterator());
                        treeWalk.addTree(new DirCacheIterator(dirCache));
                        treeWalk.addTree(initialWorkingTreeIterator);
+                       initialWorkingTreeIterator.setDirCacheIterator(treeWalk, 1);
                        Collection<TreeFilter> filters = new ArrayList<TreeFilter>(4);
 
                        if (monitor != null) {
index 8a59a700e3ca57f83e7b93b1798095f2c303ae72..06dc0bf6d0c7e19778646ebf54991d8c5f1e5d8b 100644 (file)
 package org.eclipse.jgit.treewalk;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Set;
 
 import org.eclipse.jgit.annotations.Nullable;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.attributes.Attribute;
+import org.eclipse.jgit.attributes.Attribute.State;
 import org.eclipse.jgit.attributes.Attributes;
 import org.eclipse.jgit.attributes.AttributesNode;
 import org.eclipse.jgit.attributes.AttributesNodeProvider;
 import org.eclipse.jgit.attributes.AttributesProvider;
-import org.eclipse.jgit.attributes.Attribute.State;
 import org.eclipse.jgit.dircache.DirCacheIterator;
 import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.errors.StopWalkException;
 import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.Config;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.MutableObjectId;
@@ -70,6 +73,7 @@ import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevTree;
 import org.eclipse.jgit.treewalk.filter.PathFilter;
 import org.eclipse.jgit.treewalk.filter.TreeFilter;
+import org.eclipse.jgit.util.QuotedString;
 import org.eclipse.jgit.util.RawParseUtils;
 
 /**
@@ -116,6 +120,12 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
         */
        private OperationType operationType = OperationType.CHECKOUT_OP;
 
+       /**
+        * The filter command as defined in gitattributes. The keys are
+        * filterName+"."+filterCommandType. E.g. "lfs.clean"
+        */
+       private Map<String, String> filterCommandsByNameDotType = new HashMap<String, String>();
+
        /**
         * @param operationType
         * @since 4.2
@@ -259,6 +269,8 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
        /** Cached attribute for the current entry */
        private Attributes attrs = null;
 
+       private Config config;
+
        /**
         * Create a new tree walker for a given repository.
         *
@@ -269,6 +281,7 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
         */
        public TreeWalk(final Repository repo) {
                this(repo.newObjectReader(), true);
+               config = repo.getConfig();
                attributesNodeProvider = repo.createAttributesNodeProvider();
        }
 
@@ -1308,4 +1321,66 @@ public class TreeWalk implements AutoCloseable, AttributesProvider {
                        AttributesNode defaultValue) {
                return (value == null) ? defaultValue : value;
        }
+
+       /**
+        * Inspect config and attributes to return a filtercommand applicable for
+        * the current path
+        *
+        * @param filterCommandType
+        *            which type of filterCommand should be executed. E.g. "clean",
+        *            "smudge"
+        * @return a filter command
+        * @throws IOException
+        * @since 4.2
+        */
+       public String getFilterCommand(String filterCommandType)
+                       throws IOException {
+               Attributes attributes = getAttributes();
+
+               Attribute f = attributes.get(Constants.ATTR_FILTER);
+               if (f == null) {
+                       return null;
+               }
+               String filterValue = f.getValue();
+               if (filterValue == null) {
+                       return null;
+               }
+
+               String filterCommand = getFilterCommandDefinition(filterValue,
+                               filterCommandType);
+               if (filterCommand == null) {
+                       return null;
+               }
+               return filterCommand.replaceAll("%f", //$NON-NLS-1$
+                               QuotedString.BOURNE.quote((getPathString())));
+       }
+
+       /**
+        * Get the filter command how it is defined in gitconfig. The returned
+        * string may contain "%f" which needs to be replaced by the current path
+        * before executing the filter command. These filter definitions are cached
+        * for better performance.
+        *
+        * @param filterDriverName
+        *            The name of the filter driver as it is referenced in the
+        *            gitattributes file. E.g. "lfs". For each filter driver there
+        *            may be many commands defined in the .gitconfig
+        * @param filterCommandType
+        *            The type of the filter command for a specific filter driver.
+        *            May be "clean" or "smudge".
+        * @return the definition of the command to be executed for this filter
+        *         driver and filter command
+        */
+       private String getFilterCommandDefinition(String filterDriverName,
+                       String filterCommandType) {
+               String key = filterDriverName + "." + filterCommandType; //$NON-NLS-1$
+               String filterCommand = filterCommandsByNameDotType.get(key);
+               if (filterCommand != null)
+                       return filterCommand;
+               filterCommand = config.getString(Constants.ATTR_FILTER,
+                               filterDriverName, filterCommandType);
+               if (filterCommand != null)
+                       filterCommandsByNameDotType.put(key, filterCommand);
+               return filterCommand;
+       }
 }
index 8be7f9a84cfb65e9125337a47805192bcf3ff4b2..94beeeb56fdf959253dbccc29d48cf66da976d93 100644 (file)
@@ -62,6 +62,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 
+import org.eclipse.jgit.api.errors.FilterFailedException;
 import org.eclipse.jgit.attributes.AttributesNode;
 import org.eclipse.jgit.attributes.AttributesRule;
 import org.eclipse.jgit.diff.RawText;
@@ -76,6 +77,7 @@ import org.eclipse.jgit.ignore.IgnoreNode;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig;
+import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
 import org.eclipse.jgit.lib.CoreConfig.CheckStat;
 import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.FileMode;
@@ -85,6 +87,7 @@ import org.eclipse.jgit.lib.ObjectReader;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.submodule.SubmoduleWalk;
 import org.eclipse.jgit.util.FS;
+import org.eclipse.jgit.util.FS.ExecutionResult;
 import org.eclipse.jgit.util.IO;
 import org.eclipse.jgit.util.RawParseUtils;
 import org.eclipse.jgit.util.io.EolCanonicalizingInputStream;
@@ -101,6 +104,8 @@ import org.eclipse.jgit.util.io.EolCanonicalizingInputStream;
  * @see FileTreeIterator
  */
 public abstract class WorkingTreeIterator extends AbstractTreeIterator {
+       private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024;
+
        /** An empty entry array, suitable for {@link #init(Entry[])}. */
        protected static final Entry[] EOF = {};
 
@@ -134,6 +139,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
        /** If there is a .gitignore file present, the parsed rules from it. */
        private IgnoreNode ignoreNode;
 
+       private String cleanFilterCommand;
+
        /** Repository that is the root level being iterated over */
        protected Repository repository;
 
@@ -186,6 +193,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
        protected WorkingTreeIterator(final WorkingTreeIterator p) {
                super(p);
                state = p.state;
+               repository = p.repository;
        }
 
        /**
@@ -348,7 +356,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 
        private InputStream possiblyFilteredInputStream(final Entry e,
                        final InputStream is, final long len) throws IOException {
-               if (!mightNeedCleaning()) {
+               boolean mightNeedCleaning = mightNeedCleaning();
+               if (!mightNeedCleaning) {
                        canonLen = len;
                        return is;
                }
@@ -366,7 +375,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
                        return new ByteArrayInputStream(raw, 0, n);
                }
 
-               if (isBinary(e)) {
+               // TODO: fix autocrlf causing mightneedcleaning
+               if (!mightNeedCleaning && isBinary(e)) {
                        canonLen = len;
                        return is;
                }
@@ -390,10 +400,12 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
                }
        }
 
-       private boolean mightNeedCleaning() {
+       private boolean mightNeedCleaning() throws IOException {
                switch (getOptions().getAutoCRLF()) {
                case FALSE:
                default:
+                       if (getCleanFilterCommand() != null)
+                               return true;
                        return false;
 
                case TRUE:
@@ -415,8 +427,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
                }
        }
 
-       private static ByteBuffer filterClean(byte[] src, int n)
-                       throws IOException {
+       private ByteBuffer filterClean(byte[] src, int n) throws IOException {
                InputStream in = new ByteArrayInputStream(src);
                try {
                        return IO.readWholeStream(filterClean(in), n);
@@ -425,8 +436,42 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
                }
        }
 
-       private static InputStream filterClean(InputStream in) {
-               return new EolCanonicalizingInputStream(in, true);
+       private InputStream filterClean(InputStream in) throws IOException {
+               in = handleAutoCRLF(in);
+               String filterCommand = getCleanFilterCommand();
+               if (filterCommand != null) {
+                       FS fs = repository.getFS();
+                       ProcessBuilder filterProcessBuilder = fs.runInShell(filterCommand,
+                                       new String[0]);
+                       filterProcessBuilder.directory(repository.getWorkTree());
+                       filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY,
+                                       repository.getDirectory().getAbsolutePath());
+                       ExecutionResult result;
+                       try {
+                               result = fs.execute(filterProcessBuilder, in);
+                       } catch (IOException | InterruptedException e) {
+                               throw new IOException(new FilterFailedException(e,
+                                               filterCommand, getEntryPathString()));
+                       }
+                       int rc = result.getRc();
+                       if (rc != 0) {
+                               throw new IOException(new FilterFailedException(rc,
+                                               filterCommand, getEntryPathString(),
+                                               result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE),
+                                               RawParseUtils.decode(result.getStderr()
+                                                               .toByteArray(MAX_EXCEPTION_TEXT_SIZE))));
+                       }
+                       return result.getStdout().openInputStream();
+               }
+               return in;
+       }
+
+       private InputStream handleAutoCRLF(InputStream in) {
+               AutoCRLF autoCRLF = getOptions().getAutoCRLF();
+               if (autoCRLF == AutoCRLF.TRUE || autoCRLF == AutoCRLF.INPUT) {
+                       in = new EolCanonicalizingInputStream(in, true);
+               }
+               return in;
        }
 
        /**
@@ -485,6 +530,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
                System.arraycopy(e.encodedName, 0, path, pathOffset, nameLen);
                pathLen = pathOffset + nameLen;
                canonLen = -1;
+               cleanFilterCommand = null;
        }
 
        /**
@@ -1271,4 +1317,18 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
                        }
                }
        }
+
+       /**
+        * @return the clean filter command for the current entry or
+        *         <code>null</code> if no such command is defined
+        * @throws IOException
+        * @since 4.2
+        */
+       public String getCleanFilterCommand() throws IOException {
+               if (cleanFilterCommand == null && state.walk != null) {
+                       cleanFilterCommand = state.walk
+                                       .getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN);
+               }
+               return cleanFilterCommand;
+       }
 }
index ca47f50fd9b731d1b0385f1f2a5bc41577297059..3cd5929c7f2fcac9274bf59e5a8f2e5a02d11585 100644 (file)
@@ -246,6 +246,37 @@ public abstract class TemporaryBuffer extends OutputStream {
                return out;
        }
 
+       /**
+        * Convert this buffer's contents into a contiguous byte array. If this size
+        * of the buffer exceeds the limit only return the first {@code limit} bytes
+        * <p>
+        * The buffer is only complete after {@link #close()} has been invoked.
+        *
+        * @param limit
+        *            the maximum number of bytes to be returned
+        *
+        * @return the byte array limited to {@code limit} bytes.
+        * @throws IOException
+        *             an error occurred reading from a local temporary file
+        * @throws OutOfMemoryError
+        *             the buffer cannot fit in memory
+        *
+        * @since 4.2
+        */
+       public byte[] toByteArray(int limit) throws IOException {
+               final long len = Math.min(length(), limit);
+               if (Integer.MAX_VALUE < len)
+                       throw new OutOfMemoryError(
+                                       JGitText.get().lengthExceedsMaximumArraySize);
+               final byte[] out = new byte[(int) len];
+               int outPtr = 0;
+               for (final Block b : blocks) {
+                       System.arraycopy(b.buffer, 0, out, outPtr, b.count);
+                       outPtr += b.count;
+               }
+               return out;
+       }
+
        /**
         * Send this buffer to an output stream.
         * <p>