From 5d9f595eb87fba31c2253051102116fc7876e6c0 Mon Sep 17 00:00:00 2001 From: Christian Halstrick Date: Wed, 28 Oct 2015 13:25:09 +0100 Subject: [PATCH] Add support for clean filters When filters are defined for certain paths in gitattributes make sure that clean filters are processed when adding new content to the object database. Change-Id: Iffd72914cec5b434ba4d0de232e285b7492db868 Signed-off-by: Matthias Sohn --- .../jgit/junit/RepositoryTestCase.java | 13 ++ .../org/eclipse/jgit/api/AddCommandTest.java | 188 ++++++++++++++++++ .../eclipse/jgit/internal/JGitText.properties | 2 + .../src/org/eclipse/jgit/api/AddCommand.java | 4 + .../org/eclipse/jgit/api/CommitCommand.java | 4 +- .../api/errors/FilterFailedException.java | 144 ++++++++++++++ .../org/eclipse/jgit/dircache/DirCache.java | 1 + .../org/eclipse/jgit/internal/JGitText.java | 2 + .../src/org/eclipse/jgit/lib/Constants.java | 14 ++ .../src/org/eclipse/jgit/lib/IndexDiff.java | 1 + .../org/eclipse/jgit/treewalk/TreeWalk.java | 77 ++++++- .../jgit/treewalk/WorkingTreeIterator.java | 74 ++++++- .../eclipse/jgit/util/TemporaryBuffer.java | 31 +++ 13 files changed, 546 insertions(+), 9 deletions(-) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java index 56962e869f..c649eb9086 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepositoryTestCase.java @@ -282,6 +282,19 @@ public abstract class RepositoryTestCase extends LocalDiskRepositoryTestCase { return name; } + /** + * Replaces '\' by '/' + * + * @param str + * the string in which backslashes should be replaced + * @return the resulting string with slashes + * @since 4.2 + */ + public static String slashify(String str) { + str = str.replace('\\', '/'); + return str; + } + /** * Waits until it is guaranteed that a subsequent file modification has a * younger modification timestamp than the modification timestamp of the diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java index 2abed3adc8..a5ad18d102 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java @@ -45,6 +45,7 @@ package org.eclipse.jgit.api; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; @@ -52,11 +53,13 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.PrintWriter; +import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.NoFilepatternException; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; @@ -111,6 +114,191 @@ public class AddCommandTest extends RepositoryTestCase { indexState(CONTENT)); } + @Test + public void testCleanFilter() throws IOException, + GitAPIException { + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + writeTrashFile("src/a.tmp", "foo"); + // Caution: we need a trailing '\n' since sed on mac always appends + // linefeeds if missing + writeTrashFile("src/a.txt", "foo\n"); + File script = writeTempFile("sed s/o/e/g"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "clean", + "sh " + slashify(script.getPath())); + config.save(); + + git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp") + .call(); + + assertEquals( + "[src/a.tmp, mode:100644, content:foo][src/a.txt, mode:100644, content:fee\n]", + indexState(CONTENT)); + } + + @Test + public void testCleanFilterEnvironment() + throws IOException, GitAPIException { + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + writeTrashFile("src/a.txt", "foo"); + File script = writeTempFile("echo $GIT_DIR; echo 1 >xyz"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "clean", + "sh " + slashify(script.getPath())); + config.save(); + git.add().addFilepattern("src/a.txt").call(); + + String gitDir = db.getDirectory().getAbsolutePath(); + assertEquals("[src/a.txt, mode:100644, content:" + gitDir + + "\n]", indexState(CONTENT)); + assertTrue(new File(db.getWorkTree(), "xyz").exists()); + } + + @Test + public void testMultipleCleanFilter() throws IOException, GitAPIException { + writeTrashFile(".gitattributes", + "*.txt filter=tstFilter\n*.tmp filter=tstFilter2"); + // Caution: we need a trailing '\n' since sed on mac always appends + // linefeeds if missing + writeTrashFile("src/a.tmp", "foo\n"); + writeTrashFile("src/a.txt", "foo\n"); + File script = writeTempFile("sed s/o/e/g"); + File script2 = writeTempFile("sed s/f/x/g"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "clean", + "sh " + slashify(script.getPath())); + config.setString("filter", "tstFilter2", "clean", + "sh " + slashify(script2.getPath())); + config.save(); + + git.add().addFilepattern("src/a.txt").addFilepattern("src/a.tmp") + .call(); + + assertEquals( + "[src/a.tmp, mode:100644, content:xoo\n][src/a.txt, mode:100644, content:fee\n]", + indexState(CONTENT)); + + // TODO: multiple clean filters for one file??? + } + + /** + * The path of an added file name contains ';' and afterwards malicious + * commands. Make sure when calling filter commands to properly escape the + * filenames + * + * @throws IOException + * @throws GitAPIException + */ + @Test + public void testCommandInjection() throws IOException, GitAPIException { + // Caution: we need a trailing '\n' since sed on mac always appends + // linefeeds if missing + writeTrashFile("; echo virus", "foo\n"); + File script = writeTempFile("sed s/o/e/g"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "clean", + "sh " + slashify(script.getPath()) + " %f"); + writeTrashFile(".gitattributes", "* filter=tstFilter"); + + git.add().addFilepattern("; echo virus").call(); + // Without proper escaping the content would be "feovirus". The sed + // command and the "echo virus" would contribute to the content + assertEquals("[; echo virus, mode:100644, content:fee\n]", + indexState(CONTENT)); + } + + @Test + public void testBadCleanFilter() throws IOException, GitAPIException { + writeTrashFile("a.txt", "foo"); + File script = writeTempFile("sedfoo s/o/e/g"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "clean", + "sh " + script.getPath()); + config.save(); + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + + try { + git.add().addFilepattern("a.txt").call(); + fail("Didn't received the expected exception"); + } catch (FilterFailedException e) { + assertEquals(127, e.getReturnCode()); + } + } + + @Test + public void testBadCleanFilter2() throws IOException, GitAPIException { + writeTrashFile("a.txt", "foo"); + File script = writeTempFile("sed s/o/e/g"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "clean", + "shfoo " + script.getPath()); + config.save(); + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + + try { + git.add().addFilepattern("a.txt").call(); + fail("Didn't received the expected exception"); + } catch (FilterFailedException e) { + assertEquals(127, e.getReturnCode()); + } + } + + @Test + public void testCleanFilterReturning12() throws IOException, + GitAPIException { + writeTrashFile("a.txt", "foo"); + File script = writeTempFile("exit 12"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "clean", + "sh " + slashify(script.getPath())); + config.save(); + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + + try { + git.add().addFilepattern("a.txt").call(); + fail("Didn't received the expected exception"); + } catch (FilterFailedException e) { + assertEquals(12, e.getReturnCode()); + } + } + + @Test + public void testNotApplicableFilter() throws IOException, GitAPIException { + writeTrashFile("a.txt", "foo"); + File script = writeTempFile("sed s/o/e/g"); + + Git git = new Git(db); + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "tstFilter", "something", + "sh " + script.getPath()); + config.save(); + writeTrashFile(".gitattributes", "*.txt filter=tstFilter"); + + git.add().addFilepattern("a.txt").call(); + + assertEquals("[a.txt, mode:100644, content:foo]", indexState(CONTENT)); + } + + private File writeTempFile(String body) throws IOException { + File f = File.createTempFile("AddCommandTest_", ""); + JGitTestUtil.write(f, body); + return f; + } + @Test public void testAddExistingSingleSmallFileWithNewLine() throws IOException, GitAPIException { diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 5f80b8103b..d0e1c779e4 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -281,6 +281,8 @@ fileCannotBeDeleted=File cannot be deleted: {0} fileIsTooBigForThisConvenienceMethod=File is too big for this convenience method ({0} bytes). fileIsTooLarge=File is too large: {0} fileModeNotSetForPath=FileMode not set for path {0} +filterExecutionFailed=Execution of filter command ''{0}'' on file ''{1}'' failed +filterExecutionFailedRc=Execution of filter command ''{0}'' on file ''{1}'' failed with return code ''{2}'', message on stderr: ''{3}'' findingGarbage=Finding garbage flagIsDisposed={0} is disposed. flagNotFromThis={0} not from this. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index ae297a6438..67fb342fe2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -48,6 +48,7 @@ import java.io.InputStream; import java.util.Collection; import java.util.LinkedList; +import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.api.errors.NoFilepatternException; @@ -211,6 +212,9 @@ public class AddCommand extends GitCommand { builder.commit(); setCallable(false); } catch (IOException e) { + Throwable cause = e.getCause(); + if (cause != null && cause instanceof FilterFailedException) + throw (FilterFailedException) cause; throw new JGitInternalException( JGitText.get().exceptionCaughtDuringExecutionOfAddCommand, e); } finally { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index 53e18df479..9466dab74e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -332,7 +332,9 @@ public class CommitCommand extends GitCommand { treeWalk.setOperationType(OperationType.CHECKIN_OP); int dcIdx = treeWalk .addTree(new DirCacheBuildIterator(existingBuilder)); - int fIdx = treeWalk.addTree(new FileTreeIterator(repo)); + FileTreeIterator fti = new FileTreeIterator(repo); + fti.setDirCacheIterator(treeWalk, 0); + int fIdx = treeWalk.addTree(fti); int hIdx = -1; if (headId != null) hIdx = treeWalk.addTree(rw.parseTree(headId)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java new file mode 100644 index 0000000000..fbc30ef162 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/FilterFailedException.java @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2015, Christian Halstrick and + * other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v1.0 which accompanies this + * distribution, is reproduced below, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.api.errors; + +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; + +/** + * Exception thrown when the execution of a filter command failed + * + * @since 4.2 + */ +public class FilterFailedException extends GitAPIException { + private static final long serialVersionUID = 1L; + + private String filterCommand; + + private String path; + + private byte[] stdout; + + private String stderr; + + private int rc; + + /** + * Thrown if during execution of filter command an exception occurred + * + * @param cause + * the exception + * @param filterCommand + * the command which failed + * @param path + * the path processed by the filter + */ + public FilterFailedException(Exception cause, String filterCommand, + String path) { + super(MessageFormat.format(JGitText.get().filterExecutionFailed, + filterCommand, path), cause); + this.filterCommand = filterCommand; + this.path = path; + } + + /** + * Thrown if a filter command returns a non-zero return code + * + * @param rc + * the return code + * @param filterCommand + * the command which failed + * @param path + * the path processed by the filter + * @param stdout + * the output the filter generated so far. This should be limited + * to reasonable size. + * @param stderr + * the stderr output of the filter + */ + @SuppressWarnings("boxing") + public FilterFailedException(int rc, String filterCommand, String path, + byte[] stdout, String stderr) { + super(MessageFormat.format(JGitText.get().filterExecutionFailedRc, + filterCommand, path, rc, stderr)); + this.rc = rc; + this.filterCommand = filterCommand; + this.path = path; + this.stdout = stdout; + this.stderr = stderr; + } + + /** + * @return the filterCommand + */ + public String getFilterCommand() { + return filterCommand; + } + + /** + * @return the path of the file processed by the filter command + */ + public String getPath() { + return path; + } + + /** + * @return the output generated by the filter command. Might be truncated to + * limit memory consumption. + */ + public byte[] getOutput() { + return stdout; + } + + /** + * @return the error output returned by the filter command + */ + public String getError() { + return stderr; + } + + /** + * @return the return code returned by the filter command + */ + public int getReturnCode() { + return rc; + } + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java index 387d8ce739..a3980d2126 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java @@ -983,6 +983,7 @@ public class DirCache { FileTreeIterator fIter = new FileTreeIterator(repository); walk.addTree(iIter); walk.addTree(fIter); + fIter.setDirCacheIterator(walk, 0); walk.setRecursive(true); while (walk.next()) { iIter = walk.getTree(0, DirCacheIterator.class); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 6680564ade..f6fd8a396a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -340,6 +340,8 @@ public class JGitText extends TranslationBundle { /***/ public String fileIsTooBigForThisConvenienceMethod; /***/ public String fileIsTooLarge; /***/ public String fileModeNotSetForPath; + /***/ public String filterExecutionFailed; + /***/ public String filterExecutionFailedRc; /***/ public String findingGarbage; /***/ public String flagIsDisposed; /***/ public String flagNotFromThis; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 613df37a75..1a3111ab49 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -370,6 +370,20 @@ public final class Constants { */ public static final String DOT_GIT_ATTRIBUTES = ".gitattributes"; + /** + * Key for filters in .gitattributes + * + * @since 4.2 + */ + public static final String ATTR_FILTER = "filter"; + + /** + * clean command name, used to call filter driver + * + * @since 4.2 + */ + public static final String ATTR_FILTER_TYPE_CLEAN = "clean"; + /** Name of the ignore file */ public static final String DOT_GIT_IGNORE = ".gitignore"; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java index 281cde8750..9e474f86a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java @@ -413,6 +413,7 @@ public class IndexDiff { treeWalk.addTree(new EmptyTreeIterator()); treeWalk.addTree(new DirCacheIterator(dirCache)); treeWalk.addTree(initialWorkingTreeIterator); + initialWorkingTreeIterator.setDirCacheIterator(treeWalk, 1); Collection filters = new ArrayList(4); if (monitor != null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java index 8a59a700e3..06dc0bf6d0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java @@ -45,22 +45,25 @@ package org.eclipse.jgit.treewalk; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.Attribute; +import org.eclipse.jgit.attributes.Attribute.State; import org.eclipse.jgit.attributes.Attributes; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesNodeProvider; import org.eclipse.jgit.attributes.AttributesProvider; -import org.eclipse.jgit.attributes.Attribute.State; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StopWalkException; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.MutableObjectId; @@ -70,6 +73,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.eclipse.jgit.util.QuotedString; import org.eclipse.jgit.util.RawParseUtils; /** @@ -116,6 +120,12 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { */ private OperationType operationType = OperationType.CHECKOUT_OP; + /** + * The filter command as defined in gitattributes. The keys are + * filterName+"."+filterCommandType. E.g. "lfs.clean" + */ + private Map filterCommandsByNameDotType = new HashMap(); + /** * @param operationType * @since 4.2 @@ -259,6 +269,8 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { /** Cached attribute for the current entry */ private Attributes attrs = null; + private Config config; + /** * Create a new tree walker for a given repository. * @@ -269,6 +281,7 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { */ public TreeWalk(final Repository repo) { this(repo.newObjectReader(), true); + config = repo.getConfig(); attributesNodeProvider = repo.createAttributesNodeProvider(); } @@ -1308,4 +1321,66 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { AttributesNode defaultValue) { return (value == null) ? defaultValue : value; } + + /** + * Inspect config and attributes to return a filtercommand applicable for + * the current path + * + * @param filterCommandType + * which type of filterCommand should be executed. E.g. "clean", + * "smudge" + * @return a filter command + * @throws IOException + * @since 4.2 + */ + public String getFilterCommand(String filterCommandType) + throws IOException { + Attributes attributes = getAttributes(); + + Attribute f = attributes.get(Constants.ATTR_FILTER); + if (f == null) { + return null; + } + String filterValue = f.getValue(); + if (filterValue == null) { + return null; + } + + String filterCommand = getFilterCommandDefinition(filterValue, + filterCommandType); + if (filterCommand == null) { + return null; + } + return filterCommand.replaceAll("%f", //$NON-NLS-1$ + QuotedString.BOURNE.quote((getPathString()))); + } + + /** + * Get the filter command how it is defined in gitconfig. The returned + * string may contain "%f" which needs to be replaced by the current path + * before executing the filter command. These filter definitions are cached + * for better performance. + * + * @param filterDriverName + * The name of the filter driver as it is referenced in the + * gitattributes file. E.g. "lfs". For each filter driver there + * may be many commands defined in the .gitconfig + * @param filterCommandType + * The type of the filter command for a specific filter driver. + * May be "clean" or "smudge". + * @return the definition of the command to be executed for this filter + * driver and filter command + */ + private String getFilterCommandDefinition(String filterDriverName, + String filterCommandType) { + String key = filterDriverName + "." + filterCommandType; //$NON-NLS-1$ + String filterCommand = filterCommandsByNameDotType.get(key); + if (filterCommand != null) + return filterCommand; + filterCommand = config.getString(Constants.ATTR_FILTER, + filterDriverName, filterCommandType); + if (filterCommand != null) + filterCommandsByNameDotType.put(key, filterCommand); + return filterCommand; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 8be7f9a84c..94beeeb56f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -62,6 +62,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesRule; import org.eclipse.jgit.diff.RawText; @@ -76,6 +77,7 @@ import org.eclipse.jgit.ignore.IgnoreNode; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.lib.CoreConfig.CheckStat; import org.eclipse.jgit.lib.CoreConfig.SymLinks; import org.eclipse.jgit.lib.FileMode; @@ -85,6 +87,7 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.ExecutionResult; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.io.EolCanonicalizingInputStream; @@ -101,6 +104,8 @@ import org.eclipse.jgit.util.io.EolCanonicalizingInputStream; * @see FileTreeIterator */ public abstract class WorkingTreeIterator extends AbstractTreeIterator { + private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024; + /** An empty entry array, suitable for {@link #init(Entry[])}. */ protected static final Entry[] EOF = {}; @@ -134,6 +139,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { /** If there is a .gitignore file present, the parsed rules from it. */ private IgnoreNode ignoreNode; + private String cleanFilterCommand; + /** Repository that is the root level being iterated over */ protected Repository repository; @@ -186,6 +193,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { protected WorkingTreeIterator(final WorkingTreeIterator p) { super(p); state = p.state; + repository = p.repository; } /** @@ -348,7 +356,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { private InputStream possiblyFilteredInputStream(final Entry e, final InputStream is, final long len) throws IOException { - if (!mightNeedCleaning()) { + boolean mightNeedCleaning = mightNeedCleaning(); + if (!mightNeedCleaning) { canonLen = len; return is; } @@ -366,7 +375,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return new ByteArrayInputStream(raw, 0, n); } - if (isBinary(e)) { + // TODO: fix autocrlf causing mightneedcleaning + if (!mightNeedCleaning && isBinary(e)) { canonLen = len; return is; } @@ -390,10 +400,12 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } } - private boolean mightNeedCleaning() { + private boolean mightNeedCleaning() throws IOException { switch (getOptions().getAutoCRLF()) { case FALSE: default: + if (getCleanFilterCommand() != null) + return true; return false; case TRUE: @@ -415,8 +427,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } } - private static ByteBuffer filterClean(byte[] src, int n) - throws IOException { + private ByteBuffer filterClean(byte[] src, int n) throws IOException { InputStream in = new ByteArrayInputStream(src); try { return IO.readWholeStream(filterClean(in), n); @@ -425,8 +436,42 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } } - private static InputStream filterClean(InputStream in) { - return new EolCanonicalizingInputStream(in, true); + private InputStream filterClean(InputStream in) throws IOException { + in = handleAutoCRLF(in); + String filterCommand = getCleanFilterCommand(); + if (filterCommand != null) { + FS fs = repository.getFS(); + ProcessBuilder filterProcessBuilder = fs.runInShell(filterCommand, + new String[0]); + filterProcessBuilder.directory(repository.getWorkTree()); + filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, + repository.getDirectory().getAbsolutePath()); + ExecutionResult result; + try { + result = fs.execute(filterProcessBuilder, in); + } catch (IOException | InterruptedException e) { + throw new IOException(new FilterFailedException(e, + filterCommand, getEntryPathString())); + } + int rc = result.getRc(); + if (rc != 0) { + throw new IOException(new FilterFailedException(rc, + filterCommand, getEntryPathString(), + result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE), + RawParseUtils.decode(result.getStderr() + .toByteArray(MAX_EXCEPTION_TEXT_SIZE)))); + } + return result.getStdout().openInputStream(); + } + return in; + } + + private InputStream handleAutoCRLF(InputStream in) { + AutoCRLF autoCRLF = getOptions().getAutoCRLF(); + if (autoCRLF == AutoCRLF.TRUE || autoCRLF == AutoCRLF.INPUT) { + in = new EolCanonicalizingInputStream(in, true); + } + return in; } /** @@ -485,6 +530,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { System.arraycopy(e.encodedName, 0, path, pathOffset, nameLen); pathLen = pathOffset + nameLen; canonLen = -1; + cleanFilterCommand = null; } /** @@ -1271,4 +1317,18 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } } } + + /** + * @return the clean filter command for the current entry or + * null if no such command is defined + * @throws IOException + * @since 4.2 + */ + public String getCleanFilterCommand() throws IOException { + if (cleanFilterCommand == null && state.walk != null) { + cleanFilterCommand = state.walk + .getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN); + } + return cleanFilterCommand; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java index ca47f50fd9..3cd5929c7f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java @@ -246,6 +246,37 @@ public abstract class TemporaryBuffer extends OutputStream { return out; } + /** + * Convert this buffer's contents into a contiguous byte array. If this size + * of the buffer exceeds the limit only return the first {@code limit} bytes + *

+ * The buffer is only complete after {@link #close()} has been invoked. + * + * @param limit + * the maximum number of bytes to be returned + * + * @return the byte array limited to {@code limit} bytes. + * @throws IOException + * an error occurred reading from a local temporary file + * @throws OutOfMemoryError + * the buffer cannot fit in memory + * + * @since 4.2 + */ + public byte[] toByteArray(int limit) throws IOException { + final long len = Math.min(length(), limit); + if (Integer.MAX_VALUE < len) + throw new OutOfMemoryError( + JGitText.get().lengthExceedsMaximumArraySize); + final byte[] out = new byte[(int) len]; + int outPtr = 0; + for (final Block b : blocks) { + System.arraycopy(b.buffer, 0, out, outPtr, b.count); + outPtr += b.count; + } + return out; + } + /** * Send this buffer to an output stream. *

-- 2.39.5