From b811e4399ea578a07595bac790ad619b9fcb1300 Mon Sep 17 00:00:00 2001 From: Ivan Motsch Date: Thu, 25 Feb 2016 15:39:41 +0100 Subject: [PATCH] Add EOL stream type detection to TreeWalk TreeWalk provides the new method getEolStreamType. This new method can be used with EolStreamTypeUtil in order to create a wrapped InputStream or OutputStream when reading / writing files. The implementation implements support for the git configuration options core.crlf, core.eol and the .gitattributes "text", "eol" and "binary" CQ: 10896 Bug: 486563 Change-Id: Ie4f6367afc2a6aec1de56faf95120fff0339a358 Signed-off-by: Ivan Motsch Signed-off-by: Matthias Sohn --- .../eclipse/jgit/api/EolRepositoryTest.java | 692 ++++++++++++++++++ .../jgit/api/EolStreamTypeUtilTest.java | 335 +++++++++ .../jgit/lib/DirCacheCheckoutTest.java | 3 +- ...amTest.java => AutoLFInputStreamTest.java} | 7 +- org.eclipse.jgit/.settings/.api_filters | 21 + .../org/eclipse/jgit/api/BlameCommand.java | 4 +- .../org/eclipse/jgit/api/CheckoutCommand.java | 22 +- .../eclipse/jgit/api/StashApplyCommand.java | 12 +- .../eclipse/jgit/api/StashCreateCommand.java | 4 +- .../jgit/dircache/DirCacheCheckout.java | 127 ++-- .../org/eclipse/jgit/lib/ConfigConstants.java | 7 + .../src/org/eclipse/jgit/lib/CoreConfig.java | 40 + .../jgit/treewalk/NameConflictTreeWalk.java | 13 + .../org/eclipse/jgit/treewalk/TreeWalk.java | 89 ++- .../jgit/treewalk/WorkingTreeIterator.java | 146 ++-- .../jgit/treewalk/WorkingTreeOptions.java | 14 + .../src/org/eclipse/jgit/util/Holder.java | 77 ++ .../jgit/util/io/AutoCRLFInputStream.java | 2 +- .../jgit/util/io/AutoCRLFOutputStream.java | 23 +- .../jgit/util/io/AutoLFInputStream.java | 199 +++++ .../jgit/util/io/AutoLFOutputStream.java | 200 +++++ .../util/io/EolCanonicalizingInputStream.java | 123 +--- .../jgit/util/io/EolStreamTypeUtil.java | 255 +++++++ 23 files changed, 2160 insertions(+), 255 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java rename org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/{EolCanonicalizingInputStreamTest.java => AutoLFInputStreamTest.java} (94%) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/Holder.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java new file mode 100644 index 0000000000..5dd8da57c2 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java @@ -0,0 +1,692 @@ +/* + * Copyright (C) 2015, Ivan Motsch + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.api; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.eclipse.jgit.api.ResetCommand.ResetType; +import org.eclipse.jgit.api.errors.CheckoutConflictException; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.NoFilepatternException; +import org.eclipse.jgit.attributes.Attribute; +import org.eclipse.jgit.dircache.DirCache; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.dircache.DirCacheIterator; +import org.eclipse.jgit.errors.RevisionSyntaxException; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; +import org.eclipse.jgit.lib.CoreConfig.EOL; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.treewalk.FileTreeIterator; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.util.IO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.theories.DataPoint; +import org.junit.experimental.theories.Theories; +import org.junit.runner.RunWith; + +/** + * Unit tests for end-of-line conversion and settings using core.autocrlf, * + * core.eol and the .gitattributes eol, text, binary (macro for -diff -merge + * -text) + */ +@RunWith(Theories.class) +public class EolRepositoryTest extends RepositoryTestCase { + private static final FileMode D = FileMode.TREE; + + private static final FileMode F = FileMode.REGULAR_FILE; + + @DataPoint + public static String smallContents[] = { + generateTestData(3, 1, true, false), + generateTestData(3, 1, false, true), + generateTestData(3, 1, true, true) }; + + @DataPoint + public static String hugeContents[] = { + generateTestData(1000000, 17, true, false), + generateTestData(1000000, 17, false, true), + generateTestData(1000000, 17, true, true) }; + + static String generateTestData(int size, int lineSize, boolean withCRLF, + boolean withLF) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < size; i++) { + if (i > 0 && i % lineSize == 0) { + // newline + if (withCRLF && withLF) { + // mixed + if (i % 2 == 0) + sb.append("\r\n"); + else + sb.append("\n"); + } else if (withCRLF) { + sb.append("\r\n"); + } else if (withLF) { + sb.append("\n"); + } + } + sb.append("A"); + } + return sb.toString(); + } + + public EolRepositoryTest(String[] testContent) { + CONTENT_CRLF = testContent[0]; + CONTENT_LF = testContent[1]; + CONTENT_MIXED = testContent[2]; + } + + protected String CONTENT_CRLF; + + protected String CONTENT_LF; + + protected String CONTENT_MIXED; + + private TreeWalk walk; + + /** work tree root .gitattributes */ + private File dotGitattributes; + + /** file containing CRLF */ + private File fileCRLF; + + /** file containing LF */ + private File fileLF; + + /** file containing mixed CRLF and LF */ + private File fileMixed; + + /** this values are set in {@link #collectRepositoryState()} */ + private static class ActualEntry { + private String attrs; + + private String file; + + private String index; + + private int indexContentLength; + } + + private ActualEntry entryCRLF = new ActualEntry(); + + private ActualEntry entryLF = new ActualEntry(); + + private ActualEntry entryMixed = new ActualEntry(); + + private DirCache dc; + + @Test + public void testDefaultSetup() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(null, null, null, null, "* text=auto"); + collectRepositoryState(); + assertEquals("text=auto", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + public void checkEntryContent(ActualEntry entry, String fileContent, + String indexContent) { + assertEquals(fileContent, entry.file); + assertEquals(indexContent, entry.index); + assertEquals(fileContent.length(), entry.indexContentLength); + } + + @Test + public void test_ConfigAutoCRLF_false() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.FALSE, null, null, null, "* text=auto"); + collectRepositoryState(); + assertEquals("text=auto", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_true() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.TRUE, null, null, null, "* text=auto"); + collectRepositoryState(); + assertEquals("text=auto", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_input() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.INPUT, null, null, null, "* text=auto"); + collectRepositoryState(); + assertEquals("text=auto", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigEOL_lf() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(null, EOL.LF, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigEOL_crlf() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(null, EOL.CRLF, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_ConfigEOL_native_windows() throws Exception { + String origLineSeparator = System.getProperty("line.separator", "\n"); + System.setProperty("line.separator", "\r\n"); + try { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(null, EOL.NATIVE, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + } finally { + System.setProperty("line.separator", origLineSeparator); + } + } + + @Test + public void test_ConfigEOL_native_xnix() throws Exception { + String origLineSeparator = System.getProperty("line.separator", "\n"); + System.setProperty("line.separator", "\n"); + try { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(null, EOL.NATIVE, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + } finally { + System.setProperty("line.separator", origLineSeparator); + } + } + + @Test + public void test_ConfigAutoCRLF_false_ConfigEOL_lf() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.FALSE, EOL.LF, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_false_ConfigEOL_native() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.FALSE, EOL.NATIVE, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_true_ConfigEOL_lf() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.TRUE, EOL.LF, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_switchToBranchWithTextAttributes() + throws Exception { + Git git = Git.wrap(db); + + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.FALSE, EOL.CRLF, null, null, + "file1.txt text\nfile2.txt text\nfile3.txt text"); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + + // switch to binary for file1 + dotGitattributes = createAndAddFile(git, Constants.DOT_GIT_ATTRIBUTES, + "file1.txt binary\nfile2.txt text\nfile3.txt text"); + gitCommit(git, "switchedToBinaryFor1"); + recreateWorktree(git); + collectRepositoryState(); + assertEquals("binary -diff -merge -text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + assertEquals("text", entryLF.attrs); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + assertEquals("text", entryMixed.attrs); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + + // checkout the commit which has text for file1 + gitCheckout(git, "HEAD^"); + recreateWorktree(git); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_switchToBranchWithBinaryAttributes() throws Exception { + Git git = Git.wrap(db); + + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.FALSE, EOL.LF, null, null, + "file1.txt binary\nfile2.txt binary\nfile3.txt binary"); + collectRepositoryState(); + assertEquals("binary -diff -merge -text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_CRLF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_MIXED, CONTENT_MIXED); + + // switch to text for file1 + dotGitattributes = createAndAddFile(git, Constants.DOT_GIT_ATTRIBUTES, + "file1.txt text\nfile2.txt binary\nfile3.txt binary"); + gitCommit(git, "switchedToTextFor1"); + recreateWorktree(git); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + assertEquals("binary -diff -merge -text", entryLF.attrs); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + assertEquals("binary -diff -merge -text", entryMixed.attrs); + checkEntryContent(entryMixed, CONTENT_MIXED, CONTENT_MIXED); + + // checkout the commit which has text for file1 + gitCheckout(git, "HEAD^"); + recreateWorktree(git); + collectRepositoryState(); + assertEquals("binary -diff -merge -text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_CRLF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_MIXED, CONTENT_MIXED); + } + + @Test + public void test_ConfigAutoCRLF_input_ConfigEOL_lf() throws Exception { + // for EOL to work, the text attribute must be set + setupGitAndDoHardReset(AutoCRLF.INPUT, EOL.LF, "*.txt text", null, null); + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_true_GlobalEOL_lf() throws Exception { + setupGitAndDoHardReset(AutoCRLF.TRUE, EOL.LF, "*.txt eol=lf", null, null); + collectRepositoryState(); + assertEquals("eol=lf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_false_GlobalEOL_lf() throws Exception { + setupGitAndDoHardReset(AutoCRLF.FALSE, EOL.LF, "*.txt eol=lf", null, null); + collectRepositoryState(); + assertEquals("eol=lf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_input_GlobalEOL_lf() throws Exception { + setupGitAndDoHardReset(AutoCRLF.INPUT, EOL.LF, "*.txt eol=lf", null, null); + collectRepositoryState(); + assertEquals("eol=lf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_true_GlobalEOL_crlf() throws Exception { + setupGitAndDoHardReset(AutoCRLF.TRUE, EOL.LF, "*.txt eol=crlf", null, null); + collectRepositoryState(); + assertEquals("eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_false_GlobalEOL_crlf() throws Exception { + setupGitAndDoHardReset(AutoCRLF.FALSE, EOL.LF, "*.txt eol=crlf", null, null); + collectRepositoryState(); + assertEquals("eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_input_GlobalEOL_crlf() throws Exception { + setupGitAndDoHardReset(AutoCRLF.INPUT, EOL.LF, "*.txt eol=crlf", null, null); + collectRepositoryState(); + assertEquals("eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_true_GlobalEOL_lf_InfoEOL_crlf() + throws Exception { + setupGitAndDoHardReset(AutoCRLF.TRUE, null, "*.txt eol=lf", "*.txt eol=crlf", null); + // info decides + collectRepositoryState(); + assertEquals("eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_ConfigAutoCRLF_false_GlobalEOL_crlf_InfoEOL_lf() + throws Exception { + setupGitAndDoHardReset(AutoCRLF.FALSE, null, "*.txt eol=crlf", "*.txt eol=lf", null); + // info decides + collectRepositoryState(); + assertEquals("eol=lf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void test_GlobalEOL_lf_RootEOL_crlf() throws Exception { + setupGitAndDoHardReset(null, null, "*.txt eol=lf", null, "*.txt eol=crlf"); + // root over global + collectRepositoryState(); + assertEquals("eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_GlobalEOL_lf_InfoEOL_crlf_RootEOL_lf() throws Exception { + setupGitAndDoHardReset(null, null, "*.txt eol=lf", "*.txt eol=crlf", "*.txt eol=lf"); + // info overrides all + collectRepositoryState(); + assertEquals("eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_GlobalEOL_lf_InfoEOL_crlf_RootEOL_unspec() + throws Exception { + setupGitAndDoHardReset(null, null, "*.txt eol=lf", "*.txt eol=crlf", + "*.txt text !eol"); + // info overrides all + collectRepositoryState(); + assertEquals("eol=crlf text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_CRLF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_CRLF, CONTENT_LF); + } + + @Test + public void test_GlobalEOL_lf_InfoEOL_unspec_RootEOL_crlf() + throws Exception { + setupGitAndDoHardReset(null, null, "*.txt eol=lf", "*.txt !eol", + "*.txt text eol=crlf"); + // info overrides all + collectRepositoryState(); + assertEquals("text", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_LF, CONTENT_LF); + } + + @Test + public void testBinary1() throws Exception { + setupGitAndDoHardReset(AutoCRLF.TRUE, EOL.CRLF, "*.txt text", "*.txt binary", + "*.txt eol=crlf"); + // info overrides all + collectRepositoryState(); + assertEquals("binary -diff -merge -text eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_CRLF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_MIXED, CONTENT_MIXED); + } + + @Test + public void testBinary2() throws Exception { + setupGitAndDoHardReset(AutoCRLF.TRUE, EOL.CRLF, "*.txt text eol=crlf", null, + "*.txt binary"); + // root over global + collectRepositoryState(); + assertEquals("binary -diff -merge -text eol=crlf", entryCRLF.attrs); + checkEntryContent(entryCRLF, CONTENT_CRLF, CONTENT_CRLF); + checkEntryContent(entryLF, CONTENT_LF, CONTENT_LF); + checkEntryContent(entryMixed, CONTENT_MIXED, CONTENT_MIXED); + } + + // create new repo with + // global .gitattributes + // info .git/config/info/.gitattributes + // workdir root .gitattributes + // text file lf.txt CONTENT_LF + // text file crlf.txt CONTENT_CRLF + // + // commit files (checkin) + // delete working dir files + // reset hard (checkout) + private void setupGitAndDoHardReset(AutoCRLF autoCRLF, EOL eol, + String globalAttributesContent, String infoAttributesContent, + String workDirRootAttributesContent) throws Exception { + Git git = new Git(db); + FileBasedConfig config = db.getConfig(); + if (autoCRLF != null) { + config.setEnum(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOCRLF, autoCRLF); + } + if (eol != null) { + config.setEnum(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_EOL, eol); + } + if (globalAttributesContent != null) { + File f = new File(db.getDirectory(), "global/attrs"); + write(f, globalAttributesContent); + config.setString(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_ATTRIBUTESFILE, + f.getAbsolutePath()); + + } + if (infoAttributesContent != null) { + File f = new File(db.getDirectory(), Constants.INFO_ATTRIBUTES); + write(f, infoAttributesContent); + } + config.save(); + + if (workDirRootAttributesContent != null) { + dotGitattributes = createAndAddFile(git, + Constants.DOT_GIT_ATTRIBUTES, workDirRootAttributesContent); + } else { + dotGitattributes = null; + } + + fileCRLF = createAndAddFile(git, "file1.txt", CONTENT_CRLF); + + fileLF = createAndAddFile(git, "file2.txt", CONTENT_LF); + + fileMixed = createAndAddFile(git, "file3.txt", CONTENT_MIXED); + + gitCommit(git, "addFiles"); + + recreateWorktree(git); + } + + private void recreateWorktree(Git git) + throws GitAPIException, CheckoutConflictException, + InterruptedException, IOException, NoFilepatternException { + // re-create file from the repo + for (File f : new File[] { dotGitattributes, fileCRLF, fileLF, fileMixed }) { + if (f == null) + continue; + f.delete(); + Assert.assertFalse(f.exists()); + } + gitResetHard(git); + fsTick(db.getIndexFile()); + gitAdd(git, "."); + } + + protected void gitCommit(Git git, String msg) throws GitAPIException { + git.commit().setMessage(msg).call(); + } + + protected void gitAdd(Git git, String path) throws GitAPIException { + git.add().addFilepattern(path).call(); + } + + protected void gitResetHard(Git git) throws GitAPIException { + git.reset().setMode(ResetType.HARD).call(); + } + + protected void gitCheckout(Git git, String revstr) + throws GitAPIException, RevisionSyntaxException, IOException { + git.checkout().setName(db.resolve(revstr).getName()).call(); + } + + // create a file and add it to the repo + private File createAndAddFile(Git git, String path, String content) + throws Exception { + File f; + int pos = path.lastIndexOf('/'); + if (pos < 0) { + f = writeTrashFile(path, content); + } else { + f = writeTrashFile(path.substring(0, pos), path.substring(pos + 1), + content); + } + gitAdd(git, path); + Assert.assertTrue(f.exists()); + return f; + } + + private void collectRepositoryState() throws Exception { + dc = db.readDirCache(); + walk = beginWalk(); + if (dotGitattributes != null) + collectEntryContentAndAttributes(F, ".gitattributes", null); + collectEntryContentAndAttributes(F, fileCRLF.getName(), entryCRLF); + collectEntryContentAndAttributes(F, fileLF.getName(), entryLF); + collectEntryContentAndAttributes(F, fileMixed.getName(), entryMixed); + endWalk(); + } + + private TreeWalk beginWalk() throws Exception { + TreeWalk newWalk = new TreeWalk(db); + newWalk.addTree(new FileTreeIterator(db)); + newWalk.addTree(new DirCacheIterator(db.readDirCache())); + return newWalk; + } + + private void endWalk() throws IOException { + assertFalse("Not all files tested", walk.next()); + } + + private void collectEntryContentAndAttributes(FileMode type, String pathName, + ActualEntry e) throws IOException { + assertTrue("walk has entry", walk.next()); + + assertEquals(pathName, walk.getPathString()); + assertEquals(type, walk.getFileMode(0)); + + if (e != null) { + e.attrs = ""; + for (Attribute a : walk.getAttributes().getAll()) { + e.attrs += " " + a.toString(); + } + e.attrs = e.attrs.trim(); + e.file = new String( + IO.readFully(new File(db.getWorkTree(), pathName))); + DirCacheEntry dce = dc.getEntry(pathName); + ObjectLoader open = walk.getObjectReader().open(dce.getObjectId()); + e.index = new String(open.getBytes()); + e.indexContentLength = dce.getLength(); + } + + if (D.equals(type)) + walk.enterSubtree(); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java new file mode 100644 index 0000000000..8ca1b3175d --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java @@ -0,0 +1,335 @@ +/* + * Copyright (C) 2015, Ivan Motsch + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.api; + +import static org.junit.Assert.assertArrayEquals; +import static org.eclipse.jgit.lib.CoreConfig.EolStreamType.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.io.EolStreamTypeUtil; +import org.junit.Test; + +/** + * Unit tests for end-of-line conversion streams + */ +public class EolStreamTypeUtilTest { + + @Test + public void testCheckoutDirect() throws Exception { + testCheckout(DIRECT, DIRECT, "", ""); + testCheckout(DIRECT, DIRECT, "\r", "\r"); + testCheckout(DIRECT, DIRECT, "\n", "\n"); + + testCheckout(DIRECT, DIRECT, "\r\n", "\r\n"); + testCheckout(DIRECT, DIRECT, "\n\r", "\n\r"); + + testCheckout(DIRECT, DIRECT, "\n\r\n", "\n\r\n"); + testCheckout(DIRECT, DIRECT, "\r\n\r", "\r\n\r"); + + testCheckout(DIRECT, DIRECT, "a\nb\n", "a\nb\n"); + testCheckout(DIRECT, DIRECT, "a\rb\r", "a\rb\r"); + testCheckout(DIRECT, DIRECT, "a\n\rb\n\r", "a\n\rb\n\r"); + testCheckout(DIRECT, DIRECT, "a\r\nb\r\n", "a\r\nb\r\n"); + } + + @Test + public void testCheckoutLF() throws Exception { + testCheckout(TEXT_LF, AUTO_LF, "", ""); + testCheckout(TEXT_LF, AUTO_LF, "\r", "\r"); + testCheckout(TEXT_LF, AUTO_LF, "\n", "\n"); + + testCheckout(TEXT_LF, AUTO_LF, "\r\n", "\n"); + testCheckout(TEXT_LF, AUTO_LF, "\n\r", "\n\r"); + + testCheckout(TEXT_LF, AUTO_LF, "\n\r\n", "\n\n"); + testCheckout(TEXT_LF, AUTO_LF, "\r\n\r", "\n\r"); + + testCheckout(TEXT_LF, AUTO_LF, "a\nb\n", "a\nb\n"); + testCheckout(TEXT_LF, AUTO_LF, "a\rb\r", "a\rb\r"); + testCheckout(TEXT_LF, AUTO_LF, "a\n\rb\n\r", "a\n\rb\n\r"); + testCheckout(TEXT_LF, AUTO_LF, "a\r\nb\r\n", "a\nb\n"); + } + + @Test + public void testCheckoutCRLF() throws Exception { + testCheckout(TEXT_CRLF, AUTO_CRLF, "", ""); + testCheckout(TEXT_CRLF, AUTO_CRLF, "\r", "\r"); + testCheckout(TEXT_CRLF, AUTO_CRLF, "\n", "\r\n"); + + testCheckout(TEXT_CRLF, AUTO_CRLF, "\r\n", "\r\n"); + testCheckout(TEXT_CRLF, AUTO_CRLF, "\n\r", "\r\n\r"); + + testCheckout(TEXT_CRLF, AUTO_CRLF, "\n\r\n", "\r\n\r\n"); + testCheckout(TEXT_CRLF, AUTO_CRLF, "\r\n\r", "\r\n\r"); + + testCheckout(TEXT_CRLF, AUTO_CRLF, "a\nb\n", "a\r\nb\r\n"); + testCheckout(TEXT_CRLF, AUTO_CRLF, "a\rb\r", "a\rb\r"); + testCheckout(TEXT_CRLF, AUTO_CRLF, "a\n\rb\n\r", "a\r\n\rb\r\n\r"); + testCheckout(TEXT_CRLF, AUTO_CRLF, "a\r\nb\r\n", "a\r\nb\r\n"); + } + + /** + * Test stream type detection based on stream content. + *

+ * Tests three things with the output text: + *

+ * 1) conversion if output was declared as text + *

+ * 2) conversion if output was declared as potentially text (AUTO_...) and + * is in fact text + *

+ * 3) conversion if modified output (now with binary characters) was + * declared as potentially text but now contains binary characters + *

+ * + * @param streamTypeText + * is the enum meaning that the output is definitely text (no + * binary check at all) + * @param streamTypeWithBinaryCheck + * is the enum meaning that the output may be text (binary check + * is done) + * @param output + * is a text output without binary characters + * @param expectedConversion + * is the expected converted output without binary characters + * @throws Exception + */ + private void testCheckout(EolStreamType streamTypeText, + EolStreamType streamTypeWithBinaryCheck, String output, + String expectedConversion) throws Exception { + ByteArrayOutputStream b; + byte[] outputBytes = output.getBytes(StandardCharsets.UTF_8); + byte[] expectedConversionBytes = expectedConversion + .getBytes(StandardCharsets.UTF_8); + + // test using output text and assuming it was declared TEXT + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeText)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); + + // test using ouput text and assuming it was declared AUTO, using binary + // detection + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeWithBinaryCheck)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); + + // now pollute output text with some binary bytes + outputBytes = extendWithBinaryData(outputBytes); + expectedConversionBytes = extendWithBinaryData(expectedConversionBytes); + + // again, test using output text and assuming it was declared TEXT + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeText)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); + + // again, test using ouput text and assuming it was declared AUTO, using + // binary + // detection + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeWithBinaryCheck)) { + out.write(outputBytes); + } + // expect no conversion + assertArrayEquals(outputBytes, b.toByteArray()); + } + + @Test + public void testCheckinDirect() throws Exception { + testCheckin(DIRECT, DIRECT, "", ""); + testCheckin(DIRECT, DIRECT, "\r", "\r"); + testCheckin(DIRECT, DIRECT, "\n", "\n"); + + testCheckin(DIRECT, DIRECT, "\r\n", "\r\n"); + testCheckin(DIRECT, DIRECT, "\n\r", "\n\r"); + + testCheckin(DIRECT, DIRECT, "\n\r\n", "\n\r\n"); + testCheckin(DIRECT, DIRECT, "\r\n\r", "\r\n\r"); + + testCheckin(DIRECT, DIRECT, "a\nb\n", "a\nb\n"); + testCheckin(DIRECT, DIRECT, "a\rb\r", "a\rb\r"); + testCheckin(DIRECT, DIRECT, "a\n\rb\n\r", "a\n\rb\n\r"); + testCheckin(DIRECT, DIRECT, "a\r\nb\r\n", "a\r\nb\r\n"); + } + + @Test + public void testCheckinLF() throws Exception { + testCheckin(TEXT_LF, AUTO_LF, "", ""); + testCheckin(TEXT_LF, AUTO_LF, "\r", "\r"); + testCheckin(TEXT_LF, AUTO_LF, "\n", "\n"); + + testCheckin(TEXT_LF, AUTO_LF, "\r\n", "\n"); + testCheckin(TEXT_LF, AUTO_LF, "\n\r", "\n\r"); + + testCheckin(TEXT_LF, AUTO_LF, "\n\r\n", "\n\n"); + testCheckin(TEXT_LF, AUTO_LF, "\r\n\r", "\n\r"); + + testCheckin(TEXT_LF, AUTO_LF, "a\nb\n", "a\nb\n"); + testCheckin(TEXT_LF, AUTO_LF, "a\rb\r", "a\rb\r"); + testCheckin(TEXT_LF, AUTO_LF, "a\n\rb\n\r", "a\n\rb\n\r"); + testCheckin(TEXT_LF, AUTO_LF, "a\r\nb\r\n", "a\nb\n"); + } + + @Test + public void testCheckinCRLF() throws Exception { + testCheckin(TEXT_CRLF, AUTO_CRLF, "", ""); + testCheckin(TEXT_CRLF, AUTO_CRLF, "\r", "\r"); + testCheckin(TEXT_CRLF, AUTO_CRLF, "\n", "\r\n"); + + testCheckin(TEXT_CRLF, AUTO_CRLF, "\r\n", "\r\n"); + testCheckin(TEXT_CRLF, AUTO_CRLF, "\n\r", "\r\n\r"); + + testCheckin(TEXT_CRLF, AUTO_CRLF, "\n\r\n", "\r\n\r\n"); + testCheckin(TEXT_CRLF, AUTO_CRLF, "\r\n\r", "\r\n\r"); + + testCheckin(TEXT_CRLF, AUTO_CRLF, "a\nb\n", "a\r\nb\r\n"); + testCheckin(TEXT_CRLF, AUTO_CRLF, "a\rb\r", "a\rb\r"); + testCheckin(TEXT_CRLF, AUTO_CRLF, "a\n\rb\n\r", "a\r\n\rb\r\n\r"); + testCheckin(TEXT_CRLF, AUTO_CRLF, "a\r\nb\r\n", "a\r\nb\r\n"); + } + + /** + * Test stream type detection based on stream content. + *

+ * Tests three things with the input text: + *

+ * 1) conversion if input was declared as text + *

+ * 2) conversion if input was declared as potentially text (AUTO_...) and is + * in fact text + *

+ * 3) conversion if modified input (now with binary characters) was declared + * as potentially text but now contains binary characters + *

+ * + * @param streamTypeText + * is the enum meaning that the input is definitely text (no + * binary check at all) + * @param streamTypeWithBinaryCheck + * is the enum meaning that the input may be text (binary check + * is done) + * @param input + * is a text input without binary characters + * @param expectedConversion + * is the expected converted input without binary characters + * @throws Exception + */ + private void testCheckin(EolStreamType streamTypeText, + EolStreamType streamTypeWithBinaryCheck, String input, + String expectedConversion) throws Exception { + byte[] inputBytes = input.getBytes(StandardCharsets.UTF_8); + byte[] expectedConversionBytes = expectedConversion + .getBytes(StandardCharsets.UTF_8); + + // test using input text and assuming it was declared TEXT + try (InputStream in = EolStreamTypeUtil.wrapInputStream( + new ByteArrayInputStream(inputBytes), + streamTypeText)) { + byte[] b = new byte[1024]; + int len = IO.readFully(in, b, 0); + assertArrayEquals(expectedConversionBytes, Arrays.copyOf(b, len)); + } + + // test using input text and assuming it was declared AUTO, using binary + // detection + try (InputStream in = EolStreamTypeUtil.wrapInputStream( + new ByteArrayInputStream(inputBytes), + streamTypeWithBinaryCheck)) { + byte[] b = new byte[1024]; + int len = IO.readFully(in, b, 0); + assertArrayEquals(expectedConversionBytes, Arrays.copyOf(b, len)); + } + + // now pollute input text with some binary bytes + inputBytes = extendWithBinaryData(inputBytes); + expectedConversionBytes = extendWithBinaryData(expectedConversionBytes); + + // again, test using input text and assuming it was declared TEXT + try (InputStream in = EolStreamTypeUtil.wrapInputStream( + new ByteArrayInputStream(inputBytes), streamTypeText)) { + byte[] b = new byte[1024]; + int len = IO.readFully(in, b, 0); + assertArrayEquals(expectedConversionBytes, Arrays.copyOf(b, len)); + } + + // again, test using input text and assuming it was declared AUTO, using + // binary + // detection + try (InputStream in = EolStreamTypeUtil.wrapInputStream( + new ByteArrayInputStream(inputBytes), + streamTypeWithBinaryCheck)) { + byte[] b = new byte[1024]; + int len = IO.readFully(in, b, 0); + // expect no conversion + assertArrayEquals(inputBytes, Arrays.copyOf(b, len)); + } + } + + private byte[] extendWithBinaryData(byte[] data) throws Exception { + int n = 3; + byte[] dataEx = new byte[data.length + n]; + System.arraycopy(data, 0, dataEx, 0, data.length); + for (int i = 0; i < n; i++) { + dataEx[data.length + i] = (byte) i; + } + return dataEx; + } + +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java index c1b882a656..5578c03d4a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java @@ -65,6 +65,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.NoFilepatternException; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheCheckout; +import org.eclipse.jgit.dircache.DirCacheCheckout.CheckoutMetadata; import org.eclipse.jgit.dircache.DirCacheEditor; import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -113,7 +114,7 @@ public class DirCacheCheckoutTest extends RepositoryTestCase { return dco.getRemoved(); } - private Map getUpdated() { + private Map getUpdated() { return dco.getUpdated(); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/EolCanonicalizingInputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java similarity index 94% rename from org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/EolCanonicalizingInputStreamTest.java rename to org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java index ed2a4f2102..40cac93f3b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/EolCanonicalizingInputStreamTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java @@ -1,5 +1,6 @@ /* * Copyright (C) 2010, Marc Strapetz + * Copyright (C) 2015, Ivan Motsch * and other copyright owners as documented in the project's IP log. * * This program and the accompanying materials are made available @@ -52,7 +53,7 @@ import java.io.UnsupportedEncodingException; import org.junit.Test; -public class EolCanonicalizingInputStreamTest { +public class AutoLFInputStreamTest { @Test public void testLF() throws IOException { @@ -97,7 +98,7 @@ public class EolCanonicalizingInputStreamTest { private static void test(byte[] input, byte[] expected, boolean detectBinary) throws IOException { final InputStream bis1 = new ByteArrayInputStream(input); - final InputStream cis1 = new EolCanonicalizingInputStream(bis1, detectBinary); + final InputStream cis1 = new AutoLFInputStream(bis1, detectBinary); int index1 = 0; for (int b = cis1.read(); b != -1; b = cis1.read()) { assertEquals(expected[index1], (byte) b); @@ -109,7 +110,7 @@ public class EolCanonicalizingInputStreamTest { for (int bufferSize = 1; bufferSize < 10; bufferSize++) { final byte[] buffer = new byte[bufferSize]; final InputStream bis2 = new ByteArrayInputStream(input); - final InputStream cis2 = new EolCanonicalizingInputStream(bis2, detectBinary); + final InputStream cis2 = new AutoLFInputStream(bis2, detectBinary); int read = 0; for (int readNow = cis2.read(buffer, 0, buffer.length); readNow != -1 diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index a5000dd6bd..c0dbc779f3 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -16,4 +16,25 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java index a83814eb46..d803efd649 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java @@ -66,7 +66,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.util.IO; -import org.eclipse.jgit.util.io.EolCanonicalizingInputStream; +import org.eclipse.jgit.util.io.AutoLFInputStream; /** * Blame command for building a {@link BlameResult} for a file path. @@ -248,7 +248,7 @@ public class BlameCommand extends GitCommand { rawText = new RawText(inTree); break; case TRUE: - EolCanonicalizingInputStream in = new EolCanonicalizingInputStream( + AutoLFInputStream in = new AutoLFInputStream( new FileInputStream(inTree), true); // Canonicalization should lead to same or shorter length // (CRLF to LF), so the file size on disk is an upper size bound diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java index 4f918fa357..c37c317c51 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java @@ -59,6 +59,7 @@ import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheCheckout; +import org.eclipse.jgit.dircache.DirCacheCheckout.CheckoutMetadata; import org.eclipse.jgit.dircache.DirCacheEditor; import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -68,6 +69,7 @@ import org.eclipse.jgit.errors.UnmergedPathException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; @@ -395,7 +397,8 @@ public class CheckoutCommand extends GitCommand { RefNotFoundException { DirCache dc = repo.lockDirCache(); try (RevWalk revWalk = new RevWalk(repo); - TreeWalk treeWalk = new TreeWalk(revWalk.getObjectReader())) { + TreeWalk treeWalk = new TreeWalk(repo, + revWalk.getObjectReader())) { treeWalk.setRecursive(true); if (!checkoutAllPaths) treeWalk.setFilter(PathFilterGroup.createFromStrings(paths)); @@ -426,20 +429,23 @@ public class CheckoutCommand extends GitCommand { if (path.equals(previousPath)) continue; + final EolStreamType eolStreamType = treeWalk.getEolStreamType(); editor.add(new PathEdit(path) { public void apply(DirCacheEntry ent) { int stage = ent.getStage(); if (stage > DirCacheEntry.STAGE_0) { if (checkoutStage != null) { if (stage == checkoutStage.number) - checkoutPath(ent, r); + checkoutPath(ent, r, new CheckoutMetadata( + eolStreamType, null)); } else { UnmergedPathException e = new UnmergedPathException( ent); throw new JGitInternalException(e.getMessage(), e); } } else { - checkoutPath(ent, r); + checkoutPath(ent, r, + new CheckoutMetadata(eolStreamType, null)); } } }); @@ -457,20 +463,24 @@ public class CheckoutCommand extends GitCommand { while (treeWalk.next()) { final ObjectId blobId = treeWalk.getObjectId(0); final FileMode mode = treeWalk.getFileMode(0); + final EolStreamType eolStreamType = treeWalk.getEolStreamType(); editor.add(new PathEdit(treeWalk.getPathString()) { public void apply(DirCacheEntry ent) { ent.setObjectId(blobId); ent.setFileMode(mode); - checkoutPath(ent, r); + checkoutPath(ent, r, + new CheckoutMetadata(eolStreamType, null)); } }); } editor.commit(); } - private void checkoutPath(DirCacheEntry entry, ObjectReader reader) { + private void checkoutPath(DirCacheEntry entry, ObjectReader reader, + CheckoutMetadata checkoutMetadata) { try { - DirCacheCheckout.checkoutEntry(repo, entry, reader, true); + DirCacheCheckout.checkoutEntry(repo, entry, reader, true, + checkoutMetadata); } catch (IOException e) { throw new JGitInternalException(MessageFormat.format( JGitText.get().checkoutConflictWithFile, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java index 8ef550871f..1699b9f3d7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java @@ -54,11 +54,13 @@ import org.eclipse.jgit.api.errors.WrongRepositoryStateException; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheCheckout; +import org.eclipse.jgit.dircache.DirCacheCheckout.CheckoutMetadata; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.CheckoutConflictException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; @@ -336,6 +338,7 @@ public class StashApplyCommand extends GitCommand { // Not in commit, don't create untracked continue; + final EolStreamType eolStreamType = walk.getEolStreamType(); final DirCacheEntry entry = new DirCacheEntry(walk.getRawPath()); entry.setFileMode(cIter.getEntryFileMode()); entry.setObjectIdFromRaw(cIter.idBuffer(), cIter.idOffset()); @@ -350,14 +353,17 @@ public class StashApplyCommand extends GitCommand { } } - checkoutPath(entry, reader); + checkoutPath(entry, reader, + new CheckoutMetadata(eolStreamType, null)); } } } - private void checkoutPath(DirCacheEntry entry, ObjectReader reader) { + private void checkoutPath(DirCacheEntry entry, ObjectReader reader, + CheckoutMetadata checkoutMetadata) { try { - DirCacheCheckout.checkoutEntry(repo, entry, reader, true); + DirCacheCheckout.checkoutEntry(repo, entry, reader, true, + checkoutMetadata); } catch (IOException e) { throw new JGitInternalException(MessageFormat.format( JGitText.get().checkoutConflictWithFile, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java index 2cdaf24019..ef32ac929a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java @@ -245,12 +245,14 @@ public class StashCreateCommand extends GitCommand { DirCache cache = repo.lockDirCache(); ObjectId commitId; try (ObjectInserter inserter = repo.newObjectInserter(); - TreeWalk treeWalk = new TreeWalk(reader)) { + TreeWalk treeWalk = new TreeWalk(repo, reader)) { treeWalk.setRecursive(true); treeWalk.addTree(headCommit.getTree()); treeWalk.addTree(new DirCacheIterator(cache)); treeWalk.addTree(new FileTreeIterator(repo)); + treeWalk.getTree(2, FileTreeIterator.class) + .setDirCacheIterator(treeWalk, 1); treeWalk.setFilter(AndTreeFilter.create(new SkipWorkTreeFilter( 1), new IndexDiffFilter(1, 2))); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index a1e1d15ac6..3fcaa38395 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -62,6 +62,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.CoreConfig.SymLinks; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -84,16 +85,43 @@ import org.eclipse.jgit.util.FS.ExecutionResult; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.SystemReader; -import org.eclipse.jgit.util.io.AutoCRLFOutputStream; +import org.eclipse.jgit.util.io.EolStreamTypeUtil; /** * This class handles checking out one or two trees merging with the index. */ public class DirCacheCheckout { private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024; + + /** + * Metadata used in checkout process + * + * @since 4.3 + */ + public static class CheckoutMetadata { + /** git attributes */ + public final EolStreamType eolStreamType; + + /** filter command to apply */ + public final String smudgeFilterCommand; + + /** + * @param eolStreamType + * @param smudgeFilterCommand + */ + public CheckoutMetadata(EolStreamType eolStreamType, + String smudgeFilterCommand) { + this.eolStreamType = eolStreamType; + this.smudgeFilterCommand = smudgeFilterCommand; + } + + static CheckoutMetadata EMPTY = new CheckoutMetadata( + EolStreamType.DIRECT, null); + } + private Repository repo; - private HashMap updated = new HashMap(); + private HashMap updated = new HashMap(); private ArrayList conflicts = new ArrayList(); @@ -120,7 +148,7 @@ public class DirCacheCheckout { /** * @return a list of updated paths and smudgeFilterCommands */ - public Map getUpdated() { + public Map getUpdated() { return updated; } @@ -450,11 +478,12 @@ public class DirCacheCheckout { if (file != null) removeEmptyParents(file); - for (String path : updated.keySet()) { + for (Map.Entry e : updated.entrySet()) { + String path = e.getKey(); + CheckoutMetadata meta = e.getValue(); DirCacheEntry entry = dc.getEntry(path); if (!FileMode.GITLINK.equals(entry.getRawMode())) - checkoutEntry(repo, entry, objectReader, false, - updated.get(path)); + checkoutEntry(repo, entry, objectReader, false, meta); } // commit the index builder - a new index is persisted @@ -1006,8 +1035,8 @@ public class DirCacheCheckout { private void update(String path, ObjectId mId, FileMode mode) throws IOException { if (!FileMode.TREE.equals(mode)) { - updated.put(path, - walk.getFilterCommand(Constants.ATTR_FILTER_TYPE_SMUDGE)); + updated.put(path, new CheckoutMetadata(walk.getEolStreamType(), + walk.getFilterCommand(Constants.ATTR_FILTER_TYPE_SMUDGE))); DirCacheEntry entry = new DirCacheEntry(path, DirCacheEntry.STAGE_0); entry.setObjectId(mId); @@ -1190,52 +1219,22 @@ public class DirCacheCheckout { * @param deleteRecursive * true to recursively delete final path if it exists on the file * system - * - * @throws IOException - * @since 4.2 - */ - public static void checkoutEntry(Repository repo, DirCacheEntry entry, - ObjectReader or, boolean deleteRecursive) throws IOException { - checkoutEntry(repo, entry, or, deleteRecursive, null); - } - - /** - * Updates the file in the working tree with content and mode from an entry - * in the index. The new content is first written to a new temporary file in - * the same directory as the real file. Then that new file is renamed to the - * final filename. - * - *

- * Note: if the entry path on local file system exists as a file, it - * will be deleted and if it exists as a directory, it will be deleted - * recursively, independently if has any content. - *

- * - *

- * TODO: this method works directly on File IO, we may need another - * abstraction (like WorkingTreeIterator). This way we could tell e.g. - * Eclipse that Files in the workspace got changed - *

- * - * @param repo - * repository managing the destination work tree. - * @param entry - * the entry containing new mode and content - * @param or - * object reader to use for checkout - * @param deleteRecursive - * true to recursively delete final path if it exists on the file - * system - * @param smudgeFilterCommand - * the filter command to be run for smudging the entry to be - * checked out + * @param checkoutMetadata + * containing + *
    + *
  • smudgeFilterCommand to be run for smudging the entry to be + * checked out
    • + *
  • eolStreamType used for stream conversion
    • + *
* * @throws IOException * @since 4.2 */ public static void checkoutEntry(Repository repo, DirCacheEntry entry, ObjectReader or, boolean deleteRecursive, - String smudgeFilterCommand) throws IOException { + CheckoutMetadata checkoutMetadata) throws IOException { + if (checkoutMetadata == null) + checkoutMetadata = CheckoutMetadata.EMPTY; ObjectLoader ol = or.open(entry.getObjectId()); File f = new File(repo.getWorkTree(), entry.getPathString()); File parentDir = f.getParentFile(); @@ -1257,12 +1256,19 @@ public class DirCacheCheckout { File tmpFile = File.createTempFile( "._" + f.getName(), null, parentDir); //$NON-NLS-1$ - OutputStream channel = new FileOutputStream(tmpFile); - if (opt.getAutoCRLF() == AutoCRLF.TRUE) - channel = new AutoCRLFOutputStream(channel); - if (smudgeFilterCommand != null) { - ProcessBuilder filterProcessBuilder = fs - .runInShell(smudgeFilterCommand, new String[0]); + EolStreamType nonNullEolStreamType; + if (checkoutMetadata.eolStreamType != null) { + nonNullEolStreamType = checkoutMetadata.eolStreamType; + } else if (opt.getAutoCRLF() == AutoCRLF.TRUE) { + nonNullEolStreamType = EolStreamType.AUTO_CRLF; + } else { + nonNullEolStreamType = EolStreamType.DIRECT; + } + OutputStream channel = EolStreamTypeUtil.wrapOutputStream( + new FileOutputStream(tmpFile), nonNullEolStreamType); + if (checkoutMetadata.smudgeFilterCommand != null) { + ProcessBuilder filterProcessBuilder = fs.runInShell( + checkoutMetadata.smudgeFilterCommand, new String[0]); filterProcessBuilder.directory(repo.getWorkTree()); filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, repo.getDirectory().getAbsolutePath()); @@ -1278,14 +1284,16 @@ public class DirCacheCheckout { } } catch (IOException | InterruptedException e) { throw new IOException(new FilterFailedException(e, - smudgeFilterCommand, entry.getPathString())); + checkoutMetadata.smudgeFilterCommand, + entry.getPathString())); } finally { channel.close(); } if (rc != 0) { throw new IOException(new FilterFailedException(rc, - smudgeFilterCommand, entry.getPathString(), + checkoutMetadata.smudgeFilterCommand, + entry.getPathString(), result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE), RawParseUtils.decode(result.getStderr() .toByteArray(MAX_EXCEPTION_TEXT_SIZE)))); @@ -1301,10 +1309,11 @@ public class DirCacheCheckout { // was filtered (either by autocrlf handling or smudge filters) ask the // filesystem again for the length. Otherwise the objectloader knows the // size - if (opt.getAutoCRLF() == AutoCRLF.TRUE || smudgeFilterCommand != null) { - entry.setLength(tmpFile.length()); - } else { + if (checkoutMetadata.eolStreamType == EolStreamType.DIRECT + && checkoutMetadata.smudgeFilterCommand == null) { entry.setLength(ol.getSize()); + } else { + entry.setLength(tmpFile.length()); } if (opt.isFileMode() && fs.supportsExecute()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index a89bcee730..054d193017 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -107,6 +107,13 @@ public class ConfigConstants { /** The "autocrlf" key */ public static final String CONFIG_KEY_AUTOCRLF = "autocrlf"; + /** + * The "eol" key + * + * @since 4.3 + */ + public static final String CONFIG_KEY_EOL = "eol"; + /** The "bare" key */ public static final String CONFIG_KEY_BARE = "bare"; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java index 5a7634a6f1..83efd43aa0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java @@ -75,6 +75,46 @@ public class CoreConfig { INPUT; } + /** + * Permissible values for {@code core.eol}. + *

+ * https://git-scm.com/docs/gitattributes + * + * @since 4.3 + */ + public static enum EOL { + /** checkin with LF, checkout with CRLF. */ + CRLF, + + /** checkin with LF, checkout without conversion. */ + LF, + + /** use the platform's native line ending. */ + NATIVE; + } + + /** + * EOL stream conversion protocol + * + * @since 4.3 + */ + public static enum EolStreamType { + /** convert to CRLF without binary detection */ + TEXT_CRLF, + + /** convert to LF without binary detection */ + TEXT_LF, + + /** convert to CRLF with binary detection */ + AUTO_CRLF, + + /** convert to LF with binary detection */ + AUTO_LF, + + /** do not convert */ + DIRECT; + } + /** * Permissible values for {@code core.checkstat} * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/NameConflictTreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/NameConflictTreeWalk.java index d2195a874c..b9293ebfb6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/NameConflictTreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/NameConflictTreeWalk.java @@ -101,6 +101,19 @@ public class NameConflictTreeWalk extends TreeWalk { super(repo); } + /** + * Create a new tree walker for a given repository. + * + * @param repo + * the repository the walker will obtain data from. + * @param or + * the reader the walker will obtain tree data from. + * @since 4.3 + */ + public NameConflictTreeWalk(Repository repo, final ObjectReader or) { + super(repo, or); + } + /** * Create a new tree walker for a given repository. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java index 4775e96175..aecbac11ea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java @@ -49,6 +49,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.attributes.Attributes; @@ -64,6 +65,7 @@ import org.eclipse.jgit.errors.StopWalkException; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.ObjectId; @@ -74,6 +76,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.QuotedString; import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.io.EolStreamTypeUtil; /** * Walks one or more {@link AbstractTreeIterator}s in parallel. @@ -161,7 +164,44 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { public static TreeWalk forPath(final ObjectReader reader, final String path, final AnyObjectId... trees) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { - TreeWalk tw = new TreeWalk(reader); + return forPath(null, reader, path, trees); + } + + /** + * Open a tree walk and filter to exactly one path. + *

+ * The returned tree walk is already positioned on the requested path, so + * the caller should not need to invoke {@link #next()} unless they are + * looking for a possible directory/file name conflict. + * + * @param repo + * repository to read config data and + * {@link AttributesNodeProvider} from. + * @param reader + * the reader the walker will obtain tree data from. + * @param path + * single path to advance the tree walk instance into. + * @param trees + * one or more trees to walk through, all with the same root. + * @return a new tree walk configured for exactly this one path; null if no + * path was found in any of the trees. + * @throws IOException + * reading a pack file or loose object failed. + * @throws CorruptObjectException + * an tree object could not be read as its data stream did not + * appear to be a tree, or could not be inflated. + * @throws IncorrectObjectTypeException + * an object we expected to be a tree was not a tree. + * @throws MissingObjectException + * a tree object was not found. + * @since 4.3 + */ + public static TreeWalk forPath(final @Nullable Repository repo, + final ObjectReader reader, final String path, + final AnyObjectId... trees) + throws MissingObjectException, IncorrectObjectTypeException, + CorruptObjectException, IOException { + TreeWalk tw = new TreeWalk(repo, reader); PathFilter f = PathFilter.create(path); tw.setFilter(f); tw.reset(trees); @@ -206,7 +246,7 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { final AnyObjectId... trees) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { try (ObjectReader reader = db.newObjectReader()) { - return forPath(reader, path, trees); + return forPath(db, reader, path, trees); } } @@ -282,9 +322,23 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { * when the walker is closed. */ public TreeWalk(final Repository repo) { - this(repo.newObjectReader(), true); - config = repo.getConfig(); - attributesNodeProvider = repo.createAttributesNodeProvider(); + this(repo, repo.newObjectReader(), true); + } + + /** + * Create a new tree walker for a given repository. + * + * @param repo + * the repository the walker will obtain data from. An + * ObjectReader will be created by the walker, and will be closed + * when the walker is closed. + * @param or + * the reader the walker will obtain tree data from. The reader + * is not closed when the walker is closed. + * @since 4.3 + */ + public TreeWalk(final @Nullable Repository repo, final ObjectReader or) { + this(repo, or, false); } /** @@ -295,10 +349,18 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { * is not closed when the walker is closed. */ public TreeWalk(final ObjectReader or) { - this(or, false); + this(null, or, false); } - private TreeWalk(final ObjectReader or, final boolean closeReader) { + private TreeWalk(final @Nullable Repository repo, final ObjectReader or, + final boolean closeReader) { + if (repo != null) { + config = repo.getConfig(); + attributesNodeProvider = repo.createAttributesNodeProvider(); + } else { + config = null; + attributesNodeProvider = null; + } reader = or; filter = TreeFilter.ALL; trees = NO_TREES; @@ -517,6 +579,19 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { } } + /** + * @return the EOL stream type of the current entry using the config and + * {@link #getAttributes()} Note that this method may return null if + * the {@link TreeWalk} is not based on a working tree + * @since 4.3 + */ + public @Nullable EolStreamType getEolStreamType() { + if (attributesNodeProvider == null || config == null) + return null; + return EolStreamTypeUtil.detectStreamType(operationType, + config.get(WorkingTreeOptions.KEY), getAttributes()); + } + /** Reset this walker so new tree iterators can be added to it. */ public void reset() { attrs = null; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 0d617ee7f9..ca8f9aa373 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -77,8 +77,8 @@ import org.eclipse.jgit.ignore.IgnoreNode; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; -import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.lib.CoreConfig.CheckStat; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.CoreConfig.SymLinks; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; @@ -88,10 +88,12 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.ExecutionResult; +import org.eclipse.jgit.util.Holder; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.Paths; import org.eclipse.jgit.util.RawParseUtils; -import org.eclipse.jgit.util.io.EolCanonicalizingInputStream; +import org.eclipse.jgit.util.io.AutoLFInputStream; +import org.eclipse.jgit.util.io.EolStreamTypeUtil; /** * Walks a working directory tree as part of a {@link TreeWalk}. @@ -140,7 +142,17 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { /** If there is a .gitignore file present, the parsed rules from it. */ private IgnoreNode ignoreNode; - private String cleanFilterCommand; + /** + * cached clean filter command. Use a Ref in order to distinguish between + * the ref not cached yet and the value null + */ + private Holder cleanFilterCommandHolder; + + /** + * cached eol stream type. Use a Ref in order to distinguish between the ref + * not cached yet and the value null + */ + private Holder eolStreamTypeHolder; /** Repository that is the root level being iterated over */ protected Repository repository; @@ -357,8 +369,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { private InputStream possiblyFilteredInputStream(final Entry e, final InputStream is, final long len) throws IOException { - boolean mightNeedCleaning = mightNeedCleaning(); - if (!mightNeedCleaning) { + if (getCleanFilterCommand() == null + && getEolStreamType() == EolStreamType.DIRECT) { canonLen = len; return is; } @@ -376,11 +388,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return new ByteArrayInputStream(raw, 0, n); } - // TODO: fix autocrlf causing mightneedcleaning - if (!mightNeedCleaning && isBinary(e)) { - canonLen = len; - return is; - } + if (getCleanFilterCommand() == null && isBinary(e)) { + canonLen = len; + return is; + } final InputStream lenIs = filterClean(e.openInputStream()); try { @@ -401,20 +412,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } } - private boolean mightNeedCleaning() throws IOException { - switch (getOptions().getAutoCRLF()) { - case FALSE: - default: - if (getCleanFilterCommand() != null) - return true; - return false; - - case TRUE: - case INPUT: - return true; - } - } - private static boolean isBinary(byte[] content, int sz) { return RawText.isBinary(content, sz); } @@ -467,12 +464,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return in; } - private InputStream handleAutoCRLF(InputStream in) { - AutoCRLF autoCRLF = getOptions().getAutoCRLF(); - if (autoCRLF == AutoCRLF.TRUE || autoCRLF == AutoCRLF.INPUT) { - in = new EolCanonicalizingInputStream(in, true); - } - return in; + private InputStream handleAutoCRLF(InputStream in) throws IOException { + return EolStreamTypeUtil.wrapInputStream(in, getEolStreamType()); } /** @@ -531,7 +524,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { System.arraycopy(e.encodedName, 0, path, pathOffset, nameLen); pathLen = pathOffset + nameLen; canonLen = -1; - cleanFilterCommand = null; + cleanFilterCommandHolder = null; + eolStreamTypeHolder = null; } /** @@ -594,10 +588,11 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { */ public InputStream openEntryStream() throws IOException { InputStream rawis = current().openInputStream(); - if (mightNeedCleaning()) - return filterClean(rawis); - else + if (getCleanFilterCommand() == null + && getEolStreamType() == EolStreamType.DIRECT) return rawis; + else + return filterClean(rawis); } /** @@ -971,10 +966,11 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { // Content differs: that's a real change, perhaps if (reader == null) // deprecated use, do no further checks return true; - switch (getOptions().getAutoCRLF()) { - case INPUT: - case TRUE: - InputStream dcIn = null; + + switch (getEolStreamType()) { + case DIRECT: + return true; + default: try { ObjectLoader loader = reader.open(entry.getObjectId()); if (loader == null) @@ -982,37 +978,26 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { // We need to compute the length, but only if it is not // a binary stream. - dcIn = new EolCanonicalizingInputStream( - loader.openStream(), true, true /* abort if binary */); long dcInLen; - try { + try (InputStream dcIn = new AutoLFInputStream( + loader.openStream(), true, + true /* abort if binary */)) { dcInLen = computeLength(dcIn); - } catch (EolCanonicalizingInputStream.IsBinaryException e) { + } catch (AutoLFInputStream.IsBinaryException e) { return true; - } finally { - dcIn.close(); } - dcIn = new EolCanonicalizingInputStream( - loader.openStream(), true); - byte[] autoCrLfHash = computeHash(dcIn, dcInLen); - boolean changed = getEntryObjectId().compareTo( - autoCrLfHash, 0) != 0; - return changed; + try (InputStream dcIn = new AutoLFInputStream( + loader.openStream(), true)) { + byte[] autoCrLfHash = computeHash(dcIn, dcInLen); + boolean changed = getEntryObjectId() + .compareTo(autoCrLfHash, 0) != 0; + return changed; + } } catch (IOException e) { return true; - } finally { - if (dcIn != null) - try { - dcIn.close(); - } catch (IOException e) { - // empty - } } - case FALSE: - break; } - return true; } } @@ -1308,10 +1293,43 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { * @since 4.2 */ public String getCleanFilterCommand() throws IOException { - if (cleanFilterCommand == null && state.walk != null) { - cleanFilterCommand = state.walk - .getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN); + if (cleanFilterCommandHolder == null) { + String cmd = null; + if (state.walk != null) { + cmd = state.walk + .getFilterCommand(Constants.ATTR_FILTER_TYPE_CLEAN); + } + cleanFilterCommandHolder = new Holder(cmd); + } + return cleanFilterCommandHolder.get(); + } + + /** + * @return the eol stream type for the current entry or null if + * it cannot be determined. When state or state.walk is null or the + * {@link TreeWalk} is not based on a {@link Repository} then null + * is returned. + * @throws IOException + * @since 4.3 + */ + public EolStreamType getEolStreamType() throws IOException { + if (eolStreamTypeHolder == null) { + EolStreamType type=null; + if (state.walk != null) { + type=state.walk.getEolStreamType(); + } else { + switch (getOptions().getAutoCRLF()) { + case FALSE: + type = EolStreamType.DIRECT; + break; + case TRUE: + case INPUT: + type = EolStreamType.AUTO_LF; + break; + } + } + eolStreamTypeHolder = new Holder(type); } - return cleanFilterCommand; + return eolStreamTypeHolder.get(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java index a6dccce031..a8990b1e95 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java @@ -48,6 +48,7 @@ import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Config.SectionParser; import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.lib.CoreConfig.CheckStat; +import org.eclipse.jgit.lib.CoreConfig.EOL; import org.eclipse.jgit.lib.CoreConfig.HideDotFiles; import org.eclipse.jgit.lib.CoreConfig.SymLinks; @@ -64,6 +65,8 @@ public class WorkingTreeOptions { private final AutoCRLF autoCRLF; + private final EOL eol; + private final CheckStat checkStat; private final SymLinks symlinks; @@ -75,6 +78,8 @@ public class WorkingTreeOptions { ConfigConstants.CONFIG_KEY_FILEMODE, true); autoCRLF = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_AUTOCRLF, AutoCRLF.FALSE); + eol = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_EOL, EOL.NATIVE); checkStat = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_CHECKSTAT, CheckStat.DEFAULT); symlinks = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, @@ -94,6 +99,15 @@ public class WorkingTreeOptions { return autoCRLF; } + /** + * @return how text line endings should be normalized. + * + * @since 4.3 + */ + public EOL getEOL() { + return eol; + } + /** * @return how stat data is compared * @since 3.0 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Holder.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Holder.java new file mode 100644 index 0000000000..3563e1bf1d --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Holder.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2015, Ivan Motsch + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.util; + +/** + * Holder of an object. + * + * @param + * the type of value held by this {@link Holder} + * + * @since 4.3 + */ +public class Holder { + private T value; + + /** + * @param value + * is the initial value that is {@link #set(Object)} + */ + public Holder(T value) { + set(value); + } + + /** + * @return the value held by this {@link Holder} + */ + public T get() { + return value; + } + + /** + * @param value + * to be set as new value held by this {@link Holder} + */ + public void set(T value) { + this.value = value; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFInputStream.java index 98c5477de1..30f9ce95fc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFInputStream.java @@ -50,7 +50,7 @@ import java.io.InputStream; import org.eclipse.jgit.diff.RawText; /** - * An OutputStream that expands LF to CRLF. + * An InputStream that expands LF to CRLF. * * Existing CRLF are not expanded to CRCRLF, but retained as is. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFOutputStream.java index f05da1c73c..3a72f7e1dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFOutputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoCRLFOutputStream.java @@ -50,8 +50,11 @@ import org.eclipse.jgit.diff.RawText; /** * An OutputStream that expands LF to CRLF. - *

+ * * Existing CRLF are not expanded to CRCRLF, but retained as is. + * + * A binary check on the first 8000 bytes is performed and in case of binary + * files, canonicalization is turned off (for the complete file). */ public class AutoCRLFOutputStream extends OutputStream { @@ -67,13 +70,26 @@ public class AutoCRLFOutputStream extends OutputStream { private int binbufcnt = 0; + private boolean detectBinary; + private boolean isBinary; /** * @param out */ public AutoCRLFOutputStream(OutputStream out) { + this(out, true); + } + + /** + * @param out + * @param detectBinary + * whether binaries should be detected + * @since 4.3 + */ + public AutoCRLFOutputStream(OutputStream out, boolean detectBinary) { this.out = out; + this.detectBinary = detectBinary; } @Override @@ -141,7 +157,10 @@ public class AutoCRLFOutputStream extends OutputStream { } private void decideMode() throws IOException { - isBinary = RawText.isBinary(binbuf, binbufcnt); + if (detectBinary) { + isBinary = RawText.isBinary(binbuf, binbufcnt); + detectBinary = false; + } int cachedLen = binbufcnt; binbufcnt = binbuf.length + 1; // full! write(binbuf, 0, cachedLen); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java new file mode 100644 index 0000000000..6e33f99127 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2010, 2013 Marc Strapetz + * Copyright (C) 2015, Ivan Motsch + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.util.io; + +import java.io.IOException; +import java.io.InputStream; + +import org.eclipse.jgit.diff.RawText; + +/** + * An InputStream that normalizes CRLF to LF. + * + * Existing single CR are not changed to LF, but retained as is. + * + * Optionally, a binary check on the first 8000 bytes is performed and in case + * of binary files, canonicalization is turned off (for the complete file). + *

+ * This is the former EolCanonicalizingInputStream with a new name in order to + * have same naming for all LF / CRLF streams + * + * @since 4.3 + */ +public class AutoLFInputStream extends InputStream { + private final byte[] single = new byte[1]; + + private final byte[] buf = new byte[8096]; + + private final InputStream in; + + private int cnt; + + private int ptr; + + private boolean isBinary; + + private boolean detectBinary; + + private boolean abortIfBinary; + + /** + * A special exception thrown when {@link AutoLFInputStream} is told to + * throw an exception when attempting to read a binary file. The exception + * may be thrown at any stage during reading. + * + * @since 3.3 + */ + public static class IsBinaryException extends IOException { + private static final long serialVersionUID = 1L; + + IsBinaryException() { + super(); + } + } + + /** + * Creates a new InputStream, wrapping the specified stream + * + * @param in + * raw input stream + * @param detectBinary + * whether binaries should be detected + * @since 2.0 + */ + public AutoLFInputStream(InputStream in, boolean detectBinary) { + this(in, detectBinary, false); + } + + /** + * Creates a new InputStream, wrapping the specified stream + * + * @param in + * raw input stream + * @param detectBinary + * whether binaries should be detected + * @param abortIfBinary + * throw an IOException if the file is binary + * @since 3.3 + */ + public AutoLFInputStream(InputStream in, boolean detectBinary, + boolean abortIfBinary) { + this.in = in; + this.detectBinary = detectBinary; + this.abortIfBinary = abortIfBinary; + } + + @Override + public int read() throws IOException { + final int read = read(single, 0, 1); + return read == 1 ? single[0] & 0xff : -1; + } + + @Override + public int read(byte[] bs, final int off, final int len) + throws IOException { + if (len == 0) + return 0; + + if (cnt == -1) + return -1; + + int i = off; + final int end = off + len; + + while (i < end) { + if (ptr == cnt && !fillBuffer()) { + break; + } + + byte b = buf[ptr++]; + if (isBinary || b != '\r') { + // Logic for binary files ends here + bs[i++] = b; + continue; + } + + if (ptr == cnt && !fillBuffer()) { + bs[i++] = '\r'; + break; + } + + if (buf[ptr] == '\n') { + bs[i++] = '\n'; + ptr++; + } else + bs[i++] = '\r'; + } + + return i == off ? -1 : i - off; + } + + /** + * @return true if the stream has detected as a binary so far + * @since 3.3 + */ + public boolean isBinary() { + return isBinary; + } + + @Override + public void close() throws IOException { + in.close(); + } + + private boolean fillBuffer() throws IOException { + cnt = in.read(buf, 0, buf.length); + if (cnt < 1) + return false; + if (detectBinary) { + isBinary = RawText.isBinary(buf, cnt); + detectBinary = false; + if (isBinary && abortIfBinary) + throw new IsBinaryException(); + } + ptr = 0; + return true; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java new file mode 100644 index 0000000000..c932b00f3c --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2015, Ivan Motsch + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.util.io; + +import java.io.IOException; +import java.io.OutputStream; + +import org.eclipse.jgit.diff.RawText; + +/** + * An OutputStream that reduces CRLF to LF. + * + * Existing single CR are not changed to LF, but retained as is. + * + * A binary check on the first 8000 bytes is performed and in case of binary + * files, canonicalization is turned off (for the complete file). + * + * @since 4.3 + */ +public class AutoLFOutputStream extends OutputStream { + + static final int BUFFER_SIZE = 8000; + + private final OutputStream out; + + private int buf = -1; + + private byte[] binbuf = new byte[BUFFER_SIZE]; + + private byte[] onebytebuf = new byte[1]; + + private int binbufcnt = 0; + + private boolean detectBinary; + + private boolean isBinary; + + /** + * @param out + */ + public AutoLFOutputStream(OutputStream out) { + this(out, true); + } + + /** + * @param out + * @param detectBinary + * whether binaries should be detected + */ + public AutoLFOutputStream(OutputStream out, boolean detectBinary) { + this.out = out; + this.detectBinary = detectBinary; + } + + @Override + public void write(int b) throws IOException { + onebytebuf[0] = (byte) b; + write(onebytebuf, 0, 1); + } + + @Override + public void write(byte[] b) throws IOException { + int overflow = buffer(b, 0, b.length); + if (overflow > 0) { + write(b, b.length - overflow, overflow); + } + } + + @Override + public void write(byte[] b, final int startOff, final int startLen) + throws IOException { + final int overflow = buffer(b, startOff, startLen); + if (overflow < 0) { + return; + } + final int off = startOff + startLen - overflow; + final int len = overflow; + if (len == 0) { + return; + } + int lastw = off; + if (isBinary) { + out.write(b, off, len); + return; + } + for (int i = off; i < off + len; ++i) { + final byte c = b[i]; + if (c == '\r') { + // skip write r but backlog r + if (lastw < i) { + out.write(b, lastw, i - lastw); + } + lastw = i + 1; + buf = '\r'; + } else if (c == '\n') { + if (buf == '\r') { + out.write('\n'); + lastw = i + 1; + buf = -1; + } else { + if (lastw < i + 1) { + out.write(b, lastw, i + 1 - lastw); + } + lastw = i + 1; + } + } else { + if (buf == '\r') { + out.write('\r'); + lastw = i; + } + buf = -1; + } + } + if (lastw < off + len) { + out.write(b, lastw, off + len - lastw); + } + } + + private int buffer(byte[] b, int off, int len) throws IOException { + if (binbufcnt > binbuf.length) { + return len; + } + int copy = Math.min(binbuf.length - binbufcnt, len); + System.arraycopy(b, off, binbuf, binbufcnt, copy); + binbufcnt += copy; + int remaining = len - copy; + if (remaining > 0) { + decideMode(); + } + return remaining; + } + + private void decideMode() throws IOException { + if (detectBinary) { + isBinary = RawText.isBinary(binbuf, binbufcnt); + detectBinary = false; + } + int cachedLen = binbufcnt; + binbufcnt = binbuf.length + 1; // full! + write(binbuf, 0, cachedLen); + } + + @Override + public void flush() throws IOException { + if (binbufcnt <= binbuf.length) { + decideMode(); + } + out.flush(); + } + + @Override + public void close() throws IOException { + flush(); + if (buf == '\r') { + out.write(buf); + buf = -1; + } + out.close(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolCanonicalizingInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolCanonicalizingInputStream.java index 98485e9090..ee729e893e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolCanonicalizingInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolCanonicalizingInputStream.java @@ -46,46 +46,16 @@ package org.eclipse.jgit.util.io; import java.io.IOException; import java.io.InputStream; -import org.eclipse.jgit.diff.RawText; - /** * An input stream which canonicalizes EOLs bytes on the fly to '\n'. * - * Optionally, a binary check on the first 8000 bytes is performed - * and in case of binary files, canonicalization is turned off - * (for the complete file). + * Optionally, a binary check on the first 8000 bytes is performed and in case + * of binary files, canonicalization is turned off (for the complete file). + * + * @deprecated use {@link AutoLFInputStream} instead */ -public class EolCanonicalizingInputStream extends InputStream { - private final byte[] single = new byte[1]; - - private final byte[] buf = new byte[8096]; - - private final InputStream in; - - private int cnt; - - private int ptr; - - private boolean isBinary; - - private boolean detectBinary; - - private boolean abortIfBinary; - - /** - * A special exception thrown when {@link EolCanonicalizingInputStream} is - * told to throw an exception when attempting to read a binary file. The - * exception may be thrown at any stage during reading. - * - * @since 3.3 - */ - public static class IsBinaryException extends IOException { - private static final long serialVersionUID = 1L; - - IsBinaryException() { - super(); - } - } +@Deprecated +public class EolCanonicalizingInputStream extends AutoLFInputStream { /** * Creates a new InputStream, wrapping the specified stream @@ -94,10 +64,9 @@ public class EolCanonicalizingInputStream extends InputStream { * raw input stream * @param detectBinary * whether binaries should be detected - * @since 2.0 */ public EolCanonicalizingInputStream(InputStream in, boolean detectBinary) { - this(in, detectBinary, false); + super(in, detectBinary); } /** @@ -109,83 +78,25 @@ public class EolCanonicalizingInputStream extends InputStream { * whether binaries should be detected * @param abortIfBinary * throw an IOException if the file is binary - * @since 3.3 */ public EolCanonicalizingInputStream(InputStream in, boolean detectBinary, boolean abortIfBinary) { - this.in = in; - this.detectBinary = detectBinary; - this.abortIfBinary = abortIfBinary; - } - - @Override - public int read() throws IOException { - final int read = read(single, 0, 1); - return read == 1 ? single[0] & 0xff : -1; - } - - @Override - public int read(byte[] bs, final int off, final int len) throws IOException { - if (len == 0) - return 0; - - if (cnt == -1) - return -1; - - int i = off; - final int end = off + len; - - while (i < end) { - if (ptr == cnt && !fillBuffer()) { - break; - } - - byte b = buf[ptr++]; - if (isBinary || b != '\r') { - // Logic for binary files ends here - bs[i++] = b; - continue; - } - - if (ptr == cnt && !fillBuffer()) { - bs[i++] = '\r'; - break; - } - - if (buf[ptr] == '\n') { - bs[i++] = '\n'; - ptr++; - } else - bs[i++] = '\r'; - } - - return i == off ? -1 : i - off; + super(in, detectBinary, abortIfBinary); } /** - * @return true if the stream has detected as a binary so far + * A special exception thrown when {@link AutoLFInputStream} is told to + * throw an exception when attempting to read a binary file. The exception + * may be thrown at any stage during reading. + * * @since 3.3 */ - public boolean isBinary() { - return isBinary; - } - - @Override - public void close() throws IOException { - in.close(); - } + public static class IsBinaryException extends IOException { + private static final long serialVersionUID = 1L; - private boolean fillBuffer() throws IOException { - cnt = in.read(buf, 0, buf.length); - if (cnt < 1) - return false; - if (detectBinary) { - isBinary = RawText.isBinary(buf, cnt); - detectBinary = false; - if (isBinary && abortIfBinary) - throw new IsBinaryException(); + IsBinaryException() { + super(); } - ptr = 0; - return true; } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java new file mode 100644 index 0000000000..c95992fbc2 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2015, Ivan Motsch + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.util.io; + +import java.io.InputStream; +import java.io.OutputStream; + +import org.eclipse.jgit.attributes.Attributes; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; +import org.eclipse.jgit.treewalk.TreeWalk.OperationType; +import org.eclipse.jgit.treewalk.WorkingTreeOptions; + +/** + * Utility used to create input and output stream wrappers for + * {@link EolStreamType} + * + * @since 4.3 + */ +public final class EolStreamTypeUtil { + private static final boolean FORCE_EOL_LF_ON_CHECKOUT = false; + + private EolStreamTypeUtil() { + } + + /** + * Convenience method used to detect if CRLF conversion has been configured + * using the + *

    + *
  • global repo options
    • + *
  • global attributes
    • + *
  • info attributes
    • + *
  • working tree .gitattributes
    • + * + * @param op + * is the {@link OperationType} of the current traversal + * @param options + * are the {@link Config} options with key + * {@link WorkingTreeOptions#KEY} + * @param attrs + * are the {@link Attributes} of the file for which the + * {@link EolStreamType} is to be detected + * + * @return the stream conversion {@link EolStreamType} to be performed for + * the selected {@link OperationType} + */ + public static EolStreamType detectStreamType(OperationType op, + WorkingTreeOptions options, Attributes attrs) { + switch (op) { + case CHECKIN_OP: + return checkInStreamType(options, attrs); + case CHECKOUT_OP: + return checkOutStreamType(options, attrs); + default: + throw new IllegalArgumentException("unknown OperationType " + op); //$NON-NLS-1$ + } + } + + /** + * @param in + * original stream + * @param conversion + * to be performed + * @return the converted stream depending on {@link EolStreamType} + */ + public static InputStream wrapInputStream(InputStream in, + EolStreamType conversion) { + switch (conversion) { + case TEXT_CRLF: + return new AutoCRLFInputStream(in, false); + case TEXT_LF: + return new AutoLFInputStream(in, false); + case AUTO_CRLF: + return new AutoCRLFInputStream(in, true); + case AUTO_LF: + return new AutoLFInputStream(in, true); + default: + return in; + } + } + + /** + * @param out + * original stream + * @param conversion + * to be performed + * @return the converted stream depending on {@link EolStreamType} + */ + public static OutputStream wrapOutputStream(OutputStream out, + EolStreamType conversion) { + switch (conversion) { + case TEXT_CRLF: + return new AutoCRLFOutputStream(out, false); + case AUTO_CRLF: + return new AutoCRLFOutputStream(out, true); + case TEXT_LF: + return new AutoLFOutputStream(out, false); + case AUTO_LF: + return new AutoLFOutputStream(out, true); + default: + return out; + } + } + + private static EolStreamType checkInStreamType(WorkingTreeOptions options, + Attributes attrs) { + // old git system + if (attrs.isSet("crlf")) {//$NON-NLS-1$ + return EolStreamType.TEXT_LF; + } else if (attrs.isUnset("crlf")) {//$NON-NLS-1$ + return EolStreamType.DIRECT; + } else if ("input".equals(attrs.getValue("crlf"))) {//$NON-NLS-1$ //$NON-NLS-2$ + return EolStreamType.TEXT_LF; + } + + // new git system + if (attrs.isUnset("text")) {//$NON-NLS-1$ + return EolStreamType.DIRECT; + } + String eol = attrs.getValue("eol"); //$NON-NLS-1$ + if (eol != null) + // check-in is always normalized to LF + return EolStreamType.TEXT_LF; + + if (attrs.isSet("text")) { //$NON-NLS-1$ + return EolStreamType.TEXT_LF; + } + + if ("auto".equals(attrs.getValue("text"))) { //$NON-NLS-1$ //$NON-NLS-2$ + return EolStreamType.AUTO_LF; + } + + switch (options.getAutoCRLF()) { + case TRUE: + case INPUT: + return EolStreamType.AUTO_LF; + case FALSE: + return EolStreamType.DIRECT; + } + + return EolStreamType.DIRECT; + } + + private static EolStreamType checkOutStreamType(WorkingTreeOptions options, + Attributes attrs) { + // old git system + if (attrs.isSet("crlf")) {//$NON-NLS-1$ + return FORCE_EOL_LF_ON_CHECKOUT ? EolStreamType.TEXT_LF + : EolStreamType.DIRECT; + } else if (attrs.isUnset("crlf")) {//$NON-NLS-1$ + return EolStreamType.DIRECT; + } else if ("input".equals(attrs.getValue("crlf"))) {//$NON-NLS-1$ //$NON-NLS-2$ + return EolStreamType.DIRECT; + } + + // new git system + if (attrs.isUnset("text")) {//$NON-NLS-1$ + return EolStreamType.DIRECT; + } + String eol = attrs.getValue("eol"); //$NON-NLS-1$ + if (eol != null && "crlf".equals(eol)) //$NON-NLS-1$ + return EolStreamType.TEXT_CRLF; + if (eol != null && "lf".equals(eol)) //$NON-NLS-1$ + return FORCE_EOL_LF_ON_CHECKOUT ? EolStreamType.TEXT_LF + : EolStreamType.DIRECT; + + if (attrs.isSet("text")) { //$NON-NLS-1$ + switch (options.getAutoCRLF()) { + case TRUE: + return EolStreamType.TEXT_CRLF; + default: + // no decision + } + switch (options.getEOL()) { + case CRLF: + return EolStreamType.TEXT_CRLF; + case LF: + return FORCE_EOL_LF_ON_CHECKOUT ? EolStreamType.TEXT_LF + : EolStreamType.DIRECT; + case NATIVE: + default: + return EolStreamType.DIRECT; + } + } + + if ("auto".equals(attrs.getValue("text"))) { //$NON-NLS-1$ //$NON-NLS-2$ + switch (options.getAutoCRLF()) { + case TRUE: + return EolStreamType.AUTO_CRLF; + default: + // no decision + } + switch (options.getEOL()) { + case CRLF: + return EolStreamType.AUTO_CRLF; + case LF: + return FORCE_EOL_LF_ON_CHECKOUT ? EolStreamType.TEXT_LF + : EolStreamType.DIRECT; + case NATIVE: + default: + return EolStreamType.DIRECT; + } + } + + switch (options.getAutoCRLF()) { + case TRUE: + return EolStreamType.AUTO_CRLF; + default: + // no decision + } + + return EolStreamType.DIRECT; + } + +} -- 2.39.5