From efd1cc05af7e59a24763dfedb7fc44cda151be50 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Wed, 22 Jul 2020 00:51:24 +0200 Subject: [PATCH] Keep line endings for text files committed with CR/LF on text=auto Git never converts line endings if the version in the repository is a text file with CR/LF and text=auto. See [1]: "When the file has been committed with CRLF, no conversion is done." Because the sentence just before is about converting line endings on check-in, I had understood that in commit 60cf85a [2] to mean that no conversion on check-in was to be done. However, as bug 565048 and a code inspection of the C git code showed it really means no conversion is done on check-in *or check-out*. If the text attribute is not set but core.autocrlf = true, this is the same as text=auto eol=crlf. C git does not convert on check-out even on text=auto eol=lf if the index version is a text file with CR/LF. For check-in, one has to look at the intended target, which is done in WorkingTreeIterator since commit 60cf85a. For check-out, it can be done by looking at the source and can thus be done in the AutoLFOutputStream. Additionally, provide a constructor for AutoLFInputStream to do the same; for cases where the equivalent of a check-out is done via an input stream obtained from a blob. (EGit does that in its GitBlobStorage for the Eclipse compare framework; it's more efficient than using a TemporaryBuffer and DirCacheCheckout.getContent(), and it avoids the need for a temporary file.) Adapt existing tests, and add new checkout and merge tests to verify the resulting files have the correct line endings. EGit's GitBlobStorage will need to call the new version of EolStreamTypeUtil.wrapInputStream(). [1] https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-Settostringvalueauto [2] https://git.eclipse.org/r/c/jgit/jgit/+/127324 Bug: 565048 Change-Id: If1282ef43e2abd00263541bd10a01fe1f5c619fc Signed-off-by: Thomas Wolf --- .../jgit/api/EolStreamTypeUtilTest.java | 120 +++++++---------- .../jgit/lib/DirCacheCheckoutTest.java | 117 +++++++++++----- .../org/eclipse/jgit/merge/MergerTest.java | 44 +++++- .../jgit/util/io/AutoLFInputStreamTest.java | 43 ++++-- .../jgit/util/io/AutoLFOutputStreamTest.java | 84 ++++++++++++ .../jgit/util/io/AutoLFInputStream.java | 125 ++++++++++++++++-- .../jgit/util/io/AutoLFOutputStream.java | 63 +++------ .../jgit/util/io/EolStreamTypeUtil.java | 35 ++++- 8 files changed, 448 insertions(+), 183 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java index 1e3a39aad8..2a553ce1a2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolStreamTypeUtilTest.java @@ -1,43 +1,11 @@ /* - * Copyright (C) 2015, Ivan Motsch + * Copyright (C) 2015, 2020 Ivan Motsch and others * - * This program and the accompanying materials are made available - * under the terms of the Eclipse Distribution License v1.0 which - * accompanies this distribution, is reproduced below, and is - * available at http://www.eclipse.org/org/documents/edl-v10.php + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the - * names of its contributors may be used to endorse or promote - * products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.api; @@ -90,16 +58,20 @@ public class EolStreamTypeUtilTest { testCheckout(TEXT_LF, AUTO_LF, "\r", "\r"); testCheckout(TEXT_LF, AUTO_LF, "\n", "\n"); - testCheckout(TEXT_LF, AUTO_LF, "\r\n", "\n"); + testCheckout(TEXT_LF, null, "\r\n", "\n"); + testCheckout(null, AUTO_LF, "\r\n", "\r\n"); testCheckout(TEXT_LF, AUTO_LF, "\n\r", "\n\r"); - testCheckout(TEXT_LF, AUTO_LF, "\n\r\n", "\n\n"); - testCheckout(TEXT_LF, AUTO_LF, "\r\n\r", "\n\r"); + testCheckout(TEXT_LF, null, "\n\r\n", "\n\n"); + testCheckout(null, AUTO_LF, "\n\r\n", "\n\r\n"); + testCheckout(TEXT_LF, null, "\r\n\r", "\n\r"); + testCheckout(null, AUTO_LF, "\r\n\r", "\r\n\r"); testCheckout(TEXT_LF, AUTO_LF, "a\nb\n", "a\nb\n"); testCheckout(TEXT_LF, AUTO_LF, "a\rb\r", "a\rb\r"); testCheckout(TEXT_LF, AUTO_LF, "a\n\rb\n\r", "a\n\rb\n\r"); - testCheckout(TEXT_LF, AUTO_LF, "a\r\nb\r\n", "a\nb\n"); + testCheckout(TEXT_LF, null, "a\r\nb\r\n", "a\nb\n"); + testCheckout(null, AUTO_LF, "a\r\nb\r\n", "a\r\nb\r\n"); } @Test @@ -153,45 +125,49 @@ public class EolStreamTypeUtilTest { byte[] outputBytes = output.getBytes(UTF_8); byte[] expectedConversionBytes = expectedConversion.getBytes(UTF_8); - // test using output text and assuming it was declared TEXT - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeText)) { - out.write(outputBytes); + if (streamTypeText != null) { + // test using output text and assuming it was declared TEXT + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeText)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); } - assertArrayEquals(expectedConversionBytes, b.toByteArray()); - - // test using ouput text and assuming it was declared AUTO, using binary - // detection - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeWithBinaryCheck)) { - out.write(outputBytes); + if (streamTypeWithBinaryCheck != null) { + // test using output text and assuming it was declared AUTO, using + // binary detection + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeWithBinaryCheck)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); } - assertArrayEquals(expectedConversionBytes, b.toByteArray()); - // now pollute output text with some binary bytes outputBytes = extendWithBinaryData(outputBytes); expectedConversionBytes = extendWithBinaryData(expectedConversionBytes); - // again, test using output text and assuming it was declared TEXT - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeText)) { - out.write(outputBytes); + if (streamTypeText != null) { + // again, test using output text and assuming it was declared TEXT + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeText)) { + out.write(outputBytes); + } + assertArrayEquals(expectedConversionBytes, b.toByteArray()); } - assertArrayEquals(expectedConversionBytes, b.toByteArray()); - - // again, test using ouput text and assuming it was declared AUTO, using - // binary - // detection - b = new ByteArrayOutputStream(); - try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, - streamTypeWithBinaryCheck)) { - out.write(outputBytes); + if (streamTypeWithBinaryCheck != null) { + // again, test using output text and assuming it was declared AUTO, + // using binary detection + b = new ByteArrayOutputStream(); + try (OutputStream out = EolStreamTypeUtil.wrapOutputStream(b, + streamTypeWithBinaryCheck)) { + out.write(outputBytes); + } + // expect no conversion + assertArrayEquals(outputBytes, b.toByteArray()); } - // expect no conversion - assertArrayEquals(outputBytes, b.toByteArray()); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java index a272c8f2ee..b943486b1b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/DirCacheCheckoutTest.java @@ -2,41 +2,13 @@ * Copyright (C) 2007, Dave Watson * Copyright (C) 2008-2011, Shawn O. Pearce * Copyright (C) 2008-2011, Robin Rosenberg - * Copyright (C) 2010-2011, Christian Halstrick - * and other copyright owners as documented in the project's IP log. + * Copyright (C) 2010, 2020 Christian Halstrick and others * * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v1.0 which accompanies this - * distribution, is reproduced below, and is available at - * http://www.eclipse.org/org/documents/edl-v10.php + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.lib; @@ -85,6 +57,7 @@ import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.WorkingTreeIterator; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.StringUtils; import org.junit.Assume; import org.junit.Test; @@ -284,6 +257,86 @@ public class DirCacheCheckoutTest extends RepositoryTestCase { } } + private void checkoutLineEndings(String inIndex, String expected, + String attributes) throws Exception { + try (Git git = new Git(db); + TestRepository db_t = new TestRepository<>(db)) { + BranchBuilder master = db_t.branch("master"); + master.commit().add("f", inIndex).message("m0").create(); + if (!StringUtils.isEmptyOrNull(attributes)) { + master.commit().add(".gitattributes", attributes) + .message("attributes").create(); + } + File f = new File(db.getWorkTree(), "f"); + assertFalse(f.exists()); + git.checkout().setName("master").call(); + assertTrue(f.exists()); + checkFile(f, expected); + } + } + + @Test + public void testCheckoutWithCRLF() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", null); + } + + @Test + public void testCheckoutWithCRLFAuto() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto"); + } + + @Test + public void testCheckoutWithCRLFAutoEolLf() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto eol=lf"); + } + + @Test + public void testCheckoutWithCRLFAutoEolNative() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto eol=native"); + } + + @Test + public void testCheckoutWithCRLFAutoEolCrLf() throws Exception { + checkoutLineEndings("first line\r\nsecond line\r\n", + "first line\r\nsecond line\r\n", "f text=auto eol=crlf"); + } + + @Test + public void testCheckoutWithLF() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\nsecond line\n", null); + } + + @Test + public void testCheckoutWithLFAuto() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\nsecond line\n", "f text=auto"); + } + + @Test + public void testCheckoutWithLFAutoEolLf() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\nsecond line\n", "f text=auto eol=lf"); + } + + @Test + public void testCheckoutWithLFAutoEolNative() throws Exception { + checkoutLineEndings( + "first line\nsecond line\n", "first line\nsecond line\n" + .replaceAll("\n", System.lineSeparator()), + "f text=auto eol=native"); + } + + @Test + public void testCheckoutWithLFAutoEolCrLf() throws Exception { + checkoutLineEndings("first line\nsecond line\n", + "first line\r\nsecond line\r\n", "f text=auto eol=crlf"); + } + private DirCacheCheckout resetHard(RevCommit commit) throws NoWorkTreeException, CorruptObjectException, IOException { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java index 7a244e1d8b..e2ac89be90 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012, Robin Stocker and others + * Copyright (C) 2012, 2020 Robin Stocker and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -366,6 +366,48 @@ public class MergerTest extends RepositoryTestCase { mergeResult.getMergeStatus()); } + @Theory + public void mergeConflictWithCrLfTextAuto(MergeStrategy strategy) + throws IOException, GitAPIException { + Git git = Git.wrap(db); + writeTrashFile("crlf.txt", "a crlf file\r\n"); + git.add().addFilepattern("crlf.txt").call(); + git.commit().setMessage("base").call(); + assertEquals("[crlf.txt, mode:100644, content:a crlf file\r\n]", + indexState(CONTENT)); + writeTrashFile(".gitattributes", "crlf.txt text=auto"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("attributes").call(); + + git.branchCreate().setName("brancha").call(); + + writeTrashFile("crlf.txt", "a crlf file\r\na second line\r\n"); + git.add().addFilepattern("crlf.txt").call(); + git.commit().setMessage("on master").call(); + assertEquals( + "[.gitattributes, mode:100644, content:crlf.txt text=auto]" + + "[crlf.txt, mode:100644, content:a crlf file\r\na second line\r\n]", + indexState(CONTENT)); + + git.checkout().setName("brancha").call(); + File testFile = writeTrashFile("crlf.txt", + "a crlf file\r\nanother line\r\n"); + git.add().addFilepattern("crlf.txt").call(); + git.commit().setMessage("on brancha").call(); + + MergeResult mergeResult = git.merge().setStrategy(strategy) + .include(db.resolve("master")).call(); + assertEquals(MergeResult.MergeStatus.CONFLICTING, + mergeResult.getMergeStatus()); + checkFile(testFile, + "a crlf file\r\n" // + + "<<<<<<< HEAD\n" // + + "another line\r\n" // + + "=======\n" // + + "a second line\r\n" // + + ">>>>>>> 8e9e704742f1bc8a41eac88aac4aeefd338b7384\n"); + } + @Theory public void mergeWithCrlfAutoCrlfTrue(MergeStrategy strategy) throws IOException, GitAPIException { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java index fb90461e1a..c391694fbc 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFInputStreamTest.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Marc Strapetz - * Copyright (C) 2015, Ivan Motsch and others + * Copyright (C) 2015, 2020 Ivan Motsch and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -17,7 +17,9 @@ import static org.junit.Assert.assertEquals; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.util.function.Function; +import org.eclipse.jgit.util.io.AutoLFInputStream.StreamFlag; import org.junit.Test; public class AutoLFInputStreamTest { @@ -25,47 +27,65 @@ public class AutoLFInputStreamTest { @Test public void testLF() throws IOException { final byte[] bytes = asBytes("1\n2\n3"); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testCR() throws IOException { final byte[] bytes = asBytes("1\r2\r3"); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testCRLF() throws IOException { - test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3"), false); + test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3")); } @Test public void testLFCR() throws IOException { final byte[] bytes = asBytes("1\n\r2\n\r3"); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testEmpty() throws IOException { final byte[] bytes = asBytes(""); - test(bytes, bytes, false); + test(bytes, bytes); } @Test public void testBinaryDetect() throws IOException { final byte[] bytes = asBytes("1\r\n2\r\n3\0"); - test(bytes, bytes, true); + test(bytes, bytes, StreamFlag.DETECT_BINARY); } @Test public void testBinaryDontDetect() throws IOException { - test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0"), false); + test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0")); + } + + @Test + public void testCrLf() throws IOException { + byte[] bytes = asBytes("1\r\n2\n3\r\n\r"); + test(bytes, bytes, in -> AutoLFInputStream.create(in, + StreamFlag.DETECT_BINARY, StreamFlag.FOR_CHECKOUT)); + } + + @Test + public void testCrLfDontDetect() throws IOException { + test(asBytes("1\r\n2\r\n"), asBytes("1\n2\n"), + in -> AutoLFInputStream.create(in, StreamFlag.DETECT_BINARY)); + } + + private static void test(byte[] input, byte[] expected, StreamFlag... flags) + throws IOException { + test(input, expected, in -> AutoLFInputStream.create(in, flags)); } private static void test(byte[] input, byte[] expected, - boolean detectBinary) throws IOException { + Function factory) throws IOException { try (InputStream bis1 = new ByteArrayInputStream(input); - InputStream cis1 = new AutoLFInputStream(bis1, detectBinary)) { + InputStream cis1 = factory.apply(bis1)) { int index1 = 0; for (int b = cis1.read(); b != -1; b = cis1.read()) { assertEquals(expected[index1], (byte) b); @@ -77,8 +97,7 @@ public class AutoLFInputStreamTest { for (int bufferSize = 1; bufferSize < 10; bufferSize++) { final byte[] buffer = new byte[bufferSize]; try (InputStream bis2 = new ByteArrayInputStream(input); - InputStream cis2 = new AutoLFInputStream(bis2, - detectBinary)) { + InputStream cis2 = factory.apply(bis2)) { int read = 0; for (int readNow = cis2.read(buffer, 0, diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java new file mode 100644 index 0000000000..1b7e55e457 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoLFOutputStreamTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2020, Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.util.io; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertArrayEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import org.junit.Test; + +public class AutoLFOutputStreamTest { + + @Test + public void testLF() throws IOException { + final byte[] bytes = asBytes("1\n2\n3"); + test(bytes, bytes, false); + } + + @Test + public void testCR() throws IOException { + final byte[] bytes = asBytes("1\r2\r3"); + test(bytes, bytes, false); + } + + @Test + public void testCRLFNoDetect() throws IOException { + test(asBytes("1\r\n2\r\n3"), asBytes("1\n2\n3"), false); + } + + @Test + public void testLFCR() throws IOException { + final byte[] bytes = asBytes("1\n\r2\n\r3"); + test(bytes, bytes, false); + } + + @Test + public void testEmpty() throws IOException { + final byte[] bytes = asBytes(""); + test(bytes, bytes, false); + } + + @Test + public void testBinaryDetect() throws IOException { + final byte[] bytes = asBytes("1\r\n2\r\n3\0"); + test(bytes, bytes, true); + } + + @Test + public void testBinaryDontDetect() throws IOException { + test(asBytes("1\r\n2\r\n3\0"), asBytes("1\n2\n3\0"), false); + } + + @Test + public void testCrLfDetect() throws IOException { + byte[] bytes = asBytes("1\r\n2\n3\r\n\r"); + test(bytes, bytes, true); + } + + private static void test(byte[] input, byte[] expected, + boolean detectBinary) throws IOException { + try (ByteArrayOutputStream result = new ByteArrayOutputStream(); + OutputStream out = new AutoLFOutputStream(result, + detectBinary)) { + out.write(input); + out.close(); + assertArrayEquals(expected, result.toByteArray()); + } + } + + private static byte[] asBytes(String in) { + return in.getBytes(UTF_8); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java index 8c9b1bf5cc..0e335a9dc4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, 2013 Marc Strapetz - * Copyright (C) 2015, Ivan Motsch and others + * Copyright (C) 2015, 2020 Ivan Motsch and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -13,26 +13,58 @@ package org.eclipse.jgit.util.io; import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.Set; import org.eclipse.jgit.diff.RawText; /** * An InputStream that normalizes CRLF to LF. - * - * Existing single CR are not changed to LF, but retained as is. - * - * Optionally, a binary check on the first 8000 bytes is performed and in case - * of binary files, canonicalization is turned off (for the complete file). *

- * This is the former EolCanonicalizingInputStream with a new name in order to - * have same naming for all LF / CRLF streams + * Existing single CR are not changed to LF but are retained as is. + *

+ *

+ * Optionally, a binary check on the first 8kB is performed and in case of + * binary files, canonicalization is turned off (for the complete file). If + * binary checking determines that the input is CR/LF-delimited text and the + * stream has been created for checkout, canonicalization is also turned off. + *

* * @since 4.3 */ public class AutoLFInputStream extends InputStream { + + // This is the former EolCanonicalizingInputStream with a new name in order + // to have same naming for all LF / CRLF streams. + + /** + * Flags for controlling auto-detection of binary vs. text content (for + * text=auto). + * + * @since 5.9 + */ + public enum StreamFlag { + /** + * Check the first 8kB for binary content and switch off + * canonicalization off for the whole file if so. + */ + DETECT_BINARY, + /** + * If {@link #DETECT_BINARY} is set, throw an {@link IsBinaryException} + * if binary content is detected. + */ + ABORT_IF_BINARY, + /** + * If {@link #DETECT_BINARY} is set and content is found to be CR-LF + * delimited text, switch off canonicalization. + */ + FOR_CHECKOUT + } + private final byte[] single = new byte[1]; - private final byte[] buf = new byte[8096]; + private final byte[] buf = new byte[8 * 1024]; private final InputStream in; @@ -40,11 +72,23 @@ public class AutoLFInputStream extends InputStream { private int ptr; + /** + * Set to {@code true} if no CR/LF processing is to be done: if the input is + * binary data, or CR/LF-delimited text and {@link StreamFlag#FOR_CHECKOUT} + * was given. + */ + private boolean passAsIs; + + /** + * Set to {@code true} if the input was detected to be binary data. + */ private boolean isBinary; private boolean detectBinary; - private boolean abortIfBinary; + private final boolean abortIfBinary; + + private final boolean forCheckout; /** * A special exception thrown when {@link AutoLFInputStream} is told to @@ -62,20 +106,64 @@ public class AutoLFInputStream extends InputStream { } /** - * Creates a new InputStream, wrapping the specified stream + * Factory method for creating an {@link AutoLFInputStream} with the + * specified {@link StreamFlag flags}. + * + * @param in + * raw input stream + * @param flags + * {@link StreamFlag}s controlling the stream behavior + * @return a new {@link AutoLFInputStream} + * @since 5.9 + */ + public static AutoLFInputStream create(InputStream in, + StreamFlag... flags) { + if (flags == null) { + return new AutoLFInputStream(in, null); + } + EnumSet set = EnumSet.noneOf(StreamFlag.class); + set.addAll(Arrays.asList(flags)); + return new AutoLFInputStream(in, set); + } + + /** + * Creates a new InputStream, wrapping the specified stream. + * + * @param in + * raw input stream + * @param flags + * {@link StreamFlag}s controlling the stream behavior; + * {@code null} is treated as an empty set + * @since 5.9 + */ + public AutoLFInputStream(InputStream in, Set flags) { + this.in = in; + this.detectBinary = flags != null + && flags.contains(StreamFlag.DETECT_BINARY); + this.abortIfBinary = flags != null + && flags.contains(StreamFlag.ABORT_IF_BINARY); + this.forCheckout = flags != null + && flags.contains(StreamFlag.FOR_CHECKOUT); + } + + /** + * Creates a new InputStream, wrapping the specified stream. * * @param in * raw input stream * @param detectBinary * whether binaries should be detected * @since 2.0 + * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} + * instead */ + @Deprecated public AutoLFInputStream(InputStream in, boolean detectBinary) { this(in, detectBinary, false); } /** - * Creates a new InputStream, wrapping the specified stream + * Creates a new InputStream, wrapping the specified stream. * * @param in * raw input stream @@ -84,12 +172,16 @@ public class AutoLFInputStream extends InputStream { * @param abortIfBinary * throw an IOException if the file is binary * @since 3.3 + * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} + * instead */ + @Deprecated public AutoLFInputStream(InputStream in, boolean detectBinary, boolean abortIfBinary) { this.in = in; this.detectBinary = detectBinary; this.abortIfBinary = abortIfBinary; + this.forCheckout = false; } /** {@inheritDoc} */ @@ -118,7 +210,7 @@ public class AutoLFInputStream extends InputStream { } byte b = buf[ptr++]; - if (isBinary || b != '\r') { + if (passAsIs || b != '\r') { // Logic for binary files ends here bs[i++] = b; continue; @@ -170,9 +262,14 @@ public class AutoLFInputStream extends InputStream { } if (detectBinary) { isBinary = RawText.isBinary(buf, cnt); + passAsIs = isBinary; detectBinary = false; - if (isBinary && abortIfBinary) + if (isBinary && abortIfBinary) { throw new IsBinaryException(); + } + if (!passAsIs && forCheckout) { + passAsIs = RawText.isCrLfText(buf, cnt); + } } ptr = 0; return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java index e235aa0ed4..195fdb4213 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFOutputStream.java @@ -1,43 +1,12 @@ /* * Copyright (C) 2015, Ivan Motsch + * Copyright (C) 2020, Thomas Wolf and others * - * This program and the accompanying materials are made available - * under the terms of the Eclipse Distribution License v1.0 which - * accompanies this distribution, is reproduced below, and is - * available at http://www.eclipse.org/org/documents/edl-v10.php + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials provided - * with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the - * names of its contributors may be used to endorse or promote - * products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND - * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, - * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF - * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.util.io; @@ -49,11 +18,15 @@ import org.eclipse.jgit.diff.RawText; /** * An OutputStream that reduces CRLF to LF. - * + *

* Existing single CR are not changed to LF, but retained as is. - * + *

+ *

* A binary check on the first 8000 bytes is performed and in case of binary - * files, canonicalization is turned off (for the complete file). + * files, canonicalization is turned off (for the complete file). If the binary + * check determines that the input is not binary but text with CR/LF, + * canonicalization is also turned off. + *

* * @since 4.3 */ @@ -76,9 +49,7 @@ public class AutoLFOutputStream extends OutputStream { private boolean isBinary; /** - *

* Constructor for AutoLFOutputStream. - *

* * @param out * an {@link java.io.OutputStream} object. @@ -88,9 +59,7 @@ public class AutoLFOutputStream extends OutputStream { } /** - *

* Constructor for AutoLFOutputStream. - *

* * @param out * an {@link java.io.OutputStream} object. @@ -123,14 +92,11 @@ public class AutoLFOutputStream extends OutputStream { public void write(byte[] b, int startOff, int startLen) throws IOException { final int overflow = buffer(b, startOff, startLen); - if (overflow < 0) { + if (overflow <= 0) { return; } final int off = startOff + startLen - overflow; final int len = overflow; - if (len == 0) { - return; - } int lastw = off; if (isBinary) { out.write(b, off, len); @@ -190,6 +156,9 @@ public class AutoLFOutputStream extends OutputStream { private void decideMode() throws IOException { if (detectBinary) { isBinary = RawText.isBinary(binbuf, binbufcnt); + if (!isBinary) { + isBinary = RawText.isCrLfText(binbuf, binbufcnt); + } detectBinary = false; } int cachedLen = binbufcnt; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java index c33c869b64..88ee2aee88 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015, Ivan Motsch and others + * Copyright (C) 2015, 2020 Ivan Motsch and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -12,12 +12,14 @@ package org.eclipse.jgit.util.io; import java.io.InputStream; import java.io.OutputStream; +import java.util.EnumSet; import org.eclipse.jgit.attributes.Attributes; import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.treewalk.TreeWalk.OperationType; import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.util.SystemReader; +import org.eclipse.jgit.util.io.AutoLFInputStream.StreamFlag; /** * Utility used to create input and output stream wrappers for @@ -71,7 +73,7 @@ public final class EolStreamTypeUtil { /** * Wrap the input stream depending on - * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType} + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}. * * @param in * original stream @@ -82,15 +84,38 @@ public final class EolStreamTypeUtil { */ public static InputStream wrapInputStream(InputStream in, EolStreamType conversion) { + return wrapInputStream(in, conversion, false); + } + + /** + * Wrap the input stream depending on + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}. + * + * @param in + * original stream + * @param conversion + * to be performed + * @param forCheckout + * whether the stream is for checking out from the repository + * @return the converted stream depending on + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType} + * @since 5.9 + */ + public static InputStream wrapInputStream(InputStream in, + EolStreamType conversion, boolean forCheckout) { switch (conversion) { case TEXT_CRLF: return new AutoCRLFInputStream(in, false); case TEXT_LF: - return new AutoLFInputStream(in, false); + return AutoLFInputStream.create(in); case AUTO_CRLF: return new AutoCRLFInputStream(in, true); case AUTO_LF: - return new AutoLFInputStream(in, true); + EnumSet flags = forCheckout + ? EnumSet.of(StreamFlag.DETECT_BINARY, + StreamFlag.FOR_CHECKOUT) + : EnumSet.of(StreamFlag.DETECT_BINARY); + return new AutoLFInputStream(in, flags); default: return in; } @@ -98,7 +123,7 @@ public final class EolStreamTypeUtil { /** * Wrap the output stream depending on - * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType} + * {@link org.eclipse.jgit.lib.CoreConfig.EolStreamType}. * * @param out * original stream -- 2.39.5