Fix core.autocrlf for non-normalized index

With text=auto or core.autocrlf=true, git does not normalize upon
check-in if the file in the index contains already CR/LFs. The
documentation says: "When text is set to "auto", the path is
marked for automatic end-of-line conversion. If Git decides that
the content is text, its line endings are converted to LF on
checkin. When the file has been committed with CRLF, no conversion
is done."[1]

Implement the last bit as in canonical git: check the blob in the
index for CR/LFs. For very large files, we check only the first 8000
bytes, like RawText.isBinary() and AutoLFInputStream do.

In Auto(CR)LFInputStream, ensure that the buffer is filled as much as
possible for the isBinary() check.

Regarding these content checks, there are a number of inconsistencies:

* Canonical git considers files containing lone CRs as binary.
* RawText checks the first 8000 bytes.
* Auto(CR)LFInputStream checks the first 8096 (not 8192!) bytes.

None of these are changed with this commit. It appears that canonical
git will check the whole blob, not just the first 8k bytes. Also
note: the check for CR/LF text won't work with LFS (neither in JGit
nor in git) since the blob data is not run through the smudge filter.
C.f. [2].

Two tests in AddCommandTest actually tested that normalization was
done even if the file was already committed with CR/LF.These tests
had to be adapted. I find the git documentation unclear about the
case where core.autocrlf=input, but from [3] it looks as if this
non-normalization also applies in this case.

Add new tests in CommitCommandTest testing this for the case where
the index entry is for a merge conflict. In this case, canonical git
uses the "ours" version.[4] Do the same.

[1] https://git-scm.com/docs/gitattributes
[2] https://github.com/git/git/blob/3434569fc/convert.c#L225
[3] https://github.com/git/git/blob/3434569fc/convert.c#L529
[4] https://github.com/git/git/blob/f2b6aa98b/read-cache.c#L3281

Bug: 470643
Change-Id: Ie7310539fbe6c737d78b1dcc29e34735d4616b88
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

3 files changed, 128 insertions, 6 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java
index 1a5793ce31..3fee51a885 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java
@@ -501,11 +501,11 @@ public class AddCommandTest extends RepositoryTestCase {
 					indexState(CONTENT));
 			db.getConfig().setString("core", null, "autocrlf", "true");
 			git.add().addFilepattern("a.txt").call();
-			assertEquals("[a.txt, mode:100644, content:row1\nrow2]",
+			assertEquals("[a.txt, mode:100644, content:row1\r\nrow2]",
 					indexState(CONTENT));
 			db.getConfig().setString("core", null, "autocrlf", "input");
 			git.add().addFilepattern("a.txt").call();
-			assertEquals("[a.txt, mode:100644, content:row1\nrow2]",
+			assertEquals("[a.txt, mode:100644, content:row1\r\nrow2]",
 					indexState(CONTENT));
 		}
 	}
@@ -523,19 +523,18 @@ public class AddCommandTest extends RepositoryTestCase {
 		try (PrintWriter writer = new PrintWriter(file, UTF_8.name())) {
 			writer.print(crData);
 		}
-		String lfData = data.toString().replaceAll("\r", "");
 		try (Git git = new Git(db)) {
 			db.getConfig().setString("core", null, "autocrlf", "false");
 			git.add().addFilepattern("a.txt").call();
-			assertEquals("[a.txt, mode:100644, content:" + data + "]",
+			assertEquals("[a.txt, mode:100644, content:" + crData + "]",
 					indexState(CONTENT));
 			db.getConfig().setString("core", null, "autocrlf", "true");
 			git.add().addFilepattern("a.txt").call();
-			assertEquals("[a.txt, mode:100644, content:" + lfData + "]",
+			assertEquals("[a.txt, mode:100644, content:" + crData + "]",
 					indexState(CONTENT));
 			db.getConfig().setString("core", null, "autocrlf", "input");
 			git.add().addFilepattern("a.txt").call();
-			assertEquals("[a.txt, mode:100644, content:" + lfData + "]",
+			assertEquals("[a.txt, mode:100644, content:" + crData + "]",
 					indexState(CONTENT));
 		}
 	}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java
index b76d8f987b..cd96f41a42 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java
@@ -56,6 +56,7 @@ import java.util.List;
 import java.util.TimeZone;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.eclipse.jgit.api.CherryPickResult.CherryPickStatus;
 import org.eclipse.jgit.api.errors.CanceledException;
 import org.eclipse.jgit.api.errors.EmptyCommitException;
 import org.eclipse.jgit.api.errors.WrongRepositoryStateException;
@@ -77,6 +78,7 @@ import org.eclipse.jgit.lib.ReflogEntry;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.submodule.SubmoduleWalk;
 import org.eclipse.jgit.transport.CredentialsProvider;
 import org.eclipse.jgit.treewalk.TreeWalk;
@@ -620,9 +622,100 @@ public class CommitCommandTest extends RepositoryTestCase {
 			writeTrashFile(".gitignore", "bar");
 			git.add().addFilepattern("subdir").call();
 			git.commit().setOnly("subdir").setMessage("first commit").call();
+			assertEquals("[subdir/foo, mode:100644, content:Hello World]",
+					indexState(CONTENT));
 		}
 	}
 
+	@Test
+	public void commitWithAutoCrlfAndNonNormalizedIndex() throws Exception {
+		try (Git git = new Git(db)) {
+			// Commit a file with CR/LF into the index
+			FileBasedConfig config = db.getConfig();
+			config.setString("core", null, "autocrlf", "false");
+			config.save();
+			writeTrashFile("file.txt", "line 1\r\nline 2\r\n");
+			git.add().addFilepattern("file.txt").call();
+			git.commit().setMessage("Initial").call();
+			assertEquals(
+					"[file.txt, mode:100644, content:line 1\r\nline 2\r\n]",
+					indexState(CONTENT));
+			config.setString("core", null, "autocrlf", "true");
+			config.save();
+			writeTrashFile("file.txt", "line 1\r\nline 1.5\r\nline 2\r\n");
+			writeTrashFile("file2.txt", "new\r\nfile\r\n");
+			git.add().addFilepattern("file.txt").addFilepattern("file2.txt")
+					.call();
+			git.commit().setMessage("Second").call();
+			assertEquals(
+					"[file.txt, mode:100644, content:line 1\r\nline 1.5\r\nline 2\r\n]"
+							+ "[file2.txt, mode:100644, content:new\nfile\n]",
+					indexState(CONTENT));
+			writeTrashFile("file2.txt", "new\r\nfile\r\ncontent\r\n");
+			git.add().addFilepattern("file2.txt").call();
+			git.commit().setMessage("Third").call();
+			assertEquals(
+					"[file.txt, mode:100644, content:line 1\r\nline 1.5\r\nline 2\r\n]"
+							+ "[file2.txt, mode:100644, content:new\nfile\ncontent\n]",
+					indexState(CONTENT));
+		}
+	}
+
+	private void testConflictWithAutoCrlf(String baseLf, String lf)
+			throws Exception {
+		try (Git git = new Git(db)) {
+			// Commit a file with CR/LF into the index
+			FileBasedConfig config = db.getConfig();
+			config.setString("core", null, "autocrlf", "false");
+			config.save();
+			writeTrashFile("file.txt", "foo" + baseLf);
+			git.add().addFilepattern("file.txt").call();
+			git.commit().setMessage("Initial").call();
+			// Switch to side branch
+			git.checkout().setCreateBranch(true).setName("side").call();
+			writeTrashFile("file.txt", "bar\r\n");
+			git.add().addFilepattern("file.txt").call();
+			RevCommit side = git.commit().setMessage("Side").call();
+			// Switch back to master and commit a conflict with the given lf
+			git.checkout().setName("master");
+			writeTrashFile("file.txt", "foob" + lf);
+			git.add().addFilepattern("file.txt").call();
+			git.commit().setMessage("Second").call();
+			// Switch on autocrlf=true
+			config.setString("core", null, "autocrlf", "true");
+			config.save();
+			// Cherry pick side: conflict. Resolve with CR-LF and commit.
+			CherryPickResult pick = git.cherryPick().include(side).call();
+			assertEquals("Expected a cherry-pick conflict",
+					CherryPickStatus.CONFLICTING, pick.getStatus());
+			writeTrashFile("file.txt", "foobar\r\n");
+			git.add().addFilepattern("file.txt").call();
+			git.commit().setMessage("Second").call();
+			assertEquals("[file.txt, mode:100644, content:foobar" + lf + "]",
+					indexState(CONTENT));
+		}
+	}
+
+	@Test
+	public void commitConflictWithAutoCrlfBaseCrLfOursLf() throws Exception {
+		testConflictWithAutoCrlf("\r\n", "\n");
+	}
+
+	@Test
+	public void commitConflictWithAutoCrlfBaseLfOursLf() throws Exception {
+		testConflictWithAutoCrlf("\n", "\n");
+	}
+
+	@Test
+	public void commitConflictWithAutoCrlfBasCrLfOursCrLf() throws Exception {
+		testConflictWithAutoCrlf("\r\n", "\r\n");
+	}
+
+	@Test
+	public void commitConflictWithAutoCrlfBaseLfOursCrLf() throws Exception {
+		testConflictWithAutoCrlf("\n", "\r\n");
+	}
+
 	private static void addUnmergedEntry(String file, DirCacheBuilder builder) {
 		DirCacheEntry stage1 = new DirCacheEntry(file, DirCacheEntry.STAGE_1);
 		DirCacheEntry stage2 = new DirCacheEntry(file, DirCacheEntry.STAGE_2);
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RawTextTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RawTextTest.java
index 178d62072d..5333451a96 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RawTextTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/RawTextTest.java
@@ -79,6 +79,36 @@ public class RawTextTest {
 	}
 
 	@Test
+	public void testCrLfTextYes() {
+		assertTrue(RawText
+				.isCrLfText(Constants.encodeASCII("line 1\r\nline 2\r\n")));
+	}
+
+	@Test
+	public void testCrLfTextNo() {
+		assertFalse(
+				RawText.isCrLfText(Constants.encodeASCII("line 1\nline 2\n")));
+	}
+
+	@Test
+	public void testCrLfTextBinary() {
+		assertFalse(RawText
+				.isCrLfText(Constants.encodeASCII("line 1\r\nline\0 2\r\n")));
+	}
+
+	@Test
+	public void testCrLfTextMixed() {
+		assertTrue(RawText
+				.isCrLfText(Constants.encodeASCII("line 1\nline 2\r\n")));
+	}
+
+	@Test
+	public void testCrLfTextCutShort() {
+		assertFalse(
+				RawText.isCrLfText(Constants.encodeASCII("line 1\nline 2\r")));
+	}
+
+	@Test
 	public void testEquals() {
 		final RawText a = new RawText(Constants.encodeASCII("foo-a\nfoo-b\n"));
 		final RawText b = new RawText(Constants.encodeASCII("foo-b\nfoo-c\n"));