Make the buffer size for text/binary detection configurable

The various streams used in JGit for text/binary and CR-LF detection
used different buffer sizes. Most used 8000, but one used 8KiB, and one
used 8096 (SIC!) bytes.

Considering only the first 8kB of a file/blob is not sufficient; it
may give behavior incompatible with C git. C git considers the whole
blob; since it uses memory-mapped files it can do so with acceptable
performance. Doing this in JGit would most likely incur a noticeable
performance penalty. But 8kB is a bit small; in the file in bug 576971
the limit was hit before the first CR-LF, which occurred on line 155
at offset 9759 in the file.

Make RawText.FIRST_FEW_BYTES only a default and minimum setting, and
set it to 8KiB. Make the actual buffer size configurable: provide
static methods getBufferSize() and setBuffersize(), and use
getBufferSize() throughout instead of the constant.

This enables users of the JGit library to set their own possibly larger
buffer size.

Bug: 576971
Change-Id: I447762c9a5147a521f73d2864ba59ed89f555d54
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

4 files changed, 21 insertions, 11 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java
index b694f4aaf7..a093cc78de 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterTest.java
@@ -608,7 +608,7 @@ public class DiffFormatterTest extends RepositoryTestCase {
 	public void testDiffAutoCrlfMediumFile() throws Exception {
 		String content = mediumCrLfString();
 		String expectedDiff = "diff --git a/test.txt b/test.txt\n"
-				+ "index 215c502..c10f08c 100644\n" //
+				+ "index 6d9ffed..50d7b5a 100644\n" //
 				+ "--- a/test.txt\n" //
 				+ "+++ b/test.txt\n" //
 				+ "@@ -1,4 +1,5 @@\n" //
@@ -624,7 +624,7 @@ public class DiffFormatterTest extends RepositoryTestCase {
 	public void testDiffAutoCrlfLargeFile() throws Exception {
 		String content = largeCrLfString();
 		String expectedDiff = "diff --git a/test.txt b/test.txt\n"
-				+ "index 7014942..c0487a7 100644\n" //
+				+ "index d6399a1..de26ce5 100644\n" //
 				+ "--- a/test.txt\n" //
 				+ "+++ b/test.txt\n" //
 				+ "@@ -1,4 +1,5 @@\n"
@@ -665,9 +665,9 @@ public class DiffFormatterTest extends RepositoryTestCase {
 
 	private static String largeCrLfString() {
 		String line = "012345678901234567890123456789012345678901234567\r\n";
-		StringBuilder builder = new StringBuilder(
-				2 * RawText.FIRST_FEW_BYTES);
-		while (builder.length() < 2 * RawText.FIRST_FEW_BYTES) {
+		int bufferSize = RawText.getBufferSize();
+		StringBuilder builder = new StringBuilder(2 * bufferSize);
+		while (builder.length() < 2 * bufferSize) {
 			builder.append(line);
 		}
 		return builder.toString();
@@ -677,9 +677,9 @@ public class DiffFormatterTest extends RepositoryTestCase {
 		// Create a CR-LF string longer than RawText.FIRST_FEW_BYTES whose
 		// canonical representation is shorter than RawText.FIRST_FEW_BYTES.
 		String line = "01234567\r\n"; // 10 characters
-		StringBuilder builder = new StringBuilder(
-				RawText.FIRST_FEW_BYTES + line.length());
-		while (builder.length() <= RawText.FIRST_FEW_BYTES) {
+		int bufferSize = RawText.getBufferSize();
+		StringBuilder builder = new StringBuilder(bufferSize + line.length());
+		while (builder.length() <= bufferSize) {
 			builder.append(line);
 		}
 		return builder.toString();
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java
index 6cbb4a89b2..dd8573d2bc 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java
@@ -33,6 +33,7 @@ import org.eclipse.jgit.api.RebaseResult;
 import org.eclipse.jgit.api.errors.CheckoutConflictException;
 import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.JGitInternalException;
+import org.eclipse.jgit.diff.RawText;
 import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheEditor;
 import org.eclipse.jgit.dircache.DirCacheEntry;
@@ -826,6 +827,8 @@ public class MergerTest extends RepositoryTestCase {
 		RevCommit sideCommit = git.commit().setAll(true)
 			.setMessage("modified file l 1500").call();
 
+		int originalBufferSize = RawText.getBufferSize();
+		int smallBufferSize = RawText.setBufferSize(8000);
 		try (ObjectInserter ins = db.newObjectInserter()) {
 			// Check that we don't read the large blobs.
 			ObjectInserter forbidInserter = new ObjectInserter.Filter() {
@@ -836,7 +839,8 @@ public class MergerTest extends RepositoryTestCase {
 
 				@Override
 				public ObjectReader newReader() {
-					return new BigReadForbiddenReader(super.newReader(), 8000);
+					return new BigReadForbiddenReader(super.newReader(),
+							smallBufferSize);
 				}
 			};
 
@@ -844,6 +848,8 @@ public class MergerTest extends RepositoryTestCase {
 				(ResolveMerger) strategy.newMerger(forbidInserter, db.getConfig());
 			boolean noProblems = merger.merge(masterCommit, sideCommit);
 			assertFalse(noProblems);
+		} finally {
+			RawText.setBufferSize(originalBufferSize);
 		}
 	}
 
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFInputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFInputStreamTest.java
index ae8c7ec7a7..cd4e503390 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFInputStreamTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFInputStreamTest.java
@@ -17,6 +17,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
+import org.eclipse.jgit.diff.RawText;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -38,7 +39,8 @@ public class AutoCRLFInputStreamTest {
 
 	@Test
 	public void testBoundary() throws IOException {
-		for (int i = AutoCRLFInputStream.BUFFER_SIZE - 10; i < AutoCRLFInputStream.BUFFER_SIZE + 10; i++) {
+		int boundary = RawText.getBufferSize();
+		for (int i = boundary - 10; i < boundary + 10; i++) {
 			String s1 = Strings.repeat("a", i);
 			assertNoCrLf(s1, s1);
 			String s2 = Strings.repeat("\0", i);
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFOutputStreamTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFOutputStreamTest.java
index db2f6da31e..150df08453 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFOutputStreamTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/io/AutoCRLFOutputStreamTest.java
@@ -19,6 +19,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.eclipse.jgit.diff.RawText;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -40,7 +41,8 @@ public class AutoCRLFOutputStreamTest {
 
 	@Test
 	public void testBoundary() throws IOException {
-		for (int i = AutoCRLFOutputStream.BUFFER_SIZE - 10; i < AutoCRLFOutputStream.BUFFER_SIZE + 10; i++) {
+		int bufferSize = RawText.getBufferSize();
+		for (int i = bufferSize - 10; i < bufferSize + 10; i++) {
 			String s1 = Strings.repeat("a", i);
 			assertNoCrLf(s1, s1);
 			String s2 = Strings.repeat("\0", i);