Perform automatic CRLF to LF conversion during WorkingTreeIterator

WorkingTreeIterator now optionally performs CRLF to LF conversion for
text files.  A basic framework is left in place to support enabling
(or disabling) this feature based on gitattributes, and also to
support the more generic smudge/clean filter system.  As there is
no gitattribute support yet in JGit this is left unimplemented,
but the mightNeedCleaning(), isBinary() and filterClean() methods
will provide reasonable places to plug that into in the future.

[sp: All bugs inside of WorkingTreeIterator are my fault, I wrote
     most of it while cherry-picking this patch and building it on
     top of Marc's original work.]

CQ: 4419
Bug: 301775
Change-Id: I0ca35cfbfe3f503729cbfc1d5034ad4abcd1097e
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

2 files changed, 175 insertions, 1 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java
index 1f2042d4c2..a9c3853a9d 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java
@@ -51,6 +51,7 @@ import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
 import java.text.MessageFormat;
 
 import org.eclipse.jgit.JGitText;
@@ -96,7 +97,8 @@ public class IO {
 		try {
 			final long sz = in.getChannel().size();
 			if (sz > max)
-				throw new IOException(MessageFormat.format(JGitText.get().fileIsTooLarge, path));
+				throw new IOException(MessageFormat.format(
+						JGitText.get().fileIsTooLarge, path));
 			final byte[] buf = new byte[(int) sz];
 			IO.readFully(in, buf, 0, buf.length);
 			return buf;
@@ -110,6 +112,48 @@ public class IO {
 	}
 
 	/**
+	 * Read an entire input stream into memory as a ByteBuffer.
+	 *
+	 * Note: The stream is read to its end and is not usable after calling this
+	 * method. The caller is responsible for closing the stream.
+	 *
+	 * @param in
+	 *            input stream to be read.
+	 * @param sizeHint
+	 *            a hint on the approximate number of bytes contained in the
+	 *            stream, used to allocate temporary buffers more efficiently
+	 * @return complete contents of the input stream. The ByteBuffer always has
+	 *         a writable backing array, with {@code position() == 0} and
+	 *         {@code limit()} equal to the actual length read. Callers may rely
+	 *         on obtaining the underlying array for efficient data access. If
+	 *         {@code sizeHint} was too large, the array may be over-allocated,
+	 *         resulting in {@code limit() < array().length}.
+	 * @throws IOException
+	 *             there was an error reading from the stream.
+	 */
+	public static ByteBuffer readWholeStream(InputStream in, int sizeHint)
+			throws IOException {
+		byte[] out = new byte[sizeHint];
+		int pos = 0;
+		while (pos < out.length) {
+			int read = in.read(out, pos, out.length - pos);
+			if (read < 0)
+				return ByteBuffer.wrap(out, 0, pos);
+			pos += read;
+		}
+
+		int last = in.read();
+		if (last < 0)
+			return ByteBuffer.wrap(out, 0, pos);
+
+		TemporaryBuffer.Heap tmp = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
+		tmp.write(out);
+		tmp.write(last);
+		tmp.copy(in);
+		return ByteBuffer.wrap(tmp.toByteArray());
+	}
+
+	/**
 	 * Read the entire byte array into memory, or throw an exception.
 	 *
 	 * @param fd
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolCanonicalizingInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolCanonicalizingInputStream.java
new file mode 100644
index 0000000000..4bdd2b3e55
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolCanonicalizingInputStream.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2010, Marc Strapetz <marc.strapetz@syntevo.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.util.io;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An input stream which canonicalizes EOLs bytes on the fly to '\n'.
+ *
+ * Note: Make sure to apply this InputStream only to text files!
+ */
+public class EolCanonicalizingInputStream extends InputStream {
+	private final byte[] single = new byte[1];
+
+	private final byte[] buf = new byte[8096];
+
+	private final InputStream in;
+
+	private int cnt;
+
+	private int ptr;
+
+	/**
+	 * Creates a new InputStream, wrapping the specified stream
+	 *
+	 * @param in
+	 *            raw input stream
+	 */
+	public EolCanonicalizingInputStream(InputStream in) {
+		this.in = in;
+	}
+
+	@Override
+	public int read() throws IOException {
+		final int read = read(single, 0, 1);
+		return read == 1 ? single[0] & 0xff : -1;
+	}
+
+	@Override
+	public int read(byte[] bs, int off, int len) throws IOException {
+		if (len == 0)
+			return 0;
+
+		if (cnt == -1)
+			return -1;
+
+		final int startOff = off;
+		final int end = off + len;
+
+		while (off < end) {
+			if (ptr == cnt && !fillBuffer()) {
+				break;
+			}
+
+			byte b = buf[ptr++];
+			if (b != '\r') {
+				bs[off++] = b;
+				continue;
+			}
+
+			if (ptr == cnt && !fillBuffer()) {
+				bs[off++] = '\r';
+				break;
+			}
+
+			if (buf[ptr] == '\n') {
+				bs[off++] = '\n';
+				ptr++;
+			} else
+				bs[off++] = '\r';
+		}
+
+		return startOff == off ? -1 : off - startOff;
+	}
+
+	@Override
+	public void close() throws IOException {
+		in.close();
+	}
+
+	private boolean fillBuffer() throws IOException {
+		cnt = in.read(buf, 0, buf.length);
+		if (cnt < 1)
+			return false;
+		ptr = 0;
+		return true;
+	}
+}