Handle content length in WorkingTreeIterator

Content length is computed and cached (short term) in the working
tree iterator when core.autocrlf is set.

Hopefully this is a cleaner fix than my previous attempt to make
autocrlf work.

Change-Id: I1b6bbb643101a00db94e5514b5e2b069f338907a

5 files changed, 81 insertions, 41 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java
index 206d0876ae..3da9640b49 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java
@@ -176,10 +176,12 @@ public class AddCommand extends GitCommand<DirCache> {
 									entry.setLength(sz);
 									entry.setLastModified(f
 											.getEntryLastModified());
+									long contentSize = f
+											.getEntryContentLength();
 									InputStream in = f.openEntryStream();
 									try {
 										entry.setObjectId(inserter.insert(
-												Constants.OBJ_BLOB, sz, in));
+												Constants.OBJ_BLOB, contentSize, in));
 									} finally {
 										in.close();
 									}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java
index 87edaddc60..242d11efaa 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java
@@ -357,11 +357,11 @@ public class CommitCommand extends GitCommand<RevCommit> {
 							// insert object
 							if (inserter == null)
 								inserter = repo.newObjectInserter();
-
+							long contentLength = fTree.getEntryContentLength();
 							InputStream inputStream = fTree.openEntryStream();
 							try {
 								dcEntry.setObjectId(inserter.insert(
-										Constants.OBJ_BLOB, entryLength,
+										Constants.OBJ_BLOB, contentLength,
 										inputStream));
 							} finally {
 								inputStream.close();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java
index 07d74b266d..104c22f2ae 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashCreateCommand.java
@@ -254,11 +254,11 @@ public class StashCreateCommand extends GitCommand<RevCommit> {
 						entry.setLength(wtIter.getEntryLength());
 						entry.setLastModified(wtIter.getEntryLastModified());
 						entry.setFileMode(wtIter.getEntryFileMode());
+						long contentLength = wtIter.getEntryContentLength();
 						InputStream in = wtIter.openEntryStream();
 						try {
 							entry.setObjectId(inserter.insert(
-									Constants.OBJ_BLOB,
-									wtIter.getEntryLength(), in));
+									Constants.OBJ_BLOB, contentLength, in));
 						} finally {
 							in.close();
 						}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java
index 6fdab6bf89..6e28f854a2 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java
@@ -184,9 +184,10 @@ public abstract class ContentSource {
 				@Override
 				public ObjectStream openStream() throws MissingObjectException,
 						IOException {
+					long contentLength = ptr.getEntryContentLength();
 					InputStream in = ptr.openEntryStream();
 					in = new BufferedInputStream(in);
-					return new ObjectStream.Filter(getType(), getSize(), in);
+					return new ObjectStream.Filter(getType(), contentLength, in);
 				}
 
 				@Override
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
index ac92eb5c25..955d7d8575 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
@@ -72,7 +72,6 @@ import org.eclipse.jgit.ignore.IgnoreRule;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig;
-import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Repository;
@@ -129,6 +128,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	/** Repository that is the root level being iterated over */
 	protected Repository repository;
 
+	/** Cached canonical length, initialized from {@link #idBuffer()} */
+	private long canonLen = -1;
+
 	/**
 	 * Create a new iterator with no parent.
 	 *
@@ -320,33 +322,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 				state.initializeDigestAndReadBuffer();
 
 				final long len = e.getLength();
-				if (!mightNeedCleaning())
-					return computeHash(is, len);
-
-				if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) {
-					ByteBuffer rawbuf = IO.readWholeStream(is, (int) len);
-					byte[] raw = rawbuf.array();
-					int n = rawbuf.limit();
-					if (!isBinary(raw, n)) {
-						rawbuf = filterClean(raw, n);
-						raw = rawbuf.array();
-						n = rawbuf.limit();
-					}
-					return computeHash(new ByteArrayInputStream(raw, 0, n), n);
-				}
-
-				if (isBinary(e))
-					return computeHash(is, len);
-
-				final long canonLen;
-				final InputStream lenIs = filterClean(e.openInputStream());
-				try {
-					canonLen = computeLength(lenIs);
-				} finally {
-					safeClose(lenIs);
-				}
-
-				return computeHash(filterClean(is), canonLen);
+				InputStream filteredIs = possiblyFilteredInputStream(e, is, len);
+				return computeHash(filteredIs, canonLen);
 			} finally {
 				safeClose(is);
 			}
@@ -356,6 +333,43 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		}
 	}
 
+	private InputStream possiblyFilteredInputStream(final Entry e,
+			final InputStream is, final long len) throws IOException {
+		InputStream filteredIs;
+		if (!mightNeedCleaning()) {
+			filteredIs = is;
+			canonLen = len;
+		} else {
+			if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) {
+				ByteBuffer rawbuf = IO.readWholeStream(is, (int) len);
+				byte[] raw = rawbuf.array();
+				int n = rawbuf.limit();
+				if (!isBinary(raw, n)) {
+					rawbuf = filterClean(raw, n);
+					raw = rawbuf.array();
+					n = rawbuf.limit();
+				}
+				filteredIs = new ByteArrayInputStream(raw, 0, n);
+				canonLen = n;
+			} else {
+				if (isBinary(e)) {
+					filteredIs = is;
+					canonLen = len;
+				} else {
+					final InputStream lenIs = filterClean(e
+							.openInputStream());
+					try {
+						canonLen = computeLength(lenIs);
+					} finally {
+						safeClose(lenIs);
+					}
+					filteredIs = filterClean(is);
+				}
+			}
+		}
+		return filteredIs;
+	}
+
 	private static void safeClose(final InputStream in) {
 		try {
 			in.close();
@@ -441,8 +455,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	@Override
 	public void next(final int delta) throws CorruptObjectException {
 		ptr += delta;
-		if (!eof())
+		if (!eof()) {
+			canonLen = -1;
 			parseEntry();
+		}
 	}
 
 	@Override
@@ -462,7 +478,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	}
 
 	/**
-	 * Get the byte length of this entry.
+	 * Get the raw byte length of this entry.
 	 *
 	 * @return size of this file, in bytes.
 	 */
@@ -471,6 +487,29 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	}
 
 	/**
+	 * Get the filtered input length of this entry
+	 *
+	 * @return size of the content, in bytes
+	 * @throws IOException
+	 */
+	public long getEntryContentLength() throws IOException {
+		if (canonLen == -1) {
+			long rawLen = getEntryLength();
+			if (rawLen == 0)
+				canonLen = 0;
+			InputStream is = current().openInputStream();
+			try {
+				// canonLen gets updated here
+				possiblyFilteredInputStream(current(), is, current()
+						.getLength());
+			} finally {
+				safeClose(is);
+			}
+		}
+		return canonLen;
+	}
+
+	/**
 	 * Get the last modified time of this entry.
 	 *
 	 * @return last modified time of this file, in milliseconds since the epoch
@@ -498,12 +537,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	 */
 	public InputStream openEntryStream() throws IOException {
 		InputStream rawis = current().openInputStream();
-		InputStream is;
-		if (getOptions().getAutoCRLF() != AutoCRLF.FALSE)
-			is = new EolCanonicalizingInputStream(rawis, true);
+		if (mightNeedCleaning())
+			return filterClean(rawis);
 		else
-			is = rawis;
-		return is;
+			return rawis;
 	}
 
 	/**