DiffFormatter: correctly deal with tracked files in ignored folders

In JGit 5.0, the FileTreeIterator was changed to skip ignored folders
by default. To catch tracked files inside ignored folders, the tree
walk needs to have a DirCacheIterator, and the FileTreeIterator has
to know about that DirCacheIterator via setDirCacheIterator(). (Or
the optimization has to be switched off explicitly via
setWalkIgnoredDirectories(true).)

Skipping ignored directories is an important optimization in some
cases, for instance in node.js/npm projects, where we'd otherwise
traverse the whole huge and deep hierarchy of the typically ignored
node_modules folder.

While all uses of WorkingTreeIterator in JGit had been adapted,
DiffFormatter was forgotten. To make it work correctly (again) also
for such cases, make it set up a WorkingTreeeIterator automatically,
and make sure the WorkingTreeSource can find such files, too. Also
pass the repository to the TreeWalks used inside the DiffFormatter
to pick up the correct attributes, filters, and line-ending settings.

Bug: 565081
Change-Id: Ie88ac81166dc396ba28b83313964c1712b6ca199
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

3 files changed, 37 insertions, 7 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java
index 7ae005ada8..1a41df3d0a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/ContentSource.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2010, Google Inc. and others
+ * Copyright (C) 2010, 2020 Google Inc. and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -123,7 +123,8 @@ public abstract class ContentSource {
 		WorkingTreeIterator ptr;
 
 		WorkingTreeSource(WorkingTreeIterator iterator) {
-			this.tw = new TreeWalk((ObjectReader) null);
+			this.tw = new TreeWalk(iterator.getRepository(),
+					(ObjectReader) null);
 			this.tw.setRecursive(true);
 			this.iterator = iterator;
 		}
@@ -173,6 +174,15 @@ public abstract class ContentSource {
 		private void seek(String path) throws IOException {
 			if (!path.equals(current)) {
 				iterator.reset();
+				// Possibly this iterator had an associated DirCacheIterator,
+				// but we have no access to it and thus don't know about it.
+				// We have to reset this iterator here to work without
+				// DirCacheIterator and to descend always into ignored
+				// directories. Otherwise we might not find tracked files below
+				// ignored folders. Since we're looking only for a single
+				// specific path this is not a performance problem.
+				iterator.setWalkIgnoredDirectories(true);
+				iterator.setDirCacheIterator(null, -1);
 				tw.reset();
 				tw.addTree(iterator);
 				tw.setFilter(PathFilter.create(path));
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java
index 81367eaa08..ec21954aa2 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2009, Google Inc.
- * Copyright (C) 2008-2009, Johannes E. Schindelin <johannes.schindelin@gmx.de> and others
+ * Copyright (C) 2008-2020, Johannes E. Schindelin <johannes.schindelin@gmx.de> and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -502,9 +502,18 @@ public class DiffFormatter implements AutoCloseable {
 			throws IOException {
 		assertHaveReader();
 
-		TreeWalk walk = new TreeWalk(reader);
-		walk.addTree(a);
-		walk.addTree(b);
+		TreeWalk walk = new TreeWalk(repository, reader);
+		int aIndex = walk.addTree(a);
+		int bIndex = walk.addTree(b);
+		if (repository != null) {
+			if (a instanceof WorkingTreeIterator
+					&& b instanceof DirCacheIterator) {
+				((WorkingTreeIterator) a).setDirCacheIterator(walk, bIndex);
+			} else if (b instanceof WorkingTreeIterator
+					&& a instanceof DirCacheIterator) {
+				((WorkingTreeIterator) b).setDirCacheIterator(walk, aIndex);
+			}
+		}
 		walk.setRecursive(true);
 
 		TreeFilter filter = getDiffTreeFilterFor(a, b);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
index 994af2607c..4c26dd0c40 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
@@ -2,7 +2,7 @@
  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
  * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com>
  * Copyright (C) 2010, Matthias Sohn <matthias.sohn@sap.com>
- * Copyright (C) 2012-2013, Robin Rosenberg and others
+ * Copyright (C) 2012-2020, Robin Rosenberg and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -522,6 +522,17 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 		return state.options;
 	}
 
+	/**
+	 * Retrieves the {@link Repository} this {@link WorkingTreeIterator}
+	 * operates on.
+	 *
+	 * @return the {@link Repository}
+	 * @since 5.9
+	 */
+	public Repository getRepository() {
+		return repository;
+	}
+
 	/** {@inheritDoc} */
 	@Override
 	public int idOffset() {