Merge branch 'stable-5.3'

* stable-5.3:
  Prepare 5.3.2-SNAPSHOT builds
  JGit v5.3.1.201904271842-r
  Prepare 5.2.3-SNAPSHOT builds
  JGit v5.2.2.201904231744-r
  Revert 4678f4b and provide another solution for bug 467631
  Apache MINA sshd: make sendKexInit() work also for re-keying
  Prepare 5.1.8-SNAPSHOT builds
  JGit v5.1.7.201904200442-r
  ObjectUploadListener: Add callback interface
  Prepare 4.11.9-SNAPSHOT builds
  JGit v4.11.8.201904181247-r
  Prepare 4.9.11-SNAPSHOT builds
  JGit v4.9.10.201904181027-r
  Prepare 4.7.10-SNAPSHOT builds
  JGit v4.7.9.201904161809-r
  Prepare 4.5.8-SNAPSHOT builds
  JGit v4.5.7.201904151645-r
  Remember the cause for invalidating a packfile
  Fix API problem filters
  Fix pack files scan when filesnapshot isn't modified

Change-Id: I8a8671f7767444a77b809bd66a27d776c8332736
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

16 files changed, 323 insertions, 95 deletions

diff --git a/org.eclipse.jgit.lfs.server/.settings/.api_filters b/org.eclipse.jgit.lfs.server/.settings/.api_filters
new file mode 100644
index 0000000000..6609c3d40d
--- /dev/null
+++ b/org.eclipse.jgit.lfs.server/.settings/.api_filters
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<component id="org.eclipse.jgit.lfs.server" version="2">
+    <resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener">
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.7"/>
+                <message_argument value="setCallback(ObjectUploadListener.Callback)"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener$Callback">
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.7"/>
+                <message_argument value="Callback"/>
+            </message_arguments>
+        </filter>
+    </resource>
+</component>
diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java
index c5b6a67876..3bb2899b9c 100644
--- a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java
+++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java
@@ -48,6 +48,7 @@ import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
+import java.nio.file.Path;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -87,6 +88,29 @@ public class ObjectUploadListener implements ReadListener {
 
 	private final ByteBuffer buffer = ByteBuffer.allocateDirect(8192);
 
+	private final Path path;
+
+	private long uploaded;
+
+	private Callback callback;
+
+	/**
+	 * Callback invoked after object upload completed.
+	 *
+	 * @since 5.1.7
+	 */
+	public interface Callback {
+		/**
+		 * Notified after object upload completed.
+		 *
+		 * @param path
+		 *            path to the object on the backend
+		 * @param size
+		 *            uploaded size in bytes
+		 */
+		void uploadCompleted(String path, long size);
+	}
+
 	/**
 	 * Constructor for ObjectUploadListener.
 	 *
@@ -113,10 +137,25 @@ public class ObjectUploadListener implements ReadListener {
 		this.inChannel = Channels.newChannel(in);
 		this.out = repository.getOutputStream(id);
 		this.channel = Channels.newChannel(out);
+		this.path = repository.getPath(id);
+		this.uploaded = 0L;
 		response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON);
 	}
 
 	/**
+	 * Set the callback to invoke after upload completed.
+	 *
+	 * @param callback
+	 *            the callback
+	 * @return {@code this}.
+	 * @since 5.1.7
+	 */
+	public ObjectUploadListener setCallback(Callback callback) {
+		this.callback = callback;
+		return this;
+	}
+
+	/**
 	 * {@inheritDoc}
 	 *
 	 * Writes all the received data to the output channel
@@ -126,12 +165,13 @@ public class ObjectUploadListener implements ReadListener {
 		while (in.isReady()) {
 			if (inChannel.read(buffer) > 0) {
 				buffer.flip();
-				channel.write(buffer);
+				uploaded += Integer.valueOf(channel.write(buffer)).longValue();
 				buffer.compact();
 			} else {
 				buffer.flip();
 				while (buffer.hasRemaining()) {
-					channel.write(buffer);
+					uploaded += Integer.valueOf(channel.write(buffer))
+							.longValue();
 				}
 				close();
 				return;
@@ -159,6 +199,9 @@ public class ObjectUploadListener implements ReadListener {
 			if (!response.isCommitted()) {
 				response.setStatus(HttpServletResponse.SC_OK);
 			}
+			if (callback != null) {
+				callback.uploadCompleted(path.toString(), uploaded);
+			}
 		} finally {
 			context.complete();
 		}
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java
index 9b4694c450..dcf330a2a4 100644
--- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java
+++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/JGitClientSession.java
@@ -149,10 +149,27 @@ public class JGitClientSession extends ClientSessionImpl {
 	@Override
 	protected IoWriteFuture sendIdentification(String ident)
 			throws IOException {
-		// Nothing; we do this below together with the KEX init in
-		// sendStartSsh(). Called only from the ClientSessionImpl constructor,
-		// where the return value is ignored.
-		return null;
+		StatefulProxyConnector proxy = proxyHandler;
+		if (proxy != null) {
+			try {
+				// We must not block here; the framework starts reading messages
+				// from the peer only once the initial sendKexInit() following
+				// this call to sendIdentification() has returned!
+				proxy.runWhenDone(() -> {
+					JGitClientSession.super.sendIdentification(ident);
+					return null;
+				});
+				// Called only from the ClientSessionImpl constructor, where the
+				// return value is ignored.
+				return null;
+			} catch (IOException e) {
+				throw e;
+			} catch (Exception other) {
+				throw new IOException(other.getLocalizedMessage(), other);
+			}
+		} else {
+			return super.sendIdentification(ident);
+		}
 	}
 
 	@Override
@@ -161,12 +178,13 @@ public class JGitClientSession extends ClientSessionImpl {
 		if (proxy != null) {
 			try {
 				// We must not block here; the framework starts reading messages
-				// from the peer only once sendKexInit() has returned!
+				// from the peer only once the initial sendKexInit() has
+				// returned!
 				proxy.runWhenDone(() -> {
-					sendStartSsh();
+					JGitClientSession.super.sendKexInit();
 					return null;
 				});
-				// sendKexInit() is called only from the ClientSessionImpl
+				// This is called only from the ClientSessionImpl
 				// constructor, where the return value is ignored.
 				return null;
 			} catch (IOException e) {
@@ -175,24 +193,11 @@ public class JGitClientSession extends ClientSessionImpl {
 				throw new IOException(other.getLocalizedMessage(), other);
 			}
 		} else {
-			return sendStartSsh();
+			return super.sendKexInit();
 		}
 	}
 
 	/**
-	 * Sends the initial messages starting the ssh setup: the client
-	 * identification and the KEX init message.
-	 *
-	 * @return the client's KEX seed
-	 * @throws IOException
-	 *             if something goes wrong
-	 */
-	private byte[] sendStartSsh() throws IOException {
-		super.sendIdentification(clientVersion);
-		return super.sendKexInit();
-	}
-
-	/**
 	 * {@inheritDoc}
 	 *
 	 * As long as we're still setting up the proxy connection, diverts messages
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/AbstractClientProxyConnector.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/AbstractClientProxyConnector.java
index 444fbb62ed..2e87c57332 100644
--- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/AbstractClientProxyConnector.java
+++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/AbstractClientProxyConnector.java
@@ -43,7 +43,9 @@
 package org.eclipse.jgit.internal.transport.sshd.proxy;
 
 import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
@@ -61,12 +63,12 @@ public abstract class AbstractClientProxyConnector
 	private static final long DEFAULT_PROXY_TIMEOUT_MILLIS = TimeUnit.SECONDS
 			.toMillis(30L);
 
-	/** Guards {@link #done} and {@link #startSsh}. */
+	/** Guards {@link #done} and {@link #bufferedCommands}. */
 	private Object lock = new Object();
 
 	private boolean done;
 
-	private Callable<Void> startSsh;
+	private List<Callable<Void>> bufferedCommands = new ArrayList<>();
 
 	private AtomicReference<Runnable> unregister = new AtomicReference<>();
 
@@ -173,18 +175,20 @@ public abstract class AbstractClientProxyConnector
 	 *             if starting ssh fails
 	 */
 	protected void setDone(boolean success) throws Exception {
-		Callable<Void> starter;
+		List<Callable<Void>> buffered;
 		Runnable unset = unregister.getAndSet(null);
 		if (unset != null) {
 			unset.run();
 		}
 		synchronized (lock) {
 			done = true;
-			starter = startSsh;
-			startSsh = null;
+			buffered = bufferedCommands;
+			bufferedCommands = null;
 		}
-		if (success && starter != null) {
-			starter.call();
+		if (success && buffered != null) {
+			for (Callable<Void> starter : buffered) {
+				starter.call();
+			}
 		}
 	}
 
@@ -192,7 +196,7 @@ public abstract class AbstractClientProxyConnector
 	public void runWhenDone(Callable<Void> starter) throws Exception {
 		synchronized (lock) {
 			if (!done) {
-				this.startSsh = starter;
+				bufferedCommands.add(starter);
 				return;
 			}
 		}
diff --git a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/StatefulProxyConnector.java b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/StatefulProxyConnector.java
index 0d8e0f93e5..6bd836a587 100644
--- a/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/StatefulProxyConnector.java
+++ b/org.eclipse.jgit.ssh.apache/src/org/eclipse/jgit/internal/transport/sshd/proxy/StatefulProxyConnector.java
@@ -78,12 +78,15 @@ public interface StatefulProxyConnector extends ClientProxyConnector {
 	void messageReceived(IoSession session, Readable buffer) throws Exception;
 
 	/**
-	 * Runs {@code startSsh} once the proxy connection is established.
+	 * Runs {@code command} once the proxy connection is established. May be
+	 * called multiple times; commands are run sequentially. If the proxy
+	 * connection is already established, {@code command} is executed directly
+	 * synchronously.
 	 *
-	 * @param startSsh
+	 * @param command
 	 *            operation to run
 	 * @throws Exception
 	 *             if the operation is run synchronously and throws an exception
 	 */
-	void runWhenDone(Callable<Void> startSsh) throws Exception;
+	void runWhenDone(Callable<Void> command) throws Exception;
 }
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffEntryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffEntryTest.java
index 73c230ac68..de768118bf 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffEntryTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffEntryTest.java
@@ -59,9 +59,12 @@ import org.eclipse.jgit.diff.DiffEntry.ChangeType;
 import org.eclipse.jgit.dircache.DirCache;
 import org.eclipse.jgit.dircache.DirCacheEditor;
 import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
 import org.eclipse.jgit.dircache.DirCacheEntry;
+import org.eclipse.jgit.junit.JGitTestUtil;
 import org.eclipse.jgit.junit.RepositoryTestCase;
 import org.eclipse.jgit.lib.FileMode;
+import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.revwalk.RevCommit;
 import org.eclipse.jgit.treewalk.EmptyTreeIterator;
 import org.eclipse.jgit.treewalk.FileTreeIterator;
@@ -417,4 +420,64 @@ public class DiffEntryTest extends RepositoryTestCase {
 			assertEquals(FileMode.REGULAR_FILE, diff.getOldMode());
 		}
 	}
+
+	@Test
+	public void shouldReportSubmoduleReplacedByFileMove() throws Exception {
+		// Create a submodule
+		FileRepository submoduleStandalone = createWorkRepository();
+		JGitTestUtil.writeTrashFile(submoduleStandalone, "fileInSubmodule",
+				"submodule");
+		Git submoduleStandaloneGit = Git.wrap(submoduleStandalone);
+		submoduleStandaloneGit.add().addFilepattern("fileInSubmodule").call();
+		submoduleStandaloneGit.commit().setMessage("add file to submodule")
+				.call();
+
+		Repository submodule_db = Git.wrap(db).submoduleAdd()
+				.setPath("modules/submodule")
+				.setURI(submoduleStandalone.getDirectory().toURI().toString())
+				.call();
+		File submodule_trash = submodule_db.getWorkTree();
+		addRepoToClose(submodule_db);
+		writeTrashFile("fileInRoot", "root");
+		Git rootGit = Git.wrap(db);
+		rootGit.add().addFilepattern("fileInRoot").call();
+		rootGit.commit().setMessage("add submodule and root file").call();
+		// Dummy change on fileInRoot
+		writeTrashFile("fileInRoot", "changed");
+		rootGit.add().addFilepattern("fileInRoot").call();
+		RevCommit firstCommit = rootGit.commit().setMessage("change root file")
+				.call();
+		// Remove the submodule again and move fileInRoot into that subfolder
+		rootGit.rm().setCached(true).addFilepattern("modules/submodule").call();
+		recursiveDelete(submodule_trash);
+		JGitTestUtil.deleteTrashFile(db, "fileInRoot");
+		// Move the fileInRoot file
+		writeTrashFile("modules/submodule/fileInRoot", "changed");
+		rootGit.rm().addFilepattern("fileInRoot").addFilepattern("modules/")
+				.call();
+		rootGit.add().addFilepattern("modules/").call();
+		RevCommit secondCommit = rootGit.commit()
+				.setMessage("remove submodule and move root file")
+				.call();
+		// Diff should report submodule having been deleted and file moved
+		// (deleted and added)
+		try (TreeWalk walk = new TreeWalk(db)) {
+			walk.addTree(firstCommit.getTree());
+			walk.addTree(secondCommit.getTree());
+			walk.setRecursive(true);
+			List<DiffEntry> diffs = DiffEntry.scan(walk);
+			assertEquals(3, diffs.size());
+			DiffEntry e = diffs.get(0);
+			assertEquals(DiffEntry.ChangeType.DELETE, e.getChangeType());
+			assertEquals("fileInRoot", e.getOldPath());
+			e = diffs.get(1);
+			assertEquals(DiffEntry.ChangeType.DELETE, e.getChangeType());
+			assertEquals("modules/submodule", e.getOldPath());
+			assertEquals(FileMode.GITLINK, e.getOldMode());
+			e = diffs.get(2);
+			assertEquals(DiffEntry.ChangeType.ADD, e.getChangeType());
+			assertEquals("modules/submodule/fileInRoot", e.getNewPath());
+		}
+
+	}
 }
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffSubmoduleTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffSubmoduleTest.java
index 93ada4f301..fa7f5ab522 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffSubmoduleTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffSubmoduleTest.java
@@ -43,12 +43,14 @@
 
 package org.eclipse.jgit.lib;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Set;
 
 import org.eclipse.jgit.api.CloneCommand;
@@ -128,7 +130,8 @@ public class IndexDiffSubmoduleTest extends RepositoryTestCase {
 		IndexDiff indexDiff = new IndexDiff(db2, Constants.HEAD,
 				new FileTreeIterator(db2));
 		indexDiff.setIgnoreSubmoduleMode(mode);
-		assertFalse(indexDiff.diff());
+		boolean changed = indexDiff.diff();
+		assertFalse(changed);
 	}
 
 	@Theory
@@ -263,4 +266,33 @@ public class IndexDiffSubmoduleTest extends RepositoryTestCase {
 		assertDiff(indexDiff, mode, IgnoreSubmoduleMode.ALL,
 				IgnoreSubmoduleMode.DIRTY, IgnoreSubmoduleMode.UNTRACKED);
 	}
+
+	@Theory
+	public void testSubmoduleReplacedByMovedFile(IgnoreSubmoduleMode mode)
+			throws Exception {
+		Git git = Git.wrap(db);
+		git.rm().setCached(true).addFilepattern("modules/submodule").call();
+		recursiveDelete(submodule_trash);
+		JGitTestUtil.deleteTrashFile(db, "fileInRoot");
+		// Move the fileInRoot file
+		writeTrashFile("modules/submodule/fileInRoot", "root");
+		git.rm().addFilepattern("fileInRoot").addFilepattern("modules/").call();
+		git.add().addFilepattern("modules/").call();
+		IndexDiff indexDiff = new IndexDiff(db, Constants.HEAD,
+				new FileTreeIterator(db));
+		indexDiff.setIgnoreSubmoduleMode(mode);
+		assertTrue(indexDiff.diff());
+		String[] removed = indexDiff.getRemoved().toArray(new String[0]);
+		Arrays.sort(removed);
+		if (IgnoreSubmoduleMode.ALL.equals(mode)) {
+			assertArrayEquals(new String[] { "fileInRoot" }, removed);
+		} else {
+			assertArrayEquals(
+					new String[] { "fileInRoot", "modules/submodule" },
+					removed);
+		}
+		assertEquals("[modules/submodule/fileInRoot]",
+				indexDiff.getAdded().toString());
+	}
+
 }
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/PathsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/PathsTest.java
index f78cbe2601..7542ec8910 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/PathsTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/PathsTest.java
@@ -89,29 +89,14 @@ public class PathsTest {
 				a, 0, a.length, FileMode.TREE.getBits(),
 				b, 0, b.length, FileMode.TREE.getBits()));
 		assertEquals(0, compare(
-				a, 0, a.length, FileMode.TREE.getBits(),
-				b, 0, b.length, FileMode.GITLINK.getBits()));
-		assertEquals(0, compare(
-				a, 0, a.length, FileMode.GITLINK.getBits(),
-				b, 0, b.length, FileMode.GITLINK.getBits()));
-		assertEquals(0, compare(
-				a, 0, a.length, FileMode.GITLINK.getBits(),
-				b, 0, b.length, FileMode.TREE.getBits()));
-		assertEquals(0, compare(
 				a, 0, a.length, FileMode.REGULAR_FILE.getBits(),
 				b, 0, b.length, FileMode.REGULAR_FILE.getBits()));
 		assertEquals(-47, compare(
 				a, 0, a.length, FileMode.REGULAR_FILE.getBits(),
 				b, 0, b.length, FileMode.TREE.getBits()));
-		assertEquals(0, compare(
-				a, 0, a.length, FileMode.REGULAR_FILE.getBits(),
-				b, 0, b.length, FileMode.GITLINK.getBits()));
 		assertEquals(47, compare(
 				a, 0, a.length, FileMode.TREE.getBits(),
 				b, 0, b.length, FileMode.REGULAR_FILE.getBits()));
-		assertEquals(0, compare(
-				a, 0, a.length, FileMode.GITLINK.getBits(),
-				b, 0, b.length, FileMode.REGULAR_FILE.getBits()));
 
 		assertEquals(0, compareSameName(
 				a, 0, a.length,
@@ -119,9 +104,6 @@ public class PathsTest {
 		assertEquals(0, compareSameName(
 				a, 0, a.length,
 				b, 0, b.length, FileMode.REGULAR_FILE.getBits()));
-		assertEquals(0, compareSameName(
-				a, 0, a.length,
-				b, 0, b.length, FileMode.GITLINK.getBits()));
 
 		a = Constants.encode("a.c");
 		b = Constants.encode("a");
diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters
index 9392534b13..089506c6a5 100644
--- a/org.eclipse.jgit/.settings/.api_filters
+++ b/org.eclipse.jgit/.settings/.api_filters
@@ -1,5 +1,19 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <component id="org.eclipse.jgit" version="2">
+    <resource path="src/org/eclipse/jgit/errors/PackInvalidException.java" type="org.eclipse.jgit.errors.PackInvalidException">
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="4.5.7"/>
+                <message_argument value="PackInvalidException(File, Throwable)"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="4.5.7"/>
+                <message_argument value="PackInvalidException(String, Throwable)"/>
+            </message_arguments>
+        </filter>
+    </resource>
     <resource path="src/org/eclipse/jgit/lib/Ref.java" type="org.eclipse.jgit.lib.Ref">
         <filter id="403767336">
             <message_arguments>
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java
index a83de06262..b238ab4718 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java
@@ -60,9 +60,24 @@ public class PackInvalidException extends IOException {
 	 *
 	 * @param path
 	 *            path of the invalid pack file.
+	 * @deprecated Use {@link #PackInvalidException(File, Throwable)}.
 	 */
+	@Deprecated
 	public PackInvalidException(File path) {
-		this(path.getAbsolutePath());
+		this(path, null);
+	}
+
+	/**
+	 * Construct a pack invalid error with cause.
+	 *
+	 * @param path
+	 *            path of the invalid pack file.
+	 * @param cause
+	 *            cause of the pack file becoming invalid.
+	 * @since 4.5.7
+	 */
+	public PackInvalidException(File path, Throwable cause) {
+		this(path.getAbsolutePath(), cause);
 	}
 
 	/**
@@ -70,8 +85,23 @@ public class PackInvalidException extends IOException {
 	 *
 	 * @param path
 	 *            path of the invalid pack file.
+	 * @deprecated Use {@link #PackInvalidException(String, Throwable)}.
 	 */
+	@Deprecated
 	public PackInvalidException(String path) {
-		super(MessageFormat.format(JGitText.get().packFileInvalid, path));
+		this(path, null);
+	}
+
+	/**
+	 * Construct a pack invalid error with cause.
+	 *
+	 * @param path
+	 *            path of the invalid pack file.
+	 * @param cause
+	 *            cause of the pack file becoming invalid.
+	 * @since 4.5.7
+	 */
+	public PackInvalidException(String path, Throwable cause) {
+		super(MessageFormat.format(JGitText.get().packFileInvalid, path), cause);
 	}
 }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
index 4e4a26f4e6..c7d6c584e4 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/BlockBasedFile.java
@@ -83,6 +83,9 @@ abstract class BlockBasedFile {
 	/** True once corruption has been detected that cannot be worked around. */
 	volatile boolean invalid;
 
+	/** Exception that caused the packfile to be flagged as invalid */
+	protected volatile Exception invalidatingCause;
+
 	BlockBasedFile(DfsBlockCache cache, DfsPackDescription desc, PackExt ext) {
 		this.cache = cache;
 		this.key = desc.getStreamKey(ext);
@@ -136,8 +139,9 @@ abstract class BlockBasedFile {
 
 	DfsBlock readOneBlock(long pos, DfsReader ctx, ReadableChannel rc)
 			throws IOException {
-		if (invalid)
-			throw new PackInvalidException(getFileName());
+		if (invalid) {
+			throw new PackInvalidException(getFileName(), invalidatingCause);
+		}
 
 		ctx.stats.readBlock++;
 		long start = System.nanoTime();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
index 451986e831..be1387ed0c 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java
@@ -181,7 +181,7 @@ public final class DfsPackFile extends BlockBasedFile {
 		}
 
 		if (invalid) {
-			throw new PackInvalidException(getFileName());
+			throw new PackInvalidException(getFileName(), invalidatingCause);
 		}
 
 		Repository.getGlobalListenerList()
@@ -240,6 +240,7 @@ public final class DfsPackFile extends BlockBasedFile {
 				return index;
 			} catch (IOException e) {
 				invalid = true;
+				invalidatingCause = e;
 				throw e;
 			}
 		}
@@ -703,8 +704,10 @@ public final class DfsPackFile extends BlockBasedFile {
 
 	private IOException packfileIsTruncated() {
 		invalid = true;
-		return new IOException(MessageFormat.format(
+		IOException exc = new IOException(MessageFormat.format(
 				JGitText.get().packfileIsTruncated, getFileName()));
+		invalidatingCause = exc;
+		return exc;
 	}
 
 	private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
index 3ffb8c424d..d502fea020 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
@@ -132,8 +132,6 @@ public class ObjectDirectory extends FileObjectDatabase {
 
 	private final File alternatesFile;
 
-	private final AtomicReference<PackList> packList;
-
 	private final FS fs;
 
 	private final AtomicReference<AlternateHandle[]> alternates;
@@ -146,6 +144,8 @@ public class ObjectDirectory extends FileObjectDatabase {
 
 	private Set<ObjectId> shallowCommitsIds;
 
+	final AtomicReference<PackList> packList;
+
 	/**
 	 * Initialize a reference to an on-disk object directory.
 	 *
@@ -673,13 +673,8 @@ public class ObjectDirectory extends FileObjectDatabase {
 			transientErrorCount = p.incrementTransientErrorCount();
 		}
 		if (warnTmpl != null) {
-			if (LOG.isDebugEnabled()) {
-				LOG.debug(MessageFormat.format(warnTmpl,
-						p.getPackFile().getAbsolutePath()), e);
-			} else {
-				LOG.warn(MessageFormat.format(warnTmpl,
-						p.getPackFile().getAbsolutePath()));
-			}
+			LOG.warn(MessageFormat.format(warnTmpl,
+					p.getPackFile().getAbsolutePath()), e);
 		} else {
 			if (doLogExponentialBackoff(transientErrorCount)) {
 				// Don't remove the pack from the list, as the error may be
@@ -766,7 +761,7 @@ public class ObjectDirectory extends FileObjectDatabase {
 		return InsertLooseObjectResult.FAILURE;
 	}
 
-	private boolean searchPacksAgain(PackList old) {
+	boolean searchPacksAgain(PackList old) {
 		// Whether to trust the pack folder's modification time. If set
 		// to false we will always scan the .git/objects/pack folder to
 		// check for new pack files. If set to true (default) we use the
@@ -916,7 +911,8 @@ public class ObjectDirectory extends FileObjectDatabase {
 			final String packName = base + PACK.getExtension();
 			final File packFile = new File(packDirectory, packName);
 			final PackFile oldPack = forReuse.remove(packName);
-			if (oldPack != null && oldPack.getFileSnapshot().isModified(packFile)) {
+			if (oldPack != null
+					&& !oldPack.getFileSnapshot().isModified(packFile)) {
 				list.add(oldPack);
 				continue;
 			}
@@ -1073,7 +1069,7 @@ public class ObjectDirectory extends FileObjectDatabase {
 		return new File(new File(getDirectory(), d), f);
 	}
 
-	private static final class PackList {
+	static final class PackList {
 		/** State just before reading the pack directory. */
 		final FileSnapshot snapshot;
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
index ee3c0a8dd9..d834b1a729 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
@@ -135,6 +135,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 
 	private volatile boolean invalid;
 
+	private volatile Exception invalidatingCause;
+
 	private boolean invalidBitmap;
 
 	private AtomicInteger transientErrorCount = new AtomicInteger();
@@ -184,7 +186,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 				idx = loadedIdx;
 				if (idx == null) {
 					if (invalid) {
-						throw new PackInvalidException(packFile);
+						throw new PackInvalidException(packFile, invalidatingCause);
 					}
 					try {
 						idx = PackIndex.open(extFile(INDEX));
@@ -208,6 +210,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 						throw e;
 					} catch (IOException e) {
 						invalid = true;
+						invalidatingCause = e;
 						throw e;
 					}
 				}
@@ -661,7 +664,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 
 	private void doOpen() throws IOException {
 		if (invalid) {
-			throw new PackInvalidException(packFile);
+			throw new PackInvalidException(packFile, invalidatingCause);
 		}
 		try {
 			synchronized (readLock) {
@@ -671,13 +674,13 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 			}
 		} catch (InterruptedIOException e) {
 			// don't invalidate the pack, we are interrupted from another thread
-			openFail(false);
+			openFail(false, e);
 			throw e;
 		} catch (FileNotFoundException fn) {
 			// don't invalidate the pack if opening an existing file failed
 			// since it may be related to a temporary lack of resources (e.g.
 			// max open files)
-			openFail(!packFile.exists());
+			openFail(!packFile.exists(), fn);
 			throw fn;
 		} catch (EOFException | AccessDeniedException | NoSuchFileException
 				| CorruptObjectException | NoPackSignatureException
@@ -685,20 +688,21 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 				| UnsupportedPackIndexVersionException
 				| UnsupportedPackVersionException pe) {
 			// exceptions signaling permanent problems with a pack
-			openFail(true);
+			openFail(true, pe);
 			throw pe;
 		} catch (IOException | RuntimeException ge) {
 			// generic exceptions could be transient so we should not mark the
 			// pack invalid to avoid false MissingObjectExceptions
-			openFail(false);
+			openFail(false, ge);
 			throw ge;
 		}
 	}
 
-	private void openFail(boolean invalidate) {
+	private void openFail(boolean invalidate, Exception cause) {
 		activeWindows = 0;
 		activeCopyRawData = 0;
 		invalid = invalidate;
+		invalidatingCause = cause;
 		doClose();
 	}
 
@@ -725,7 +729,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 				// Detect the situation and throw a proper exception so that can be properly
 				// managed by the main packfile search loop and the Git client won't receive
 				// any failures.
-				throw new PackInvalidException(packFile);
+				throw new PackInvalidException(packFile, invalidatingCause);
 			}
 			if (length < pos + size)
 				size = (int) (length - pos);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java
index 94b9ddc188..f37c310752 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java
@@ -47,7 +47,11 @@
 
 package org.eclipse.jgit.lib;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.DirectoryIteratorException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -261,6 +265,8 @@ public class IndexDiff {
 
 	private Set<String> missing = new HashSet<>();
 
+	private Set<String> missingSubmodules = new HashSet<>();
+
 	private Set<String> modified = new HashSet<>();
 
 	private Set<String> untracked = new HashSet<>();
@@ -501,9 +507,15 @@ public class IndexDiff {
 				if (dirCacheIterator != null) {
 					if (workingTreeIterator == null) {
 						// in index, not in workdir => missing
-						if (!isEntryGitLink(dirCacheIterator)
-								|| ignoreSubmoduleMode != IgnoreSubmoduleMode.ALL)
-							missing.add(treeWalk.getPathString());
+						boolean isGitLink = isEntryGitLink(dirCacheIterator);
+						if (!isGitLink
+								|| ignoreSubmoduleMode != IgnoreSubmoduleMode.ALL) {
+							String path = treeWalk.getPathString();
+							missing.add(path);
+							if (isGitLink) {
+								missingSubmodules.add(path);
+							}
+						}
 					} else {
 						if (workingTreeIterator.isModified(
 								dirCacheIterator.getDirCacheEntry(), true,
@@ -543,8 +555,8 @@ public class IndexDiff {
 							smw.getPath()), e);
 				}
 				try (Repository subRepo = smw.getRepository()) {
+					String subRepoPath = smw.getPath();
 					if (subRepo != null) {
-						String subRepoPath = smw.getPath();
 						ObjectId subHead = subRepo.resolve("HEAD"); //$NON-NLS-1$
 						if (subHead != null
 								&& !subHead.equals(smw.getObjectId())) {
@@ -573,6 +585,21 @@ public class IndexDiff {
 								recordFileMode(subRepoPath, FileMode.GITLINK);
 							}
 						}
+					} else if (missingSubmodules.remove(subRepoPath)) {
+						// If the directory is there and empty but the submodule
+						// repository in .git/modules doesn't exist yet it isn't
+						// "missing".
+						File gitDir = new File(
+								new File(repository.getDirectory(),
+										Constants.MODULES),
+								subRepoPath);
+						if (!gitDir.isDirectory()) {
+							File dir = SubmoduleWalk.getSubmoduleDirectory(
+									repository, subRepoPath);
+							if (dir.isDirectory() && !hasFiles(dir)) {
+								missing.remove(subRepoPath);
+							}
+						}
 					}
 				}
 			}
@@ -592,6 +619,15 @@ public class IndexDiff {
 			return true;
 	}
 
+	private boolean hasFiles(File directory) {
+		try (DirectoryStream<java.nio.file.Path> dir = Files
+				.newDirectoryStream(directory.toPath())) {
+			return dir.iterator().hasNext();
+		} catch (DirectoryIteratorException | IOException e) {
+			return false;
+		}
+	}
+
 	private void recordFileMode(String path, FileMode mode) {
 		Set<String> values = fileModes.get(mode);
 		if (path != null) {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Paths.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Paths.java
index be1b95e0d4..6be7ddbe12 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/util/Paths.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Paths.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016, 2018 Google Inc.
+ * Copyright (C) 2016, Google Inc.
  * and other copyright owners as documented in the project's IP log.
  *
  * This program and the accompanying materials are made available
@@ -43,7 +43,6 @@
 
 package org.eclipse.jgit.util;
 
-import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK;
 import static org.eclipse.jgit.lib.FileMode.TYPE_MASK;
 import static org.eclipse.jgit.lib.FileMode.TYPE_TREE;
 
@@ -107,7 +106,7 @@ public class Paths {
 				aPath, aPos, aEnd, aMode,
 				bPath, bPos, bEnd, bMode);
 		if (cmp == 0) {
-			cmp = modeCompare(aMode, bMode);
+			cmp = lastPathChar(aMode) - lastPathChar(bMode);
 		}
 		return cmp;
 	}
@@ -184,15 +183,6 @@ public class Paths {
 		return 0;
 	}
 
-	private static int modeCompare(int aMode, int bMode) {
-		if ((aMode & TYPE_MASK) == TYPE_GITLINK
-				|| (bMode & TYPE_MASK) == TYPE_GITLINK) {
-			// Git links can be equal to files or folders
-			return 0;
-		}
-		return lastPathChar(aMode) - lastPathChar(bMode);
-	}
-
 	private Paths() {
 	}
 }