Retry loose object read upon "Stale file handle" exception

When reading loose objects over NFS it is possible that the OS syscall
would fail with ESTALE errors: This happens when the open file
descriptor no longer refers to a valid file.

Notoriously it is possible to hit this scenario when git data is shared
among multiple clients, for example by multiple gerrit instances in HA.

If one of the two clients performs a GC operation that would cause the
packing and then the pruning of loose objects, the other client might
still hold a reference to those objects, which would cause an exception
to bubble up the stack.

The Linux NFS FAQ[1] (at point A.10), suggests that the proper way to
handle such ESTALE scenarios is to:

"[...] close the file or directory where the error occurred, and reopen
it so the NFS client can resolve the pathname again and retrieve the new
file handle."

In case of a stale file handle exception, we now attempt to read the
loose object again (up to 5 times), until we either succeed or encounter
a FileNotFoundException, in which case the search can continue to
Packfiles and alternates.

The limit of 5 provides an arbitrary upper bounds that is consistent to
the one chosen when handling stale file handles for packed-refs
files (see [2] for context).

[1] http://nfs.sourceforge.net/
[2] https://git.eclipse.org/r/c/jgit/jgit/+/54350

Bug: 573791
Change-Id: I9950002f772bbd8afeb9c6108391923be9d0ef51

1 files changed, 41 insertions, 0 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java
index d269457eb1..a0bc63a698 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java
@@ -44,8 +44,12 @@ package org.eclipse.jgit.internal.storage.file;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 
 import java.io.File;
 import java.io.IOException;
@@ -69,6 +73,7 @@ import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.util.FS;
 import org.junit.Assume;
 import org.junit.Test;
+import org.mockito.Mockito;
 
 public class ObjectDirectoryTest extends RepositoryTestCase {
 
@@ -195,6 +200,42 @@ public class ObjectDirectoryTest extends RepositoryTestCase {
 	}
 
 	@Test
+	public void testOpenLooseObjectSuppressStaleFileHandleException()
+			throws Exception {
+		ObjectId id = ObjectId
+				.fromString("873fb8d667d05436d728c52b1d7a09528e6eb59b");
+		WindowCursor curs = new WindowCursor(db.getObjectDatabase());
+
+		ObjectDirectory mock = mock(ObjectDirectory.class);
+		UnpackedObjectCache unpackedObjectCacheMock = mock(
+				UnpackedObjectCache.class);
+
+		Mockito.when(mock.getObjectLoader(any(), any(), any()))
+				.thenThrow(new IOException("Stale File Handle"));
+		Mockito.when(mock.openLooseObject(curs, id)).thenCallRealMethod();
+		Mockito.when(mock.unpackedObjectCache())
+				.thenReturn(unpackedObjectCacheMock);
+
+		assertNull(mock.openLooseObject(curs, id));
+		verify(unpackedObjectCacheMock).remove(id);
+	}
+
+	@Test
+	public void testOpenLooseObjectPropagatesIOExceptions() throws Exception {
+		ObjectId id = ObjectId
+				.fromString("873fb8d667d05436d728c52b1d7a09528e6eb59b");
+		WindowCursor curs = new WindowCursor(db.getObjectDatabase());
+
+		ObjectDirectory mock = mock(ObjectDirectory.class);
+
+		Mockito.when(mock.getObjectLoader(any(), any(), any()))
+				.thenThrow(new IOException("some IO failure"));
+		Mockito.when(mock.openLooseObject(curs, id)).thenCallRealMethod();
+
+		assertThrows(IOException.class, () -> mock.openLooseObject(curs, id));
+	}
+
+	@Test
 	public void testShallowFileCorrupt() throws Exception {
 		FileRepository repository = createBareRepository();
 		ObjectDirectory dir = repository.getObjectDatabase();