Refresh 'objects' dir and retry if a loose object is not found

A new loose object may not be immediately visible on a NFS
client if it was created on another client. Refreshing the
'objects' dir and trying again can help work around the NFS
behavior.

Here's an E2E problem that this change can help fix. Consider
a Gerrit multi-primary setup with repositories based on NFS.
Add a new patch-set to an existing change and then immediately
fetch the new patch-set of that change. If the fetch is handled
by a Gerrit primary different that the one which created the
patch-set, then we sometimes run into a MissingObjectException
that causes the fetch to fail.

Bug: 581317
Change-Id: Iccc6676c68ef13a1e8b2ff52b3eeca790a89a13d
Signed-off-by: Kaushik Lingarkar <quic_kaushikl@quicinc.com>

2 files changed, 65 insertions, 8 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java
index 33621a1e9f..b9af83d24d 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java
@@ -14,6 +14,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.StandardCopyOption;
@@ -24,6 +25,8 @@ import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.internal.storage.file.FileObjectDatabase.InsertLooseObjectResult;
 import org.eclipse.jgit.lib.AbbreviatedObjectId;
 import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.Config;
+import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -52,15 +55,22 @@ class LooseObjects {
 
 	private final UnpackedObjectCache unpackedObjectCache;
 
+	private final boolean trustFolderStat;
+
 	/**
 	 * Initialize a reference to an on-disk object directory.
 	 *
+	 * @param config
+	 *            configuration for the loose objects handler.
 	 * @param dir
 	 *            the location of the <code>objects</code> directory.
 	 */
-	LooseObjects(File dir) {
+	LooseObjects(Config config, File dir) {
 		directory = dir;
 		unpackedObjectCache = new UnpackedObjectCache();
+		trustFolderStat = config.getBoolean(
+				ConfigConstants.CONFIG_CORE_SECTION,
+				ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true);
 	}
 
 	/**
@@ -98,6 +108,19 @@ class LooseObjects {
 	 * @return {@code true} if the specified object is stored as a loose object.
 	 */
 	boolean has(AnyObjectId objectId) {
+		boolean exists = hasWithoutRefresh(objectId);
+		if (trustFolderStat || exists) {
+			return exists;
+		}
+		try (InputStream stream = Files.newInputStream(directory.toPath())) {
+			// refresh directory to work around NFS caching issue
+		} catch (IOException e) {
+			return false;
+		}
+		return hasWithoutRefresh(objectId);
+	}
+
+	private boolean hasWithoutRefresh(AnyObjectId objectId) {
 		return fileFor(objectId).exists();
 	}
 
@@ -183,6 +206,22 @@ class LooseObjects {
 	 */
 	ObjectLoader getObjectLoader(WindowCursor curs, File path, AnyObjectId id)
 			throws IOException {
+		try {
+			return getObjectLoaderWithoutRefresh(curs, path, id);
+		} catch (FileNotFoundException e) {
+			if (trustFolderStat) {
+				throw e;
+			}
+			try (InputStream stream = Files
+					.newInputStream(directory.toPath())) {
+				// refresh directory to work around NFS caching issues
+			}
+			return getObjectLoaderWithoutRefresh(curs, path, id);
+		}
+	}
+
+	private ObjectLoader getObjectLoaderWithoutRefresh(WindowCursor curs,
+			File path, AnyObjectId id) throws IOException {
 		try (FileInputStream in = new FileInputStream(path)) {
 			unpackedObjectCache().add(id);
 			return UnpackedObject.open(in, path, id, curs);
@@ -203,16 +242,34 @@ class LooseObjects {
 	}
 
 	long getSize(WindowCursor curs, AnyObjectId id) throws IOException {
+		try {
+			return getSizeWithoutRefresh(curs, id);
+		} catch (FileNotFoundException noFile) {
+			try {
+				if (trustFolderStat) {
+					throw noFile;
+				}
+				try (InputStream stream = Files
+						.newInputStream(directory.toPath())) {
+					// refresh directory to work around NFS caching issue
+				}
+				return getSizeWithoutRefresh(curs, id);
+			} catch (FileNotFoundException e) {
+				if (fileFor(id).exists()) {
+					throw noFile;
+				}
+				unpackedObjectCache().remove(id);
+				return -1;
+			}
+		}
+	}
+
+	private long getSizeWithoutRefresh(WindowCursor curs, AnyObjectId id)
+			throws IOException {
 		File f = fileFor(id);
 		try (FileInputStream in = new FileInputStream(f)) {
 			unpackedObjectCache().add(id);
 			return UnpackedObject.getSize(in, id, curs);
-		} catch (FileNotFoundException noFile) {
-			if (f.exists()) {
-				throw noFile;
-			}
-			unpackedObjectCache().remove(id);
-			return -1;
 		}
 	}
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
index 06c8cad3ac..531fd78cd0 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java
@@ -120,7 +120,7 @@ public class ObjectDirectory extends FileObjectDatabase {
 		File packDirectory = new File(objects, "pack"); //$NON-NLS-1$
 		File preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$
 		alternatesFile = new File(objects, Constants.INFO_ALTERNATES);
-		loose = new LooseObjects(objects);
+		loose = new LooseObjects(config, objects);
 		packed = new PackDirectory(config, packDirectory);
 		preserved = new PackDirectory(config, preservedDirectory);
 		this.fs = fs;