Move reachability checker generation into the ObjectReader object

Reachability checkers are retrieved from RevWalk and ObjectWalk objects:
* RevWalk.createReachabilityChecker()
* ObjectWalk.createObjectReachabilityChecker()

Since RevWalks and ObjectWalks are themselves directly instantiated
in hundreds of places (e.g. UploadPack...) overriding them in a
consistent way requires overloading 100s of methods, which isn't
feasible. Moving reachability checker generation to a more central
place solves that problem.

The ObjectReader object seems a good place from which to get
reachability checkers, because reachability checkers return
information about relationships between objects. ObjectDatabases
delegate many operations to ObjectReaders, and reachability bitmaps
are attached to ObjectReaders.

The Bitmapped and Pedestrian reachability checker objects were
package private in the org.eclipse.jgit.revwalk package. This change
makes them public and moves them to the
org.eclipse.jgit.internal.revwalk package. Corresponding tests are
also moved.

Motivation:
1) Reachability checking algorithms need to scale. One of the
   internal Android repositories has ~2.4 million refs/changes/*
   references, causing bad long tail performance in reachability
   checks.
2) Reachability check performance is impacted by repository
   topography: number of refs, number of objects, amounts of
   related vs. unrelated history.
3) Reachability check performance is also affected by per-branch
   access (Gerrit branch permissions) since different users can
   see different branches.
4) Reachability check performance isn't affected by any state in a
   RevWalk or ObjectWalk.

I don't yet know if a single algorithm will work for all cases in #2
and #3. We may need to evolve the ReachabilityChecker interfaces
over time to solve the Gerrit branch permissions case, or use
Gerrit-specific identity information to solve that in an efficient
way.

This change takes the existing public API and moves it to the
ObjectReader/whole repository level, which is where we can do
consistent customizations for #2 and #3. We intend to upstream the
best of whatever works, but anticipate the need for multiple rounds
of experimentation.

Change-Id: I9185feff43551fb387957c436112d5250486833d
Signed-off-by: Terry Parker <tparker@google.com>

org.eclipse.jgit.test/META-INF/MANIFEST.MF

@@ -34,6 +34,7 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)",
  org.eclipse.jgit.ignore.internal;version="[5.11.0,5.12.0)",
  org.eclipse.jgit.internal;version="[5.11.0,5.12.0)",
  org.eclipse.jgit.internal.fsck;version="[5.11.0,5.12.0)",
+ org.eclipse.jgit.internal.revwalk;version="[5.11.0,5.12.0)",
  org.eclipse.jgit.internal.storage.dfs;version="[5.11.0,5.12.0)",
  org.eclipse.jgit.internal.storage.file;version="[5.11.0,5.12.0)",
  org.eclipse.jgit.internal.storage.io;version="[5.11.0,5.12.0)",

org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityTest.java → org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityTest.java

@@ -7,11 +7,12 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import org.eclipse.jgit.internal.storage.file.FileRepository;
 import org.eclipse.jgit.internal.storage.file.GC;
 import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.revwalk.ObjectReachabilityChecker;
 
 public class BitmappedObjectReachabilityTest
 	extends ObjectReachabilityTestCase {

org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/BitmappedReachabilityCheckerTest.java → org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityCheckerTest.java

@@ -7,13 +7,14 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import static org.junit.Assert.assertNotNull;
 
 import org.eclipse.jgit.internal.storage.file.FileRepository;
 import org.eclipse.jgit.internal.storage.file.GC;
 import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.revwalk.ReachabilityChecker;
 
 public class BitmappedReachabilityCheckerTest
 		extends ReachabilityCheckerTestCase {

org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ObjectReachabilityTestCase.java → org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ObjectReachabilityTestCase.java

@@ -7,7 +7,7 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -20,6 +20,10 @@ import org.eclipse.jgit.internal.storage.file.FileRepository;
 import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
 import org.eclipse.jgit.junit.TestRepository;
 import org.eclipse.jgit.junit.TestRepository.CommitBuilder;
+import org.eclipse.jgit.revwalk.ObjectReachabilityChecker;
+import org.eclipse.jgit.revwalk.RevBlob;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevObject;
 import org.junit.Before;
 import org.junit.Test;
 

org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityTest.java → org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityTest.java

@@ -7,10 +7,11 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import org.eclipse.jgit.internal.storage.file.FileRepository;
 import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.revwalk.ObjectReachabilityChecker;
 
 public class PedestrianObjectReachabilityTest
 		extends ObjectReachabilityTestCase {

org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/PedestrianReachabilityCheckerTest.java → org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityCheckerTest.java

@@ -7,10 +7,11 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import org.eclipse.jgit.internal.storage.file.FileRepository;
 import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.revwalk.ReachabilityChecker;
 
 public class PedestrianReachabilityCheckerTest
 		extends ReachabilityCheckerTestCase {

org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/ReachabilityCheckerTestCase.java → org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/revwalk/ReachabilityCheckerTestCase.java

@@ -7,7 +7,7 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -19,6 +19,8 @@ import java.util.stream.Stream;
 import org.eclipse.jgit.internal.storage.file.FileRepository;
 import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
 import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.revwalk.ReachabilityChecker;
+import org.eclipse.jgit.revwalk.RevCommit;
 import org.junit.Before;
 import org.junit.Test;
 

org.eclipse.jgit/.settings/.api_filters

@@ -8,4 +8,20 @@
             </message_arguments>
         </filter>
     </resource>
+    <resource path="src/org/eclipse/jgit/revwalk/ObjectWalk.java" type="org.eclipse.jgit.revwalk.ObjectWalk">
+        <filter id="421654647">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.revwalk.ObjectWalk"/>
+                <message_argument value="createObjectReachabilityChecker()"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/revwalk/RevWalk.java" type="org.eclipse.jgit.revwalk.RevWalk">
+        <filter id="421654647">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.revwalk.RevWalk"/>
+                <message_argument value="createReachabilityChecker()"/>
+            </message_arguments>
+        </filter>
+    </resource>
 </component>

org.eclipse.jgit/META-INF/MANIFEST.MF

@@ -72,7 +72,8 @@ Export-Package: org.eclipse.jgit.annotations;version="5.11.0",
    org.eclipse.jgit.http.test",
  org.eclipse.jgit.internal.fsck;version="5.11.0";
   x-friends:="org.eclipse.jgit.test",
- org.eclipse.jgit.internal.revwalk;version="5.11.0";x-internal:=true,
+ org.eclipse.jgit.internal.revwalk;version="5.11.0";
+  x-friends:="org.eclipse.jgit.test",
  org.eclipse.jgit.internal.storage.dfs;version="5.11.0";
   x-friends:="org.eclipse.jgit.test,
    org.eclipse.jgit.http.server,

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java → org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityChecker.java

@@ -7,7 +7,7 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -21,12 +21,16 @@ import java.util.stream.Stream;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
+import org.eclipse.jgit.revwalk.BitmapWalker;
+import org.eclipse.jgit.revwalk.ObjectReachabilityChecker;
+import org.eclipse.jgit.revwalk.ObjectWalk;
+import org.eclipse.jgit.revwalk.RevObject;
 
 /**
  * Checks if all objects are reachable from certain starting points using
  * bitmaps.
  */
-class BitmappedObjectReachabilityChecker
+public class BitmappedObjectReachabilityChecker
 		implements ObjectReachabilityChecker {
 
 	private final ObjectWalk walk;

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java → org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityChecker.java

@@ -7,7 +7,7 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -23,12 +23,17 @@ import org.eclipse.jgit.lib.BitmapIndex;
 import org.eclipse.jgit.lib.BitmapIndex.Bitmap;
 import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
 import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.revwalk.ReachabilityChecker;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevFlag;
+import org.eclipse.jgit.revwalk.RevSort;
+import org.eclipse.jgit.revwalk.RevWalk;
 import org.eclipse.jgit.revwalk.filter.RevFilter;
 
 /**
  * Checks the reachability using bitmaps.
  */
-class BitmappedReachabilityChecker implements ReachabilityChecker {
+public class BitmappedReachabilityChecker implements ReachabilityChecker {
 
 	private final RevWalk walk;
 
@@ -42,7 +47,7 @@ class BitmappedReachabilityChecker implements ReachabilityChecker {
 	 * @throws IOException
 	 *             if the index or the object reader cannot be opened.
 	 */
-	BitmappedReachabilityChecker(RevWalk walk)
+	public BitmappedReachabilityChecker(RevWalk walk)
 			throws IOException {
 		this.walk = walk;
 		if (walk.getObjectReader().getBitmapIndex() == null) {

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java → org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityChecker.java

@@ -7,7 +7,7 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import java.io.IOException;
 import java.io.InvalidObjectException;
@@ -17,12 +17,18 @@ import java.util.Optional;
 import java.util.stream.Stream;
 
 import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.revwalk.ObjectReachabilityChecker;
+import org.eclipse.jgit.revwalk.ObjectWalk;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevObject;
+import org.eclipse.jgit.revwalk.RevSort;
 
 /**
  * Checks if all objects are reachable from certain starting points doing a
  * walk.
  */
-class PedestrianObjectReachabilityChecker implements ObjectReachabilityChecker {
+public class PedestrianObjectReachabilityChecker
+		implements ObjectReachabilityChecker {
 	private final ObjectWalk walk;
 
 	/**
@@ -31,7 +37,7 @@ class PedestrianObjectReachabilityChecker implements ObjectReachabilityChecker {
 	 * @param walk
 	 *            ObjectWalk instance to reuse. Caller retains ownership.
 	 */
-	PedestrianObjectReachabilityChecker(ObjectWalk walk) {
+	public PedestrianObjectReachabilityChecker(ObjectWalk walk) {
 		this.walk = walk;
 	}
 

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianReachabilityChecker.java → org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityChecker.java

@@ -7,7 +7,7 @@
  *
  * SPDX-License-Identifier: BSD-3-Clause
  */
-package org.eclipse.jgit.revwalk;
+package org.eclipse.jgit.internal.revwalk;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -17,12 +17,16 @@ import java.util.stream.Stream;
 
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.revwalk.ReachabilityChecker;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevSort;
+import org.eclipse.jgit.revwalk.RevWalk;
 
 /**
  * Checks the reachability walking the graph from the starters towards the
  * target.
  */
-class PedestrianReachabilityChecker implements ReachabilityChecker {
+public class PedestrianReachabilityChecker implements ReachabilityChecker {
 
 	private final boolean topoSort;
 

org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java

@@ -17,9 +17,18 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 
+import org.eclipse.jgit.annotations.NonNull;
 import org.eclipse.jgit.annotations.Nullable;
 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
 import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.internal.revwalk.BitmappedObjectReachabilityChecker;
+import org.eclipse.jgit.internal.revwalk.BitmappedReachabilityChecker;
+import org.eclipse.jgit.internal.revwalk.PedestrianObjectReachabilityChecker;
+import org.eclipse.jgit.internal.revwalk.PedestrianReachabilityChecker;
+import org.eclipse.jgit.revwalk.ObjectReachabilityChecker;
+import org.eclipse.jgit.revwalk.ObjectWalk;
+import org.eclipse.jgit.revwalk.ReachabilityChecker;
+import org.eclipse.jgit.revwalk.RevWalk;
 
 /**
  * Reads an {@link org.eclipse.jgit.lib.ObjectDatabase} for a single thread.
@@ -407,6 +416,54 @@ public abstract class ObjectReader implements AutoCloseable {
 		return null;
 	}
 
+	/**
+	 * Create a reachability checker that will use bitmaps if possible.
+	 *
+	 * @param rw
+	 *            revwalk for use by the reachability checker
+	 * @return the most efficient reachability checker for this repository.
+	 * @throws IOException
+	 *             if it cannot open any of the underlying indices.
+	 *
+	 * @since 5.11
+	 */
+	@NonNull
+	public ReachabilityChecker createReachabilityChecker(RevWalk rw)
+			throws IOException {
+		if (getBitmapIndex() != null) {
+			return new BitmappedReachabilityChecker(rw);
+		}
+
+		return new PedestrianReachabilityChecker(true, rw);
+	}
+
+	/**
+	 * Create an object reachability checker that will use bitmaps if possible.
+	 *
+	 * This reachability checker accepts any object as target. For checks
+	 * exclusively between commits, use
+	 * {@link #createReachabilityChecker(RevWalk)}.
+	 *
+	 * @param ow
+	 *            objectwalk for use by the reachability checker
+	 * @return the most efficient object reachability checker for this
+	 *         repository.
+	 *
+	 * @throws IOException
+	 *             if it cannot open any of the underlying indices.
+	 *
+	 * @since 5.11
+	 */
+	@NonNull
+	public ObjectReachabilityChecker createObjectReachabilityChecker(
+			ObjectWalk ow) throws IOException {
+		if (getBitmapIndex() != null) {
+			return new BitmappedObjectReachabilityChecker(ow);
+		}
+
+		return new PedestrianObjectReachabilityChecker(ow);
+	}
+
 	/**
 	 * Get the {@link org.eclipse.jgit.lib.ObjectInserter} from which this
 	 * reader was created using {@code inserter.newReader()}

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java

@@ -172,14 +172,14 @@ public class ObjectWalk extends RevWalk {
 	 *             when the index fails to load.
 	 *
 	 * @since 5.8
+	 * @deprecated use
+	 *             {@code ObjectReader#createObjectReachabilityChecker(ObjectWalk)}
+	 *             instead.
 	 */
-	public ObjectReachabilityChecker createObjectReachabilityChecker()
+	@Deprecated
+	public final ObjectReachabilityChecker createObjectReachabilityChecker()
 			throws IOException {
-		if (reader.getBitmapIndex() != null) {
-			return new BitmappedObjectReachabilityChecker(this);
-		}
-
-		return new PedestrianObjectReachabilityChecker(this);
+		return reader.createObjectReachabilityChecker(this);
 	}
 
 	/**

org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java

@@ -236,13 +236,13 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable {
 	 *             if it cannot open any of the underlying indices.
 	 *
 	 * @since 5.4
+	 * @deprecated use {@code ObjectReader#createReachabilityChecker(RevWalk)}
+	 *             instead.
 	 */
-	public ReachabilityChecker createReachabilityChecker() throws IOException {
-		if (reader.getBitmapIndex() != null) {
-			return new BitmappedReachabilityChecker(this);
-		}
-
-		return new PedestrianReachabilityChecker(true, this);
+	@Deprecated
+	public final ReachabilityChecker createReachabilityChecker()
+			throws IOException {
+		return reader.createReachabilityChecker(this);
 	}
 
 	/**

org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java

@@ -1959,8 +1959,8 @@ public class UploadPack {
 							.map(objId -> objectIdToRevObject(objWalk, objId))
 							.filter(Objects::nonNull); // Ignore missing tips
 
-					ObjectReachabilityChecker reachabilityChecker = objWalk
-							.createObjectReachabilityChecker();
+					ObjectReachabilityChecker reachabilityChecker = reader
+							.createObjectReachabilityChecker(objWalk);
 					Optional<RevObject> unreachable = reachabilityChecker
 							.areAllReachable(wantsAsObjs, startersAsObjs);
 					if (unreachable.isPresent()) {
@@ -1971,8 +1971,8 @@ public class UploadPack {
 			}
 
 			// All wants are commits, we can use ReachabilityChecker
-			ReachabilityChecker reachabilityChecker = walk
-					.createReachabilityChecker();
+			ReachabilityChecker reachabilityChecker = reader
+					.createReachabilityChecker(walk);
 
 			Stream<RevCommit> reachableCommits = importantRefsFirst(visibleRefs)
 					.map(UploadPack::refToObjectId)