Merge branch 'stable-5.3' into stable-5.4

* stable-5.3:
  Prepare 5.3.6-SNAPSHOT builds
  JGit v5.3.5.201909031855-r
  Prepare 5.1.12-SNAPSHOT builds
  JGit v5.1.11.201909031202-r
  Prepare 4.11.10-SNAPSHOT builds
  JGit v4.11.9.201909030838-r
  Bazel: Update bazlets to the latest master revision
  Bazel: Remove FileTreeIteratorWithTimeControl from BUILD file
  BatchRefUpdate: repro racy atomic update, and fix it
  Delete unused FileTreeIteratorWithTimeControl
  Fix RacyGitTests#testRacyGitDetection
  Change RacyGitTests to create a racy git situation in a stable way
  Silence API warnings

Change-Id: Icfd5a612055feeff04733c0d2c44837ab554a2c4
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

6 files changed, 279 insertions, 86 deletions

diff --git a/WORKSPACE b/WORKSPACE
index 9fb3eedef6..63adf052a3 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -15,7 +15,7 @@ versions.check(minimum_bazel_version = "0.26.1")
 
 load("//tools:bazlets.bzl", "load_bazlets")
 
-load_bazlets(commit = "8528a0df69dadf6311d8d3f81c1b693afda8bcf1")
+load_bazlets(commit = "09a035e98077dce549d5f6a7472d06c4b8f792d2")
 
 load(
     "@com_googlesource_gerrit_bazlets//tools:maven_jar.bzl",
diff --git a/org.eclipse.jgit.lfs.server/.settings/.api_filters b/org.eclipse.jgit.lfs.server/.settings/.api_filters
new file mode 100644
index 0000000000..6609c3d40d
--- /dev/null
+++ b/org.eclipse.jgit.lfs.server/.settings/.api_filters
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<component id="org.eclipse.jgit.lfs.server" version="2">
+    <resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener">
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.7"/>
+                <message_argument value="setCallback(ObjectUploadListener.Callback)"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener$Callback">
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.7"/>
+                <message_argument value="Callback"/>
+            </message_arguments>
+        </filter>
+    </resource>
+</component>
diff --git a/org.eclipse.jgit.lfs/.settings/.api_filters b/org.eclipse.jgit.lfs/.settings/.api_filters
deleted file mode 100644
index b1d5e9ce94..0000000000
--- a/org.eclipse.jgit.lfs/.settings/.api_filters
+++ /dev/null
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<component id="org.eclipse.jgit.lfs" version="2">
-    <resource path="META-INF/MANIFEST.MF">
-        <filter id="924844039">
-            <message_arguments>
-                <message_argument value="5.4.3"/>
-                <message_argument value="5.4.0"/>
-            </message_arguments>
-        </filter>
-    </resource>
-</component>
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java
index 501b788f89..79cf4d5aff 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java
@@ -43,6 +43,7 @@
 
 package org.eclipse.jgit.internal.storage.file;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.eclipse.jgit.internal.storage.file.BatchRefUpdateTest.Result.LOCK_FAILURE;
@@ -64,6 +65,7 @@ import static org.junit.Assume.assumeTrue;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -162,6 +164,33 @@ public class BatchRefUpdateTest extends LocalDiskRepositoryTestCase {
 	}
 
 	@Test
+	public void packedRefsFileIsSorted() throws IOException {
+		assumeTrue(atomic);
+
+		for (int i = 0; i < 2; i++) {
+			BatchRefUpdate bu = diskRepo.getRefDatabase().newBatchUpdate();
+			String b1  = String.format("refs/heads/a%d",i);
+			String b2  = String.format("refs/heads/b%d",i);
+			bu.setAtomic(atomic);
+			ReceiveCommand c1 = new ReceiveCommand(ObjectId.zeroId(), A, b1);
+			ReceiveCommand c2 = new ReceiveCommand(ObjectId.zeroId(), B, b2);
+			bu.addCommand(c1, c2);
+			try (RevWalk rw = new RevWalk(diskRepo)) {
+				bu.execute(rw, NullProgressMonitor.INSTANCE);
+			}
+			assertEquals(c1.getResult(), ReceiveCommand.Result.OK);
+			assertEquals(c2.getResult(), ReceiveCommand.Result.OK);
+		}
+
+		File packed = new File(diskRepo.getDirectory(), "packed-refs");
+		String packedStr = new String(Files.readAllBytes(packed.toPath()), UTF_8);
+
+		int a2 = packedStr.indexOf("refs/heads/a1");
+		int b1 = packedStr.indexOf("refs/heads/b0");
+		assertTrue(a2 <  b1);
+	}
+
+	@Test
 	public void simpleNoForce() throws IOException {
 		writeLooseRef("refs/heads/master", A);
 		writeLooseRef("refs/heads/masters", B);
diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters
index fc5bda2086..842fc873d4 100644
--- a/org.eclipse.jgit/.settings/.api_filters
+++ b/org.eclipse.jgit/.settings/.api_filters
@@ -1,13 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <component id="org.eclipse.jgit" version="2">
-    <resource path="META-INF/MANIFEST.MF">
-        <filter id="924844039">
-            <message_arguments>
-                <message_argument value="5.4.3"/>
-                <message_argument value="5.4.0"/>
-            </message_arguments>
-        </filter>
-    </resource>
     <resource path="src/org/eclipse/jgit/dircache/DirCacheEntry.java" type="org.eclipse.jgit.dircache.DirCacheEntry">
         <filter id="1142947843">
             <message_arguments>
@@ -28,6 +20,20 @@
             </message_arguments>
         </filter>
     </resource>
+    <resource path="src/org/eclipse/jgit/errors/PackInvalidException.java" type="org.eclipse.jgit.errors.PackInvalidException">
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="4.5.7"/>
+                <message_argument value="PackInvalidException(File, Throwable)"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="4.5.7"/>
+                <message_argument value="PackInvalidException(String, Throwable)"/>
+            </message_arguments>
+        </filter>
+    </resource>
     <resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants">
         <filter id="1142947843">
             <message_arguments>
@@ -48,6 +54,136 @@
             </message_arguments>
         </filter>
     </resource>
+    <resource path="src/org/eclipse/jgit/lib/Ref.java" type="org.eclipse.jgit.lib.Ref">
+        <filter id="403767336">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.lib.Ref"/>
+                <message_argument value="UNDEFINED_UPDATE_INDEX"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/lib/Repository.java" type="org.eclipse.jgit.lib.Repository">
+        <filter id="336695337">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.lib.Repository"/>
+                <message_argument value="getIdentifier()"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/revwalk/ObjectWalk.java" type="org.eclipse.jgit.revwalk.ObjectWalk">
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.revwalk.ObjectWalk"/>
+                <message_argument value="SIMPLE_VISITATION_POLICY"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/storage/pack/PackConfig.java" type="org.eclipse.jgit.storage.pack.PackConfig">
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.storage.pack.PackConfig"/>
+                <message_argument value="DEFAULT_MINSIZE_PREVENT_RACY_PACK"/>
+            </message_arguments>
+        </filter>
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.storage.pack.PackConfig"/>
+                <message_argument value="DEFAULT_WAIT_PREVENT_RACY_PACK"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="DEFAULT_MINSIZE_PREVENT_RACY_PACK"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="DEFAULT_WAIT_PREVENT_RACY_PACK"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="doWaitPreventRacyPack(long)"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="getMinSizePreventRacyPack()"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="isWaitPreventRacyPack()"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="setMinSizePreventRacyPack(long)"/>
+            </message_arguments>
+        </filter>
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="setWaitPreventRacyPack(boolean)"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/storage/pack/PackStatistics.java" type="org.eclipse.jgit.storage.pack.PackStatistics$Accumulator">
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.storage.pack.PackStatistics.Accumulator"/>
+                <message_argument value="treesTraversed"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/transport/HttpConfig.java" type="org.eclipse.jgit.transport.HttpConfig">
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.HttpConfig"/>
+                <message_argument value="COOKIE_FILE_CACHE_LIMIT_KEY"/>
+            </message_arguments>
+        </filter>
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.HttpConfig"/>
+                <message_argument value="COOKIE_FILE_KEY"/>
+            </message_arguments>
+        </filter>
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.HttpConfig"/>
+                <message_argument value="SAVE_COOKIES_KEY"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/transport/Transport.java" type="org.eclipse.jgit.transport.Transport">
+        <filter id="421654647">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.Transport"/>
+                <message_argument value="getFilterBlobLimit()"/>
+            </message_arguments>
+        </filter>
+        <filter id="421654647">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.Transport"/>
+                <message_argument value="setFilterBlobLimit(long)"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/transport/UploadPack.java" type="org.eclipse.jgit.transport.UploadPack">
+        <filter id="421654647">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.transport.UploadPack"/>
+                <message_argument value="getFilterBlobLimit()"/>
+            </message_arguments>
+        </filter>
+    </resource>
     <resource path="src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java" type="org.eclipse.jgit.treewalk.WorkingTreeIterator">
         <filter id="1142947843">
             <message_arguments>
@@ -71,12 +207,6 @@
         </filter>
     </resource>
     <resource path="src/org/eclipse/jgit/util/FS.java" type="org.eclipse.jgit.util.FS">
-        <filter id="338792546">
-            <message_arguments>
-                <message_argument value="org.eclipse.jgit.util.FS"/>
-                <message_argument value="getFsTimerResolution(Path)"/>
-            </message_arguments>
-        </filter>
         <filter id="1142947843">
             <message_arguments>
                 <message_argument value="5.1.9"/>
@@ -124,6 +254,34 @@
             </message_arguments>
         </filter>
     </resource>
+    <resource path="src/org/eclipse/jgit/util/FileUtils.java" type="org.eclipse.jgit.util.FileUtils">
+        <filter id="1142947843">
+            <message_arguments>
+                <message_argument value="5.1.8"/>
+                <message_argument value="touch(Path)"/>
+            </message_arguments>
+        </filter>
+    </resource>
+    <resource path="src/org/eclipse/jgit/util/HttpSupport.java" type="org.eclipse.jgit.util.HttpSupport">
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.util.HttpSupport"/>
+                <message_argument value="HDR_COOKIE"/>
+            </message_arguments>
+        </filter>
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.util.HttpSupport"/>
+                <message_argument value="HDR_SET_COOKIE"/>
+            </message_arguments>
+        </filter>
+        <filter id="336658481">
+            <message_arguments>
+                <message_argument value="org.eclipse.jgit.util.HttpSupport"/>
+                <message_argument value="HDR_SET_COOKIE2"/>
+            </message_arguments>
+        </filter>
+    </resource>
     <resource path="src/org/eclipse/jgit/util/SimpleLruCache.java" type="org.eclipse.jgit.util.SimpleLruCache">
         <filter id="1109393411">
             <message_arguments>
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java
index 200c63c17a..e45b53ea68 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackedBatchRefUpdate.java
@@ -51,10 +51,10 @@ import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_RE
 
 import java.io.IOException;
 import java.text.MessageFormat;
-import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -364,65 +364,72 @@ class PackedBatchRefUpdate extends BatchRefUpdate {
 
 	private static RefList<Ref> applyUpdates(RevWalk walk, RefList<Ref> refs,
 			List<ReceiveCommand> commands) throws IOException {
-		int nDeletes = 0;
-		List<ReceiveCommand> adds = new ArrayList<>(commands.size());
+		// Construct a new RefList by merging the old list with the updates.
+		// This assumes that each ref occurs at most once as a ReceiveCommand.
+		Collections.sort(commands, new Comparator<ReceiveCommand>() {
+			@Override
+			public int compare(ReceiveCommand a, ReceiveCommand b) {
+				return a.getRefName().compareTo(b.getRefName());
+			}
+		});
+
+		int delta = 0;
 		for (ReceiveCommand c : commands) {
-			if (c.getType() == ReceiveCommand.Type.CREATE) {
-				adds.add(c);
-			} else if (c.getType() == ReceiveCommand.Type.DELETE) {
-				nDeletes++;
+			switch (c.getType()) {
+			case DELETE:
+				delta--;
+				break;
+			case CREATE:
+				delta++;
+				break;
+			default:
 			}
 		}
-		int addIdx = 0;
-
-		// Construct a new RefList by linearly scanning the old list, and merging in
-		// any updates.
-		Map<String, ReceiveCommand> byName = byName(commands);
-		RefList.Builder<Ref> b =
-				new RefList.Builder<>(refs.size() - nDeletes + adds.size());
-		for (Ref ref : refs) {
-			String name = ref.getName();
-			ReceiveCommand cmd = byName.remove(name);
-			if (cmd == null) {
-				b.add(ref);
-				continue;
-			}
-			if (!cmd.getOldId().equals(ref.getObjectId())) {
-				lockFailure(cmd, commands);
-				return null;
+
+		RefList.Builder<Ref> b = new RefList.Builder<>(refs.size() + delta);
+		int refIdx = 0;
+		int cmdIdx = 0;
+		while (refIdx < refs.size() || cmdIdx < commands.size()) {
+			Ref ref = (refIdx < refs.size()) ? refs.get(refIdx) : null;
+			ReceiveCommand cmd = (cmdIdx < commands.size())
+					? commands.get(cmdIdx)
+					: null;
+			int cmp = 0;
+			if (ref != null && cmd != null) {
+				cmp = ref.getName().compareTo(cmd.getRefName());
+			} else if (ref == null) {
+				cmp = 1;
+			} else if (cmd == null) {
+				cmp = -1;
 			}
 
-			// Consume any adds between the last and current ref.
-			while (addIdx < adds.size()) {
-				ReceiveCommand currAdd = adds.get(addIdx);
-				if (currAdd.getRefName().compareTo(name) < 0) {
-					b.add(peeledRef(walk, currAdd));
-					byName.remove(currAdd.getRefName());
-				} else {
-					break;
+			if (cmp < 0) {
+				b.add(ref);
+				refIdx++;
+			} else if (cmp > 0) {
+				assert cmd != null;
+				if (cmd.getType() != ReceiveCommand.Type.CREATE) {
+					lockFailure(cmd, commands);
+					return null;
 				}
-				addIdx++;
-			}
 
-			if (cmd.getType() != ReceiveCommand.Type.DELETE) {
 				b.add(peeledRef(walk, cmd));
-			}
-		}
-
-		// All remaining adds are valid, since the refs didn't exist.
-		while (addIdx < adds.size()) {
-			ReceiveCommand cmd = adds.get(addIdx++);
-			byName.remove(cmd.getRefName());
-			b.add(peeledRef(walk, cmd));
-		}
+				cmdIdx++;
+			} else {
+				assert cmd != null;
+				assert ref != null;
+				if (!cmd.getOldId().equals(ref.getObjectId())) {
+					lockFailure(cmd, commands);
+					return null;
+				}
 
-		// Any remaining updates/deletes do not correspond to any existing refs, so
-		// they are lock failures.
-		if (!byName.isEmpty()) {
-			lockFailure(byName.values().iterator().next(), commands);
-			return null;
+				if (cmd.getType() != ReceiveCommand.Type.DELETE) {
+					b.add(peeledRef(walk, cmd));
+				}
+				cmdIdx++;
+				refIdx++;
+			}
 		}
-
 		return b.toRefList();
 	}
 
@@ -501,15 +508,6 @@ class PackedBatchRefUpdate extends BatchRefUpdate {
 		}
 	}
 
-	private static Map<String, ReceiveCommand> byName(
-			List<ReceiveCommand> commands) {
-		Map<String, ReceiveCommand> ret = new LinkedHashMap<>();
-		for (ReceiveCommand cmd : commands) {
-			ret.put(cmd.getRefName(), cmd);
-		}
-		return ret;
-	}
-
 	private static Ref peeledRef(RevWalk walk, ReceiveCommand cmd)
 			throws IOException {
 		ObjectId newId = cmd.getNewId().copy();