Record the estimated size of the pack files.

The Compacter and Garbage Collector will record the estimated size of
the newly going to be created compact, gc or garbage packs. This
information can be used by the clients to better make a call on how to
actually store the pack based on the approximated expected size.

Added a new protected method DfsObjDatabase.newPack(PackSource
packSource, long estimatedPackSize), so that the clients can override
this method to make use of the estimatedPackSize while creating a new
PackDescription object. The default implementation of this method is
equivalent to
newPack(packSource).setEstimatedPackSize(estimatedPackSize). I didn't
make it abstract because that would force all the existing sub classes
of DfsObjDatabase to implement this method. Due to this default
implementation, the estimatedPackSize is added to DfsPackDescription
using a setter instead of a constructor parameter (even though
constructor parameter would be a better choice as this value is set only
during the object creation).

Change-Id: Iade1122633ea774c2e842178a6a6cbb4a57b598b
Signed-off-by: Thirumala Reddy Mutchukota <thirumala@google.com>

2 files changed, 385 insertions, 0 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
index c7125bb9fe..188b723e6e 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java
@@ -1,8 +1,10 @@
 package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
+import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST;
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -169,6 +171,245 @@ public class DfsGarbageCollectorTest {
 		}
 	}
 
+	@Test
+	public void testEstimateGcPackSizeInNewRepo() throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("master", commit1);
+
+		// Packs start out as INSERT.
+		long inputPacksSize = 32;
+		assertEquals(2, odb.getPacks().length);
+		for (DfsPackFile pack : odb.getPacks()) {
+			assertEquals(INSERT, pack.getPackDescription().getPackSource());
+			inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
+		}
+
+		gcNoTtl();
+
+		// INSERT packs are combined into a single GC pack.
+		assertEquals(1, odb.getPacks().length);
+		DfsPackFile pack = odb.getPacks()[0];
+		assertEquals(GC, pack.getPackDescription().getPackSource());
+		assertEquals(inputPacksSize,
+				pack.getPackDescription().getEstimatedPackSize());
+	}
+
+	@Test
+	public void testEstimateGcPackSizeWithAnExistingGcPack() throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("master", commit1);
+
+		gcNoTtl();
+
+		RevCommit commit2 = commit().message("2").parent(commit1).create();
+		git.update("master", commit2);
+
+		// There will be one INSERT pack and one GC pack.
+		assertEquals(2, odb.getPacks().length);
+		boolean gcPackFound = false;
+		boolean insertPackFound = false;
+		long inputPacksSize = 32;
+		for (DfsPackFile pack : odb.getPacks()) {
+			DfsPackDescription d = pack.getPackDescription();
+			if (d.getPackSource() == GC) {
+				gcPackFound = true;
+			} else if (d.getPackSource() == INSERT) {
+				insertPackFound = true;
+			} else {
+				fail("unexpected " + d.getPackSource());
+			}
+			inputPacksSize += d.getFileSize(PACK) - 32;
+		}
+		assertTrue(gcPackFound);
+		assertTrue(insertPackFound);
+
+		gcNoTtl();
+
+		// INSERT pack is combined into the GC pack.
+		DfsPackFile pack = odb.getPacks()[0];
+		assertEquals(GC, pack.getPackDescription().getPackSource());
+		assertEquals(inputPacksSize,
+				pack.getPackDescription().getEstimatedPackSize());
+	}
+
+	@Test
+	public void testEstimateGcRestPackSizeInNewRepo() throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("refs/notes/note1", commit1);
+
+		// Packs start out as INSERT.
+		long inputPacksSize = 32;
+		assertEquals(2, odb.getPacks().length);
+		for (DfsPackFile pack : odb.getPacks()) {
+			assertEquals(INSERT, pack.getPackDescription().getPackSource());
+			inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
+		}
+
+		gcNoTtl();
+
+		// INSERT packs are combined into a single GC_REST pack.
+		assertEquals(1, odb.getPacks().length);
+		DfsPackFile pack = odb.getPacks()[0];
+		assertEquals(GC_REST, pack.getPackDescription().getPackSource());
+		assertEquals(inputPacksSize,
+				pack.getPackDescription().getEstimatedPackSize());
+	}
+
+	@Test
+	public void testEstimateGcRestPackSizeWithAnExistingGcPack()
+			throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("refs/notes/note1", commit1);
+
+		gcNoTtl();
+
+		RevCommit commit2 = commit().message("2").parent(commit1).create();
+		git.update("refs/notes/note2", commit2);
+
+		// There will be one INSERT pack and one GC_REST pack.
+		assertEquals(2, odb.getPacks().length);
+		boolean gcRestPackFound = false;
+		boolean insertPackFound = false;
+		long inputPacksSize = 32;
+		for (DfsPackFile pack : odb.getPacks()) {
+			DfsPackDescription d = pack.getPackDescription();
+			if (d.getPackSource() == GC_REST) {
+				gcRestPackFound = true;
+			} else if (d.getPackSource() == INSERT) {
+				insertPackFound = true;
+			} else {
+				fail("unexpected " + d.getPackSource());
+			}
+			inputPacksSize += d.getFileSize(PACK) - 32;
+		}
+		assertTrue(gcRestPackFound);
+		assertTrue(insertPackFound);
+
+		gcNoTtl();
+
+		// INSERT pack is combined into the GC_REST pack.
+		DfsPackFile pack = odb.getPacks()[0];
+		assertEquals(GC_REST, pack.getPackDescription().getPackSource());
+		assertEquals(inputPacksSize,
+				pack.getPackDescription().getEstimatedPackSize());
+	}
+
+	@Test
+	public void testEstimateGcPackSizesWithGcAndGcRestPacks() throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		git.update("head", commit0);
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("refs/notes/note1", commit1);
+
+		gcNoTtl();
+
+		RevCommit commit2 = commit().message("2").parent(commit1).create();
+		git.update("refs/notes/note2", commit2);
+
+		// There will be one INSERT, one GC and one GC_REST packs.
+		assertEquals(3, odb.getPacks().length);
+		boolean gcPackFound = false;
+		boolean gcRestPackFound = false;
+		boolean insertPackFound = false;
+		long gcPackSize = 0;
+		long gcRestPackSize = 0;
+		long insertPackSize = 0;
+		for (DfsPackFile pack : odb.getPacks()) {
+			DfsPackDescription d = pack.getPackDescription();
+			if (d.getPackSource() == GC) {
+				gcPackFound = true;
+				gcPackSize = d.getFileSize(PACK);
+			} else if (d.getPackSource() == GC_REST) {
+				gcRestPackFound = true;
+				gcRestPackSize = d.getFileSize(PACK);
+			} else if (d.getPackSource() == INSERT) {
+				insertPackFound = true;
+				insertPackSize = d.getFileSize(PACK);
+			} else {
+				fail("unexpected " + d.getPackSource());
+			}
+		}
+		assertTrue(gcPackFound);
+		assertTrue(gcRestPackFound);
+		assertTrue(insertPackFound);
+
+		gcNoTtl();
+
+		// In this test INSERT pack would be combined into the GC_REST pack.
+		// But, as there is no good heuristic to know whether the new packs will
+		// be combined into a GC pack or GC_REST packs, the new pick size is
+		// considered while estimating both the GC and GC_REST packs.
+		assertEquals(2, odb.getPacks().length);
+		gcPackFound = false;
+		gcRestPackFound = false;
+		for (DfsPackFile pack : odb.getPacks()) {
+			DfsPackDescription d = pack.getPackDescription();
+			if (d.getPackSource() == GC) {
+				gcPackFound = true;
+				assertEquals(gcPackSize + insertPackSize - 32,
+						pack.getPackDescription().getEstimatedPackSize());
+			} else if (d.getPackSource() == GC_REST) {
+				gcRestPackFound = true;
+				assertEquals(gcRestPackSize + insertPackSize - 32,
+						pack.getPackDescription().getEstimatedPackSize());
+			} else {
+				fail("unexpected " + d.getPackSource());
+			}
+		}
+		assertTrue(gcPackFound);
+		assertTrue(gcRestPackFound);
+	}
+
+	@Test
+	public void testEstimateUnreachableGarbagePackSize() throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("master", commit0);
+
+		assertTrue("commit0 reachable", isReachable(repo, commit0));
+		assertFalse("commit1 garbage", isReachable(repo, commit1));
+
+		// Packs start out as INSERT.
+		long packSize0 = 0;
+		long packSize1 = 0;
+		assertEquals(2, odb.getPacks().length);
+		for (DfsPackFile pack : odb.getPacks()) {
+			DfsPackDescription d = pack.getPackDescription();
+			assertEquals(INSERT, d.getPackSource());
+			if (isObjectInPack(commit0, pack)) {
+				packSize0 = d.getFileSize(PACK);
+			} else if (isObjectInPack(commit1, pack)) {
+				packSize1 = d.getFileSize(PACK);
+			} else {
+				fail("expected object not found in the pack");
+			}
+		}
+
+		gcNoTtl();
+
+		assertEquals(2, odb.getPacks().length);
+		for (DfsPackFile pack : odb.getPacks()) {
+			DfsPackDescription d = pack.getPackDescription();
+			if (d.getPackSource() == GC) {
+				// Even though just commit0 will end up in GC pack, because
+				// there is no good way to know that up front, both the pack
+				// sizes are considered while computing the estimated size of GC
+				// pack.
+				assertEquals(packSize0 + packSize1 - 32,
+						d.getEstimatedPackSize());
+			} else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
+				// commit1 is moved to UNREACHABLE_GARBAGE pack.
+				assertEquals(packSize1, d.getEstimatedPackSize());
+			} else {
+				fail("unexpected " + d.getPackSource());
+			}
+		}
+	}
+
 	private TestRepository<InMemoryRepository>.CommitBuilder commit() {
 		return git.commit();
 	}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java
new file mode 100644
index 0000000000..db5b24a9b5
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.dfs;
+
+import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
+import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.eclipse.jgit.junit.TestRepository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DfsPackCompacterTest {
+	private TestRepository<InMemoryRepository> git;
+	private InMemoryRepository repo;
+	private DfsObjDatabase odb;
+
+	@Before
+	public void setUp() throws IOException {
+		DfsRepositoryDescription desc = new DfsRepositoryDescription("test");
+		git = new TestRepository<>(new InMemoryRepository(desc));
+		repo = git.getRepository();
+		odb = repo.getObjectDatabase();
+	}
+
+	@Test
+	public void testEstimateCompactPackSizeInNewRepo() throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("master", commit1);
+
+		// Packs start out as INSERT.
+		long inputPacksSize = 32;
+		assertEquals(2, odb.getPacks().length);
+		for (DfsPackFile pack : odb.getPacks()) {
+			assertEquals(INSERT, pack.getPackDescription().getPackSource());
+			inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
+		}
+
+		compact();
+
+		// INSERT packs are compacted into a single COMPACT pack.
+		assertEquals(1, odb.getPacks().length);
+		DfsPackFile pack = odb.getPacks()[0];
+		assertEquals(COMPACT, pack.getPackDescription().getPackSource());
+		assertEquals(inputPacksSize,
+				pack.getPackDescription().getEstimatedPackSize());
+	}
+
+	@Test
+	public void testEstimateGcPackSizeWithAnExistingGcPack() throws Exception {
+		RevCommit commit0 = commit().message("0").create();
+		RevCommit commit1 = commit().message("1").parent(commit0).create();
+		git.update("master", commit1);
+
+		compact();
+
+		RevCommit commit2 = commit().message("2").parent(commit1).create();
+		git.update("master", commit2);
+
+		// There will be one INSERT pack and one COMPACT pack.
+		assertEquals(2, odb.getPacks().length);
+		boolean compactPackFound = false;
+		boolean insertPackFound = false;
+		long inputPacksSize = 32;
+		for (DfsPackFile pack : odb.getPacks()) {
+			DfsPackDescription packDescription = pack.getPackDescription();
+			if (packDescription.getPackSource() == COMPACT) {
+				compactPackFound = true;
+			}
+			if (packDescription.getPackSource() == INSERT) {
+				insertPackFound = true;
+			}
+			inputPacksSize += packDescription.getFileSize(PACK) - 32;
+		}
+		assertTrue(compactPackFound);
+		assertTrue(insertPackFound);
+
+		compact();
+
+		// INSERT pack is combined into the COMPACT pack.
+		DfsPackFile pack = odb.getPacks()[0];
+		assertEquals(COMPACT, pack.getPackDescription().getPackSource());
+		assertEquals(inputPacksSize,
+				pack.getPackDescription().getEstimatedPackSize());
+	}
+
+	private TestRepository<InMemoryRepository>.CommitBuilder commit() {
+		return git.commit();
+	}
+
+	private void compact() throws IOException {
+		DfsPackCompactor compactor = new DfsPackCompactor(repo);
+		compactor.autoAdd();
+		compactor.compact(null);
+		odb.clearCache();
+	}
+}