From c9f55032a281011f8db15afab33f43e850ec413b Mon Sep 17 00:00:00 2001 From: Thirumala Reddy Mutchukota Date: Fri, 20 Jan 2017 14:31:01 -0800 Subject: [PATCH] Record the estimated size of the pack files. The Compacter and Garbage Collector will record the estimated size of the newly going to be created compact, gc or garbage packs. This information can be used by the clients to better make a call on how to actually store the pack based on the approximated expected size. Added a new protected method DfsObjDatabase.newPack(PackSource packSource, long estimatedPackSize), so that the clients can override this method to make use of the estimatedPackSize while creating a new PackDescription object. The default implementation of this method is equivalent to newPack(packSource).setEstimatedPackSize(estimatedPackSize). I didn't make it abstract because that would force all the existing sub classes of DfsObjDatabase to implement this method. Due to this default implementation, the estimatedPackSize is added to DfsPackDescription using a setter instead of a constructor parameter (even though constructor parameter would be a better choice as this value is set only during the object creation). Change-Id: Iade1122633ea774c2e842178a6a6cbb4a57b598b Signed-off-by: Thirumala Reddy Mutchukota --- .../storage/dfs/DfsGarbageCollectorTest.java | 241 ++++++++++++++++++ .../storage/dfs/DfsPackCompacterTest.java | 144 +++++++++++ .../storage/dfs/DfsGarbageCollector.java | 44 +++- .../internal/storage/dfs/DfsObjDatabase.java | 26 ++ .../storage/dfs/DfsPackCompactor.java | 14 +- .../storage/dfs/DfsPackDescription.java | 21 ++ 6 files changed, 482 insertions(+), 8 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index c7125bb9fe..188b723e6e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -1,8 +1,10 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; +import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -169,6 +171,245 @@ public class DfsGarbageCollectorTest { } } + @Test + public void testEstimateGcPackSizeInNewRepo() throws Exception { + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + // Packs start out as INSERT. + long inputPacksSize = 32; + assertEquals(2, odb.getPacks().length); + for (DfsPackFile pack : odb.getPacks()) { + assertEquals(INSERT, pack.getPackDescription().getPackSource()); + inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32; + } + + gcNoTtl(); + + // INSERT packs are combined into a single GC pack. + assertEquals(1, odb.getPacks().length); + DfsPackFile pack = odb.getPacks()[0]; + assertEquals(GC, pack.getPackDescription().getPackSource()); + assertEquals(inputPacksSize, + pack.getPackDescription().getEstimatedPackSize()); + } + + @Test + public void testEstimateGcPackSizeWithAnExistingGcPack() throws Exception { + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + gcNoTtl(); + + RevCommit commit2 = commit().message("2").parent(commit1).create(); + git.update("master", commit2); + + // There will be one INSERT pack and one GC pack. + assertEquals(2, odb.getPacks().length); + boolean gcPackFound = false; + boolean insertPackFound = false; + long inputPacksSize = 32; + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription d = pack.getPackDescription(); + if (d.getPackSource() == GC) { + gcPackFound = true; + } else if (d.getPackSource() == INSERT) { + insertPackFound = true; + } else { + fail("unexpected " + d.getPackSource()); + } + inputPacksSize += d.getFileSize(PACK) - 32; + } + assertTrue(gcPackFound); + assertTrue(insertPackFound); + + gcNoTtl(); + + // INSERT pack is combined into the GC pack. + DfsPackFile pack = odb.getPacks()[0]; + assertEquals(GC, pack.getPackDescription().getPackSource()); + assertEquals(inputPacksSize, + pack.getPackDescription().getEstimatedPackSize()); + } + + @Test + public void testEstimateGcRestPackSizeInNewRepo() throws Exception { + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("refs/notes/note1", commit1); + + // Packs start out as INSERT. + long inputPacksSize = 32; + assertEquals(2, odb.getPacks().length); + for (DfsPackFile pack : odb.getPacks()) { + assertEquals(INSERT, pack.getPackDescription().getPackSource()); + inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32; + } + + gcNoTtl(); + + // INSERT packs are combined into a single GC_REST pack. + assertEquals(1, odb.getPacks().length); + DfsPackFile pack = odb.getPacks()[0]; + assertEquals(GC_REST, pack.getPackDescription().getPackSource()); + assertEquals(inputPacksSize, + pack.getPackDescription().getEstimatedPackSize()); + } + + @Test + public void testEstimateGcRestPackSizeWithAnExistingGcPack() + throws Exception { + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("refs/notes/note1", commit1); + + gcNoTtl(); + + RevCommit commit2 = commit().message("2").parent(commit1).create(); + git.update("refs/notes/note2", commit2); + + // There will be one INSERT pack and one GC_REST pack. + assertEquals(2, odb.getPacks().length); + boolean gcRestPackFound = false; + boolean insertPackFound = false; + long inputPacksSize = 32; + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription d = pack.getPackDescription(); + if (d.getPackSource() == GC_REST) { + gcRestPackFound = true; + } else if (d.getPackSource() == INSERT) { + insertPackFound = true; + } else { + fail("unexpected " + d.getPackSource()); + } + inputPacksSize += d.getFileSize(PACK) - 32; + } + assertTrue(gcRestPackFound); + assertTrue(insertPackFound); + + gcNoTtl(); + + // INSERT pack is combined into the GC_REST pack. + DfsPackFile pack = odb.getPacks()[0]; + assertEquals(GC_REST, pack.getPackDescription().getPackSource()); + assertEquals(inputPacksSize, + pack.getPackDescription().getEstimatedPackSize()); + } + + @Test + public void testEstimateGcPackSizesWithGcAndGcRestPacks() throws Exception { + RevCommit commit0 = commit().message("0").create(); + git.update("head", commit0); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("refs/notes/note1", commit1); + + gcNoTtl(); + + RevCommit commit2 = commit().message("2").parent(commit1).create(); + git.update("refs/notes/note2", commit2); + + // There will be one INSERT, one GC and one GC_REST packs. + assertEquals(3, odb.getPacks().length); + boolean gcPackFound = false; + boolean gcRestPackFound = false; + boolean insertPackFound = false; + long gcPackSize = 0; + long gcRestPackSize = 0; + long insertPackSize = 0; + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription d = pack.getPackDescription(); + if (d.getPackSource() == GC) { + gcPackFound = true; + gcPackSize = d.getFileSize(PACK); + } else if (d.getPackSource() == GC_REST) { + gcRestPackFound = true; + gcRestPackSize = d.getFileSize(PACK); + } else if (d.getPackSource() == INSERT) { + insertPackFound = true; + insertPackSize = d.getFileSize(PACK); + } else { + fail("unexpected " + d.getPackSource()); + } + } + assertTrue(gcPackFound); + assertTrue(gcRestPackFound); + assertTrue(insertPackFound); + + gcNoTtl(); + + // In this test INSERT pack would be combined into the GC_REST pack. + // But, as there is no good heuristic to know whether the new packs will + // be combined into a GC pack or GC_REST packs, the new pick size is + // considered while estimating both the GC and GC_REST packs. + assertEquals(2, odb.getPacks().length); + gcPackFound = false; + gcRestPackFound = false; + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription d = pack.getPackDescription(); + if (d.getPackSource() == GC) { + gcPackFound = true; + assertEquals(gcPackSize + insertPackSize - 32, + pack.getPackDescription().getEstimatedPackSize()); + } else if (d.getPackSource() == GC_REST) { + gcRestPackFound = true; + assertEquals(gcRestPackSize + insertPackSize - 32, + pack.getPackDescription().getEstimatedPackSize()); + } else { + fail("unexpected " + d.getPackSource()); + } + } + assertTrue(gcPackFound); + assertTrue(gcRestPackFound); + } + + @Test + public void testEstimateUnreachableGarbagePackSize() throws Exception { + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit0); + + assertTrue("commit0 reachable", isReachable(repo, commit0)); + assertFalse("commit1 garbage", isReachable(repo, commit1)); + + // Packs start out as INSERT. + long packSize0 = 0; + long packSize1 = 0; + assertEquals(2, odb.getPacks().length); + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription d = pack.getPackDescription(); + assertEquals(INSERT, d.getPackSource()); + if (isObjectInPack(commit0, pack)) { + packSize0 = d.getFileSize(PACK); + } else if (isObjectInPack(commit1, pack)) { + packSize1 = d.getFileSize(PACK); + } else { + fail("expected object not found in the pack"); + } + } + + gcNoTtl(); + + assertEquals(2, odb.getPacks().length); + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription d = pack.getPackDescription(); + if (d.getPackSource() == GC) { + // Even though just commit0 will end up in GC pack, because + // there is no good way to know that up front, both the pack + // sizes are considered while computing the estimated size of GC + // pack. + assertEquals(packSize0 + packSize1 - 32, + d.getEstimatedPackSize()); + } else if (d.getPackSource() == UNREACHABLE_GARBAGE) { + // commit1 is moved to UNREACHABLE_GARBAGE pack. + assertEquals(packSize1, d.getEstimatedPackSize()); + } else { + fail("unexpected " + d.getPackSource()); + } + } + } + private TestRepository.CommitBuilder commit() { return git.commit(); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java new file mode 100644 index 0000000000..db5b24a9b5 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; +import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.Before; +import org.junit.Test; + +public class DfsPackCompacterTest { + private TestRepository git; + private InMemoryRepository repo; + private DfsObjDatabase odb; + + @Before + public void setUp() throws IOException { + DfsRepositoryDescription desc = new DfsRepositoryDescription("test"); + git = new TestRepository<>(new InMemoryRepository(desc)); + repo = git.getRepository(); + odb = repo.getObjectDatabase(); + } + + @Test + public void testEstimateCompactPackSizeInNewRepo() throws Exception { + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + // Packs start out as INSERT. + long inputPacksSize = 32; + assertEquals(2, odb.getPacks().length); + for (DfsPackFile pack : odb.getPacks()) { + assertEquals(INSERT, pack.getPackDescription().getPackSource()); + inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32; + } + + compact(); + + // INSERT packs are compacted into a single COMPACT pack. + assertEquals(1, odb.getPacks().length); + DfsPackFile pack = odb.getPacks()[0]; + assertEquals(COMPACT, pack.getPackDescription().getPackSource()); + assertEquals(inputPacksSize, + pack.getPackDescription().getEstimatedPackSize()); + } + + @Test + public void testEstimateGcPackSizeWithAnExistingGcPack() throws Exception { + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + compact(); + + RevCommit commit2 = commit().message("2").parent(commit1).create(); + git.update("master", commit2); + + // There will be one INSERT pack and one COMPACT pack. + assertEquals(2, odb.getPacks().length); + boolean compactPackFound = false; + boolean insertPackFound = false; + long inputPacksSize = 32; + for (DfsPackFile pack : odb.getPacks()) { + DfsPackDescription packDescription = pack.getPackDescription(); + if (packDescription.getPackSource() == COMPACT) { + compactPackFound = true; + } + if (packDescription.getPackSource() == INSERT) { + insertPackFound = true; + } + inputPacksSize += packDescription.getFileSize(PACK) - 32; + } + assertTrue(compactPackFound); + assertTrue(insertPackFound); + + compact(); + + // INSERT pack is combined into the COMPACT pack. + DfsPackFile pack = odb.getPacks()[0]; + assertEquals(COMPACT, pack.getPackDescription().getPackSource()); + assertEquals(inputPacksSize, + pack.getPackDescription().getEstimatedPackSize()); + } + + private TestRepository.CommitBuilder commit() { + return git.commit(); + } + + private void compact() throws IOException { + DfsPackCompactor compactor = new DfsPackCompactor(repo); + compactor.autoAdd(); + compactor.compact(null); + odb.clearCache(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index f7d078fa3c..56ff1b5c2e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -43,9 +43,12 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; +import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; +import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; @@ -55,6 +58,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -63,6 +67,7 @@ import java.util.concurrent.TimeUnit; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.internal.storage.file.PackReverseIndex; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; @@ -390,7 +395,8 @@ public class DfsGarbageCollector { pw.setTagTargets(tagTargets); pw.preparePack(pm, allHeads, PackWriter.NONE); if (0 < pw.getObjectCount()) - writePack(GC, pw, pm); + writePack(GC, pw, pm, + estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC)); } } @@ -403,7 +409,8 @@ public class DfsGarbageCollector { pw.excludeObjects(packedObjs); pw.preparePack(pm, nonHeads, allHeads); if (0 < pw.getObjectCount()) - writePack(GC_REST, pw, pm); + writePack(GC_REST, pw, pm, + estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC_REST)); } } @@ -416,7 +423,7 @@ public class DfsGarbageCollector { pw.excludeObjects(packedObjs); pw.preparePack(pm, txnHeads, PackWriter.NONE); if (0 < pw.getObjectCount()) - writePack(GC_TXN, pw, pm); + writePack(GC_TXN, pw, pm, 0 /* unknown pack size */); } } @@ -432,21 +439,29 @@ public class DfsGarbageCollector { pw.setDeltaBaseAsOffset(true); pw.setReuseDeltaCommits(true); pm.beginTask(JGitText.get().findingGarbage, objectsBefore()); + long estimatedPackSize = 12 + 20; // header and trailer sizes. for (DfsPackFile oldPack : packsBefore) { PackIndex oldIdx = oldPack.getPackIndex(ctx); + PackReverseIndex oldRevIdx = oldPack.getReverseIdx(ctx); + long maxOffset = oldPack.getPackDescription().getFileSize(PACK) + - 20; // pack size - trailer size. for (PackIndex.MutableEntry ent : oldIdx) { pm.update(1); ObjectId id = ent.toObjectId(); if (pool.lookupOrNull(id) != null || anyPackHas(id)) continue; - int type = oldPack.getObjectType(ctx, ent.getOffset()); + long offset = ent.getOffset(); + int type = oldPack.getObjectType(ctx, offset); pw.addObject(pool.lookupAny(id, type)); + long objSize = oldRevIdx.findNextOffset(offset, maxOffset) + - offset; + estimatedPackSize += objSize; } } pm.endTask(); if (0 < pw.getObjectCount()) - writePack(UNREACHABLE_GARBAGE, pw, pm); + writePack(UNREACHABLE_GARBAGE, pw, pm, estimatedPackSize); } } @@ -479,9 +494,24 @@ public class DfsGarbageCollector { return pw; } + private long estimateGcPackSize(PackSource first, PackSource... rest) { + EnumSet sourceSet = EnumSet.of(first, rest); + // Every pack file contains 12 bytes of header and 20 bytes of trailer. + // Include the final pack file header and trailer size here and ignore + // the same from individual pack files. + long size = 32; + for (DfsPackDescription pack : getSourcePacks()) { + if (sourceSet.contains(pack.getPackSource())) { + size += pack.getFileSize(PACK) - 32; + } + } + return size; + } + private DfsPackDescription writePack(PackSource source, PackWriter pw, - ProgressMonitor pm) throws IOException { - DfsPackDescription pack = repo.getObjectDatabase().newPack(source); + ProgressMonitor pm, long estimatedPackSize) throws IOException { + DfsPackDescription pack = repo.getObjectDatabase().newPack(source, + estimatedPackSize); newPackDesc.add(pack); try (DfsOutputStream out = objdb.writeFile(pack, PACK)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index b1d6c0dd19..b68527544a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -265,6 +265,32 @@ public abstract class DfsObjDatabase extends ObjectDatabase { protected abstract DfsPackDescription newPack(PackSource source) throws IOException; + /** + * Generate a new unique name for a pack file. + * + *

+ * Default implementation of this method would be equivalent to + * {@code newPack(source).setEstimatedPackSize(estimatedPackSize)}. But the + * clients can override this method to use the given + * {@code estomatedPackSize} value more efficiently in the process of + * creating a new {@link DfsPackDescription} object. + * + * @param source + * where the pack stream is created. + * @param estimatedPackSize + * the estimated size of the pack. + * @return a unique name for the pack file. Must not collide with any other + * pack file name in the same DFS. + * @throws IOException + * a new unique pack description cannot be generated. + */ + protected DfsPackDescription newPack(PackSource source, + long estimatedPackSize) throws IOException { + DfsPackDescription pack = newPack(source); + pack.setEstimatedPackSize(estimatedPackSize); + return pack; + } + /** * Commit a pack and index pair that was written to the DFS. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index 11aef7feaf..ba3b39360b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -224,7 +224,8 @@ public class DfsPackCompactor { } boolean rollback = true; - DfsPackDescription pack = objdb.newPack(COMPACT); + DfsPackDescription pack = objdb.newPack(COMPACT, + estimatePackSize()); try { writePack(objdb, pack, pw, pm); writeIndex(objdb, pack, pw); @@ -251,6 +252,17 @@ public class DfsPackCompactor { } } + private long estimatePackSize() { + // Every pack file contains 12 bytes of header and 20 bytes of trailer. + // Include the final pack file header and trailer size here and ignore + // the same from individual pack files. + long size = 32; + for (DfsPackFile pack : srcPacks) { + size += pack.getPackDescription().getFileSize(PACK) - 32; + } + return size; + } + /** @return all of the source packs that fed into this compaction. */ public List getSourcePacks() { return toPrune(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java index 2b9d0e55c7..0411bbe5e4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java @@ -81,6 +81,8 @@ public class DfsPackDescription implements Comparable { private int indexVersion; + private long estimatedPackSize; + /** * Initialize a description by pack name and repository. *

@@ -189,6 +191,25 @@ public class DfsPackDescription implements Comparable { return size == null ? 0 : size.longValue(); } + /** + * @param estimatedPackSize + * estimated size of the .pack file in bytes. If 0 the pack file + * size is unknown. + * @return {@code this} + */ + public DfsPackDescription setEstimatedPackSize(long estimatedPackSize) { + this.estimatedPackSize = Math.max(0, estimatedPackSize); + return this; + } + + /** + * @return estimated size of the .pack file in bytes. If 0 the pack file + * size is unknown. + */ + public long getEstimatedPackSize() { + return estimatedPackSize; + } + /** @return number of objects in the pack. */ public long getObjectCount() { return objectCount; -- 2.39.5