From 4902b2baf8dde2f6249eb0aeb7f1164921ce5a86 Mon Sep 17 00:00:00 2001 From: Jacek Centkowski Date: Fri, 20 Sep 2024 08:47:13 +0200 Subject: [PATCH] Add `numberOfPackFilesAfterBitmap` to RepoStatistics Introduce a `numberOfPackFilesAfterBitmap` that contains the number of packfiles created since the latest bitmap generation. Notes: * the `repo.getObjectDatabase().getPacks()` that obtains the list of packs (in the existing `getStatistics` function) uses `PackDirectory.scanPacks` that boils down to call to `PackDirectory.scanPacksImpl` which is sorting packs prior returning them therefore the `numberOfPackFilesAfterBitmap` is just all packs before the one that has bitmap attached * the improved version of `packAndPrune` function (one that skips non-existent packfiles) was introduced for testing Change-Id: I608011462f104fc002ac527aa405f492a8a4b0c2 --- ...rOfPackFilesAfterBitmapStatisticsTest.java | 180 ++++++++++++++++++ .../jgit/internal/storage/file/GC.java | 10 + 2 files changed, 190 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java new file mode 100644 index 0000000000..820f0c768d --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2024 Jacek Centkowski and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.StreamSupport; + +import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.FileUtils; +import org.junit.Test; + +public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { + @Test + public void testShouldReportZeroObjectsForInitializedRepo() + throws IOException { + assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportAllPackFilesWhenNoGcWasPerformed() + throws Exception { + packAndPrune(); + long result = gc.getStatistics().numberOfPackFilesAfterBitmap; + + assertEquals(repo.getObjectDatabase().getPacks().size(), result); + } + + @Test + public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { + // given + addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportNewObjectsAfterGcWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & gc + parent = addCommit(parent); + gc.gc().get(); + assertEquals(2L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + private void packAndPrune() throws Exception { + try (SkipNonExistingFilesTestRepository testRepo = new SkipNonExistingFilesTestRepository( + repo)) { + testRepo.packAndPrune(); + } + } + + private RevCommit addCommit(RevCommit parent) throws Exception { + PersonIdent ident = new PersonIdent("repo-metrics", "repo@metrics.com"); + TestRepository.CommitBuilder builder = tr.commit() + .author(ident); + if (parent != null) { + builder.parent(parent); + } + RevCommit commit = builder.create(); + tr.update("master", commit); + parent = commit; + return parent; + } + + private long repositoryBitmapFiles() throws IOException { + return StreamSupport + .stream(Files + .newDirectoryStream(repo.getObjectDatabase() + .getPackDirectory().toPath(), "pack-*.bitmap") + .spliterator(), false) + .count(); + } + + /** + * The TestRepository has a {@link TestRepository#packAndPrune()} function + * but it fails in the last step after GC was performed as it doesn't + * SKIP_MISSING files. In order to circumvent it was copied and improved + * here. + */ + private static class SkipNonExistingFilesTestRepository + extends TestRepository { + private final FileRepository repo; + + private SkipNonExistingFilesTestRepository(FileRepository db) throws IOException { + super(db); + repo = db; + } + + @Override + public void packAndPrune() throws Exception { + ObjectDirectory odb = repo.getObjectDatabase(); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + + final PackFile pack, idx; + try (PackWriter pw = new PackWriter(repo)) { + Set all = new HashSet<>(); + for (Ref r : repo.getRefDatabase().getRefs()) + all.add(r.getObjectId()); + pw.preparePack(m, all, PackWriter.NONE); + + pack = new PackFile(odb.getPackDirectory(), pw.computeName(), + PackExt.PACK); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(pack))) { + pw.writePack(m, m, out); + } + pack.setReadOnly(); + + idx = pack.create(PackExt.INDEX); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(idx))) { + pw.writeIndex(out); + } + idx.setReadOnly(); + } + + odb.openPack(pack); + updateServerInfo(); + + // alternative packAndPrune implementation that skips missing files + // after GC. + for (Pack p : odb.getPacks()) { + for (MutableEntry e : p) + FileUtils.delete(odb.fileFor(e.toObjectId()), + FileUtils.SKIP_MISSING); + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 4fafc5a088..34b5ec01e1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1508,6 +1508,12 @@ public class GC { */ public long numberOfPackFiles; + /** + * The number of pack files that were created after the last bitmap + * generation. + */ + public long numberOfPackFilesAfterBitmap; + /** * The number of objects stored as loose objects. */ @@ -1543,6 +1549,8 @@ public class GC { final StringBuilder b = new StringBuilder(); b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$ + b.append(", numberOfPackFilesAfterBitmap=") //$NON-NLS-1$ + .append(numberOfPackFilesAfterBitmap); b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$ b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$ b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$ @@ -1569,6 +1577,8 @@ public class GC { ret.sizeOfPackedObjects += p.getPackFile().length(); if (p.getBitmapIndex() != null) ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); + else + ret.numberOfPackFilesAfterBitmap++; } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); -- 2.39.5