diff options
author | Matthias Sohn <matthias.sohn@sap.com> | 2024-11-12 00:06:18 +0100 |
---|---|---|
committer | Matthias Sohn <matthias.sohn@sap.com> | 2024-11-12 00:06:18 +0100 |
commit | 9f474f9445d170de651a0cb24c368fcadbbf943d (patch) | |
tree | 43f77d02536195ee7165d8b55eff931362586d35 /org.eclipse.jgit.test/tst | |
parent | 1342502ad0954526589437fd309ef70a36ee3099 (diff) | |
parent | 0dfc30951b5753b3966051c70e5b1c3860fd9fc5 (diff) | |
download | jgit-9f474f9445d170de651a0cb24c368fcadbbf943d.tar.gz jgit-9f474f9445d170de651a0cb24c368fcadbbf943d.zip |
Merge branch 'master' into stable-7.1
* master:
errorprone: Disable javadoc checks in tests
Rename numberOfPackFilesAfterBitmap to numberOfPackFilesSinceBitmap
Replace custom encoder Constants#encodeASCII by JDK implementation
Replace custom encoder `Constants#encode` by JDK implementation
DfsGarbageCollector: #setReflogExpire with Instant instead of Date
ssh: Minor simplification in SerialRangeSet
DfsBlockCacheConfig: propagate hotmap configs to pack ext cache configs
SystemReader: Offer methods with java.time API
Add `numberOfPackFilesAfterBitmap` to RepoStatistics
Enhance CommitBuilder#parent to tolerate null parent
GPG: use BC PGP secret key parsing out of the box
[errorprone] bc: Remove unused SExprParser#parseSecretKey
Update bouncycastle to 1.79
Update bytebuddy to 1.15.10
DfsPackCompactor: write object size index
[errorprone] BaseRepositoryBuilder: Use #split(sep, limit)
[errorprone] Remove deprecated security manager
[errorprone] RefDatabase: #getConflictingNames immutable return
DfsGarbageCollector: Add setter for reflog expiration time.
[errorprone] SeparateClassloadertTestRunner: use #split(String,int)
[errorprone] HttpConnection: Add missing summary in java
[errorprone] PackWriter: Fix javadoc tag in new #writeIndex method
[errorprone] ByteArraySet: Add summary fragment to javadoc
[errorprone] util.Stats: Add summary fragment to javadoc
DfsInserter: Read minBytesForObjectSizeIndex only from repo config
PackWriter: make PackWriter.writeIndex() take a PackIndexWriter
dfs: update getBlockCacheStats to return a List of BlockCacheStats
Update mockito to 5.14.2
Update bytebuddy to 1.15.7
Remove unnecessary argument handler in MergeBase.java
Replace custom encoder Constants#encodeASCII by JDK implementation
Change-Id: I0f84a69af152936f66bbcd2c8d9190ad159e7878
Diffstat (limited to 'org.eclipse.jgit.test/tst')
9 files changed, 325 insertions, 340 deletions
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerMissingPermissionsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerMissingPermissionsTest.java deleted file mode 100644 index d0fbdbd090..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerMissingPermissionsTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2019 Alex Jitianu <alex_jitianu@sync.ro> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.api; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.Policy; -import java.util.Collections; - -import org.eclipse.jgit.junit.RepositoryTestCase; -import org.eclipse.jgit.util.FileUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Tests that using a SecurityManager does not result in errors logged. - */ -public class SecurityManagerMissingPermissionsTest extends RepositoryTestCase { - - /** - * Collects all logging sent to the logging system. - */ - private final ByteArrayOutputStream errorOutput = new ByteArrayOutputStream(); - - private SecurityManager originalSecurityManager; - - private PrintStream defaultErrorOutput; - - @Override - @Before - public void setUp() throws Exception { - originalSecurityManager = System.getSecurityManager(); - - // slf4j-simple logs to System.err, redirect it to enable asserting - // logged errors - defaultErrorOutput = System.err; - System.setErr(new PrintStream(errorOutput)); - - refreshPolicyAllPermission(Policy.getPolicy()); - System.setSecurityManager(new SecurityManager()); - super.setUp(); - } - - /** - * If a SecurityManager is active a lot of {@link java.io.FilePermission} - * errors are thrown and logged while initializing a repository. - * - * @throws Exception - */ - @Test - public void testCreateNewRepos_MissingPermissions() throws Exception { - File wcTree = new File(getTemporaryDirectory(), - "CreateNewRepositoryTest_testCreateNewRepos"); - - File marker = new File(getTemporaryDirectory(), "marker"); - Files.write(marker.toPath(), Collections.singletonList("Can write")); - assertTrue("Can write in test directory", marker.isFile()); - FileUtils.delete(marker); - assertFalse("Can delete in test direcory", marker.exists()); - - Git git = Git.init().setBare(false) - .setDirectory(new File(wcTree.getAbsolutePath())).call(); - - addRepoToClose(git.getRepository()); - - assertEquals("", errorOutput.toString()); - } - - @Override - @After - public void tearDown() throws Exception { - System.setSecurityManager(originalSecurityManager); - System.setErr(defaultErrorOutput); - super.tearDown(); - } - - /** - * Refresh the Java Security Policy. - * - * @param policy - * the policy object - * - * @throws IOException - * if the temporary file that contains the policy could not be - * created - */ - private static void refreshPolicyAllPermission(Policy policy) - throws IOException { - // Starting with an all permissions policy. - String policyString = "grant { permission java.security.AllPermission; };"; - - // Do not use TemporaryFilesFactory, it will create a dependency cycle - Path policyFile = Files.createTempFile("testpolicy", ".txt"); - - try { - Files.write(policyFile, Collections.singletonList(policyString)); - System.setProperty("java.security.policy", - policyFile.toUri().toURL().toString()); - policy.refresh(); - } finally { - try { - Files.delete(policyFile); - } catch (IOException e) { - // Do not log; the test tests for no logging having occurred - e.printStackTrace(); - } - } - } - -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerTest.java deleted file mode 100644 index 2b930a1133..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2019 Nail Samatov <sanail@yandex.ru> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.api; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FilePermission; -import java.io.IOException; -import java.lang.reflect.ReflectPermission; -import java.nio.file.Files; -import java.security.Permission; -import java.security.SecurityPermission; -import java.util.ArrayList; -import java.util.List; -import java.util.PropertyPermission; -import java.util.logging.LoggingPermission; - -import javax.security.auth.AuthPermission; - -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.junit.JGitTestUtil; -import org.eclipse.jgit.junit.MockSystemReader; -import org.eclipse.jgit.junit.SeparateClassloaderTestRunner; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.SystemReader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * <p> - * Tests if jgit works if SecurityManager is enabled. - * </p> - * - * <p> - * Note: JGit's classes shouldn't be used before SecurityManager is configured. - * If you use some JGit's class before SecurityManager is replaced then part of - * the code can be invoked outside of our custom SecurityManager and this test - * becomes useless. - * </p> - * - * <p> - * For example the class {@link org.eclipse.jgit.util.FS} is used widely in jgit - * sources. It contains DETECTED static field. At the first usage of the class - * FS the field DETECTED is initialized and during initialization many system - * operations that SecurityManager can forbid are invoked. - * </p> - * - * <p> - * For this reason this test doesn't extend LocalDiskRepositoryTestCase (it uses - * JGit's classes in setUp() method) and other JGit's utility classes. It's done - * to affect SecurityManager as less as possible. - * </p> - * - * <p> - * We use SeparateClassloaderTestRunner to isolate FS.DETECTED field - * initialization between different tests run. - * </p> - */ -@RunWith(SeparateClassloaderTestRunner.class) -public class SecurityManagerTest { - private File root; - - private SecurityManager originalSecurityManager; - - private List<Permission> permissions = new ArrayList<>(); - - @Before - public void setUp() throws Exception { - // Create working directory - SystemReader.setInstance(new MockSystemReader()); - root = Files.createTempDirectory("jgit-security").toFile(); - - // Add system permissions - permissions.add(new RuntimePermission("*")); - permissions.add(new SecurityPermission("*")); - permissions.add(new AuthPermission("*")); - permissions.add(new ReflectPermission("*")); - permissions.add(new PropertyPermission("*", "read,write")); - permissions.add(new LoggingPermission("control", null)); - - permissions.add(new FilePermission( - System.getProperty("java.home") + "/-", "read")); - - String tempDir = System.getProperty("java.io.tmpdir"); - permissions.add(new FilePermission(tempDir, "read,write,delete")); - permissions - .add(new FilePermission(tempDir + "/-", "read,write,delete")); - - // Add permissions to dependent jar files. - String classPath = System.getProperty("java.class.path"); - if (classPath != null) { - for (String path : classPath.split(File.pathSeparator)) { - permissions.add(new FilePermission(path, "read")); - } - } - // Add permissions to jgit class files. - String jgitSourcesRoot = new File(System.getProperty("user.dir")) - .getParent(); - permissions.add(new FilePermission(jgitSourcesRoot + "/-", "read")); - - // Add permissions to working dir for jgit. Our git repositories will be - // initialized and cloned here. - permissions.add(new FilePermission(root.getPath() + "/-", - "read,write,delete,execute")); - - // Replace Security Manager - originalSecurityManager = System.getSecurityManager(); - System.setSecurityManager(new SecurityManager() { - - @Override - public void checkPermission(Permission requested) { - for (Permission permission : permissions) { - if (permission.implies(requested)) { - return; - } - } - - super.checkPermission(requested); - } - }); - } - - @After - public void tearDown() throws Exception { - System.setSecurityManager(originalSecurityManager); - - // Note: don't use this method before security manager is replaced in - // setUp() method. The method uses FS.DETECTED internally and can affect - // the test. - FileUtils.delete(root, FileUtils.RECURSIVE | FileUtils.RETRY); - } - - @Test - public void testInitAndClone() throws IOException, GitAPIException { - File remote = new File(root, "remote"); - File local = new File(root, "local"); - - try (Git git = Git.init().setDirectory(remote).call()) { - JGitTestUtil.write(new File(remote, "hello.txt"), "Hello world!"); - git.add().addFilepattern(".").call(); - git.commit().setMessage("Initial commit").call(); - } - - try (Git git = Git.cloneRepository().setURI(remote.toURI().toString()) - .setDirectory(local).call()) { - assertTrue(new File(local, ".git").exists()); - - JGitTestUtil.write(new File(local, "hi.txt"), "Hi!"); - git.add().addFilepattern(".").call(); - RevCommit commit1 = git.commit().setMessage("Commit on local repo") - .call(); - assertEquals("Commit on local repo", commit1.getFullMessage()); - assertNotNull(TreeWalk.forPath(git.getRepository(), "hello.txt", - commit1.getTree())); - } - - } - -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java index ac769498e2..2c4b432a01 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java @@ -24,9 +24,10 @@ public class AggregatedBlockCacheStatsTest { @Test public void getName() { BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", List.of()); + .fromStatsList(List.of()); - assertThat(aggregatedBlockCacheStats.getName(), equalTo("name")); + assertThat(aggregatedBlockCacheStats.getName(), + equalTo(AggregatedBlockCacheStats.class.getName())); } @Test @@ -46,8 +47,7 @@ public class AggregatedBlockCacheStatsTest { currentSizes[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getCurrentSize(), currentSizes); @@ -73,8 +73,7 @@ public class AggregatedBlockCacheStatsTest { hitCounts[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getHitCount(), hitCounts); } @@ -99,8 +98,7 @@ public class AggregatedBlockCacheStatsTest { missCounts[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getMissCount(), missCounts); } @@ -131,8 +129,7 @@ public class AggregatedBlockCacheStatsTest { totalRequestCounts[PackExt.INDEX.getPosition()] = 14; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getTotalRequestCount(), totalRequestCounts); @@ -160,8 +157,7 @@ public class AggregatedBlockCacheStatsTest { hitRatios[PackExt.INDEX.getPosition()] = 0; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("Name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getHitRatio(), hitRatios); } @@ -186,8 +182,7 @@ public class AggregatedBlockCacheStatsTest { evictions[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("Name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getEvictions(), evictions); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java index cb68bbc515..2e2f86bf80 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java @@ -1,8 +1,14 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DEFAULT_NAME; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isA; + +import java.util.List; import org.junit.Test; @@ -30,7 +36,8 @@ public class ClockBlockCacheTableTest { ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( createBlockCacheConfig()); - assertThat(cacheTable.getBlockCacheStats().getName(), + assertThat(cacheTable.getBlockCacheStats(), hasSize(1)); + assertThat(cacheTable.getBlockCacheStats().get(0).getName(), equalTo(DEFAULT_NAME)); } @@ -39,7 +46,18 @@ public class ClockBlockCacheTableTest { ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( createBlockCacheConfig().setName(NAME)); - assertThat(cacheTable.getBlockCacheStats().getName(), equalTo(NAME)); + assertThat(cacheTable.getBlockCacheStats(), hasSize(1)); + assertThat(cacheTable.getBlockCacheStats().get(0).getName(), + equalTo(NAME)); + } + + @Test + public void getAllBlockCacheStats() { + ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( + createBlockCacheConfig()); + + List<BlockCacheStats> blockCacheStats = cacheTable.getBlockCacheStats(); + assertThat(blockCacheStats, contains(isA(BlockCacheStats.class))); } private static DfsBlockCacheConfig createBlockCacheConfig() { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java index 65774e6d6c..afa3179cde 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java @@ -180,6 +180,76 @@ public class DfsBlockCacheConfigTest { } @Test + public void fromConfig_withExistingCacheHotMap_configWithPackExtConfigsHasHotMaps() { + Config config = new Config(); + addPackExtConfigEntry(config, "pack", List.of(PackExt.PACK), + /* blockLimit= */ 20 * 512, /* blockSize= */ 512); + + addPackExtConfigEntry(config, "bitmap", List.of(PackExt.BITMAP_INDEX), + /* blockLimit= */ 25 * 1024, /* blockSize= */ 1024); + + addPackExtConfigEntry(config, "index", + List.of(PackExt.INDEX, PackExt.OBJECT_SIZE_INDEX, + PackExt.REVERSE_INDEX), + /* blockLimit= */ 30 * 1024, /* blockSize= */ 1024); + + Map<PackExt, Integer> cacheHotMap = Map.of(PackExt.PACK, 1, + PackExt.BITMAP_INDEX, 2, PackExt.INDEX, 3, PackExt.REFTABLE, 4); + + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig(); + cacheConfig.setCacheHotMap(cacheHotMap); + cacheConfig.fromConfig(config); + + var configs = cacheConfig.getPackExtCacheConfigurations(); + assertThat(cacheConfig.getCacheHotMap(), is(cacheHotMap)); + assertThat(configs, hasSize(3)); + var packConfig = getConfigForExt(configs, PackExt.PACK); + assertThat(packConfig.getCacheHotMap(), is(Map.of(PackExt.PACK, 1))); + + var bitmapConfig = getConfigForExt(configs, PackExt.BITMAP_INDEX); + assertThat(bitmapConfig.getCacheHotMap(), + is(Map.of(PackExt.BITMAP_INDEX, 2))); + + var indexConfig = getConfigForExt(configs, PackExt.INDEX); + assertThat(indexConfig.getCacheHotMap(), is(Map.of(PackExt.INDEX, 3))); + } + + @Test + public void setCacheHotMap_configWithPackExtConfigs_setsHotMaps() { + Config config = new Config(); + addPackExtConfigEntry(config, "pack", List.of(PackExt.PACK), + /* blockLimit= */ 20 * 512, /* blockSize= */ 512); + + addPackExtConfigEntry(config, "bitmap", List.of(PackExt.BITMAP_INDEX), + /* blockLimit= */ 25 * 1024, /* blockSize= */ 1024); + + addPackExtConfigEntry(config, "index", + List.of(PackExt.INDEX, PackExt.OBJECT_SIZE_INDEX, + PackExt.REVERSE_INDEX), + /* blockLimit= */ 30 * 1024, /* blockSize= */ 1024); + + Map<PackExt, Integer> cacheHotMap = Map.of(PackExt.PACK, 1, + PackExt.BITMAP_INDEX, 2, PackExt.INDEX, 3, PackExt.REFTABLE, 4); + + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig() + .fromConfig(config); + cacheConfig.setCacheHotMap(cacheHotMap); + + var configs = cacheConfig.getPackExtCacheConfigurations(); + assertThat(cacheConfig.getCacheHotMap(), is(cacheHotMap)); + assertThat(configs, hasSize(3)); + var packConfig = getConfigForExt(configs, PackExt.PACK); + assertThat(packConfig.getCacheHotMap(), is(Map.of(PackExt.PACK, 1))); + + var bitmapConfig = getConfigForExt(configs, PackExt.BITMAP_INDEX); + assertThat(bitmapConfig.getCacheHotMap(), + is(Map.of(PackExt.BITMAP_INDEX, 2))); + + var indexConfig = getConfigForExt(configs, PackExt.INDEX); + assertThat(indexConfig.getCacheHotMap(), is(Map.of(PackExt.INDEX, 3))); + } + + @Test public void fromConfigs_baseConfigOnly_nameSetFromConfigDfsSubSection() { Config config = new Config(); @@ -291,6 +361,7 @@ public class DfsBlockCacheConfigTest { " Name: dfs.pack", " BlockLimit: " + 20 * 512, " BlockSize: 512", " StreamRatio: 0.3", " ConcurrencyLevel: 32", + " CacheHotMapEntry: " + PackExt.PACK + " : " + 10, " PackExts: " + List.of(PackExt.PACK)))); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index 2be11d32ea..f9fbfe8db0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -6,6 +6,7 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.IN import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -15,8 +16,11 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; +import java.util.Date; +import java.util.GregorianCalendar; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; @@ -24,6 +28,7 @@ import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.reftable.LogCursor; import org.eclipse.jgit.internal.storage.reftable.RefCursor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableReader; @@ -37,6 +42,7 @@ import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevBlob; @@ -44,6 +50,7 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.GitDateParser; import org.eclipse.jgit.util.SystemReader; import org.junit.After; import org.junit.Before; @@ -1275,6 +1282,90 @@ public class DfsGarbageCollectorTest { bitmapIndex.getXorBitmapCount() > 0); } + @Test + public void gitGCWithRefLogExpire() throws Exception { + String master = "refs/heads/master"; + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update(master, commit1); + DfsGarbageCollector gc = new DfsGarbageCollector(repo); + gc.setReftableConfig(new ReftableConfig()); + run(gc); + DfsPackDescription t1 = odb.newPack(INSERT); + Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, + "refs/heads/next", commit0.copy()); + long currentDay = new Date().getTime(); + GregorianCalendar cal = new GregorianCalendar(SystemReader + .getInstance().getTimeZone(), SystemReader.getInstance() + .getLocale()); + long ten_days_ago = GitDateParser.parse("10 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ; + long twenty_days_ago = GitDateParser.parse("20 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ; + long thirty_days_ago = GitDateParser.parse("30 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ;; + long fifty_days_ago = GitDateParser.parse("50 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ; + PersonIdent who2 = new PersonIdent("J.Author", "authemail", currentDay, -8 * 60); + PersonIdent who3 = new PersonIdent("J.Author", "authemail", ten_days_ago, -8 * 60); + PersonIdent who4 = new PersonIdent("J.Author", "authemail", twenty_days_ago, -8 * 60); + PersonIdent who5 = new PersonIdent("J.Author", "authemail", thirty_days_ago, -8 * 60); + PersonIdent who6 = new PersonIdent("J.Author", "authemail", fifty_days_ago, -8 * 60); + + try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) { + ReftableWriter w = new ReftableWriter(out); + w.setMinUpdateIndex(42); + w.setMaxUpdateIndex(42); + w.begin(); + w.sortAndWriteRefs(Collections.singleton(next)); + w.writeLog("refs/heads/branch", 1, who2, ObjectId.zeroId(),id(2), "Branch Message"); + w.writeLog("refs/heads/branch1", 2, who3, ObjectId.zeroId(),id(3), "Branch Message1"); + w.writeLog("refs/heads/branch2", 2, who4, ObjectId.zeroId(),id(4), "Branch Message2"); + w.writeLog("refs/heads/branch3", 2, who5, ObjectId.zeroId(),id(5), "Branch Message3"); + w.writeLog("refs/heads/branch4", 2, who6, ObjectId.zeroId(),id(6), "Branch Message4"); + w.finish(); + t1.addFileExt(REFTABLE); + t1.setReftableStats(w.getStats()); + } + odb.commitPack(Collections.singleton(t1), null); + + gc = new DfsGarbageCollector(repo); + gc.setReftableConfig(new ReftableConfig()); + // Expire ref log entries older than 30 days + gc.setRefLogExpire(Instant.ofEpochMilli(thirty_days_ago)); + run(gc); + + // Single GC pack present with all objects. + assertEquals(1, odb.getPacks().length); + DfsPackFile pack = odb.getPacks()[0]; + DfsPackDescription desc = pack.getPackDescription(); + + DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc); + try (DfsReader ctx = odb.newReader(); + ReftableReader rr = table.open(ctx); + RefCursor rc = rr.allRefs(); + LogCursor lc = rr.allLogs()) { + assertTrue(rc.next()); + assertEquals(master, rc.getRef().getName()); + assertEquals(commit1, rc.getRef().getObjectId()); + assertTrue(rc.next()); + assertEquals(next.getName(), rc.getRef().getName()); + assertEquals(commit0, rc.getRef().getObjectId()); + assertFalse(rc.next()); + assertTrue(lc.next()); + assertEquals(lc.getRefName(),"refs/heads/branch"); + assertTrue(lc.next()); + assertEquals(lc.getRefName(),"refs/heads/branch1"); + assertTrue(lc.next()); + assertEquals(lc.getRefName(),"refs/heads/branch2"); + // Old entries are purged + assertFalse(lc.next()); + + } + + } + + private RevCommit commitChain(RevCommit parent, int length) throws Exception { for (int i = 0; i < length; i++) { @@ -1364,4 +1455,12 @@ public class DfsGarbageCollectorTest { } return cnt; } + private static ObjectId id(int i) { + byte[] buf = new byte[OBJECT_ID_LENGTH]; + buf[0] = (byte) (i & 0xff); + buf[1] = (byte) ((i >>> 8) & 0xff); + buf[2] = (byte) ((i >>> 16) & 0xff); + buf[3] = (byte) (i >>> 24); + return ObjectId.fromRaw(buf); + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java index c516e30f50..c3b6aa85a2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java @@ -12,13 +12,18 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.revwalk.RevCommit; import org.junit.Before; import org.junit.Test; @@ -98,6 +103,40 @@ public class DfsPackCompacterTest { pack.getPackDescription().getEstimatedPackSize()); } + @Test + public void testObjectSizeIndexWritten() throws Exception { + writeObjectSizeIndex(repo, true); + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + compact(); + + Optional<DfsPackFile> compactPack = Arrays.stream(odb.getPacks()) + .filter(pack -> pack.getPackDescription() + .getPackSource() == COMPACT) + .findFirst(); + assertTrue(compactPack.isPresent()); + assertTrue(compactPack.get().getPackDescription().hasFileExt(OBJECT_SIZE_INDEX)); + } + + @Test + public void testObjectSizeIndexNotWritten() throws Exception { + writeObjectSizeIndex(repo, false); + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + compact(); + + Optional<DfsPackFile> compactPack = Arrays.stream(odb.getPacks()) + .filter(pack -> pack.getPackDescription() + .getPackSource() == COMPACT) + .findFirst(); + assertTrue(compactPack.isPresent()); + assertFalse(compactPack.get().getPackDescription().hasFileExt(OBJECT_SIZE_INDEX)); + } + private TestRepository<InMemoryRepository>.CommitBuilder commit() { return git.commit(); } @@ -108,4 +147,9 @@ public class DfsPackCompacterTest { compactor.compact(null); odb.clearCache(); } + + private static void writeObjectSizeIndex(DfsRepository repo, boolean should) { + repo.getConfig().setInt(ConfigConstants.CONFIG_PACK_SECTION, null, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, should ? 0 : -1); + } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java index c5c964bcab..e7627bc4ab 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java @@ -10,8 +10,10 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.sameInstance; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotNull; @@ -398,15 +400,51 @@ public class PackExtBlockCacheTableTest { } @Test - public void getBlockCacheStats_getName_returnsPackExtCacheTableName() { - DfsBlockCacheStats packStats = new DfsBlockCacheStats(); - PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( - cacheTableWithStats(/* name= */ "defaultName", packStats), - Map.of(PackExt.PACK, cacheTableWithStats(/* name= */ "packName", - packStats))); + public void getAllBlockCacheStats() { + String defaultTableName = "default table"; + DfsBlockCacheStats defaultStats = new DfsBlockCacheStats( + defaultTableName); + incrementCounter(4, + () -> defaultStats.incrementHit(new TestKey(PackExt.REFTABLE))); + + String packTableName = "pack table"; + DfsBlockCacheStats packStats = new DfsBlockCacheStats(packTableName); + incrementCounter(5, + () -> packStats.incrementHit(new TestKey(PackExt.PACK))); - assertThat(tables.getBlockCacheStats().getName(), - equalTo("defaultName,packName")); + String bitmapTableName = "bitmap table"; + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats( + bitmapTableName); + incrementCounter(6, () -> bitmapStats + .incrementHit(new TestKey(PackExt.BITMAP_INDEX))); + + DfsBlockCacheTable defaultTable = cacheTableWithStats(defaultStats); + DfsBlockCacheTable packTable = cacheTableWithStats(packStats); + DfsBlockCacheTable bitmapTable = cacheTableWithStats(bitmapStats); + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(defaultTable, Map.of(PackExt.PACK, packTable, + PackExt.BITMAP_INDEX, bitmapTable)); + + List<BlockCacheStats> statsList = tables.getBlockCacheStats(); + assertThat(statsList, hasSize(3)); + + long[] defaultTableHitCounts = createEmptyStatsArray(); + defaultTableHitCounts[PackExt.REFTABLE.getPosition()] = 4; + assertArrayEquals( + getCacheStatsByName(statsList, defaultTableName).getHitCount(), + defaultTableHitCounts); + + long[] packTableHitCounts = createEmptyStatsArray(); + packTableHitCounts[PackExt.PACK.getPosition()] = 5; + assertArrayEquals( + getCacheStatsByName(statsList, packTableName).getHitCount(), + packTableHitCounts); + + long[] bitmapHitCounts = createEmptyStatsArray(); + bitmapHitCounts[PackExt.BITMAP_INDEX.getPosition()] = 6; + assertArrayEquals( + getCacheStatsByName(statsList, bitmapTableName).getHitCount(), + bitmapHitCounts); } @Test @@ -431,7 +469,8 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getCurrentSize(), + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getCurrentSize(), currentSizes); } @@ -460,7 +499,9 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getHitCount(), hitCounts); + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getHitCount(), + hitCounts); } @Test @@ -488,7 +529,8 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getMissCount(), + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getMissCount(), missCounts); } @@ -523,8 +565,9 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getTotalRequestCount(), - totalRequestCounts); + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()) + .getTotalRequestCount(), totalRequestCounts); } @Test @@ -554,7 +597,9 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getHitRatio(), hitRatios); + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getHitRatio(), + hitRatios); } @Test @@ -582,10 +627,21 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getEvictions(), + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getEvictions(), evictions); } + private BlockCacheStats getCacheStatsByName( + List<BlockCacheStats> blockCacheStats, String name) { + for (BlockCacheStats entry : blockCacheStats) { + if (entry.getName().equals(name)) { + return entry; + } + } + return null; + } + private static void incrementCounter(int amount, Runnable fn) { for (int i = 0; i < amount; i++) { fn.run(); @@ -597,15 +653,16 @@ public class PackExtBlockCacheTableTest { } private static DfsBlockCacheTable cacheTableWithStats( - DfsBlockCacheStats dfsBlockCacheStats) { + BlockCacheStats dfsBlockCacheStats) { return cacheTableWithStats(CACHE_NAME, dfsBlockCacheStats); } private static DfsBlockCacheTable cacheTableWithStats(String name, - DfsBlockCacheStats dfsBlockCacheStats) { + BlockCacheStats dfsBlockCacheStats) { DfsBlockCacheTable cacheTable = mock(DfsBlockCacheTable.class); when(cacheTable.getName()).thenReturn(name); - when(cacheTable.getBlockCacheStats()).thenReturn(dfsBlockCacheStats); + when(cacheTable.getBlockCacheStats()) + .thenReturn(List.of(dfsBlockCacheStats)); return cacheTable; } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java index 820f0c768d..42b99ae512 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java @@ -33,18 +33,18 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.util.FileUtils; import org.junit.Test; -public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { +public class GcNumberOfPackFilesSinceBitmapStatisticsTest extends GcTestCase { @Test public void testShouldReportZeroObjectsForInitializedRepo() throws IOException { - assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); } @Test public void testShouldReportAllPackFilesWhenNoGcWasPerformed() throws Exception { packAndPrune(); - long result = gc.getStatistics().numberOfPackFilesAfterBitmap; + long result = gc.getStatistics().numberOfPackFilesSinceBitmap; assertEquals(repo.getObjectDatabase().getPacks().size(), result); } @@ -55,11 +55,11 @@ public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { addCommit(null); gc.gc().get(); assertEquals(1L, repositoryBitmapFiles()); - assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); } @Test - public void testShouldReportNewObjectsAfterGcWhenRepositoryProgresses() + public void testShouldReportNewObjectsSinceGcWhenRepositoryProgresses() throws Exception { // commit & gc RevCommit parent = addCommit(null); @@ -70,7 +70,7 @@ public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { addCommit(parent); packAndPrune(); - assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); } @Test @@ -90,7 +90,7 @@ public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { addCommit(parent); packAndPrune(); - assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); } private void packAndPrune() throws Exception { |