package org.eclipse.jgit.internal.storage.dfs;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_CACHE_PREFIX;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_LIMIT;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_SIZE;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThrows;
+import java.util.List;
+import java.util.stream.Collectors;
+
import org.eclipse.jgit.internal.JGitText;
+import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DfsBlockCachePackExtConfig;
+import org.eclipse.jgit.internal.storage.pack.PackExt;
+import org.eclipse.jgit.lib.Config;
import org.junit.Test;
public class DfsBlockCacheConfigTest {
assertThat(config.getBlockSize(), is(65536));
}
+
+ @Test
+ public void fromConfigs() {
+ Config config = new Config();
+ config.setLong(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION,
+ CONFIG_KEY_BLOCK_LIMIT, 50 * 1024);
+ config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION,
+ CONFIG_KEY_BLOCK_SIZE, 1024);
+ config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION,
+ CONFIG_KEY_CONCURRENCY_LEVEL, 3);
+ config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION,
+ CONFIG_KEY_STREAM_RATIO, "0.5");
+
+ DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig()
+ .fromConfig(config);
+ assertThat(cacheConfig.getBlockLimit(), is(50L * 1024L));
+ assertThat(cacheConfig.getBlockSize(), is(1024));
+ assertThat(cacheConfig.getConcurrencyLevel(), is(3));
+ assertThat(cacheConfig.getStreamRatio(), closeTo(0.5, 0.0001));
+ }
+
+ @Test
+ public void fromConfig_blockLimitNotAMultipleOfBlockSize_throws() {
+ Config config = new Config();
+ config.setLong(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION,
+ CONFIG_KEY_BLOCK_LIMIT, 1025);
+ config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION,
+ CONFIG_KEY_BLOCK_SIZE, 1024);
+
+ assertThrows(IllegalArgumentException.class,
+ () -> new DfsBlockCacheConfig().fromConfig(config));
+ }
+
+ @Test
+ public void fromConfig_streamRatioInvalidFormat_throws() {
+ Config config = new Config();
+ config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION,
+ CONFIG_KEY_STREAM_RATIO, "0.a5");
+
+ assertThrows(IllegalArgumentException.class,
+ () -> new DfsBlockCacheConfig().fromConfig(config));
+ }
+
+ @Test
+ public void fromConfig_generatesDfsBlockCachePackExtConfigs() {
+ Config config = new Config();
+ addPackExtConfigEntry(config, "pack", List.of(PackExt.PACK),
+ /* blockLimit= */ 20 * 512, /* blockSize= */ 512);
+
+ addPackExtConfigEntry(config, "bitmap", List.of(PackExt.BITMAP_INDEX),
+ /* blockLimit= */ 25 * 1024, /* blockSize= */ 1024);
+
+ addPackExtConfigEntry(config, "index",
+ List.of(PackExt.INDEX, PackExt.OBJECT_SIZE_INDEX,
+ PackExt.REVERSE_INDEX),
+ /* blockLimit= */ 30 * 1024, /* blockSize= */ 1024);
+
+ DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig()
+ .fromConfig(config);
+ var configs = cacheConfig.getPackExtCacheConfigurations();
+ assertThat(configs, hasSize(3));
+ var packConfig = getConfigForExt(configs, PackExt.PACK);
+ assertThat(packConfig.getBlockLimit(), is(20L * 512L));
+ assertThat(packConfig.getBlockSize(), is(512));
+
+ var bitmapConfig = getConfigForExt(configs, PackExt.BITMAP_INDEX);
+ assertThat(bitmapConfig.getBlockLimit(), is(25L * 1024L));
+ assertThat(bitmapConfig.getBlockSize(), is(1024));
+
+ var indexConfig = getConfigForExt(configs, PackExt.INDEX);
+ assertThat(indexConfig.getBlockLimit(), is(30L * 1024L));
+ assertThat(indexConfig.getBlockSize(), is(1024));
+ assertThat(getConfigForExt(configs, PackExt.OBJECT_SIZE_INDEX),
+ is(indexConfig));
+ assertThat(getConfigForExt(configs, PackExt.REVERSE_INDEX),
+ is(indexConfig));
+ }
+
+ @Test
+ public void fromConfigs_dfsBlockCachePackExtConfigWithDuplicateExtensions_throws() {
+ Config config = new Config();
+ config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack1",
+ CONFIG_KEY_PACK_EXTENSIONS, PackExt.PACK.name());
+
+ config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack2",
+ CONFIG_KEY_PACK_EXTENSIONS, PackExt.PACK.name());
+
+ assertThrows(IllegalArgumentException.class,
+ () -> new DfsBlockCacheConfig().fromConfig(config));
+ }
+
+ @Test
+ public void fromConfigs_dfsBlockCachePackExtConfigWithEmptyExtensions_throws() {
+ Config config = new Config();
+ config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack1",
+ CONFIG_KEY_PACK_EXTENSIONS, "");
+
+ assertThrows(IllegalArgumentException.class,
+ () -> new DfsBlockCacheConfig().fromConfig(config));
+ }
+
+ @Test
+ public void fromConfigs_dfsBlockCachePackExtConfigWithNoExtensions_throws() {
+ Config config = new Config();
+ config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack1",
+ CONFIG_KEY_BLOCK_SIZE, 0);
+
+ assertThrows(IllegalArgumentException.class,
+ () -> new DfsBlockCacheConfig().fromConfig(config));
+ }
+
+ @Test
+ public void fromConfigs_dfsBlockCachePackExtConfigWithUnknownExtensions_throws() {
+ Config config = new Config();
+ config.setString(CONFIG_CORE_SECTION,
+ CONFIG_DFS_CACHE_PREFIX + "unknownExt",
+ CONFIG_KEY_PACK_EXTENSIONS, "NotAKnownExt");
+
+ assertThrows(IllegalArgumentException.class,
+ () -> new DfsBlockCacheConfig().fromConfig(config));
+ }
+
+ private static void addPackExtConfigEntry(Config config, String configName,
+ List<PackExt> packExts, long blockLimit, int blockSize) {
+ String packExtConfigName = CONFIG_DFS_CACHE_PREFIX + configName;
+ config.setString(CONFIG_CORE_SECTION, packExtConfigName,
+ CONFIG_KEY_PACK_EXTENSIONS, packExts.stream().map(PackExt::name)
+ .collect(Collectors.joining(" ")));
+ config.setLong(CONFIG_CORE_SECTION, packExtConfigName,
+ CONFIG_KEY_BLOCK_LIMIT, blockLimit);
+ config.setInt(CONFIG_CORE_SECTION, packExtConfigName,
+ CONFIG_KEY_BLOCK_SIZE, blockSize);
+ }
+
+ private static DfsBlockCacheConfig getConfigForExt(
+ List<DfsBlockCachePackExtConfig> configs, PackExt packExt) {
+ for (DfsBlockCachePackExtConfig config : configs) {
+ if (config.getPackExts().contains(packExt)) {
+ return config.getPackExtCacheConfiguration();
+ }
+ }
+ return null;
+ }
}
downloadCancelled=Download cancelled
downloadCancelledDuringIndexing=Download cancelled during indexing
duplicateAdvertisementsOf=duplicate advertisements of {0}
+duplicatePackExtensionsSet=Attempting to configure duplicate pack extensions: {0}.{1}.{2} contains {3}
duplicateRef=Duplicate ref: {0}
duplicateRefAttribute=Duplicate ref attribute: {0}
duplicateRemoteRefUpdateIsIllegal=Duplicate remote ref update is illegal. Affected remote name: {0}
noMergeHeadSpecified=No merge head specified
nonBareLinkFilesNotSupported=Link files are not supported with nonbare repos
nonCommitToHeads=Cannot point a branch to a non-commit object
+noPackExtConfigurationGiven=No PackExt configuration given
+noPackExtGivenForConfiguration=No PackExt given for configuration
noPathAttributesFound=No Attributes found for {0}.
noSuchRef=no such ref
noSuchRefKnown=no such ref: {0}
unknownObjectInIndex=unknown object {0} found in index but not in pack file
unknownObjectType=Unknown object type {0}.
unknownObjectType2=unknown
+unknownPackExtension=Unknown pack extension: {0}.{1}.{2}={3}
unknownPositionEncoding=Unknown position encoding %s
unknownRefStorageFormat=Unknown ref storage format "{0}"
unknownRepositoryFormat=Unknown repository format
/***/ public String downloadCancelled;
/***/ public String downloadCancelledDuringIndexing;
/***/ public String duplicateAdvertisementsOf;
+ /***/ public String duplicatePackExtensionsSet;
/***/ public String duplicateRef;
/***/ public String duplicateRefAttribute;
/***/ public String duplicateRemoteRefUpdateIsIllegal;
/***/ public String noMergeHeadSpecified;
/***/ public String nonBareLinkFilesNotSupported;
/***/ public String nonCommitToHeads;
+ /***/ public String noPackExtConfigurationGiven;
+ /***/ public String noPackExtGivenForConfiguration;
/***/ public String noPathAttributesFound;
/***/ public String noSuchRef;
/***/ public String noSuchRefKnown;
/***/ public String unknownObjectInIndex;
/***/ public String unknownObjectType;
/***/ public String unknownObjectType2;
+ /***/ public String unknownPackExtension;
/***/ public String unknownPositionEncoding;
/***/ public String unknownRefStorageFormat;
/***/ public String unknownRepositoryFormat;
package org.eclipse.jgit.internal.storage.dfs;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_CACHE_PREFIX;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_LIMIT;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_SIZE;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO;
import java.text.MessageFormat;
import java.time.Duration;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.function.Consumer;
+import java.util.stream.Collectors;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.pack.PackExt;
public static final int DEFAULT_CACHE_HOT_MAX = 1;
private long blockLimit;
+
private int blockSize;
+
private double streamRatio;
+
private int concurrencyLevel;
private Consumer<Long> refLock;
+
private Map<PackExt, Integer> cacheHotMap;
private IndexEventConsumer indexEventConsumer;
+ private List<DfsBlockCachePackExtConfig> packExtCacheConfigurations;
+
/**
* Create a default configuration.
*/
setStreamRatio(0.30);
setConcurrencyLevel(32);
cacheHotMap = Collections.emptyMap();
+ packExtCacheConfigurations = Collections.emptyList();
}
/**
* Set maximum number bytes of heap memory to dedicate to caching pack file
* data.
* <p>
- * It is strongly recommended to set the block limit to be an integer multiple
- * of the block size. This constraint is not enforced by this method (since
- * it may be called before {@link #setBlockSize(int)}), but it is enforced by
- * {@link #fromConfig(Config)}.
+ * It is strongly recommended to set the block limit to be an integer
+ * multiple of the block size. This constraint is not enforced by this
+ * method (since it may be called before {@link #setBlockSize(int)}), but it
+ * is enforced by {@link #fromConfig(Config)}.
*
* @param newLimit
* maximum number bytes of heap memory to dedicate to caching
*/
public DfsBlockCacheConfig setBlockLimit(long newLimit) {
if (newLimit <= 0) {
- throw new IllegalArgumentException(MessageFormat.format(
- JGitText.get().blockLimitNotPositive,
- Long.valueOf(newLimit)));
+ throw new IllegalArgumentException(
+ MessageFormat.format(JGitText.get().blockLimitNotPositive,
+ Long.valueOf(newLimit)));
}
blockLimit = newLimit;
return this;
return this;
}
+ /**
+ * Get the list of pack ext cache configs.
+ *
+ * @return the list of pack ext cache configs.
+ */
+ List<DfsBlockCachePackExtConfig> getPackExtCacheConfigurations() {
+ return packExtCacheConfigurations;
+ }
+
/**
* Update properties by setting fields from the configuration.
* <p>
* If a property is not defined in the configuration, then it is left
* unmodified.
* <p>
- * Enforces certain constraints on the combination of settings in the config,
- * for example that the block limit is a multiple of the block size.
+ * Enforces certain constraints on the combination of settings in the
+ * config, for example that the block limit is a multiple of the block size.
*
* @param rc
* configuration to read properties from.
* @return {@code this}
*/
public DfsBlockCacheConfig fromConfig(Config rc) {
- long cfgBlockLimit = rc.getLong(
- CONFIG_CORE_SECTION,
- CONFIG_DFS_SECTION,
- CONFIG_KEY_BLOCK_LIMIT,
- getBlockLimit());
- int cfgBlockSize = rc.getInt(
- CONFIG_CORE_SECTION,
- CONFIG_DFS_SECTION,
- CONFIG_KEY_BLOCK_SIZE,
+ fromConfig(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, rc);
+ loadPackExtConfigs(rc);
+ return this;
+ }
+
+ private void fromConfig(String section, String subSection, Config rc) {
+ long cfgBlockLimit = rc.getLong(section, subSection,
+ CONFIG_KEY_BLOCK_LIMIT, getBlockLimit());
+ int cfgBlockSize = rc.getInt(section, subSection, CONFIG_KEY_BLOCK_SIZE,
getBlockSize());
if (cfgBlockLimit % cfgBlockSize != 0) {
throw new IllegalArgumentException(MessageFormat.format(
JGitText.get().blockLimitNotMultipleOfBlockSize,
- Long.valueOf(cfgBlockLimit),
- Long.valueOf(cfgBlockSize)));
+ Long.valueOf(cfgBlockLimit), Long.valueOf(cfgBlockSize)));
}
setBlockLimit(cfgBlockLimit);
setBlockSize(cfgBlockSize);
- setConcurrencyLevel(rc.getInt(
- CONFIG_CORE_SECTION,
- CONFIG_DFS_SECTION,
- CONFIG_KEY_CONCURRENCY_LEVEL,
- getConcurrencyLevel()));
+ setConcurrencyLevel(rc.getInt(section, subSection,
+ CONFIG_KEY_CONCURRENCY_LEVEL, getConcurrencyLevel()));
- String v = rc.getString(
- CONFIG_CORE_SECTION,
- CONFIG_DFS_SECTION,
- CONFIG_KEY_STREAM_RATIO);
+ String v = rc.getString(section, subSection, CONFIG_KEY_STREAM_RATIO);
if (v != null) {
try {
setStreamRatio(Double.parseDouble(v));
} catch (NumberFormatException e) {
throw new IllegalArgumentException(MessageFormat.format(
- JGitText.get().enumValueNotSupported3,
- CONFIG_CORE_SECTION,
- CONFIG_DFS_SECTION,
- CONFIG_KEY_STREAM_RATIO, v), e);
+ JGitText.get().enumValueNotSupported3, section,
+ subSection, CONFIG_KEY_STREAM_RATIO, v), e);
}
}
- return this;
+ }
+
+ private void loadPackExtConfigs(Config config) {
+ List<String> subSections = config.getSubsections(CONFIG_CORE_SECTION)
+ .stream()
+ .filter(section -> section.startsWith(CONFIG_DFS_CACHE_PREFIX))
+ .collect(Collectors.toList());
+ if (subSections.size() == 0) {
+ return;
+ }
+ ArrayList<DfsBlockCachePackExtConfig> cacheConfigs = new ArrayList<>();
+ Set<PackExt> extensionsSeen = new HashSet<>();
+ for (String subSection : subSections) {
+ var cacheConfig = DfsBlockCachePackExtConfig.fromConfig(config,
+ CONFIG_CORE_SECTION, subSection);
+ Set<PackExt> packExtsDuplicates = intersection(extensionsSeen,
+ cacheConfig.packExts);
+ if (packExtsDuplicates.size() > 0) {
+ String duplicatePackExts = packExtsDuplicates.stream()
+ .map(PackExt::toString)
+ .collect(Collectors.joining(","));
+ throw new IllegalArgumentException(MessageFormat.format(
+ JGitText.get().duplicatePackExtensionsSet,
+ CONFIG_CORE_SECTION, subSection,
+ CONFIG_KEY_PACK_EXTENSIONS, duplicatePackExts));
+ }
+ extensionsSeen.addAll(cacheConfig.packExts);
+ cacheConfigs.add(cacheConfig);
+ }
+ packExtCacheConfigurations = cacheConfigs;
+ }
+
+ private static <T> Set<T> intersection(Set<T> first, Set<T> second) {
+ Set<T> ret = new HashSet<>();
+ for (T entry : second) {
+ if (first.contains(entry)) {
+ ret.add(entry);
+ }
+ }
+ return ret;
}
/** Consumer of DfsBlockCache loading and eviction events for indexes. */
return false;
}
}
+
+ /**
+ * A configuration for a single cache table storing 1 or more Pack
+ * extensions.
+ * <p>
+ * The current pack ext cache tables implementation supports the same
+ * parameters the ClockBlockCacheTable (current default implementation).
+ * <p>
+ * Configuration falls back to the defaults coded values defined in the
+ * {@link DfsBlockCacheConfig} when not set on each cache table
+ * configuration and NOT the values of the basic dfs section.
+ * <p>
+ * <code>
+ *
+ * Format:
+ * [core "dfs.packCache"]
+ * packExtensions = "PACK"
+ * blockSize = 512
+ * blockLimit = 100
+ * concurrencyLevel = 5
+ *
+ * [core "dfs.multipleExtensionCache"]
+ * packExtensions = "INDEX REFTABLE BITMAP_INDEX"
+ * blockSize = 512
+ * blockLimit = 100
+ * concurrencyLevel = 5
+ * </code>
+ */
+ static class DfsBlockCachePackExtConfig {
+ // Set of pack extensions that will map to the cache instance.
+ private final EnumSet<PackExt> packExts;
+
+ // Configuration for the cache instance.
+ private final DfsBlockCacheConfig packExtCacheConfiguration;
+
+ private DfsBlockCachePackExtConfig(EnumSet<PackExt> packExts,
+ DfsBlockCacheConfig packExtCacheConfiguration) {
+ this.packExts = packExts;
+ this.packExtCacheConfiguration = packExtCacheConfiguration;
+ }
+
+ Set<PackExt> getPackExts() {
+ return packExts;
+ }
+
+ DfsBlockCacheConfig getPackExtCacheConfiguration() {
+ return packExtCacheConfiguration;
+ }
+
+ private static DfsBlockCachePackExtConfig fromConfig(Config config,
+ String section, String subSection) {
+ String packExtensions = config.getString(section, subSection,
+ CONFIG_KEY_PACK_EXTENSIONS);
+ if (packExtensions == null) {
+ throw new IllegalArgumentException(
+ JGitText.get().noPackExtGivenForConfiguration);
+ }
+ String[] extensions = packExtensions.split(" ", -1);
+ Set<PackExt> packExts = new HashSet<>(extensions.length);
+ for (String extension : extensions) {
+ try {
+ packExts.add(PackExt.valueOf(extension));
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(MessageFormat.format(
+ JGitText.get().unknownPackExtension, section,
+ subSection, CONFIG_KEY_PACK_EXTENSIONS, extension),
+ e);
+ }
+ }
+
+ DfsBlockCacheConfig dfsBlockCacheConfig = new DfsBlockCacheConfig();
+ dfsBlockCacheConfig.fromConfig(section, subSection, config);
+ return new DfsBlockCachePackExtConfig(EnumSet.copyOf(packExts),
+ dfsBlockCacheConfig);
+ }
+
+ }
}
\ No newline at end of file
/** The "dfs" section */
public static final String CONFIG_DFS_SECTION = "dfs";
+ /** The dfs cache subsection prefix */
+ public static final String CONFIG_DFS_CACHE_PREFIX = "dfs.";
+
/**
* The "receive" section
* @since 4.6
/** The "deltaBaseCacheLimit" key */
public static final String CONFIG_KEY_DELTA_BASE_CACHE_LIMIT = "deltaBaseCacheLimit";
+ /**
+ * The "packExtensions" key
+ *
+ * @since 7.0
+ **/
+ public static final String CONFIG_KEY_PACK_EXTENSIONS = "packExtensions";
+
/**
* The "symlinks" key
* @since 3.3
projects / jgit.git / commitdiff