DfsBlockCacheConfig: support configurations for dfs cache tables per extensions

Parse configurations for tables containing a set of extensions,
defined in [core "dfs.*"] sections.

Parse configurations for cache tables according to configurations
defined in [core "dfs.*"] git config sections for sets of
extensions. The current [core "dfs"] is the default to any
extension not listed in any other table.

Configuration falls back to the defaults defined in the
DfsBlockCacheConfig.java file when not set on each cache
table configuration.

Sample format for individual cache tables:
In this example:
1. PACK types would go to the "default" table
2. INDEX and BITMAP_INDEX types would go to the
   "multipleExtensionCache" table
3. REFTABLE types would go to the "reftableCache" table

[core "dfs"] // Configuration for the "default" cache table.
  blockSize = 512
  blockLimit = 100
  concurrencyLevel = 5
  (...)

[core "dfs.multipleExtensionCache"]
  packExtensions = "INDEX BITMAP_INDEX"
  blockSize = 512
  blockLimit = 100
  concurrencyLevel = 5
  (...)

[core "dfs.reftableCache"]
  packExtensions = "REFTABLE"
  blockSize = 512
  blockLimit = 100
  concurrencyLevel = 5
  (...)

Change-Id: I0e534e6d78b684832e3d3d269cee2590aa0f1911

4 files changed, 183 insertions, 34 deletions

diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
index 19c90086aa..9d12facb33 100644
--- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
+++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
@@ -285,6 +285,7 @@ DIRCUnrecognizedExtendedFlags=Unrecognized extended flags: {0}
 downloadCancelled=Download cancelled
 downloadCancelledDuringIndexing=Download cancelled during indexing
 duplicateAdvertisementsOf=duplicate advertisements of {0}
+duplicatePackExtensionsSet=Attempting to configure duplicate pack extensions: {0}.{1}.{2} contains {3}
 duplicateRef=Duplicate ref: {0}
 duplicateRefAttribute=Duplicate ref attribute: {0}
 duplicateRemoteRefUpdateIsIllegal=Duplicate remote ref update is illegal. Affected remote name: {0}
@@ -539,6 +540,8 @@ noMergeBase=No merge base could be determined. Reason={0}. {1}
 noMergeHeadSpecified=No merge head specified
 nonBareLinkFilesNotSupported=Link files are not supported with nonbare repos
 nonCommitToHeads=Cannot point a branch to a non-commit object
+noPackExtConfigurationGiven=No PackExt configuration given
+noPackExtGivenForConfiguration=No PackExt given for configuration
 noPathAttributesFound=No Attributes found for {0}.
 noSuchRef=no such ref
 noSuchRefKnown=no such ref: {0}
@@ -829,6 +832,7 @@ unknownObject=unknown object
 unknownObjectInIndex=unknown object {0} found in index but not in pack file
 unknownObjectType=Unknown object type {0}.
 unknownObjectType2=unknown
+unknownPackExtension=Unknown pack extension: {0}.{1}.{2}={3}
 unknownPositionEncoding=Unknown position encoding %s
 unknownRefStorageFormat=Unknown ref storage format "{0}"
 unknownRepositoryFormat=Unknown repository format
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
index 700b54a7a6..311d9c22aa 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
@@ -315,6 +315,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String downloadCancelled;
 	/***/ public String downloadCancelledDuringIndexing;
 	/***/ public String duplicateAdvertisementsOf;
+	/***/ public String duplicatePackExtensionsSet;
 	/***/ public String duplicateRef;
 	/***/ public String duplicateRefAttribute;
 	/***/ public String duplicateRemoteRefUpdateIsIllegal;
@@ -569,6 +570,8 @@ public class JGitText extends TranslationBundle {
 	/***/ public String noMergeHeadSpecified;
 	/***/ public String nonBareLinkFilesNotSupported;
 	/***/ public String nonCommitToHeads;
+	/***/ public String noPackExtConfigurationGiven;
+	/***/ public String noPackExtGivenForConfiguration;
 	/***/ public String noPathAttributesFound;
 	/***/ public String noSuchRef;
 	/***/ public String noSuchRefKnown;
@@ -858,6 +861,7 @@ public class JGitText extends TranslationBundle {
 	/***/ public String unknownObjectInIndex;
 	/***/ public String unknownObjectType;
 	/***/ public String unknownObjectType2;
+	/***/ public String unknownPackExtension;
 	/***/ public String unknownPositionEncoding;
 	/***/ public String unknownRefStorageFormat;
 	/***/ public String unknownRepositoryFormat;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java
index 77273cec61..fa86701de8 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java
@@ -11,17 +11,25 @@
 package org.eclipse.jgit.internal.storage.dfs;
 
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_CACHE_PREFIX;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_LIMIT;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_SIZE;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO;
 
 import java.text.MessageFormat;
 import java.time.Duration;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Consumer;
+import java.util.stream.Collectors;
 
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.internal.storage.pack.PackExt;
@@ -42,15 +50,21 @@ public class DfsBlockCacheConfig {
 	public static final int DEFAULT_CACHE_HOT_MAX = 1;
 
 	private long blockLimit;
+
 	private int blockSize;
+
 	private double streamRatio;
+
 	private int concurrencyLevel;
 
 	private Consumer<Long> refLock;
+
 	private Map<PackExt, Integer> cacheHotMap;
 
 	private IndexEventConsumer indexEventConsumer;
 
+	private List<DfsBlockCachePackExtConfig> packExtCacheConfigurations;
+
 	/**
 	 * Create a default configuration.
 	 */
@@ -60,6 +74,7 @@ public class DfsBlockCacheConfig {
 		setStreamRatio(0.30);
 		setConcurrencyLevel(32);
 		cacheHotMap = Collections.emptyMap();
+		packExtCacheConfigurations = Collections.emptyList();
 	}
 
 	/**
@@ -77,10 +92,10 @@ public class DfsBlockCacheConfig {
 	 * Set maximum number bytes of heap memory to dedicate to caching pack file
 	 * data.
 	 * <p>
-	 * It is strongly recommended to set the block limit to be an integer multiple
-	 * of the block size. This constraint is not enforced by this method (since
-	 * it may be called before {@link #setBlockSize(int)}), but it is enforced by
-	 * {@link #fromConfig(Config)}.
+	 * It is strongly recommended to set the block limit to be an integer
+	 * multiple of the block size. This constraint is not enforced by this
+	 * method (since it may be called before {@link #setBlockSize(int)}), but it
+	 * is enforced by {@link #fromConfig(Config)}.
 	 *
 	 * @param newLimit
 	 *            maximum number bytes of heap memory to dedicate to caching
@@ -89,9 +104,9 @@ public class DfsBlockCacheConfig {
 	 */
 	public DfsBlockCacheConfig setBlockLimit(long newLimit) {
 		if (newLimit <= 0) {
-			throw new IllegalArgumentException(MessageFormat.format(
-					JGitText.get().blockLimitNotPositive,
-					Long.valueOf(newLimit)));
+			throw new IllegalArgumentException(
+					MessageFormat.format(JGitText.get().blockLimitNotPositive,
+							Long.valueOf(newLimit)));
 		}
 		blockLimit = newLimit;
 		return this;
@@ -240,61 +255,100 @@ public class DfsBlockCacheConfig {
 	}
 
 	/**
+	 * Get the list of pack ext cache configs.
+	 *
+	 * @return the list of pack ext cache configs.
+	 */
+	List<DfsBlockCachePackExtConfig> getPackExtCacheConfigurations() {
+		return packExtCacheConfigurations;
+	}
+
+	/**
 	 * Update properties by setting fields from the configuration.
 	 * <p>
 	 * If a property is not defined in the configuration, then it is left
 	 * unmodified.
 	 * <p>
-	 * Enforces certain constraints on the combination of settings in the config,
-	 * for example that the block limit is a multiple of the block size.
+	 * Enforces certain constraints on the combination of settings in the
+	 * config, for example that the block limit is a multiple of the block size.
 	 *
 	 * @param rc
 	 *            configuration to read properties from.
 	 * @return {@code this}
 	 */
 	public DfsBlockCacheConfig fromConfig(Config rc) {
-		long cfgBlockLimit = rc.getLong(
-				CONFIG_CORE_SECTION,
-				CONFIG_DFS_SECTION,
-				CONFIG_KEY_BLOCK_LIMIT,
-				getBlockLimit());
-		int cfgBlockSize = rc.getInt(
-				CONFIG_CORE_SECTION,
-				CONFIG_DFS_SECTION,
-				CONFIG_KEY_BLOCK_SIZE,
+		fromConfig(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, rc);
+		loadPackExtConfigs(rc);
+		return this;
+	}
+
+	private void fromConfig(String section, String subSection, Config rc) {
+		long cfgBlockLimit = rc.getLong(section, subSection,
+				CONFIG_KEY_BLOCK_LIMIT, getBlockLimit());
+		int cfgBlockSize = rc.getInt(section, subSection, CONFIG_KEY_BLOCK_SIZE,
 				getBlockSize());
 		if (cfgBlockLimit % cfgBlockSize != 0) {
 			throw new IllegalArgumentException(MessageFormat.format(
 					JGitText.get().blockLimitNotMultipleOfBlockSize,
-					Long.valueOf(cfgBlockLimit),
-					Long.valueOf(cfgBlockSize)));
+					Long.valueOf(cfgBlockLimit), Long.valueOf(cfgBlockSize)));
 		}
 
 		setBlockLimit(cfgBlockLimit);
 		setBlockSize(cfgBlockSize);
 
-		setConcurrencyLevel(rc.getInt(
-				CONFIG_CORE_SECTION,
-				CONFIG_DFS_SECTION,
-				CONFIG_KEY_CONCURRENCY_LEVEL,
-				getConcurrencyLevel()));
+		setConcurrencyLevel(rc.getInt(section, subSection,
+				CONFIG_KEY_CONCURRENCY_LEVEL, getConcurrencyLevel()));
 
-		String v = rc.getString(
-				CONFIG_CORE_SECTION,
-				CONFIG_DFS_SECTION,
-				CONFIG_KEY_STREAM_RATIO);
+		String v = rc.getString(section, subSection, CONFIG_KEY_STREAM_RATIO);
 		if (v != null) {
 			try {
 				setStreamRatio(Double.parseDouble(v));
 			} catch (NumberFormatException e) {
 				throw new IllegalArgumentException(MessageFormat.format(
-						JGitText.get().enumValueNotSupported3,
-						CONFIG_CORE_SECTION,
-						CONFIG_DFS_SECTION,
-						CONFIG_KEY_STREAM_RATIO, v), e);
+						JGitText.get().enumValueNotSupported3, section,
+						subSection, CONFIG_KEY_STREAM_RATIO, v), e);
 			}
 		}
-		return this;
+	}
+
+	private void loadPackExtConfigs(Config config) {
+		List<String> subSections = config.getSubsections(CONFIG_CORE_SECTION)
+				.stream()
+				.filter(section -> section.startsWith(CONFIG_DFS_CACHE_PREFIX))
+				.collect(Collectors.toList());
+		if (subSections.size() == 0) {
+			return;
+		}
+		ArrayList<DfsBlockCachePackExtConfig> cacheConfigs = new ArrayList<>();
+		Set<PackExt> extensionsSeen = new HashSet<>();
+		for (String subSection : subSections) {
+			var cacheConfig = DfsBlockCachePackExtConfig.fromConfig(config,
+					CONFIG_CORE_SECTION, subSection);
+			Set<PackExt> packExtsDuplicates = intersection(extensionsSeen,
+					cacheConfig.packExts);
+			if (packExtsDuplicates.size() > 0) {
+				String duplicatePackExts = packExtsDuplicates.stream()
+						.map(PackExt::toString)
+						.collect(Collectors.joining(","));
+				throw new IllegalArgumentException(MessageFormat.format(
+						JGitText.get().duplicatePackExtensionsSet,
+						CONFIG_CORE_SECTION, subSection,
+						CONFIG_KEY_PACK_EXTENSIONS, duplicatePackExts));
+			}
+			extensionsSeen.addAll(cacheConfig.packExts);
+			cacheConfigs.add(cacheConfig);
+		}
+		packExtCacheConfigurations = cacheConfigs;
+	}
+
+	private static <T> Set<T> intersection(Set<T> first, Set<T> second) {
+		Set<T> ret = new HashSet<>();
+		for (T entry : second) {
+			if (first.contains(entry)) {
+				ret.add(entry);
+			}
+		}
+		return ret;
 	}
 
 	/** Consumer of DfsBlockCache loading and eviction events for indexes. */
@@ -346,4 +400,81 @@ public class DfsBlockCacheConfig {
 			return false;
 		}
 	}
+
+	/**
+	 * A configuration for a single cache table storing 1 or more Pack
+	 * extensions.
+	 * <p>
+	 * The current pack ext cache tables implementation supports the same
+	 * parameters the ClockBlockCacheTable (current default implementation).
+	 * <p>
+	 * Configuration falls back to the defaults coded values defined in the
+	 * {@link DfsBlockCacheConfig} when not set on each cache table
+	 * configuration and NOT the values of the basic dfs section.
+	 * <p>
+	 * <code>
+	 *
+	 * Format:
+	 * [core "dfs.packCache"]
+	 *   packExtensions = "PACK"
+	 *   blockSize = 512
+	 *   blockLimit = 100
+	 *   concurrencyLevel = 5
+	 *
+	 * [core "dfs.multipleExtensionCache"]
+	 *   packExtensions = "INDEX REFTABLE BITMAP_INDEX"
+	 *   blockSize = 512
+	 *   blockLimit = 100
+	 *   concurrencyLevel = 5
+	 * </code>
+	 */
+	static class DfsBlockCachePackExtConfig {
+		// Set of pack extensions that will map to the cache instance.
+		private final EnumSet<PackExt> packExts;
+
+		// Configuration for the cache instance.
+		private final DfsBlockCacheConfig packExtCacheConfiguration;
+
+		private DfsBlockCachePackExtConfig(EnumSet<PackExt> packExts,
+				DfsBlockCacheConfig packExtCacheConfiguration) {
+			this.packExts = packExts;
+			this.packExtCacheConfiguration = packExtCacheConfiguration;
+		}
+
+		Set<PackExt> getPackExts() {
+			return packExts;
+		}
+
+		DfsBlockCacheConfig getPackExtCacheConfiguration() {
+			return packExtCacheConfiguration;
+		}
+
+		private static DfsBlockCachePackExtConfig fromConfig(Config config,
+				String section, String subSection) {
+			String packExtensions = config.getString(section, subSection,
+					CONFIG_KEY_PACK_EXTENSIONS);
+			if (packExtensions == null) {
+				throw new IllegalArgumentException(
+						JGitText.get().noPackExtGivenForConfiguration);
+			}
+			String[] extensions = packExtensions.split(" ", -1);
+			Set<PackExt> packExts = new HashSet<>(extensions.length);
+			for (String extension : extensions) {
+				try {
+					packExts.add(PackExt.valueOf(extension));
+				} catch (IllegalArgumentException e) {
+					throw new IllegalArgumentException(MessageFormat.format(
+							JGitText.get().unknownPackExtension, section,
+							subSection, CONFIG_KEY_PACK_EXTENSIONS, extension),
+							e);
+				}
+			}
+
+			DfsBlockCacheConfig dfsBlockCacheConfig = new DfsBlockCacheConfig();
+			dfsBlockCacheConfig.fromConfig(section, subSection, config);
+			return new DfsBlockCachePackExtConfig(EnumSet.copyOf(packExts),
+					dfsBlockCacheConfig);
+		}
+
+	}
 }
\ No newline at end of file
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
index 0edf3c5ad0..61db6a0762 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
@@ -77,6 +77,9 @@ public final class ConfigConstants {
 	/** The "dfs" section */
 	public static final String CONFIG_DFS_SECTION = "dfs";
 
+	/** The dfs cache subsection prefix */
+	public static final String CONFIG_DFS_CACHE_PREFIX = "dfs.";
+
 	/**
 	 * The "receive" section
 	 * @since 4.6
@@ -332,6 +335,13 @@ public final class ConfigConstants {
 	public static final String CONFIG_KEY_DELTA_BASE_CACHE_LIMIT = "deltaBaseCacheLimit";
 
 	/**
+	 * The "packExtensions" key
+	 *
+	 * @since 7.0
+	 **/
+	public static final String CONFIG_KEY_PACK_EXTENSIONS = "packExtensions";
+
+	/**
 	 * The "symlinks" key
 	 * @since 3.3
 	 */