From 596c445af22ed9b6e0b7e35de44c127fcb8ecf7d Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 4 Jan 2022 11:10:04 -0800 Subject: [PATCH] PackConfig: add entry for minimum size to index The object size index can have up to #(blobs-in-repo) entries, taking a relevant amount of memory. Let operators configure the threshold size to include objects in the size index. The index will include objects with size *at or above* this value (with -1 for none). This is more effective for the filter-by-size case. Lowering the threshold adds more objects to the index. This improves performance at the cost of memory/storage space. For the object-size case, more calls will use the index instead of reading IO. For the filter-by-size case, lower threshold means better granularity (if ObjectReader#isSmallerThan is implemented based only on the index). Change-Id: I6ccd9334adbbc2abf95fde51dbbfc85b8230ade0 --- Documentation/config-options.md | 1 + .../org/eclipse/jgit/lib/ConfigConstants.java | 7 +++ .../eclipse/jgit/storage/pack/PackConfig.java | 57 +++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 612b8456ae..754accf787 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -99,6 +99,7 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `pack.deltaCompression` | `true` | ⃞ | Whether the writer will create new deltas on the fly. `true` if the pack writer will create a new delta when either `pack.reuseDeltas` is false, or no suitable delta is available for reuse. | | `pack.depth` | `50` | ✅ | Maximum depth of delta chain set up for the pack writer. | | `pack.indexVersion` | `2` | ✅ | Pack index file format version. | +| `pack.minBytesForObjSizeIndex` | `-1` | ⃞ | Minimum size of an object (inclusive, in bytes) to be included in the size index. -1 to disable the object size index. | | `pack.minSizePreventRacyPack` | `100 MiB` | ⃞ | Minimum packfile size for which we wait before opening a newly written pack to prevent its lastModified timestamp could be racy if `pack.waitPreventRacyPack` is `true`. | | `pack.preserveOldPacks` | `false` | ⃞ | Whether to preserve old packs in a preserved directory. | | `prunePreserved`, only via API of PackConfig | `false` | ⃞ | Whether to remove preserved pack files in a preserved directory. | diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index ed4bf315e2..608a4842a4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -822,6 +822,13 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_WINDOW_MEMORY = "windowmemory"; + /** + * the "pack.minBytesForObjSizeIndex" key + * + * @since 6.5 + */ + public static final String CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX = "minBytesForObjSizeIndex"; + /** * The "feature" section * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index a10f6cf88a..2d0d4ee700 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -36,6 +36,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_THREADS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYPACK; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; import java.time.Duration; @@ -234,6 +235,15 @@ public class PackConfig { */ public static final String[] DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES = new String[0]; + /** + * Default minimum size for an object to be included in the size index: + * {@value} + * + * @see #setMinBytesForObjSizeIndex(int) + * @since 6.5 + */ + public static final int DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX = -1; + /** * Default max time to spend during the search for reuse phase. This * optimization is disabled by default: {@value} @@ -302,6 +312,8 @@ public class PackConfig { private boolean singlePack; + private int minBytesForObjSizeIndex = DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX; + /** * Create a default configuration. */ @@ -369,6 +381,7 @@ public class PackConfig { this.cutDeltaChains = cfg.cutDeltaChains; this.singlePack = cfg.singlePack; this.searchForReuseTimeout = cfg.searchForReuseTimeout; + this.minBytesForObjSizeIndex = cfg.minBytesForObjSizeIndex; } /** @@ -1189,6 +1202,45 @@ public class PackConfig { searchForReuseTimeout = timeout; } + /** + * Minimum size of an object (inclusive) to be added in the object size + * index. + * + * A negative value disables the writing of the object size index. + * + * @return minimum size an object must have to be included in the object + * index. + * @since 6.5 + */ + public int getMinBytesForObjSizeIndex() { + return minBytesForObjSizeIndex; + } + + /** + * Set minimum size an object must have to be included in the object size + * index. + * + * A negative value disables the object index. + * + * @param minBytesForObjSizeIndex + * minimum size (inclusive) of an object to be included in the + * object size index. -1 disables the index. + * @since 6.5 + */ + public void setMinBytesForObjSizeIndex(int minBytesForObjSizeIndex) { + this.minBytesForObjSizeIndex = minBytesForObjSizeIndex; + } + + /** + * Should writers add an object size index when writing a pack. + * + * @return true to write an object-size index with the pack + * @since 6.5 + */ + public boolean isWriteObjSizeIndex() { + return this.minBytesForObjSizeIndex >= 0; + } + /** * Update properties by setting fields from the configuration. * @@ -1267,6 +1319,9 @@ public class PackConfig { setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION, CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK, getMinSizePreventRacyPack())); + setMinBytesForObjSizeIndex(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, + DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX)); } /** {@inheritDoc} */ @@ -1302,6 +1357,8 @@ public class PackConfig { b.append(", searchForReuseTimeout") //$NON-NLS-1$ .append(getSearchForReuseTimeout()); b.append(", singlePack=").append(getSinglePack()); //$NON-NLS-1$ + b.append(", minBytesForObjSizeIndex=") //$NON-NLS-1$ + .append(getMinBytesForObjSizeIndex()); return b.toString(); } } -- 2.39.5