PackConfig: add entry for minimum size to index

The object size index can have up to #(blobs-in-repo) entries, taking
a relevant amount of memory. Let operators configure the threshold size
to include objects in the size index.

The index will include objects with size *at or above* this
value (with -1 for none). This is more effective for the
filter-by-size case.

Lowering the threshold adds more objects to the index. This improves
performance at the cost of memory/storage space. For the object-size
case, more calls will use the index instead of reading IO. For the
filter-by-size case, lower threshold means better granularity (if
ObjectReader#isSmallerThan is implemented based only on the index).

Change-Id: I6ccd9334adbbc2abf95fde51dbbfc85b8230ade0

diff --git a/Documentation/config-options.md b/Documentation/config-options.md
index 612b8456aefa9eaa120634396423e5026339480a..754accf7878613a5e2d8521a506e13da9c078068 100644 (file)
--- a/Documentation/config-options.md
+++ b/Documentation/config-options.md
@@ -99,6 +99,7 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS
 | `pack.deltaCompression` | `true` | ⃞ | Whether the writer will create new deltas on the fly. `true` if the pack writer will create a new delta when either `pack.reuseDeltas` is false, or no suitable delta is available for reuse. |
 | `pack.depth` | `50` | ✅ | Maximum depth of delta chain set up for the pack writer. |
 | `pack.indexVersion` | `2` | ✅ | Pack index file format version. |
+| `pack.minBytesForObjSizeIndex` | `-1` | ⃞ | Minimum size of an object (inclusive, in bytes) to be included in the size index. -1 to disable the object size index. |
 | `pack.minSizePreventRacyPack` | `100 MiB` | ⃞ | Minimum packfile size for which we wait before opening a newly written pack to prevent its lastModified timestamp could be racy if `pack.waitPreventRacyPack` is `true`. |
 | `pack.preserveOldPacks` | `false` | ⃞ | Whether to preserve old packs in a preserved directory. |
 | `prunePreserved`, only via API of PackConfig | `false` | ⃞ | Whether to remove preserved pack files in a preserved directory. |

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
index ed4bf315e2bb31072d042a2d11c22de45783aa9b..608a4842a4d71f998cfa01995636569b511b3d82 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
@@ -822,6 +822,13 @@ public final class ConfigConstants {
         */
        public static final String CONFIG_KEY_WINDOW_MEMORY = "windowmemory";
 
+       /**
+        * the "pack.minBytesForObjSizeIndex" key
+        *
+        * @since 6.5
+        */
+       public static final String CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX = "minBytesForObjSizeIndex";
+
        /**
         * The "feature" section
         *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java
index a10f6cf88a75116d7d1fe24dd99210b7d3d0213b..2d0d4ee70051bd232b3c32a016ebbc8d76d33028 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java
@@ -36,6 +36,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_THREADS;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYPACK;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY;
+import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX;
 import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION;
 
 import java.time.Duration;
@@ -234,6 +235,15 @@ public class PackConfig {
         */
        public static final String[] DEFAULT_BITMAP_EXCLUDED_REFS_PREFIXES = new String[0];
 
+       /**
+        * Default minimum size for an object to be included in the size index:
+        * {@value}
+        *
+        * @see #setMinBytesForObjSizeIndex(int)
+        * @since 6.5
+        */
+       public static final int DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX = -1;
+
        /**
         * Default max time to spend during the search for reuse phase. This
         * optimization is disabled by default: {@value}
@@ -302,6 +312,8 @@ public class PackConfig {
 
        private boolean singlePack;
 
+       private int minBytesForObjSizeIndex = DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX;
+
        /**
         * Create a default configuration.
         */
@@ -369,6 +381,7 @@ public class PackConfig {
                this.cutDeltaChains = cfg.cutDeltaChains;
                this.singlePack = cfg.singlePack;
                this.searchForReuseTimeout = cfg.searchForReuseTimeout;
+               this.minBytesForObjSizeIndex = cfg.minBytesForObjSizeIndex;
        }
 
        /**
@@ -1189,6 +1202,45 @@ public class PackConfig {
                searchForReuseTimeout = timeout;
        }
 
+       /**
+        * Minimum size of an object (inclusive) to be added in the object size
+        * index.
+        *
+        * A negative value disables the writing of the object size index.
+        *
+        * @return minimum size an object must have to be included in the object
+        *         index.
+        * @since 6.5
+        */
+       public int getMinBytesForObjSizeIndex() {
+               return minBytesForObjSizeIndex;
+       }
+
+       /**
+        * Set minimum size an object must have to be included in the object size
+        * index.
+        *
+        * A negative value disables the object index.
+        *
+        * @param minBytesForObjSizeIndex
+        *            minimum size (inclusive) of an object to be included in the
+        *            object size index. -1 disables the index.
+        * @since 6.5
+        */
+       public void setMinBytesForObjSizeIndex(int minBytesForObjSizeIndex) {
+               this.minBytesForObjSizeIndex = minBytesForObjSizeIndex;
+       }
+
+       /**
+        * Should writers add an object size index when writing a pack.
+        *
+        * @return true to write an object-size index with the pack
+        * @since 6.5
+        */
+       public boolean isWriteObjSizeIndex() {
+               return this.minBytesForObjSizeIndex >= 0;
+       }
+
        /**
         * Update properties by setting fields from the configuration.
         *
@@ -1267,6 +1319,9 @@ public class PackConfig {
                setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION,
                                CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK,
                                getMinSizePreventRacyPack()));
+               setMinBytesForObjSizeIndex(rc.getInt(CONFIG_PACK_SECTION,
+                               CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX,
+                               DEFAULT_MIN_BYTES_FOR_OBJ_SIZE_INDEX));
        }
 
        /** {@inheritDoc} */
@@ -1302,6 +1357,8 @@ public class PackConfig {
                b.append(", searchForReuseTimeout") //$NON-NLS-1$
                                .append(getSearchForReuseTimeout());
                b.append(", singlePack=").append(getSinglePack()); //$NON-NLS-1$
+               b.append(", minBytesForObjSizeIndex=") //$NON-NLS-1$
+                               .append(getMinBytesForObjSizeIndex());
                return b.toString();
        }
 }