From 1a06179ea707ab088b6543df77be5cf0ea44c497 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Wed, 28 Jul 2010 10:45:27 -0700 Subject: Move PackWriter configuration to PackConfig This refactoring permits applications to configure global per-process settings for all packing and easily pass it through to per-request PackWriters, ensuring that the process configuration overrides the repository specific settings. For example this might help in a daemon environment where the server wants to cap the resources used to serve a dynamic upload pack request, even though the repository's own pack.* settings might be configured to be more aggressive. This allows fast but less bandwidth efficient serving of clients, while still retaining good compression through a cron managed `git gc`. Change-Id: I58cc5e01b48924b1a99f79aa96c8150cdfc50846 Signed-off-by: Shawn O. Pearce --- .../src/org/eclipse/jgit/lib/Config.java | 15 + .../org/eclipse/jgit/storage/pack/DeltaCache.java | 6 +- .../org/eclipse/jgit/storage/pack/DeltaTask.java | 8 +- .../org/eclipse/jgit/storage/pack/DeltaWindow.java | 16 +- .../org/eclipse/jgit/storage/pack/PackConfig.java | 588 +++++++++++++++++++-- .../org/eclipse/jgit/storage/pack/PackWriter.java | 485 ++--------------- .../jgit/storage/pack/ThreadSafeDeltaCache.java | 4 +- 7 files changed, 637 insertions(+), 485 deletions(-) (limited to 'org.eclipse.jgit') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java index ccb2516917..2e1ab9a07b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java @@ -216,6 +216,21 @@ public class Config { , section, name)); } + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @param defaultValue + * default value to return if no value was present. + * @return an integer value from the configuration, or defaultValue. + */ + public long getLong(String section, String name, long defaultValue) { + return getLong(section, null, name, defaultValue); + } + /** * Obtain an integer value from the configuration. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaCache.java index b6a7436f16..93eab19e2f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaCache.java @@ -55,9 +55,9 @@ class DeltaCache { private long used; - DeltaCache(PackWriter pw) { - size = pw.getDeltaCacheSize(); - entryLimit = pw.getDeltaCacheLimit(); + DeltaCache(PackConfig pc) { + size = pc.getDeltaCacheSize(); + entryLimit = pc.getDeltaCacheLimit(); queue = new ReferenceQueue(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaTask.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaTask.java index 11bb3efa79..5e551e9d49 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaTask.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaTask.java @@ -49,7 +49,7 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ProgressMonitor; final class DeltaTask implements Callable { - private final PackWriter writer; + private final PackConfig config; private final ObjectReader templateReader; @@ -63,9 +63,9 @@ final class DeltaTask implements Callable { private final ObjectToPack[] list; - DeltaTask(PackWriter writer, ObjectReader reader, DeltaCache dc, + DeltaTask(PackConfig config, ObjectReader reader, DeltaCache dc, ProgressMonitor pm, int batchSize, int start, ObjectToPack[] list) { - this.writer = writer; + this.config = config; this.templateReader = reader; this.dc = dc; this.pm = pm; @@ -78,7 +78,7 @@ final class DeltaTask implements Callable { final ObjectReader or = templateReader.newReader(); try { DeltaWindow dw; - dw = new DeltaWindow(writer, dc, or); + dw = new DeltaWindow(config, dc, or); dw.search(pm, list, start, batchSize); } finally { or.release(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java index 6521e6d3ec..c961056508 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/DeltaWindow.java @@ -60,7 +60,7 @@ class DeltaWindow { private static final int NEXT_SRC = 1; - private final PackWriter writer; + private final PackConfig config; private final DeltaCache deltaCache; @@ -101,8 +101,8 @@ class DeltaWindow { /** Used to compress cached deltas. */ private Deflater deflater; - DeltaWindow(PackWriter pw, DeltaCache dc, ObjectReader or) { - writer = pw; + DeltaWindow(PackConfig pc, DeltaCache dc, ObjectReader or) { + config = pc; deltaCache = dc; reader = or; @@ -117,12 +117,12 @@ class DeltaWindow { // PackWriter has a minimum of 2 for the window size, but then // users might complain that JGit is creating a bigger pack file. // - window = new DeltaWindowEntry[pw.getDeltaSearchWindowSize() + 1]; + window = new DeltaWindowEntry[config.getDeltaSearchWindowSize() + 1]; for (int i = 0; i < window.length; i++) window[i] = new DeltaWindowEntry(); - maxMemory = pw.getDeltaSearchMemoryLimit(); - maxDepth = pw.getMaxDeltaDepth(); + maxMemory = config.getDeltaSearchMemoryLimit(); + maxDepth = config.getMaxDeltaDepth(); } void search(ProgressMonitor monitor, ObjectToPack[] toSearch, int off, @@ -442,7 +442,7 @@ class DeltaWindow { IncorrectObjectTypeException, IOException, LargeObjectException { byte[] buf = ent.buffer; if (buf == null) { - buf = writer.buffer(reader, ent.object); + buf = PackWriter.buffer(config, reader, ent.object); if (0 < maxMemory) loaded += buf.length; ent.buffer = buf; @@ -452,7 +452,7 @@ class DeltaWindow { private Deflater deflater() { if (deflater == null) - deflater = new Deflater(writer.getCompressionLevel()); + deflater = new Deflater(config.getCompressionLevel()); else deflater.reset(); return deflater; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 814ab8f291..355ed8a3f7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -1,5 +1,6 @@ /* - * Copyright (C) 2010, Google Inc. + * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, Marek Zawirski * and other copyright owners as documented in the project's IP log. * * This program and the accompanying materials are made available @@ -43,51 +44,572 @@ package org.eclipse.jgit.storage.pack; -import static java.util.zip.Deflater.DEFAULT_COMPRESSION; +import java.util.concurrent.Executor; +import java.util.zip.Deflater; + import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.Config.SectionParser; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.PackIndexWriter; + +/** + * Configuration used by a {@link PackWriter} when constructing the stream. + * + * A configuration may be modified once created, but should not be modified + * while it is being used by a PackWriter. If a configuration is not modified it + * is safe to share the same configuration instance between multiple concurrent + * threads executing different PackWriters. + */ +public class PackConfig { + /** + * Default value of deltas reuse option: {@value} + * + * @see #setReuseDeltas(boolean) + */ + public static final boolean DEFAULT_REUSE_DELTAS = true; + + /** + * Default value of objects reuse option: {@value} + * + * @see #setReuseObjects(boolean) + */ + public static final boolean DEFAULT_REUSE_OBJECTS = true; + + /** + * Default value of delta compress option: {@value} + * + * @see #setDeltaCompress(boolean) + */ + public static final boolean DEFAULT_DELTA_COMPRESS = true; + + /** + * Default value of delta base as offset option: {@value} + * + * @see #setDeltaBaseAsOffset(boolean) + */ + public static final boolean DEFAULT_DELTA_BASE_AS_OFFSET = false; + + /** + * Default value of maximum delta chain depth: {@value} + * + * @see #setMaxDeltaDepth(int) + */ + public static final int DEFAULT_MAX_DELTA_DEPTH = 50; + + /** + * Default window size during packing: {@value} + * + * @see #setDeltaSearchWindowSize(int) + */ + public static final int DEFAULT_DELTA_SEARCH_WINDOW_SIZE = 10; + + /** + * Default big file threshold: {@value} + * + * @see #setBigFileThreshold(long) + */ + public static final long DEFAULT_BIG_FILE_THRESHOLD = 50 * 1024 * 1024; + + /** + * Default delta cache size: {@value} + * + * @see #setDeltaCacheSize(long) + */ + public static final long DEFAULT_DELTA_CACHE_SIZE = 50 * 1024 * 1024; + + /** + * Default delta cache limit: {@value} + * + * @see #setDeltaCacheLimit(int) + */ + public static final int DEFAULT_DELTA_CACHE_LIMIT = 100; + + /** + * Default index version: {@value} + * + * @see #setIndexVersion(int) + */ + public static final int DEFAULT_INDEX_VERSION = 2; + + + private int compressionLevel = Deflater.DEFAULT_COMPRESSION; + + private boolean reuseDeltas = DEFAULT_REUSE_DELTAS; + + private boolean reuseObjects = DEFAULT_REUSE_OBJECTS; + + private boolean deltaBaseAsOffset = DEFAULT_DELTA_BASE_AS_OFFSET; + + private boolean deltaCompress = DEFAULT_DELTA_COMPRESS; + + private int maxDeltaDepth = DEFAULT_MAX_DELTA_DEPTH; -class PackConfig { - /** Key for {@link Config#get(SectionParser)}. */ - static final Config.SectionParser KEY = new SectionParser() { - public PackConfig parse(final Config cfg) { - return new PackConfig(cfg); - } - }; + private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE; - final int deltaWindow; + private long deltaSearchMemoryLimit; - final long deltaWindowMemory; + private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE; - final int deltaDepth; + private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT; - final long deltaCacheSize; + private long bigFileThreshold = DEFAULT_BIG_FILE_THRESHOLD; - final int deltaCacheLimit; + private int threads; - final int compression; + private Executor executor; - final int indexVersion; + private int indexVersion = DEFAULT_INDEX_VERSION; - final long bigFileThreshold; - final int threads; + /** Create a default configuration. */ + public PackConfig() { + // Fields are initialized to defaults. + } + + /** + * Create a configuration honoring the repository's settings. + * + * @param db + * the repository to read settings from. The repository is not + * retained by the new configuration, instead its settings are + * copied during the constructor. + */ + public PackConfig(Repository db) { + fromConfig(db.getConfig()); + } + + /** + * Create a configuration honoring settings in a {@link Config}. + * + * @param cfg + * the source to read settings from. The source is not retained + * by the new configuration, instead its settings are copied + * during the constructor. + */ + public PackConfig(Config cfg) { + fromConfig(cfg); + } + + /** + * Check whether to reuse deltas existing in repository. + * + * Default setting: {@value #DEFAULT_REUSE_DELTAS} + * + * @return true if object is configured to reuse deltas; false otherwise. + */ + public boolean isReuseDeltas() { + return reuseDeltas; + } + + /** + * Set reuse deltas configuration option for the writer. + * + * When enabled, writer will search for delta representation of object in + * repository and use it if possible. Normally, only deltas with base to + * another object existing in set of objects to pack will be used. The + * exception however is thin-packs where the base object may exist on the + * other side. + * + * When raw delta data is directly copied from a pack file, its checksum is + * computed to verify the data is not corrupt. + * + * Default setting: {@value #DEFAULT_REUSE_DELTAS} + * + * @param reuseDeltas + * boolean indicating whether or not try to reuse deltas. + */ + public void setReuseDeltas(boolean reuseDeltas) { + this.reuseDeltas = reuseDeltas; + } + + /** + * Checks whether to reuse existing objects representation in repository. + * + * Default setting: {@value #DEFAULT_REUSE_OBJECTS} + * + * @return true if writer is configured to reuse objects representation from + * pack; false otherwise. + */ + public boolean isReuseObjects() { + return reuseObjects; + } + + /** + * Set reuse objects configuration option for the writer. + * + * If enabled, writer searches for compressed representation in a pack file. + * If possible, compressed data is directly copied from such a pack file. + * Data checksum is verified. + * + * Default setting: {@value #DEFAULT_REUSE_OBJECTS} + * + * @param reuseObjects + * boolean indicating whether or not writer should reuse existing + * objects representation. + */ + public void setReuseObjects(boolean reuseObjects) { + this.reuseObjects = reuseObjects; + } + + /** + * True if writer can use offsets to point to a delta base. + * + * If true the writer may choose to use an offset to point to a delta base + * in the same pack, this is a newer style of reference that saves space. + * False if the writer has to use the older (and more compatible style) of + * storing the full ObjectId of the delta base. + * + * Default setting: {@value #DEFAULT_DELTA_BASE_AS_OFFSET} + * + * @return true if delta base is stored as an offset; false if it is stored + * as an ObjectId. + */ + public boolean isDeltaBaseAsOffset() { + return deltaBaseAsOffset; + } + + /** + * Set writer delta base format. + * + * Delta base can be written as an offset in a pack file (new approach + * reducing file size) or as an object id (legacy approach, compatible with + * old readers). + * + * Default setting: {@value #DEFAULT_DELTA_BASE_AS_OFFSET} + * + * @param deltaBaseAsOffset + * boolean indicating whether delta base can be stored as an + * offset. + */ + public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) { + this.deltaBaseAsOffset = deltaBaseAsOffset; + } + + /** + * Check whether the writer will create new deltas on the fly. + * + * Default setting: {@value #DEFAULT_DELTA_COMPRESS} + * + * @return true if the writer will create a new delta when either + * {@link #isReuseDeltas()} is false, or no suitable delta is + * available for reuse. + */ + public boolean isDeltaCompress() { + return deltaCompress; + } + + /** + * Set whether or not the writer will create new deltas on the fly. + * + * Default setting: {@value #DEFAULT_DELTA_COMPRESS} + * + * @param deltaCompress + * true to create deltas when {@link #isReuseDeltas()} is false, + * or when a suitable delta isn't available for reuse. Set to + * false to write whole objects instead. + */ + public void setDeltaCompress(boolean deltaCompress) { + this.deltaCompress = deltaCompress; + } + + /** + * Get maximum depth of delta chain set up for the writer. + * + * Generated chains are not longer than this value. + * + * Default setting: {@value #DEFAULT_MAX_DELTA_DEPTH} + * + * @return maximum delta chain depth. + */ + public int getMaxDeltaDepth() { + return maxDeltaDepth; + } + + /** + * Set up maximum depth of delta chain for the writer. + * + * Generated chains are not longer than this value. Too low value causes low + * compression level, while too big makes unpacking (reading) longer. + * + * Default setting: {@value #DEFAULT_MAX_DELTA_DEPTH} + * + * @param maxDeltaDepth + * maximum delta chain depth. + */ + public void setMaxDeltaDepth(int maxDeltaDepth) { + this.maxDeltaDepth = maxDeltaDepth; + } + + /** + * Get the number of objects to try when looking for a delta base. + * + * This limit is per thread, if 4 threads are used the actual memory used + * will be 4 times this value. + * + * Default setting: {@value #DEFAULT_DELTA_SEARCH_WINDOW_SIZE} + * + * @return the object count to be searched. + */ + public int getDeltaSearchWindowSize() { + return deltaSearchWindowSize; + } + + /** + * Set the number of objects considered when searching for a delta base. + * + * Default setting: {@value #DEFAULT_DELTA_SEARCH_WINDOW_SIZE} + * + * @param objectCount + * number of objects to search at once. Must be at least 2. + */ + public void setDeltaSearchWindowSize(int objectCount) { + if (objectCount <= 2) + setDeltaCompress(false); + else + deltaSearchWindowSize = objectCount; + } + + /** + * Get maximum number of bytes to put into the delta search window. + * + * Default setting is 0, for an unlimited amount of memory usage. Actual + * memory used is the lower limit of either this setting, or the sum of + * space used by at most {@link #getDeltaSearchWindowSize()} objects. + * + * This limit is per thread, if 4 threads are used the actual memory limit + * will be 4 times this value. + * + * @return the memory limit. + */ + public long getDeltaSearchMemoryLimit() { + return deltaSearchMemoryLimit; + } + + /** + * Set the maximum number of bytes to put into the delta search window. + * + * Default setting is 0, for an unlimited amount of memory usage. If the + * memory limit is reached before {@link #getDeltaSearchWindowSize()} the + * window size is temporarily lowered. + * + * @param memoryLimit + * Maximum number of bytes to load at once, 0 for unlimited. + */ + public void setDeltaSearchMemoryLimit(long memoryLimit) { + deltaSearchMemoryLimit = memoryLimit; + } + + /** + * Get the size of the in-memory delta cache. + * + * This limit is for the entire writer, even if multiple threads are used. + * + * Default setting: {@value #DEFAULT_DELTA_CACHE_SIZE} + * + * @return maximum number of bytes worth of delta data to cache in memory. + * If 0 the cache is infinite in size (up to the JVM heap limit + * anyway). A very tiny size such as 1 indicates the cache is + * effectively disabled. + */ + public long getDeltaCacheSize() { + return deltaCacheSize; + } + + /** + * Set the maximum number of bytes of delta data to cache. + * + * During delta search, up to this many bytes worth of small or hard to + * compute deltas will be stored in memory. This cache speeds up writing by + * allowing the cached entry to simply be dumped to the output stream. + * + * Default setting: {@value #DEFAULT_DELTA_CACHE_SIZE} + * + * @param size + * number of bytes to cache. Set to 0 to enable an infinite + * cache, set to 1 (an impossible size for any delta) to disable + * the cache. + */ + public void setDeltaCacheSize(long size) { + deltaCacheSize = size; + } + + /** + * Maximum size in bytes of a delta to cache. + * + * Default setting: {@value #DEFAULT_DELTA_CACHE_LIMIT} + * + * @return maximum size (in bytes) of a delta that should be cached. + */ + public int getDeltaCacheLimit() { + return deltaCacheLimit; + } + + /** + * Set the maximum size of a delta that should be cached. + * + * During delta search, any delta smaller than this size will be cached, up + * to the {@link #getDeltaCacheSize()} maximum limit. This speeds up writing + * by allowing these cached deltas to be output as-is. + * + * Default setting: {@value #DEFAULT_DELTA_CACHE_LIMIT} + * + * @param size + * maximum size (in bytes) of a delta to be cached. + */ + public void setDeltaCacheLimit(int size) { + deltaCacheLimit = size; + } + + /** + * Get the maximum file size that will be delta compressed. + * + * Files bigger than this setting will not be delta compressed, as they are + * more than likely already highly compressed binary data files that do not + * delta compress well, such as MPEG videos. + * + * Default setting: {@value #DEFAULT_BIG_FILE_THRESHOLD} + * + * @return the configured big file threshold. + */ + public long getBigFileThreshold() { + return bigFileThreshold; + } + + /** + * Set the maximum file size that should be considered for deltas. + * + * Default setting: {@value #DEFAULT_BIG_FILE_THRESHOLD} + * + * @param bigFileThreshold + * the limit, in bytes. + */ + public void setBigFileThreshold(long bigFileThreshold) { + this.bigFileThreshold = bigFileThreshold; + } + + /** + * Get the compression level applied to objects in the pack. + * + * Default setting: {@value java.util.zip.Deflater#DEFAULT_COMPRESSION} + * + * @return current compression level, see {@link java.util.zip.Deflater}. + */ + public int getCompressionLevel() { + return compressionLevel; + } + + /** + * Set the compression level applied to objects in the pack. + * + * Default setting: {@value java.util.zip.Deflater#DEFAULT_COMPRESSION} + * + * @param level + * compression level, must be a valid level recognized by the + * {@link java.util.zip.Deflater} class. + */ + public void setCompressionLevel(int level) { + compressionLevel = level; + } + + /** + * Get the number of threads used during delta compression. + * + * Default setting: 0 (auto-detect processors) + * + * @return number of threads used for delta compression. 0 will auto-detect + * the threads to the number of available processors. + */ + public int getThreads() { + return threads; + } + + /** + * Set the number of threads to use for delta compression. + * + * During delta compression, if there are enough objects to be considered + * the writer will start up concurrent threads and allow them to compress + * different sections of the repository concurrently. + * + * An application thread pool can be set by {@link #setExecutor(Executor)}. + * If not set a temporary pool will be created by the writer, and torn down + * automatically when compression is over. + * + * Default setting: 0 (auto-detect processors) + * + * @param threads + * number of threads to use. If <= 0 the number of available + * processors for this JVM is used. + */ + public void setThreads(int threads) { + this.threads = threads; + } + + /** @return the preferred thread pool to execute delta search on. */ + public Executor getExecutor() { + return executor; + } + + /** + * Set the executor to use when using threads. + * + * During delta compression if the executor is non-null jobs will be queued + * up on it to perform delta compression in parallel. Aside from setting the + * executor, the caller must set {@link #setThreads(int)} to enable threaded + * delta search. + * + * @param executor + * executor to use for threads. Set to null to create a temporary + * executor just for the writer. + */ + public void setExecutor(Executor executor) { + this.executor = executor; + } + + /** + * Get the pack index file format version this instance creates. + * + * Default setting: {@value #DEFAULT_INDEX_VERSION} + * + * @return the index version, the special version 0 designates the oldest + * (most compatible) format available for the objects. + * @see PackIndexWriter + */ + public int getIndexVersion() { + return indexVersion; + } - private PackConfig(Config rc) { - deltaWindow = rc.getInt("pack", "window", PackWriter.DEFAULT_DELTA_SEARCH_WINDOW_SIZE); - deltaWindowMemory = rc.getLong("pack", null, "windowmemory", 0); - deltaCacheSize = rc.getLong("pack", null, "deltacachesize", PackWriter.DEFAULT_DELTA_CACHE_SIZE); - deltaCacheLimit = rc.getInt("pack", "deltacachelimit", PackWriter.DEFAULT_DELTA_CACHE_LIMIT); - deltaDepth = rc.getInt("pack", "depth", PackWriter.DEFAULT_MAX_DELTA_DEPTH); - compression = compression(rc); - indexVersion = rc.getInt("pack", "indexversion", 2); - bigFileThreshold = rc.getLong("core", null, "bigfilethreshold", PackWriter.DEFAULT_BIG_FILE_THRESHOLD); - threads = rc.getInt("pack", "threads", 0); + /** + * Set the pack index file format version this instance will create. + * + * Default setting: {@value #DEFAULT_INDEX_VERSION} + * + * @param version + * the version to write. The special version 0 designates the + * oldest (most compatible) format available for the objects. + * @see PackIndexWriter + */ + public void setIndexVersion(final int version) { + indexVersion = version; } - private static int compression(Config rc) { - if (rc.getString("pack", null, "compression") != null) - return rc.getInt("pack", "compression", DEFAULT_COMPRESSION); - return rc.getInt("core", "compression", DEFAULT_COMPRESSION); + /** + * Update properties by setting fields from the configuration. + * + * If a property's corresponding variable is not defined in the supplied + * configuration, then it is left unmodified. + * + * @param rc + * configuration to read properties from. + */ + public void fromConfig(final Config rc) { + setMaxDeltaDepth(rc.getInt("pack", "depth", getMaxDeltaDepth())); + setDeltaSearchWindowSize(rc.getInt("pack", "window", getDeltaSearchWindowSize())); + setDeltaSearchMemoryLimit(rc.getLong("pack", "windowmemory", getDeltaSearchMemoryLimit())); + setDeltaCacheSize(rc.getLong("pack", "deltacachesize", getDeltaCacheSize())); + setDeltaCacheLimit(rc.getInt("pack", "deltacachelimit", getDeltaCacheLimit())); + setCompressionLevel(rc.getInt("pack", "compression", + rc.getInt("core", "compression", getCompressionLevel()))); + setIndexVersion(rc.getInt("pack", "indexversion", getIndexVersion())); + setBigFileThreshold(rc.getLong("core", "bigfilethreshold", getBigFileThreshold())); + setThreads(rc.getInt("pack", "threads", getThreads())); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java index b2a6697d14..df5594cf44 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackWriter.java @@ -75,7 +75,6 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -127,47 +126,6 @@ import org.eclipse.jgit.util.TemporaryBuffer; *

*/ public class PackWriter { - /** - * Default value of deltas reuse option. - * - * @see #setReuseDeltas(boolean) - */ - public static final boolean DEFAULT_REUSE_DELTAS = true; - - /** - * Default value of objects reuse option. - * - * @see #setReuseObjects(boolean) - */ - public static final boolean DEFAULT_REUSE_OBJECTS = true; - - /** - * Default value of delta base as offset option. - * - * @see #setDeltaBaseAsOffset(boolean) - */ - public static final boolean DEFAULT_DELTA_BASE_AS_OFFSET = false; - - /** - * Default value of maximum delta chain depth. - * - * @see #setMaxDeltaDepth(int) - */ - public static final int DEFAULT_MAX_DELTA_DEPTH = 50; - - /** - * Default window size during packing. - * - * @see #setDeltaSearchWindowSize(int) - */ - public static final int DEFAULT_DELTA_SEARCH_WINDOW_SIZE = 10; - - static final long DEFAULT_BIG_FILE_THRESHOLD = 50 * 1024 * 1024; - - static final long DEFAULT_DELTA_CACHE_SIZE = 50 * 1024 * 1024; - - static final int DEFAULT_DELTA_CACHE_LIMIT = 100; - private static final int PACK_VERSION_GENERATED = 2; @SuppressWarnings("unchecked") @@ -185,8 +143,6 @@ public class PackWriter { // edge objects for thin packs private final ObjectIdSubclassMap edgeObjects = new ObjectIdSubclassMap(); - private int compressionLevel; - private Deflater myDeflater; private final ObjectReader reader; @@ -194,35 +150,15 @@ public class PackWriter { /** {@link #reader} recast to the reuse interface, if it supports it. */ private final ObjectReuseAsIs reuseSupport; + private final PackConfig config; + private List sortedByName; private byte packcsum[]; - private boolean reuseDeltas = DEFAULT_REUSE_DELTAS; - - private boolean reuseObjects = DEFAULT_REUSE_OBJECTS; + private boolean deltaBaseAsOffset; - private boolean deltaBaseAsOffset = DEFAULT_DELTA_BASE_AS_OFFSET; - - private boolean deltaCompress = true; - - private int maxDeltaDepth = DEFAULT_MAX_DELTA_DEPTH; - - private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE; - - private long deltaSearchMemoryLimit; - - private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE; - - private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT; - - private int indexVersion; - - private long bigFileThreshold = DEFAULT_BIG_FILE_THRESHOLD; - - private int threads = 1; - - private Executor executor; + private boolean reuseDeltas; private boolean thin; @@ -251,7 +187,7 @@ public class PackWriter { * reader to read from the repository with. */ public PackWriter(final ObjectReader reader) { - this(null, reader); + this(new PackConfig(), reader); } /** @@ -266,105 +202,38 @@ public class PackWriter { * reader to read from the repository with. */ public PackWriter(final Repository repo, final ObjectReader reader) { - this.reader = reader; - if (reader instanceof ObjectReuseAsIs) - reuseSupport = ((ObjectReuseAsIs) reader); - else - reuseSupport = null; - - final PackConfig pc = configOf(repo).get(PackConfig.KEY); - deltaSearchWindowSize = pc.deltaWindow; - deltaSearchMemoryLimit = pc.deltaWindowMemory; - deltaCacheSize = pc.deltaCacheSize; - deltaCacheLimit = pc.deltaCacheLimit; - maxDeltaDepth = pc.deltaDepth; - compressionLevel = pc.compression; - indexVersion = pc.indexVersion; - bigFileThreshold = pc.bigFileThreshold; - threads = pc.threads; - } - - private static Config configOf(final Repository repo) { - if (repo == null) - return new Config(); - return repo.getConfig(); - } - - /** - * Check whether object is configured to reuse deltas existing in - * repository. - *

- * Default setting: {@link #DEFAULT_REUSE_DELTAS} - *

- * - * @return true if object is configured to reuse deltas; false otherwise. - */ - public boolean isReuseDeltas() { - return reuseDeltas; + this(new PackConfig(repo), reader); } /** - * Set reuse deltas configuration option for this writer. When enabled, - * writer will search for delta representation of object in repository and - * use it if possible. Normally, only deltas with base to another object - * existing in set of objects to pack will be used. Exception is however - * thin-pack (see - * {@link #preparePack(ProgressMonitor, Collection, Collection)} and - * {@link #preparePack(Iterator)}) where base object must exist on other - * side machine. + * Create writer with a specified configuration. *

- * When raw delta data is directly copied from a pack file, checksum is - * computed to verify data. - *

- *

- * Default setting: {@link #DEFAULT_REUSE_DELTAS} - *

- * - * @param reuseDeltas - * boolean indicating whether or not try to reuse deltas. - */ - public void setReuseDeltas(boolean reuseDeltas) { - this.reuseDeltas = reuseDeltas; - } - - /** - * Checks whether object is configured to reuse existing objects - * representation in repository. - *

- * Default setting: {@link #DEFAULT_REUSE_OBJECTS} - *

+ * Objects for packing are specified in {@link #preparePack(Iterator)} or + * {@link #preparePack(ProgressMonitor, Collection, Collection)}. * - * @return true if writer is configured to reuse objects representation from - * pack; false otherwise. + * @param config + * configuration for the pack writer. + * @param reader + * reader to read from the repository with. */ - public boolean isReuseObjects() { - return reuseObjects; - } + public PackWriter(final PackConfig config, final ObjectReader reader) { + this.config = config; + this.reader = reader; + if (reader instanceof ObjectReuseAsIs) + reuseSupport = ((ObjectReuseAsIs) reader); + else + reuseSupport = null; - /** - * Set reuse objects configuration option for this writer. If enabled, - * writer searches for representation in a pack file. If possible, - * compressed data is directly copied from such a pack file. Data checksum - * is verified. - *

- * Default setting: {@link #DEFAULT_REUSE_OBJECTS} - *

- * - * @param reuseObjects - * boolean indicating whether or not writer should reuse existing - * objects representation. - */ - public void setReuseObjects(boolean reuseObjects) { - this.reuseObjects = reuseObjects; + deltaBaseAsOffset = config.isDeltaBaseAsOffset(); + reuseDeltas = config.isReuseDeltas(); } /** * Check whether writer can store delta base as an offset (new style * reducing pack size) or should store it as an object id (legacy style, * compatible with old readers). - *

- * Default setting: {@link #DEFAULT_DELTA_BASE_AS_OFFSET} - *

+ * + * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET} * * @return true if delta base is stored as an offset; false if it is stored * as an object id. @@ -377,9 +246,8 @@ public class PackWriter { * Set writer delta base format. Delta base can be written as an offset in a * pack file (new approach reducing file size) or as an object id (legacy * approach, compatible with old readers). - *

- * Default setting: {@link #DEFAULT_DELTA_BASE_AS_OFFSET} - *

+ * + * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET} * * @param deltaBaseAsOffset * boolean indicating whether delta base can be stored as an @@ -389,255 +257,6 @@ public class PackWriter { this.deltaBaseAsOffset = deltaBaseAsOffset; } - /** - * Check whether the writer will create new deltas on the fly. - *

- * Default setting: true - *

- * - * @return true if the writer will create a new delta when either - * {@link #isReuseDeltas()} is false, or no suitable delta is - * available for reuse. - */ - public boolean isDeltaCompress() { - return deltaCompress; - } - - /** - * Set whether or not the writer will create new deltas on the fly. - * - * @param deltaCompress - * true to create deltas when {@link #isReuseDeltas()} is false, - * or when a suitable delta isn't available for reuse. Set to - * false to write whole objects instead. - */ - public void setDeltaCompress(boolean deltaCompress) { - this.deltaCompress = deltaCompress; - } - - /** - * Get maximum depth of delta chain set up for this writer. Generated chains - * are not longer than this value. - *

- * Default setting: {@link #DEFAULT_MAX_DELTA_DEPTH} - *

- * - * @return maximum delta chain depth. - */ - public int getMaxDeltaDepth() { - return maxDeltaDepth; - } - - /** - * Set up maximum depth of delta chain for this writer. Generated chains are - * not longer than this value. Too low value causes low compression level, - * while too big makes unpacking (reading) longer. - *

- * Default setting: {@link #DEFAULT_MAX_DELTA_DEPTH} - *

- * - * @param maxDeltaDepth - * maximum delta chain depth. - */ - public void setMaxDeltaDepth(int maxDeltaDepth) { - this.maxDeltaDepth = maxDeltaDepth; - } - - /** - * Get the number of objects to try when looking for a delta base. - *

- * This limit is per thread, if 4 threads are used the actual memory - * used will be 4 times this value. - * - * @return the object count to be searched. - */ - public int getDeltaSearchWindowSize() { - return deltaSearchWindowSize; - } - - /** - * Set the number of objects considered when searching for a delta base. - *

- * Default setting: {@link #DEFAULT_DELTA_SEARCH_WINDOW_SIZE} - *

- * - * @param objectCount - * number of objects to search at once. Must be at least 2. - */ - public void setDeltaSearchWindowSize(int objectCount) { - if (objectCount <= 2) - setDeltaCompress(false); - else - deltaSearchWindowSize = objectCount; - } - - /** - * Get maximum number of bytes to put into the delta search window. - *

- * Default setting is 0, for an unlimited amount of memory usage. Actual - * memory used is the lower limit of either this setting, or the sum of - * space used by at most {@link #getDeltaSearchWindowSize()} objects. - *

- * This limit is per thread, if 4 threads are used the actual memory - * limit will be 4 times this value. - * - * @return the memory limit. - */ - public long getDeltaSearchMemoryLimit() { - return deltaSearchMemoryLimit; - } - - /** - * Set the maximum number of bytes to put into the delta search window. - *

- * Default setting is 0, for an unlimited amount of memory usage. If the - * memory limit is reached before {@link #getDeltaSearchWindowSize()} the - * window size is temporarily lowered. - * - * @param memoryLimit - * Maximum number of bytes to load at once, 0 for unlimited. - */ - public void setDeltaSearchMemoryLimit(long memoryLimit) { - deltaSearchMemoryLimit = memoryLimit; - } - - /** - * Get the size of the in-memory delta cache. - *

- * This limit is for the entire writer, even if multiple threads are used. - * - * @return maximum number of bytes worth of delta data to cache in memory. - * If 0 the cache is infinite in size (up to the JVM heap limit - * anyway). A very tiny size such as 1 indicates the cache is - * effectively disabled. - */ - public long getDeltaCacheSize() { - return deltaCacheSize; - } - - /** - * Set the maximum number of bytes of delta data to cache. - *

- * During delta search, up to this many bytes worth of small or hard to - * compute deltas will be stored in memory. This cache speeds up writing by - * allowing the cached entry to simply be dumped to the output stream. - * - * @param size - * number of bytes to cache. Set to 0 to enable an infinite - * cache, set to 1 (an impossible size for any delta) to disable - * the cache. - */ - public void setDeltaCacheSize(long size) { - deltaCacheSize = size; - } - - /** - * Maximum size in bytes of a delta to cache. - * - * @return maximum size (in bytes) of a delta that should be cached. - */ - public int getDeltaCacheLimit() { - return deltaCacheLimit; - } - - /** - * Set the maximum size of a delta that should be cached. - *

- * During delta search, any delta smaller than this size will be cached, up - * to the {@link #getDeltaCacheSize()} maximum limit. This speeds up writing - * by allowing these cached deltas to be output as-is. - * - * @param size - * maximum size (in bytes) of a delta to be cached. - */ - public void setDeltaCacheLimit(int size) { - deltaCacheLimit = size; - } - - /** - * Get the maximum file size that will be delta compressed. - *

- * Files bigger than this setting will not be delta compressed, as they are - * more than likely already highly compressed binary data files that do not - * delta compress well, such as MPEG videos. - * - * @return the configured big file threshold. - */ - public long getBigFileThreshold() { - return bigFileThreshold; - } - - /** - * Set the maximum file size that should be considered for deltas. - * - * @param bigFileThreshold - * the limit, in bytes. - */ - public void setBigFileThreshold(long bigFileThreshold) { - this.bigFileThreshold = bigFileThreshold; - } - - /** - * Get the compression level applied to objects in the pack. - * - * @return current compression level, see {@link java.util.zip.Deflater}. - */ - public int getCompressionLevel() { - return compressionLevel; - } - - /** - * Set the compression level applied to objects in the pack. - * - * @param level - * compression level, must be a valid level recognized by the - * {@link java.util.zip.Deflater} class. Typically this setting - * is {@link java.util.zip.Deflater#BEST_SPEED}. - */ - public void setCompressionLevel(int level) { - compressionLevel = level; - } - - /** @return number of threads used for delta compression. */ - public int getThreads() { - return threads; - } - - /** - * Set the number of threads to use for delta compression. - *

- * During delta compression, if there are enough objects to be considered - * the writer will start up concurrent threads and allow them to compress - * different sections of the repository concurrently. - *

- * An application thread pool can be set by {@link #setExecutor(Executor)}. - * If not set a temporary pool will be created by the writer, and torn down - * automatically when compression is over. - * - * @param threads - * number of threads to use. If <= 0 the number of available - * processors for this JVM is used. - */ - public void setThread(int threads) { - this.threads = threads; - } - - /** - * Set the executor to use when using threads. - *

- * During delta compression if the executor is non-null jobs will be queued - * up on it to perform delta compression in parallel. Aside from setting the - * executor, the caller must set {@link #setThread(int)} to enable threaded - * delta search. - * - * @param executor - * executor to use for threads. Set to null to create a temporary - * executor just for this writer. - */ - public void setExecutor(Executor executor) { - this.executor = executor; - } - /** @return true if this writer is producing a thin pack. */ public boolean isThin() { return thin; @@ -677,18 +296,6 @@ public class PackWriter { ignoreMissingUninteresting = ignore; } - /** - * Set the pack index file format version this instance will create. - * - * @param version - * the version to write. The special version 0 designates the - * oldest (most compatible) format available for the objects. - * @see PackIndexWriter - */ - public void setIndexVersion(final int version) { - indexVersion = version; - } - /** * Returns objects number in a pack file that was created by this writer. * @@ -817,6 +424,7 @@ public class PackWriter { public void writeIndex(final OutputStream indexStream) throws IOException { final List list = sortByName(); final PackIndexWriter iw; + int indexVersion = config.getIndexVersion(); if (indexVersion <= 0) iw = PackIndexWriter.createOldestPossible(indexStream, list); else @@ -868,9 +476,9 @@ public class PackWriter { if (writeMonitor == null) writeMonitor = NullProgressMonitor.INSTANCE; - if ((reuseDeltas || reuseObjects) && reuseSupport != null) + if ((reuseDeltas || config.isReuseObjects()) && reuseSupport != null) searchForReuse(); - if (deltaCompress) + if (config.isDeltaCompress()) searchForDeltas(compressMonitor); final PackOutputStream out = new PackOutputStream(writeMonitor, @@ -980,7 +588,7 @@ public class PackWriter { // If its too big for us to handle, skip over it. // - if (bigFileThreshold <= sz || Integer.MAX_VALUE <= sz) + if (config.getBigFileThreshold() <= sz || Integer.MAX_VALUE <= sz) return false; // If its too tiny for the delta compression to work, skip it. @@ -996,17 +604,18 @@ public class PackWriter { final ObjectToPack[] list, final int cnt) throws MissingObjectException, IncorrectObjectTypeException, LargeObjectException, IOException { + int threads = config.getThreads(); if (threads == 0) threads = Runtime.getRuntime().availableProcessors(); - if (threads <= 1 || cnt <= 2 * getDeltaSearchWindowSize()) { - DeltaCache dc = new DeltaCache(this); - DeltaWindow dw = new DeltaWindow(this, dc, reader); + if (threads <= 1 || cnt <= 2 * config.getDeltaSearchWindowSize()) { + DeltaCache dc = new DeltaCache(config); + DeltaWindow dw = new DeltaWindow(config, dc, reader); dw.search(monitor, list, 0, cnt); return; } - final DeltaCache dc = new ThreadSafeDeltaCache(this); + final DeltaCache dc = new ThreadSafeDeltaCache(config); final ProgressMonitor pm = new ThreadSafeProgressMonitor(monitor); // Guess at the size of batch we want. Because we don't really @@ -1015,8 +624,8 @@ public class PackWriter { // are a bit smaller. // int estSize = cnt / (threads * 2); - if (estSize < 2 * getDeltaSearchWindowSize()) - estSize = 2 * getDeltaSearchWindowSize(); + if (estSize < 2 * config.getDeltaSearchWindowSize()) + estSize = 2 * config.getDeltaSearchWindowSize(); final List myTasks = new ArrayList(threads * 2); for (int i = 0; i < cnt;) { @@ -1043,9 +652,10 @@ public class PackWriter { batchSize = end - start; } i += batchSize; - myTasks.add(new DeltaTask(this, reader, dc, pm, batchSize, start, list)); + myTasks.add(new DeltaTask(config, reader, dc, pm, batchSize, start, list)); } + final Executor executor = config.getExecutor(); final List errors = Collections .synchronizedList(new ArrayList()); if (executor instanceof ExecutorService) { @@ -1269,8 +879,8 @@ public class PackWriter { private TemporaryBuffer.Heap delta(final ObjectToPack otp) throws IOException { - DeltaIndex index = new DeltaIndex(buffer(reader, otp.getDeltaBaseId())); - byte[] res = buffer(reader, otp); + DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId())); + byte[] res = buffer(otp); // We never would have proposed this pair if the delta would be // larger than the unpacked version of the object. So using it @@ -1281,7 +891,12 @@ public class PackWriter { return delta; } - byte[] buffer(ObjectReader or, AnyObjectId objId) throws IOException { + private byte[] buffer(AnyObjectId objId) throws IOException { + return buffer(config, reader, objId); + } + + static byte[] buffer(PackConfig config, ObjectReader or, AnyObjectId objId) + throws IOException { ObjectLoader ldr = or.open(objId); if (!ldr.isLarge()) return ldr.getCachedBytes(); @@ -1294,7 +909,7 @@ public class PackWriter { // If it really is too big to work with, abort out now. // long sz = ldr.getSize(); - if (getBigFileThreshold() <= sz || Integer.MAX_VALUE < sz) + if (config.getBigFileThreshold() <= sz || Integer.MAX_VALUE < sz) throw new LargeObjectException(objId.copy()); // Its considered to be large by the loader, but we really @@ -1321,7 +936,7 @@ public class PackWriter { private Deflater deflater() { if (myDeflater == null) - myDeflater = new Deflater(compressionLevel); + myDeflater = new Deflater(config.getCompressionLevel()); return myDeflater; } @@ -1477,7 +1092,7 @@ public class PackWriter { otp.clearDeltaBase(); otp.clearReuseAsIs(); } - } else if (nFmt == PACK_WHOLE && reuseObjects) { + } else if (nFmt == PACK_WHOLE && config.isReuseObjects()) { otp.clearDeltaBase(); otp.setReuseAsIs(); otp.setWeight(nWeight); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ThreadSafeDeltaCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ThreadSafeDeltaCache.java index 141289190a..2492a05e1c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ThreadSafeDeltaCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/ThreadSafeDeltaCache.java @@ -48,8 +48,8 @@ import java.util.concurrent.locks.ReentrantLock; class ThreadSafeDeltaCache extends DeltaCache { private final ReentrantLock lock; - ThreadSafeDeltaCache(PackWriter pw) { - super(pw); + ThreadSafeDeltaCache(PackConfig pc) { + super(pc); lock = new ReentrantLock(); } -- cgit v1.2.3