import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.file.PackIndex.MutableEntry;
+import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.storage.pack.PackWriter;
import org.eclipse.jgit.transport.IndexPack;
import org.eclipse.jgit.util.JGitTestUtil;
private static final List<RevObject> EMPTY_LIST_REVS = Collections
.<RevObject> emptyList();
+ private PackConfig config;
+
private PackWriter writer;
private ByteArrayOutputStream os;
packBase = new File(trash, "tmp_pack");
packFile = new File(trash, "tmp_pack.pack");
indexFile = new File(trash, "tmp_pack.idx");
- writer = new PackWriter(db);
+ config = new PackConfig(db);
+ }
+
+ public void tearDown() throws Exception {
+ if (writer != null)
+ writer.release();
+ super.tearDown();
}
/**
* Test constructor for exceptions, default settings, initialization.
*/
public void testContructor() {
+ writer = new PackWriter(config, db.newObjectReader());
assertEquals(false, writer.isDeltaBaseAsOffset());
- assertEquals(true, writer.isReuseDeltas());
- assertEquals(true, writer.isReuseObjects());
+ assertEquals(true, config.isReuseDeltas());
+ assertEquals(true, config.isReuseObjects());
assertEquals(0, writer.getObjectsNumber());
}
* Change default settings and verify them.
*/
public void testModifySettings() {
+ config.setReuseDeltas(false);
+ config.setReuseObjects(false);
+ config.setDeltaBaseAsOffset(false);
+ assertEquals(false, config.isReuseDeltas());
+ assertEquals(false, config.isReuseObjects());
+ assertEquals(false, config.isDeltaBaseAsOffset());
+
+ writer = new PackWriter(config, db.newObjectReader());
writer.setDeltaBaseAsOffset(true);
- writer.setReuseDeltas(false);
- writer.setReuseObjects(false);
-
assertEquals(true, writer.isDeltaBaseAsOffset());
- assertEquals(false, writer.isReuseDeltas());
- assertEquals(false, writer.isReuseObjects());
+ assertEquals(false, config.isDeltaBaseAsOffset());
}
/**
* @throws IOException
*/
public void testWritePack1() throws IOException {
- writer.setReuseDeltas(false);
+ config.setReuseDeltas(false);
writeVerifyPack1();
}
* @throws IOException
*/
public void testWritePack1NoObjectReuse() throws IOException {
- writer.setReuseDeltas(false);
- writer.setReuseObjects(false);
+ config.setReuseDeltas(false);
+ config.setReuseObjects(false);
writeVerifyPack1();
}
* @throws IOException
*/
public void testWritePack2DeltasReuseOffsets() throws IOException {
- writer.setDeltaBaseAsOffset(true);
+ config.setDeltaBaseAsOffset(true);
writeVerifyPack2(true);
}
*
*/
public void testWritePack3() throws MissingObjectException, IOException {
- writer.setReuseDeltas(false);
+ config.setReuseDeltas(false);
final ObjectId forcedOrder[] = new ObjectId[] {
ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
}
public void testWriteIndex() throws Exception {
- writer.setIndexVersion(2);
+ config.setIndexVersion(2);
writeVerifyPack4(false);
// Validate that IndexPack came up with the right CRC32 value.
}
private void writeVerifyPack2(boolean deltaReuse) throws IOException {
- writer.setReuseDeltas(deltaReuse);
+ config.setReuseDeltas(deltaReuse);
final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
interestings.add(ObjectId
.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
final boolean ignoreMissingUninteresting)
throws MissingObjectException, IOException {
NullProgressMonitor m = NullProgressMonitor.INSTANCE;
+ writer = new PackWriter(config, db.newObjectReader());
writer.setThin(thin);
writer.setIgnoreMissingUninteresting(ignoreMissingUninteresting);
writer.preparePack(m, interestings, uninterestings);
private void createVerifyOpenPack(final Iterator<RevObject> objectSource)
throws MissingObjectException, IOException {
NullProgressMonitor m = NullProgressMonitor.INSTANCE;
+ writer = new PackWriter(config, db.newObjectReader());
writer.preparePack(objectSource);
writer.writePack(m, m, os);
writer.release();
, section, name));
}
+ /**
+ * Obtain an integer value from the configuration.
+ *
+ * @param section
+ * section the key is grouped within.
+ * @param name
+ * name of the key to get.
+ * @param defaultValue
+ * default value to return if no value was present.
+ * @return an integer value from the configuration, or defaultValue.
+ */
+ public long getLong(String section, String name, long defaultValue) {
+ return getLong(section, null, name, defaultValue);
+ }
+
/**
* Obtain an integer value from the configuration.
*
private long used;
- DeltaCache(PackWriter pw) {
- size = pw.getDeltaCacheSize();
- entryLimit = pw.getDeltaCacheLimit();
+ DeltaCache(PackConfig pc) {
+ size = pc.getDeltaCacheSize();
+ entryLimit = pc.getDeltaCacheLimit();
queue = new ReferenceQueue<byte[]>();
}
import org.eclipse.jgit.lib.ProgressMonitor;
final class DeltaTask implements Callable<Object> {
- private final PackWriter writer;
+ private final PackConfig config;
private final ObjectReader templateReader;
private final ObjectToPack[] list;
- DeltaTask(PackWriter writer, ObjectReader reader, DeltaCache dc,
+ DeltaTask(PackConfig config, ObjectReader reader, DeltaCache dc,
ProgressMonitor pm, int batchSize, int start, ObjectToPack[] list) {
- this.writer = writer;
+ this.config = config;
this.templateReader = reader;
this.dc = dc;
this.pm = pm;
final ObjectReader or = templateReader.newReader();
try {
DeltaWindow dw;
- dw = new DeltaWindow(writer, dc, or);
+ dw = new DeltaWindow(config, dc, or);
dw.search(pm, list, start, batchSize);
} finally {
or.release();
private static final int NEXT_SRC = 1;
- private final PackWriter writer;
+ private final PackConfig config;
private final DeltaCache deltaCache;
/** Used to compress cached deltas. */
private Deflater deflater;
- DeltaWindow(PackWriter pw, DeltaCache dc, ObjectReader or) {
- writer = pw;
+ DeltaWindow(PackConfig pc, DeltaCache dc, ObjectReader or) {
+ config = pc;
deltaCache = dc;
reader = or;
// PackWriter has a minimum of 2 for the window size, but then
// users might complain that JGit is creating a bigger pack file.
//
- window = new DeltaWindowEntry[pw.getDeltaSearchWindowSize() + 1];
+ window = new DeltaWindowEntry[config.getDeltaSearchWindowSize() + 1];
for (int i = 0; i < window.length; i++)
window[i] = new DeltaWindowEntry();
- maxMemory = pw.getDeltaSearchMemoryLimit();
- maxDepth = pw.getMaxDeltaDepth();
+ maxMemory = config.getDeltaSearchMemoryLimit();
+ maxDepth = config.getMaxDeltaDepth();
}
void search(ProgressMonitor monitor, ObjectToPack[] toSearch, int off,
IncorrectObjectTypeException, IOException, LargeObjectException {
byte[] buf = ent.buffer;
if (buf == null) {
- buf = writer.buffer(reader, ent.object);
+ buf = PackWriter.buffer(config, reader, ent.object);
if (0 < maxMemory)
loaded += buf.length;
ent.buffer = buf;
private Deflater deflater() {
if (deflater == null)
- deflater = new Deflater(writer.getCompressionLevel());
+ deflater = new Deflater(config.getCompressionLevel());
else
deflater.reset();
return deflater;
/*
- * Copyright (C) 2010, Google Inc.
+ * Copyright (C) 2008-2010, Google Inc.
+ * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
* and other copyright owners as documented in the project's IP log.
*
* This program and the accompanying materials are made available
package org.eclipse.jgit.storage.pack;
-import static java.util.zip.Deflater.DEFAULT_COMPRESSION;
+import java.util.concurrent.Executor;
+import java.util.zip.Deflater;
+
import org.eclipse.jgit.lib.Config;
-import org.eclipse.jgit.lib.Config.SectionParser;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.storage.file.PackIndexWriter;
+
+/**
+ * Configuration used by a {@link PackWriter} when constructing the stream.
+ *
+ * A configuration may be modified once created, but should not be modified
+ * while it is being used by a PackWriter. If a configuration is not modified it
+ * is safe to share the same configuration instance between multiple concurrent
+ * threads executing different PackWriters.
+ */
+public class PackConfig {
+ /**
+ * Default value of deltas reuse option: {@value}
+ *
+ * @see #setReuseDeltas(boolean)
+ */
+ public static final boolean DEFAULT_REUSE_DELTAS = true;
+
+ /**
+ * Default value of objects reuse option: {@value}
+ *
+ * @see #setReuseObjects(boolean)
+ */
+ public static final boolean DEFAULT_REUSE_OBJECTS = true;
+
+ /**
+ * Default value of delta compress option: {@value}
+ *
+ * @see #setDeltaCompress(boolean)
+ */
+ public static final boolean DEFAULT_DELTA_COMPRESS = true;
+
+ /**
+ * Default value of delta base as offset option: {@value}
+ *
+ * @see #setDeltaBaseAsOffset(boolean)
+ */
+ public static final boolean DEFAULT_DELTA_BASE_AS_OFFSET = false;
+
+ /**
+ * Default value of maximum delta chain depth: {@value}
+ *
+ * @see #setMaxDeltaDepth(int)
+ */
+ public static final int DEFAULT_MAX_DELTA_DEPTH = 50;
+
+ /**
+ * Default window size during packing: {@value}
+ *
+ * @see #setDeltaSearchWindowSize(int)
+ */
+ public static final int DEFAULT_DELTA_SEARCH_WINDOW_SIZE = 10;
+
+ /**
+ * Default big file threshold: {@value}
+ *
+ * @see #setBigFileThreshold(long)
+ */
+ public static final long DEFAULT_BIG_FILE_THRESHOLD = 50 * 1024 * 1024;
+
+ /**
+ * Default delta cache size: {@value}
+ *
+ * @see #setDeltaCacheSize(long)
+ */
+ public static final long DEFAULT_DELTA_CACHE_SIZE = 50 * 1024 * 1024;
+
+ /**
+ * Default delta cache limit: {@value}
+ *
+ * @see #setDeltaCacheLimit(int)
+ */
+ public static final int DEFAULT_DELTA_CACHE_LIMIT = 100;
+
+ /**
+ * Default index version: {@value}
+ *
+ * @see #setIndexVersion(int)
+ */
+ public static final int DEFAULT_INDEX_VERSION = 2;
+
+
+ private int compressionLevel = Deflater.DEFAULT_COMPRESSION;
+
+ private boolean reuseDeltas = DEFAULT_REUSE_DELTAS;
+
+ private boolean reuseObjects = DEFAULT_REUSE_OBJECTS;
+
+ private boolean deltaBaseAsOffset = DEFAULT_DELTA_BASE_AS_OFFSET;
+
+ private boolean deltaCompress = DEFAULT_DELTA_COMPRESS;
+
+ private int maxDeltaDepth = DEFAULT_MAX_DELTA_DEPTH;
-class PackConfig {
- /** Key for {@link Config#get(SectionParser)}. */
- static final Config.SectionParser<PackConfig> KEY = new SectionParser<PackConfig>() {
- public PackConfig parse(final Config cfg) {
- return new PackConfig(cfg);
- }
- };
+ private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE;
- final int deltaWindow;
+ private long deltaSearchMemoryLimit;
- final long deltaWindowMemory;
+ private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE;
- final int deltaDepth;
+ private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT;
- final long deltaCacheSize;
+ private long bigFileThreshold = DEFAULT_BIG_FILE_THRESHOLD;
- final int deltaCacheLimit;
+ private int threads;
- final int compression;
+ private Executor executor;
- final int indexVersion;
+ private int indexVersion = DEFAULT_INDEX_VERSION;
- final long bigFileThreshold;
- final int threads;
+ /** Create a default configuration. */
+ public PackConfig() {
+ // Fields are initialized to defaults.
+ }
+
+ /**
+ * Create a configuration honoring the repository's settings.
+ *
+ * @param db
+ * the repository to read settings from. The repository is not
+ * retained by the new configuration, instead its settings are
+ * copied during the constructor.
+ */
+ public PackConfig(Repository db) {
+ fromConfig(db.getConfig());
+ }
+
+ /**
+ * Create a configuration honoring settings in a {@link Config}.
+ *
+ * @param cfg
+ * the source to read settings from. The source is not retained
+ * by the new configuration, instead its settings are copied
+ * during the constructor.
+ */
+ public PackConfig(Config cfg) {
+ fromConfig(cfg);
+ }
+
+ /**
+ * Check whether to reuse deltas existing in repository.
+ *
+ * Default setting: {@value #DEFAULT_REUSE_DELTAS}
+ *
+ * @return true if object is configured to reuse deltas; false otherwise.
+ */
+ public boolean isReuseDeltas() {
+ return reuseDeltas;
+ }
+
+ /**
+ * Set reuse deltas configuration option for the writer.
+ *
+ * When enabled, writer will search for delta representation of object in
+ * repository and use it if possible. Normally, only deltas with base to
+ * another object existing in set of objects to pack will be used. The
+ * exception however is thin-packs where the base object may exist on the
+ * other side.
+ *
+ * When raw delta data is directly copied from a pack file, its checksum is
+ * computed to verify the data is not corrupt.
+ *
+ * Default setting: {@value #DEFAULT_REUSE_DELTAS}
+ *
+ * @param reuseDeltas
+ * boolean indicating whether or not try to reuse deltas.
+ */
+ public void setReuseDeltas(boolean reuseDeltas) {
+ this.reuseDeltas = reuseDeltas;
+ }
+
+ /**
+ * Checks whether to reuse existing objects representation in repository.
+ *
+ * Default setting: {@value #DEFAULT_REUSE_OBJECTS}
+ *
+ * @return true if writer is configured to reuse objects representation from
+ * pack; false otherwise.
+ */
+ public boolean isReuseObjects() {
+ return reuseObjects;
+ }
+
+ /**
+ * Set reuse objects configuration option for the writer.
+ *
+ * If enabled, writer searches for compressed representation in a pack file.
+ * If possible, compressed data is directly copied from such a pack file.
+ * Data checksum is verified.
+ *
+ * Default setting: {@value #DEFAULT_REUSE_OBJECTS}
+ *
+ * @param reuseObjects
+ * boolean indicating whether or not writer should reuse existing
+ * objects representation.
+ */
+ public void setReuseObjects(boolean reuseObjects) {
+ this.reuseObjects = reuseObjects;
+ }
+
+ /**
+ * True if writer can use offsets to point to a delta base.
+ *
+ * If true the writer may choose to use an offset to point to a delta base
+ * in the same pack, this is a newer style of reference that saves space.
+ * False if the writer has to use the older (and more compatible style) of
+ * storing the full ObjectId of the delta base.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_BASE_AS_OFFSET}
+ *
+ * @return true if delta base is stored as an offset; false if it is stored
+ * as an ObjectId.
+ */
+ public boolean isDeltaBaseAsOffset() {
+ return deltaBaseAsOffset;
+ }
+
+ /**
+ * Set writer delta base format.
+ *
+ * Delta base can be written as an offset in a pack file (new approach
+ * reducing file size) or as an object id (legacy approach, compatible with
+ * old readers).
+ *
+ * Default setting: {@value #DEFAULT_DELTA_BASE_AS_OFFSET}
+ *
+ * @param deltaBaseAsOffset
+ * boolean indicating whether delta base can be stored as an
+ * offset.
+ */
+ public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) {
+ this.deltaBaseAsOffset = deltaBaseAsOffset;
+ }
+
+ /**
+ * Check whether the writer will create new deltas on the fly.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_COMPRESS}
+ *
+ * @return true if the writer will create a new delta when either
+ * {@link #isReuseDeltas()} is false, or no suitable delta is
+ * available for reuse.
+ */
+ public boolean isDeltaCompress() {
+ return deltaCompress;
+ }
+
+ /**
+ * Set whether or not the writer will create new deltas on the fly.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_COMPRESS}
+ *
+ * @param deltaCompress
+ * true to create deltas when {@link #isReuseDeltas()} is false,
+ * or when a suitable delta isn't available for reuse. Set to
+ * false to write whole objects instead.
+ */
+ public void setDeltaCompress(boolean deltaCompress) {
+ this.deltaCompress = deltaCompress;
+ }
+
+ /**
+ * Get maximum depth of delta chain set up for the writer.
+ *
+ * Generated chains are not longer than this value.
+ *
+ * Default setting: {@value #DEFAULT_MAX_DELTA_DEPTH}
+ *
+ * @return maximum delta chain depth.
+ */
+ public int getMaxDeltaDepth() {
+ return maxDeltaDepth;
+ }
+
+ /**
+ * Set up maximum depth of delta chain for the writer.
+ *
+ * Generated chains are not longer than this value. Too low value causes low
+ * compression level, while too big makes unpacking (reading) longer.
+ *
+ * Default setting: {@value #DEFAULT_MAX_DELTA_DEPTH}
+ *
+ * @param maxDeltaDepth
+ * maximum delta chain depth.
+ */
+ public void setMaxDeltaDepth(int maxDeltaDepth) {
+ this.maxDeltaDepth = maxDeltaDepth;
+ }
+
+ /**
+ * Get the number of objects to try when looking for a delta base.
+ *
+ * This limit is per thread, if 4 threads are used the actual memory used
+ * will be 4 times this value.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_SEARCH_WINDOW_SIZE}
+ *
+ * @return the object count to be searched.
+ */
+ public int getDeltaSearchWindowSize() {
+ return deltaSearchWindowSize;
+ }
+
+ /**
+ * Set the number of objects considered when searching for a delta base.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_SEARCH_WINDOW_SIZE}
+ *
+ * @param objectCount
+ * number of objects to search at once. Must be at least 2.
+ */
+ public void setDeltaSearchWindowSize(int objectCount) {
+ if (objectCount <= 2)
+ setDeltaCompress(false);
+ else
+ deltaSearchWindowSize = objectCount;
+ }
+
+ /**
+ * Get maximum number of bytes to put into the delta search window.
+ *
+ * Default setting is 0, for an unlimited amount of memory usage. Actual
+ * memory used is the lower limit of either this setting, or the sum of
+ * space used by at most {@link #getDeltaSearchWindowSize()} objects.
+ *
+ * This limit is per thread, if 4 threads are used the actual memory limit
+ * will be 4 times this value.
+ *
+ * @return the memory limit.
+ */
+ public long getDeltaSearchMemoryLimit() {
+ return deltaSearchMemoryLimit;
+ }
+
+ /**
+ * Set the maximum number of bytes to put into the delta search window.
+ *
+ * Default setting is 0, for an unlimited amount of memory usage. If the
+ * memory limit is reached before {@link #getDeltaSearchWindowSize()} the
+ * window size is temporarily lowered.
+ *
+ * @param memoryLimit
+ * Maximum number of bytes to load at once, 0 for unlimited.
+ */
+ public void setDeltaSearchMemoryLimit(long memoryLimit) {
+ deltaSearchMemoryLimit = memoryLimit;
+ }
+
+ /**
+ * Get the size of the in-memory delta cache.
+ *
+ * This limit is for the entire writer, even if multiple threads are used.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_CACHE_SIZE}
+ *
+ * @return maximum number of bytes worth of delta data to cache in memory.
+ * If 0 the cache is infinite in size (up to the JVM heap limit
+ * anyway). A very tiny size such as 1 indicates the cache is
+ * effectively disabled.
+ */
+ public long getDeltaCacheSize() {
+ return deltaCacheSize;
+ }
+
+ /**
+ * Set the maximum number of bytes of delta data to cache.
+ *
+ * During delta search, up to this many bytes worth of small or hard to
+ * compute deltas will be stored in memory. This cache speeds up writing by
+ * allowing the cached entry to simply be dumped to the output stream.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_CACHE_SIZE}
+ *
+ * @param size
+ * number of bytes to cache. Set to 0 to enable an infinite
+ * cache, set to 1 (an impossible size for any delta) to disable
+ * the cache.
+ */
+ public void setDeltaCacheSize(long size) {
+ deltaCacheSize = size;
+ }
+
+ /**
+ * Maximum size in bytes of a delta to cache.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_CACHE_LIMIT}
+ *
+ * @return maximum size (in bytes) of a delta that should be cached.
+ */
+ public int getDeltaCacheLimit() {
+ return deltaCacheLimit;
+ }
+
+ /**
+ * Set the maximum size of a delta that should be cached.
+ *
+ * During delta search, any delta smaller than this size will be cached, up
+ * to the {@link #getDeltaCacheSize()} maximum limit. This speeds up writing
+ * by allowing these cached deltas to be output as-is.
+ *
+ * Default setting: {@value #DEFAULT_DELTA_CACHE_LIMIT}
+ *
+ * @param size
+ * maximum size (in bytes) of a delta to be cached.
+ */
+ public void setDeltaCacheLimit(int size) {
+ deltaCacheLimit = size;
+ }
+
+ /**
+ * Get the maximum file size that will be delta compressed.
+ *
+ * Files bigger than this setting will not be delta compressed, as they are
+ * more than likely already highly compressed binary data files that do not
+ * delta compress well, such as MPEG videos.
+ *
+ * Default setting: {@value #DEFAULT_BIG_FILE_THRESHOLD}
+ *
+ * @return the configured big file threshold.
+ */
+ public long getBigFileThreshold() {
+ return bigFileThreshold;
+ }
+
+ /**
+ * Set the maximum file size that should be considered for deltas.
+ *
+ * Default setting: {@value #DEFAULT_BIG_FILE_THRESHOLD}
+ *
+ * @param bigFileThreshold
+ * the limit, in bytes.
+ */
+ public void setBigFileThreshold(long bigFileThreshold) {
+ this.bigFileThreshold = bigFileThreshold;
+ }
+
+ /**
+ * Get the compression level applied to objects in the pack.
+ *
+ * Default setting: {@value java.util.zip.Deflater#DEFAULT_COMPRESSION}
+ *
+ * @return current compression level, see {@link java.util.zip.Deflater}.
+ */
+ public int getCompressionLevel() {
+ return compressionLevel;
+ }
+
+ /**
+ * Set the compression level applied to objects in the pack.
+ *
+ * Default setting: {@value java.util.zip.Deflater#DEFAULT_COMPRESSION}
+ *
+ * @param level
+ * compression level, must be a valid level recognized by the
+ * {@link java.util.zip.Deflater} class.
+ */
+ public void setCompressionLevel(int level) {
+ compressionLevel = level;
+ }
+
+ /**
+ * Get the number of threads used during delta compression.
+ *
+ * Default setting: 0 (auto-detect processors)
+ *
+ * @return number of threads used for delta compression. 0 will auto-detect
+ * the threads to the number of available processors.
+ */
+ public int getThreads() {
+ return threads;
+ }
+
+ /**
+ * Set the number of threads to use for delta compression.
+ *
+ * During delta compression, if there are enough objects to be considered
+ * the writer will start up concurrent threads and allow them to compress
+ * different sections of the repository concurrently.
+ *
+ * An application thread pool can be set by {@link #setExecutor(Executor)}.
+ * If not set a temporary pool will be created by the writer, and torn down
+ * automatically when compression is over.
+ *
+ * Default setting: 0 (auto-detect processors)
+ *
+ * @param threads
+ * number of threads to use. If <= 0 the number of available
+ * processors for this JVM is used.
+ */
+ public void setThreads(int threads) {
+ this.threads = threads;
+ }
+
+ /** @return the preferred thread pool to execute delta search on. */
+ public Executor getExecutor() {
+ return executor;
+ }
+
+ /**
+ * Set the executor to use when using threads.
+ *
+ * During delta compression if the executor is non-null jobs will be queued
+ * up on it to perform delta compression in parallel. Aside from setting the
+ * executor, the caller must set {@link #setThreads(int)} to enable threaded
+ * delta search.
+ *
+ * @param executor
+ * executor to use for threads. Set to null to create a temporary
+ * executor just for the writer.
+ */
+ public void setExecutor(Executor executor) {
+ this.executor = executor;
+ }
+
+ /**
+ * Get the pack index file format version this instance creates.
+ *
+ * Default setting: {@value #DEFAULT_INDEX_VERSION}
+ *
+ * @return the index version, the special version 0 designates the oldest
+ * (most compatible) format available for the objects.
+ * @see PackIndexWriter
+ */
+ public int getIndexVersion() {
+ return indexVersion;
+ }
- private PackConfig(Config rc) {
- deltaWindow = rc.getInt("pack", "window", PackWriter.DEFAULT_DELTA_SEARCH_WINDOW_SIZE);
- deltaWindowMemory = rc.getLong("pack", null, "windowmemory", 0);
- deltaCacheSize = rc.getLong("pack", null, "deltacachesize", PackWriter.DEFAULT_DELTA_CACHE_SIZE);
- deltaCacheLimit = rc.getInt("pack", "deltacachelimit", PackWriter.DEFAULT_DELTA_CACHE_LIMIT);
- deltaDepth = rc.getInt("pack", "depth", PackWriter.DEFAULT_MAX_DELTA_DEPTH);
- compression = compression(rc);
- indexVersion = rc.getInt("pack", "indexversion", 2);
- bigFileThreshold = rc.getLong("core", null, "bigfilethreshold", PackWriter.DEFAULT_BIG_FILE_THRESHOLD);
- threads = rc.getInt("pack", "threads", 0);
+ /**
+ * Set the pack index file format version this instance will create.
+ *
+ * Default setting: {@value #DEFAULT_INDEX_VERSION}
+ *
+ * @param version
+ * the version to write. The special version 0 designates the
+ * oldest (most compatible) format available for the objects.
+ * @see PackIndexWriter
+ */
+ public void setIndexVersion(final int version) {
+ indexVersion = version;
}
- private static int compression(Config rc) {
- if (rc.getString("pack", null, "compression") != null)
- return rc.getInt("pack", "compression", DEFAULT_COMPRESSION);
- return rc.getInt("core", "compression", DEFAULT_COMPRESSION);
+ /**
+ * Update properties by setting fields from the configuration.
+ *
+ * If a property's corresponding variable is not defined in the supplied
+ * configuration, then it is left unmodified.
+ *
+ * @param rc
+ * configuration to read properties from.
+ */
+ public void fromConfig(final Config rc) {
+ setMaxDeltaDepth(rc.getInt("pack", "depth", getMaxDeltaDepth()));
+ setDeltaSearchWindowSize(rc.getInt("pack", "window", getDeltaSearchWindowSize()));
+ setDeltaSearchMemoryLimit(rc.getLong("pack", "windowmemory", getDeltaSearchMemoryLimit()));
+ setDeltaCacheSize(rc.getLong("pack", "deltacachesize", getDeltaCacheSize()));
+ setDeltaCacheLimit(rc.getInt("pack", "deltacachelimit", getDeltaCacheLimit()));
+ setCompressionLevel(rc.getInt("pack", "compression",
+ rc.getInt("core", "compression", getCompressionLevel())));
+ setIndexVersion(rc.getInt("pack", "indexversion", getIndexVersion()));
+ setBigFileThreshold(rc.getLong("core", "bigfilethreshold", getBigFileThreshold()));
+ setThreads(rc.getInt("pack", "threads", getThreads()));
}
}
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
import org.eclipse.jgit.lib.AnyObjectId;
-import org.eclipse.jgit.lib.Config;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.NullProgressMonitor;
import org.eclipse.jgit.lib.ObjectId;
* </p>
*/
public class PackWriter {
- /**
- * Default value of deltas reuse option.
- *
- * @see #setReuseDeltas(boolean)
- */
- public static final boolean DEFAULT_REUSE_DELTAS = true;
-
- /**
- * Default value of objects reuse option.
- *
- * @see #setReuseObjects(boolean)
- */
- public static final boolean DEFAULT_REUSE_OBJECTS = true;
-
- /**
- * Default value of delta base as offset option.
- *
- * @see #setDeltaBaseAsOffset(boolean)
- */
- public static final boolean DEFAULT_DELTA_BASE_AS_OFFSET = false;
-
- /**
- * Default value of maximum delta chain depth.
- *
- * @see #setMaxDeltaDepth(int)
- */
- public static final int DEFAULT_MAX_DELTA_DEPTH = 50;
-
- /**
- * Default window size during packing.
- *
- * @see #setDeltaSearchWindowSize(int)
- */
- public static final int DEFAULT_DELTA_SEARCH_WINDOW_SIZE = 10;
-
- static final long DEFAULT_BIG_FILE_THRESHOLD = 50 * 1024 * 1024;
-
- static final long DEFAULT_DELTA_CACHE_SIZE = 50 * 1024 * 1024;
-
- static final int DEFAULT_DELTA_CACHE_LIMIT = 100;
-
private static final int PACK_VERSION_GENERATED = 2;
@SuppressWarnings("unchecked")
// edge objects for thin packs
private final ObjectIdSubclassMap<ObjectToPack> edgeObjects = new ObjectIdSubclassMap<ObjectToPack>();
- private int compressionLevel;
-
private Deflater myDeflater;
private final ObjectReader reader;
/** {@link #reader} recast to the reuse interface, if it supports it. */
private final ObjectReuseAsIs reuseSupport;
+ private final PackConfig config;
+
private List<ObjectToPack> sortedByName;
private byte packcsum[];
- private boolean reuseDeltas = DEFAULT_REUSE_DELTAS;
-
- private boolean reuseObjects = DEFAULT_REUSE_OBJECTS;
+ private boolean deltaBaseAsOffset;
- private boolean deltaBaseAsOffset = DEFAULT_DELTA_BASE_AS_OFFSET;
-
- private boolean deltaCompress = true;
-
- private int maxDeltaDepth = DEFAULT_MAX_DELTA_DEPTH;
-
- private int deltaSearchWindowSize = DEFAULT_DELTA_SEARCH_WINDOW_SIZE;
-
- private long deltaSearchMemoryLimit;
-
- private long deltaCacheSize = DEFAULT_DELTA_CACHE_SIZE;
-
- private int deltaCacheLimit = DEFAULT_DELTA_CACHE_LIMIT;
-
- private int indexVersion;
-
- private long bigFileThreshold = DEFAULT_BIG_FILE_THRESHOLD;
-
- private int threads = 1;
-
- private Executor executor;
+ private boolean reuseDeltas;
private boolean thin;
* reader to read from the repository with.
*/
public PackWriter(final ObjectReader reader) {
- this(null, reader);
+ this(new PackConfig(), reader);
}
/**
* reader to read from the repository with.
*/
public PackWriter(final Repository repo, final ObjectReader reader) {
- this.reader = reader;
- if (reader instanceof ObjectReuseAsIs)
- reuseSupport = ((ObjectReuseAsIs) reader);
- else
- reuseSupport = null;
-
- final PackConfig pc = configOf(repo).get(PackConfig.KEY);
- deltaSearchWindowSize = pc.deltaWindow;
- deltaSearchMemoryLimit = pc.deltaWindowMemory;
- deltaCacheSize = pc.deltaCacheSize;
- deltaCacheLimit = pc.deltaCacheLimit;
- maxDeltaDepth = pc.deltaDepth;
- compressionLevel = pc.compression;
- indexVersion = pc.indexVersion;
- bigFileThreshold = pc.bigFileThreshold;
- threads = pc.threads;
- }
-
- private static Config configOf(final Repository repo) {
- if (repo == null)
- return new Config();
- return repo.getConfig();
- }
-
- /**
- * Check whether object is configured to reuse deltas existing in
- * repository.
- * <p>
- * Default setting: {@link #DEFAULT_REUSE_DELTAS}
- * </p>
- *
- * @return true if object is configured to reuse deltas; false otherwise.
- */
- public boolean isReuseDeltas() {
- return reuseDeltas;
+ this(new PackConfig(repo), reader);
}
/**
- * Set reuse deltas configuration option for this writer. When enabled,
- * writer will search for delta representation of object in repository and
- * use it if possible. Normally, only deltas with base to another object
- * existing in set of objects to pack will be used. Exception is however
- * thin-pack (see
- * {@link #preparePack(ProgressMonitor, Collection, Collection)} and
- * {@link #preparePack(Iterator)}) where base object must exist on other
- * side machine.
+ * Create writer with a specified configuration.
* <p>
- * When raw delta data is directly copied from a pack file, checksum is
- * computed to verify data.
- * </p>
- * <p>
- * Default setting: {@link #DEFAULT_REUSE_DELTAS}
- * </p>
- *
- * @param reuseDeltas
- * boolean indicating whether or not try to reuse deltas.
- */
- public void setReuseDeltas(boolean reuseDeltas) {
- this.reuseDeltas = reuseDeltas;
- }
-
- /**
- * Checks whether object is configured to reuse existing objects
- * representation in repository.
- * <p>
- * Default setting: {@link #DEFAULT_REUSE_OBJECTS}
- * </p>
+ * Objects for packing are specified in {@link #preparePack(Iterator)} or
+ * {@link #preparePack(ProgressMonitor, Collection, Collection)}.
*
- * @return true if writer is configured to reuse objects representation from
- * pack; false otherwise.
+ * @param config
+ * configuration for the pack writer.
+ * @param reader
+ * reader to read from the repository with.
*/
- public boolean isReuseObjects() {
- return reuseObjects;
- }
+ public PackWriter(final PackConfig config, final ObjectReader reader) {
+ this.config = config;
+ this.reader = reader;
+ if (reader instanceof ObjectReuseAsIs)
+ reuseSupport = ((ObjectReuseAsIs) reader);
+ else
+ reuseSupport = null;
- /**
- * Set reuse objects configuration option for this writer. If enabled,
- * writer searches for representation in a pack file. If possible,
- * compressed data is directly copied from such a pack file. Data checksum
- * is verified.
- * <p>
- * Default setting: {@link #DEFAULT_REUSE_OBJECTS}
- * </p>
- *
- * @param reuseObjects
- * boolean indicating whether or not writer should reuse existing
- * objects representation.
- */
- public void setReuseObjects(boolean reuseObjects) {
- this.reuseObjects = reuseObjects;
+ deltaBaseAsOffset = config.isDeltaBaseAsOffset();
+ reuseDeltas = config.isReuseDeltas();
}
/**
* Check whether writer can store delta base as an offset (new style
* reducing pack size) or should store it as an object id (legacy style,
* compatible with old readers).
- * <p>
- * Default setting: {@link #DEFAULT_DELTA_BASE_AS_OFFSET}
- * </p>
+ *
+ * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET}
*
* @return true if delta base is stored as an offset; false if it is stored
* as an object id.
* Set writer delta base format. Delta base can be written as an offset in a
* pack file (new approach reducing file size) or as an object id (legacy
* approach, compatible with old readers).
- * <p>
- * Default setting: {@link #DEFAULT_DELTA_BASE_AS_OFFSET}
- * </p>
+ *
+ * Default setting: {@value PackConfig#DEFAULT_DELTA_BASE_AS_OFFSET}
*
* @param deltaBaseAsOffset
* boolean indicating whether delta base can be stored as an
this.deltaBaseAsOffset = deltaBaseAsOffset;
}
- /**
- * Check whether the writer will create new deltas on the fly.
- * <p>
- * Default setting: true
- * </p>
- *
- * @return true if the writer will create a new delta when either
- * {@link #isReuseDeltas()} is false, or no suitable delta is
- * available for reuse.
- */
- public boolean isDeltaCompress() {
- return deltaCompress;
- }
-
- /**
- * Set whether or not the writer will create new deltas on the fly.
- *
- * @param deltaCompress
- * true to create deltas when {@link #isReuseDeltas()} is false,
- * or when a suitable delta isn't available for reuse. Set to
- * false to write whole objects instead.
- */
- public void setDeltaCompress(boolean deltaCompress) {
- this.deltaCompress = deltaCompress;
- }
-
- /**
- * Get maximum depth of delta chain set up for this writer. Generated chains
- * are not longer than this value.
- * <p>
- * Default setting: {@link #DEFAULT_MAX_DELTA_DEPTH}
- * </p>
- *
- * @return maximum delta chain depth.
- */
- public int getMaxDeltaDepth() {
- return maxDeltaDepth;
- }
-
- /**
- * Set up maximum depth of delta chain for this writer. Generated chains are
- * not longer than this value. Too low value causes low compression level,
- * while too big makes unpacking (reading) longer.
- * <p>
- * Default setting: {@link #DEFAULT_MAX_DELTA_DEPTH}
- * </p>
- *
- * @param maxDeltaDepth
- * maximum delta chain depth.
- */
- public void setMaxDeltaDepth(int maxDeltaDepth) {
- this.maxDeltaDepth = maxDeltaDepth;
- }
-
- /**
- * Get the number of objects to try when looking for a delta base.
- * <p>
- * This limit is per thread, if 4 threads are used the actual memory
- * used will be 4 times this value.
- *
- * @return the object count to be searched.
- */
- public int getDeltaSearchWindowSize() {
- return deltaSearchWindowSize;
- }
-
- /**
- * Set the number of objects considered when searching for a delta base.
- * <p>
- * Default setting: {@link #DEFAULT_DELTA_SEARCH_WINDOW_SIZE}
- * </p>
- *
- * @param objectCount
- * number of objects to search at once. Must be at least 2.
- */
- public void setDeltaSearchWindowSize(int objectCount) {
- if (objectCount <= 2)
- setDeltaCompress(false);
- else
- deltaSearchWindowSize = objectCount;
- }
-
- /**
- * Get maximum number of bytes to put into the delta search window.
- * <p>
- * Default setting is 0, for an unlimited amount of memory usage. Actual
- * memory used is the lower limit of either this setting, or the sum of
- * space used by at most {@link #getDeltaSearchWindowSize()} objects.
- * <p>
- * This limit is per thread, if 4 threads are used the actual memory
- * limit will be 4 times this value.
- *
- * @return the memory limit.
- */
- public long getDeltaSearchMemoryLimit() {
- return deltaSearchMemoryLimit;
- }
-
- /**
- * Set the maximum number of bytes to put into the delta search window.
- * <p>
- * Default setting is 0, for an unlimited amount of memory usage. If the
- * memory limit is reached before {@link #getDeltaSearchWindowSize()} the
- * window size is temporarily lowered.
- *
- * @param memoryLimit
- * Maximum number of bytes to load at once, 0 for unlimited.
- */
- public void setDeltaSearchMemoryLimit(long memoryLimit) {
- deltaSearchMemoryLimit = memoryLimit;
- }
-
- /**
- * Get the size of the in-memory delta cache.
- * <p>
- * This limit is for the entire writer, even if multiple threads are used.
- *
- * @return maximum number of bytes worth of delta data to cache in memory.
- * If 0 the cache is infinite in size (up to the JVM heap limit
- * anyway). A very tiny size such as 1 indicates the cache is
- * effectively disabled.
- */
- public long getDeltaCacheSize() {
- return deltaCacheSize;
- }
-
- /**
- * Set the maximum number of bytes of delta data to cache.
- * <p>
- * During delta search, up to this many bytes worth of small or hard to
- * compute deltas will be stored in memory. This cache speeds up writing by
- * allowing the cached entry to simply be dumped to the output stream.
- *
- * @param size
- * number of bytes to cache. Set to 0 to enable an infinite
- * cache, set to 1 (an impossible size for any delta) to disable
- * the cache.
- */
- public void setDeltaCacheSize(long size) {
- deltaCacheSize = size;
- }
-
- /**
- * Maximum size in bytes of a delta to cache.
- *
- * @return maximum size (in bytes) of a delta that should be cached.
- */
- public int getDeltaCacheLimit() {
- return deltaCacheLimit;
- }
-
- /**
- * Set the maximum size of a delta that should be cached.
- * <p>
- * During delta search, any delta smaller than this size will be cached, up
- * to the {@link #getDeltaCacheSize()} maximum limit. This speeds up writing
- * by allowing these cached deltas to be output as-is.
- *
- * @param size
- * maximum size (in bytes) of a delta to be cached.
- */
- public void setDeltaCacheLimit(int size) {
- deltaCacheLimit = size;
- }
-
- /**
- * Get the maximum file size that will be delta compressed.
- * <p>
- * Files bigger than this setting will not be delta compressed, as they are
- * more than likely already highly compressed binary data files that do not
- * delta compress well, such as MPEG videos.
- *
- * @return the configured big file threshold.
- */
- public long getBigFileThreshold() {
- return bigFileThreshold;
- }
-
- /**
- * Set the maximum file size that should be considered for deltas.
- *
- * @param bigFileThreshold
- * the limit, in bytes.
- */
- public void setBigFileThreshold(long bigFileThreshold) {
- this.bigFileThreshold = bigFileThreshold;
- }
-
- /**
- * Get the compression level applied to objects in the pack.
- *
- * @return current compression level, see {@link java.util.zip.Deflater}.
- */
- public int getCompressionLevel() {
- return compressionLevel;
- }
-
- /**
- * Set the compression level applied to objects in the pack.
- *
- * @param level
- * compression level, must be a valid level recognized by the
- * {@link java.util.zip.Deflater} class. Typically this setting
- * is {@link java.util.zip.Deflater#BEST_SPEED}.
- */
- public void setCompressionLevel(int level) {
- compressionLevel = level;
- }
-
- /** @return number of threads used for delta compression. */
- public int getThreads() {
- return threads;
- }
-
- /**
- * Set the number of threads to use for delta compression.
- * <p>
- * During delta compression, if there are enough objects to be considered
- * the writer will start up concurrent threads and allow them to compress
- * different sections of the repository concurrently.
- * <p>
- * An application thread pool can be set by {@link #setExecutor(Executor)}.
- * If not set a temporary pool will be created by the writer, and torn down
- * automatically when compression is over.
- *
- * @param threads
- * number of threads to use. If <= 0 the number of available
- * processors for this JVM is used.
- */
- public void setThread(int threads) {
- this.threads = threads;
- }
-
- /**
- * Set the executor to use when using threads.
- * <p>
- * During delta compression if the executor is non-null jobs will be queued
- * up on it to perform delta compression in parallel. Aside from setting the
- * executor, the caller must set {@link #setThread(int)} to enable threaded
- * delta search.
- *
- * @param executor
- * executor to use for threads. Set to null to create a temporary
- * executor just for this writer.
- */
- public void setExecutor(Executor executor) {
- this.executor = executor;
- }
-
/** @return true if this writer is producing a thin pack. */
public boolean isThin() {
return thin;
ignoreMissingUninteresting = ignore;
}
- /**
- * Set the pack index file format version this instance will create.
- *
- * @param version
- * the version to write. The special version 0 designates the
- * oldest (most compatible) format available for the objects.
- * @see PackIndexWriter
- */
- public void setIndexVersion(final int version) {
- indexVersion = version;
- }
-
/**
* Returns objects number in a pack file that was created by this writer.
*
public void writeIndex(final OutputStream indexStream) throws IOException {
final List<ObjectToPack> list = sortByName();
final PackIndexWriter iw;
+ int indexVersion = config.getIndexVersion();
if (indexVersion <= 0)
iw = PackIndexWriter.createOldestPossible(indexStream, list);
else
if (writeMonitor == null)
writeMonitor = NullProgressMonitor.INSTANCE;
- if ((reuseDeltas || reuseObjects) && reuseSupport != null)
+ if ((reuseDeltas || config.isReuseObjects()) && reuseSupport != null)
searchForReuse();
- if (deltaCompress)
+ if (config.isDeltaCompress())
searchForDeltas(compressMonitor);
final PackOutputStream out = new PackOutputStream(writeMonitor,
// If its too big for us to handle, skip over it.
//
- if (bigFileThreshold <= sz || Integer.MAX_VALUE <= sz)
+ if (config.getBigFileThreshold() <= sz || Integer.MAX_VALUE <= sz)
return false;
// If its too tiny for the delta compression to work, skip it.
final ObjectToPack[] list, final int cnt)
throws MissingObjectException, IncorrectObjectTypeException,
LargeObjectException, IOException {
+ int threads = config.getThreads();
if (threads == 0)
threads = Runtime.getRuntime().availableProcessors();
- if (threads <= 1 || cnt <= 2 * getDeltaSearchWindowSize()) {
- DeltaCache dc = new DeltaCache(this);
- DeltaWindow dw = new DeltaWindow(this, dc, reader);
+ if (threads <= 1 || cnt <= 2 * config.getDeltaSearchWindowSize()) {
+ DeltaCache dc = new DeltaCache(config);
+ DeltaWindow dw = new DeltaWindow(config, dc, reader);
dw.search(monitor, list, 0, cnt);
return;
}
- final DeltaCache dc = new ThreadSafeDeltaCache(this);
+ final DeltaCache dc = new ThreadSafeDeltaCache(config);
final ProgressMonitor pm = new ThreadSafeProgressMonitor(monitor);
// Guess at the size of batch we want. Because we don't really
// are a bit smaller.
//
int estSize = cnt / (threads * 2);
- if (estSize < 2 * getDeltaSearchWindowSize())
- estSize = 2 * getDeltaSearchWindowSize();
+ if (estSize < 2 * config.getDeltaSearchWindowSize())
+ estSize = 2 * config.getDeltaSearchWindowSize();
final List<DeltaTask> myTasks = new ArrayList<DeltaTask>(threads * 2);
for (int i = 0; i < cnt;) {
batchSize = end - start;
}
i += batchSize;
- myTasks.add(new DeltaTask(this, reader, dc, pm, batchSize, start, list));
+ myTasks.add(new DeltaTask(config, reader, dc, pm, batchSize, start, list));
}
+ final Executor executor = config.getExecutor();
final List<Throwable> errors = Collections
.synchronizedList(new ArrayList<Throwable>());
if (executor instanceof ExecutorService) {
private TemporaryBuffer.Heap delta(final ObjectToPack otp)
throws IOException {
- DeltaIndex index = new DeltaIndex(buffer(reader, otp.getDeltaBaseId()));
- byte[] res = buffer(reader, otp);
+ DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId()));
+ byte[] res = buffer(otp);
// We never would have proposed this pair if the delta would be
// larger than the unpacked version of the object. So using it
return delta;
}
- byte[] buffer(ObjectReader or, AnyObjectId objId) throws IOException {
+ private byte[] buffer(AnyObjectId objId) throws IOException {
+ return buffer(config, reader, objId);
+ }
+
+ static byte[] buffer(PackConfig config, ObjectReader or, AnyObjectId objId)
+ throws IOException {
ObjectLoader ldr = or.open(objId);
if (!ldr.isLarge())
return ldr.getCachedBytes();
// If it really is too big to work with, abort out now.
//
long sz = ldr.getSize();
- if (getBigFileThreshold() <= sz || Integer.MAX_VALUE < sz)
+ if (config.getBigFileThreshold() <= sz || Integer.MAX_VALUE < sz)
throw new LargeObjectException(objId.copy());
// Its considered to be large by the loader, but we really
private Deflater deflater() {
if (myDeflater == null)
- myDeflater = new Deflater(compressionLevel);
+ myDeflater = new Deflater(config.getCompressionLevel());
return myDeflater;
}
otp.clearDeltaBase();
otp.clearReuseAsIs();
}
- } else if (nFmt == PACK_WHOLE && reuseObjects) {
+ } else if (nFmt == PACK_WHOLE && config.isReuseObjects()) {
otp.clearDeltaBase();
otp.setReuseAsIs();
otp.setWeight(nWeight);
class ThreadSafeDeltaCache extends DeltaCache {
private final ReentrantLock lock;
- ThreadSafeDeltaCache(PackWriter pw) {
- super(pw);
+ ThreadSafeDeltaCache(PackConfig pc) {
+ super(pc);
lock = new ReentrantLock();
}
projects / jgit.git / commitdiff