diff options
author | David Pursehouse <david.pursehouse@gmail.com> | 2020-01-31 09:12:08 +0900 |
---|---|---|
committer | Matthias Sohn <matthias.sohn@sap.com> | 2020-02-01 01:09:46 +0100 |
commit | f57a7bd2306d81ce244621c45a0e569096d8e797 (patch) | |
tree | 0799a949f5f9a2be5aaf30404b58bd2979490fdf /org.eclipse.jgit/src/org/eclipse | |
parent | 0d962eaa78223c09c9bd38bc50440bf82ff03f70 (diff) | |
parent | 7d5300023e867ae1e259bc5ec3839dff2bc3fab8 (diff) | |
download | jgit-f57a7bd2306d81ce244621c45a0e569096d8e797.tar.gz jgit-f57a7bd2306d81ce244621c45a0e569096d8e797.zip |
Merge branch 'stable-5.1' into stable-5.2
* stable-5.1:
Fix string format parameter for invalidRefAdvertisementLine
WindowCache: add metric for cached bytes per repository
pgm daemon: fallback to user and system config if no config specified
WindowCache: add option to use strong refs to reference ByteWindows
Change-Id: I741059a1d0d5950ab5bc16ec70352655ee926a24
Signed-off-by: David Pursehouse <david.pursehouse@gmail.com>
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
Diffstat (limited to 'org.eclipse.jgit/src/org/eclipse')
4 files changed, 378 insertions, 55 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 797507dd11..3b4f662b90 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -47,11 +47,16 @@ package org.eclipse.jgit.internal.storage.file; import java.io.IOException; import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; +import java.util.Collections; +import java.util.Map; import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.atomic.LongAdder; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.internal.JGitText; @@ -85,9 +90,16 @@ import org.eclipse.jgit.util.Monitoring; * comprised of roughly 10% of the cache, and evicting the oldest accessed entry * within that window. * <p> - * Entities created by the cache are held under SoftReferences, permitting the + * Entities created by the cache are held under SoftReferences if option + * {@code core.packedGitUseStrongRefs} is set to {@code false} in the git config + * (this is the default) or by calling + * {@link WindowCacheConfig#setPackedGitUseStrongRefs(boolean)}, permitting the * Java runtime's garbage collector to evict entries when heap memory gets low. * Most JREs implement a loose least recently used algorithm for this eviction. + * When this option is set to {@code true} strong references are used which + * means that Java gc cannot evict the WindowCache to reclaim memory. On the + * other hand this provides more predictable performance since the cache isn't + * flushed when used heap comes close to the maximum heap size. * <p> * The internal hash table does not expand at runtime, instead it is fixed in * size at cache creation time. The internal lock table used to gate load @@ -104,19 +116,19 @@ import org.eclipse.jgit.util.Monitoring; * for a given <code>(PackFile,position)</code> tuple.</li> * <li>For every <code>load()</code> invocation there is exactly one * {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a - * SoftReference around the cached entity.</li> + * SoftReference or a StrongReference around the cached entity.</li> * <li>For every Reference created by <code>createRef()</code> there will be - * exactly one call to {@link #clear(Ref)} to cleanup any resources associated + * exactly one call to {@link #clear(PageRef)} to cleanup any resources associated * with the (now expired) cached entity.</li> * </ul> * <p> * Therefore, it is safe to perform resource accounting increments during the * {@link #load(PackFile, long)} or * {@link #createRef(PackFile, long, ByteWindow)} methods, and matching - * decrements during {@link #clear(Ref)}. Implementors may need to override + * decrements during {@link #clear(PageRef)}. Implementors may need to override * {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional * accounting information into an implementation specific - * {@link org.eclipse.jgit.internal.storage.file.WindowCache.Ref} subclass, as + * {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as * the cached entity may have already been evicted by the JRE's garbage * collector. * <p> @@ -176,18 +188,21 @@ public class WindowCache { /** * Record files opened by cache * - * @param count + * @param delta * delta of number of files opened by cache */ - void recordOpenFiles(int count); + void recordOpenFiles(int delta); /** * Record cached bytes * - * @param count + * @param pack + * pack file the bytes are read from + * + * @param delta * delta of cached bytes */ - void recordOpenBytes(int count); + void recordOpenBytes(PackFile pack, int delta); /** * Returns a snapshot of this recorder's stats. Note that this may be an @@ -209,6 +224,7 @@ public class WindowCache { private final LongAdder evictionCount; private final LongAdder openFileCount; private final LongAdder openByteCount; + private final Map<String, LongAdder> openByteCountPerRepository; /** * Constructs an instance with all counts initialized to zero. @@ -222,6 +238,7 @@ public class WindowCache { evictionCount = new LongAdder(); openFileCount = new LongAdder(); openByteCount = new LongAdder(); + openByteCountPerRepository = new ConcurrentHashMap<>(); } @Override @@ -252,13 +269,28 @@ public class WindowCache { } @Override - public void recordOpenFiles(int count) { - openFileCount.add(count); + public void recordOpenFiles(int delta) { + openFileCount.add(delta); } @Override - public void recordOpenBytes(int count) { - openByteCount.add(count); + public void recordOpenBytes(PackFile pack, int delta) { + openByteCount.add(delta); + String repositoryId = repositoryId(pack); + LongAdder la = openByteCountPerRepository + .computeIfAbsent(repositoryId, k -> new LongAdder()); + la.add(delta); + if (delta < 0) { + openByteCountPerRepository.computeIfPresent(repositoryId, + (k, v) -> v.longValue() == 0 ? null : v); + } + } + + private static String repositoryId(PackFile pack) { + // use repository's gitdir since packfile doesn't know its + // repository + return pack.getPackFile().getParentFile().getParentFile() + .getParent(); } @Override @@ -315,6 +347,15 @@ public class WindowCache { totalLoadTime.reset(); evictionCount.reset(); } + + @Override + public Map<String, Long> getOpenByteCountPerRepository() { + return Collections.unmodifiableMap( + openByteCountPerRepository.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> Long.valueOf(e.getValue().sum()), + (u, v) -> v))); + } } private static final int bits(int newSize) { @@ -390,8 +431,8 @@ public class WindowCache { cache.removeAll(pack); } - /** ReferenceQueue to cleanup released and garbage collected windows. */ - private final ReferenceQueue<ByteWindow> queue; + /** cleanup released and/or garbage collected windows. */ + private final CleanupQueue queue; /** Number of entries in {@link #table}. */ private final int tableSize; @@ -425,6 +466,8 @@ public class WindowCache { private final StatsRecorderImpl mbean; + private boolean useStrongRefs; + private WindowCache(WindowCacheConfig cfg) { tableSize = tableSize(cfg); final int lockCount = lockCount(cfg); @@ -433,7 +476,6 @@ public class WindowCache { if (lockCount < 1) throw new IllegalArgumentException(JGitText.get().lockCountMustBeGreaterOrEqual1); - queue = new ReferenceQueue<>(); clock = new AtomicLong(1); table = new AtomicReferenceArray<>(tableSize); locks = new Lock[lockCount]; @@ -455,6 +497,9 @@ public class WindowCache { mmap = cfg.isPackedGitMMAP(); windowSizeShift = bits(cfg.getPackedGitWindowSize()); windowSize = 1 << windowSizeShift; + useStrongRefs = cfg.isPackedGitUseStrongRefs(); + queue = useStrongRefs ? new StrongCleanupQueue(this) + : new SoftCleanupQueue(this); mbean = new StatsRecorderImpl(); statsRecorder = mbean; @@ -503,16 +548,18 @@ public class WindowCache { } } - private Ref createRef(PackFile p, long o, ByteWindow v) { - final Ref ref = new Ref(p, o, v, queue); - statsRecorder.recordOpenBytes(ref.size); + private PageRef<ByteWindow> createRef(PackFile p, long o, ByteWindow v) { + final PageRef<ByteWindow> ref = useStrongRefs + ? new StrongRef(p, o, v, queue) + : new SoftRef(p, o, v, (SoftCleanupQueue) queue); + statsRecorder.recordOpenBytes(ref.getPack(), ref.getSize()); return ref; } - private void clear(Ref ref) { - statsRecorder.recordOpenBytes(-ref.size); + private void clear(PageRef<ByteWindow> ref) { + statsRecorder.recordOpenBytes(ref.getPack(), -ref.getSize()); statsRecorder.recordEvictions(1); - close(ref.pack); + close(ref.getPack()); } private void close(PackFile pack) { @@ -577,7 +624,7 @@ public class WindowCache { } v = load(pack, position); - final Ref ref = createRef(pack, position, v); + final PageRef<ByteWindow> ref = createRef(pack, position, v); hit(ref); for (;;) { final Entry n = new Entry(clean(e2), ref); @@ -601,8 +648,8 @@ public class WindowCache { private ByteWindow scan(Entry n, PackFile pack, long position) { for (; n != null; n = n.next) { - final Ref r = n.ref; - if (r.pack == pack && r.position == position) { + final PageRef<ByteWindow> r = n.ref; + if (r.getPack() == pack && r.getPosition() == position) { final ByteWindow v = r.get(); if (v != null) { hit(r); @@ -615,7 +662,7 @@ public class WindowCache { return null; } - private void hit(Ref r) { + private void hit(PageRef r) { // We don't need to be 100% accurate here. Its sufficient that at least // one thread performs the increment. Any other concurrent access at // exactly the same time can simply use the same clock value. @@ -625,7 +672,7 @@ public class WindowCache { // final long c = clock.get(); clock.compareAndSet(c, c + 1); - r.lastAccess = c; + r.setLastAccess(c); } private void evict() { @@ -639,7 +686,8 @@ public class WindowCache { for (Entry e = table.get(ptr); e != null; e = e.next) { if (e.dead) continue; - if (old == null || e.ref.lastAccess < old.ref.lastAccess) { + if (old == null || e.ref.getLastAccess() < old.ref + .getLastAccess()) { old = e; slot = ptr; } @@ -659,7 +707,7 @@ public class WindowCache { * <p> * This is a last-ditch effort to clear out the cache, such as before it * gets replaced by another cache that is configured differently. This - * method tries to force every cached entry through {@link #clear(Ref)} to + * method tries to force every cached entry through {@link #clear(PageRef)} to * ensure that resources are correctly accounted for and cleaned up by the * subclass. A concurrent reader loading entries while this method is * running may cause resource accounting failures. @@ -692,7 +740,7 @@ public class WindowCache { final Entry e1 = table.get(s); boolean hasDead = false; for (Entry e = e1; e != null; e = e.next) { - if (e.ref.pack == pack) { + if (e.ref.getPack() == pack) { e.kill(); hasDead = true; } else if (e.dead) @@ -705,20 +753,7 @@ public class WindowCache { } private void gc() { - Ref r; - while ((r = (Ref) queue.poll()) != null) { - clear(r); - - final int s = slot(r.pack, r.position); - final Entry e1 = table.get(s); - for (Entry n = e1; n != null; n = n.next) { - if (n.ref == r) { - n.dead = true; - table.compareAndSet(s, e1, clean(e1)); - break; - } - } - } + queue.gc(); } private int slot(PackFile pack, long position) { @@ -731,7 +766,7 @@ public class WindowCache { private static Entry clean(Entry top) { while (top != null && top.dead) { - top.ref.enqueue(); + top.ref.kill(); top = top.next; } if (top == null) @@ -745,7 +780,7 @@ public class WindowCache { final Entry next; /** The referenced object. */ - final Ref ref; + final PageRef<ByteWindow> ref; /** * Marked true when ref.get() returns null and the ref is dead. @@ -756,34 +791,275 @@ public class WindowCache { */ volatile boolean dead; - Entry(Entry n, Ref r) { + Entry(Entry n, PageRef<ByteWindow> r) { next = n; ref = r; } final void kill() { dead = true; - ref.enqueue(); + ref.kill(); } } + private static interface PageRef<T> { + /** + * Returns this reference object's referent. If this reference object + * has been cleared, either by the program or by the garbage collector, + * then this method returns <code>null</code>. + * + * @return The object to which this reference refers, or + * <code>null</code> if this reference object has been cleared + */ + T get(); + + /** + * Kill this ref + * + * @return <code>true</code> if this reference object was successfully + * killed; <code>false</code> if it was already killed + */ + boolean kill(); + + /** + * Get the packfile the referenced cache page is allocated for + * + * @return the packfile the referenced cache page is allocated for + */ + PackFile getPack(); + + /** + * Get the position of the referenced cache page in the packfile + * + * @return the position of the referenced cache page in the packfile + */ + long getPosition(); + + /** + * Get size of cache page + * + * @return size of cache page + */ + int getSize(); + + /** + * Get pseudo time of last access to this cache page + * + * @return pseudo time of last access to this cache page + */ + long getLastAccess(); + + /** + * Set pseudo time of last access to this cache page + * + * @param time + * pseudo time of last access to this cache page + */ + void setLastAccess(long time); + + /** + * Whether this is a strong reference. + * @return {@code true} if this is a strong reference + */ + boolean isStrongRef(); + } + /** A soft reference wrapped around a cached object. */ - private static class Ref extends SoftReference<ByteWindow> { - final PackFile pack; + private static class SoftRef extends SoftReference<ByteWindow> + implements PageRef<ByteWindow> { + private final PackFile pack; - final long position; + private final long position; - final int size; + private final int size; - long lastAccess; + private long lastAccess; - protected Ref(final PackFile pack, final long position, - final ByteWindow v, final ReferenceQueue<ByteWindow> queue) { + protected SoftRef(final PackFile pack, final long position, + final ByteWindow v, final SoftCleanupQueue queue) { super(v, queue); this.pack = pack; this.position = position; this.size = v.size(); } + + @Override + public PackFile getPack() { + return pack; + } + + @Override + public long getPosition() { + return position; + } + + @Override + public int getSize() { + return size; + } + + @Override + public long getLastAccess() { + return lastAccess; + } + + @Override + public void setLastAccess(long time) { + this.lastAccess = time; + } + + @Override + public boolean kill() { + return enqueue(); + } + + @Override + public boolean isStrongRef() { + return false; + } + } + + /** A strong reference wrapped around a cached object. */ + private static class StrongRef implements PageRef<ByteWindow> { + private ByteWindow referent; + + private final PackFile pack; + + private final long position; + + private final int size; + + private long lastAccess; + + private CleanupQueue queue; + + protected StrongRef(final PackFile pack, final long position, + final ByteWindow v, final CleanupQueue queue) { + this.pack = pack; + this.position = position; + this.referent = v; + this.size = v.size(); + this.queue = queue; + } + + @Override + public PackFile getPack() { + return pack; + } + + @Override + public long getPosition() { + return position; + } + + @Override + public int getSize() { + return size; + } + + @Override + public long getLastAccess() { + return lastAccess; + } + + @Override + public void setLastAccess(long time) { + this.lastAccess = time; + } + + @Override + public ByteWindow get() { + return referent; + } + + @Override + public boolean kill() { + if (referent == null) { + return false; + } + referent = null; + return queue.enqueue(this); + } + + @Override + public boolean isStrongRef() { + return true; + } + } + + private static interface CleanupQueue { + boolean enqueue(PageRef<ByteWindow> r); + void gc(); + } + + private static class SoftCleanupQueue extends ReferenceQueue<ByteWindow> + implements CleanupQueue { + private final WindowCache wc; + + SoftCleanupQueue(WindowCache cache) { + this.wc = cache; + } + + @Override + public boolean enqueue(PageRef<ByteWindow> r) { + // no need to explicitly add soft references which are enqueued by + // the JVM + return false; + } + + @Override + public void gc() { + SoftRef r; + while ((r = (SoftRef) poll()) != null) { + wc.clear(r); + + final int s = wc.slot(r.getPack(), r.getPosition()); + final Entry e1 = wc.table.get(s); + for (Entry n = e1; n != null; n = n.next) { + if (n.ref == r) { + n.dead = true; + wc.table.compareAndSet(s, e1, clean(e1)); + break; + } + } + } + } + } + + private static class StrongCleanupQueue implements CleanupQueue { + private final WindowCache wc; + + private final ConcurrentLinkedQueue<PageRef<ByteWindow>> queue = new ConcurrentLinkedQueue<>(); + + StrongCleanupQueue(WindowCache wc) { + this.wc = wc; + } + + @Override + public boolean enqueue(PageRef<ByteWindow> r) { + if (queue.contains(r)) { + return false; + } + return queue.add(r); + } + + @Override + public void gc() { + PageRef<ByteWindow> r; + while ((r = queue.poll()) != null) { + wc.clear(r); + + final int s = wc.slot(r.getPack(), r.getPosition()); + final Entry e1 = wc.table.get(s); + for (Entry n = e1; n != null; n = n.next) { + if (n.ref == r) { + n.dead = true; + wc.table.compareAndSet(s, e1, clean(e1)); + break; + } + } + } + } } private static final class Lock { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 9d1c73b057..c72395fb69 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -254,6 +254,12 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_PACKED_GIT_OPENFILES = "packedgitopenfiles"; + /** + * The "packedGitUseStrongRefs" key + * @since 5.1.13 + */ + public static final String CONFIG_KEY_PACKED_GIT_USE_STRONGREFS = "packedgitusestrongrefs"; + /** The "remote" key */ public static final String CONFIG_KEY_REMOTE = "remote"; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java index c04b677df1..f066ee18c2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java @@ -50,6 +50,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_MMAP; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_OPENFILES; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_WINDOWSIZE; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_TRESHOLD; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_GIT_USE_STRONGREFS; import org.eclipse.jgit.internal.storage.file.WindowCache; import org.eclipse.jgit.lib.Config; @@ -69,6 +70,8 @@ public class WindowCacheConfig { private long packedGitLimit; + private boolean useStrongRefs; + private int packedGitWindowSize; private boolean packedGitMMAP; @@ -83,6 +86,7 @@ public class WindowCacheConfig { public WindowCacheConfig() { packedGitOpenFiles = 128; packedGitLimit = 10 * MB; + useStrongRefs = false; packedGitWindowSize = 8 * KB; packedGitMMAP = false; deltaBaseCacheLimit = 10 * MB; @@ -134,6 +138,31 @@ public class WindowCacheConfig { } /** + * Get whether the window cache should use strong references or + * SoftReferences + * + * @return {@code true} if the window cache should use strong references, + * otherwise it will use {@link java.lang.ref.SoftReference}s + * @since 5.1.13 + */ + public boolean isPackedGitUseStrongRefs() { + return useStrongRefs; + } + + /** + * Set if the cache should use strong refs or soft refs + * + * @param useStrongRefs + * if @{code true} the cache strongly references cache pages + * otherwise it uses {@link java.lang.ref.SoftReference}s which + * can be evicted by the Java gc if heap is almost full + * @since 5.1.13 + */ + public void setPackedGitUseStrongRefs(boolean useStrongRefs) { + this.useStrongRefs = useStrongRefs; + } + + /** * Get size in bytes of a single window mapped or read in from the pack * file. * @@ -235,6 +264,9 @@ public class WindowCacheConfig { * @since 3.0 */ public WindowCacheConfig fromConfig(Config rc) { + setPackedGitUseStrongRefs(rc.getBoolean(CONFIG_CORE_SECTION, + CONFIG_KEY_PACKED_GIT_USE_STRONGREFS, + isPackedGitUseStrongRefs())); setPackedGitOpenFiles(rc.getInt(CONFIG_CORE_SECTION, null, CONFIG_KEY_PACKED_GIT_OPENFILES, getPackedGitOpenFiles())); setPackedGitLimit(rc.getLong(CONFIG_CORE_SECTION, null, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java index b7f6394df6..65f8dae342 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java @@ -42,6 +42,8 @@ */ package org.eclipse.jgit.storage.file; +import java.util.Map; + import javax.management.MXBean; import org.eclipse.jgit.internal.storage.file.WindowCache; @@ -226,6 +228,13 @@ public interface WindowCacheStats { long getOpenByteCount(); /** + * Number of bytes cached per repository + * + * @return number of bytes cached per repository + */ + Map<String, Long> getOpenByteCountPerRepository(); + + /** * Reset counters. Does not reset open bytes and open files counters. */ void resetCounters(); |