diff options
Diffstat (limited to 'org.eclipse.jgit/src')
54 files changed, 1643 insertions, 644 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java index 3dc53ec248..5bc035a46a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java @@ -714,6 +714,16 @@ public class Git implements AutoCloseable { } /** + * Return a command object to execute a {@code PackRefs} command + * + * @return a {@link org.eclipse.jgit.api.PackRefsCommand} + * @since 7.1 + */ + public PackRefsCommand packRefs() { + return new PackRefsCommand(repo); + } + + /** * Return a command object to find human-readable names of revisions. * * @return a {@link org.eclipse.jgit.api.NameRevCommand}. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java new file mode 100644 index 0000000000..29a69c5ac4 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.api; + +import java.io.IOException; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Repository; + +/** + * Optimize storage of references. + * + * @since 7.1 + */ +public class PackRefsCommand extends GitCommand<String> { + private ProgressMonitor monitor; + + private boolean all; + + /** + * Creates a new {@link PackRefsCommand} instance with default values. + * + * @param repo + * the repository this command will be used on + */ + public PackRefsCommand(Repository repo) { + super(repo); + this.monitor = NullProgressMonitor.INSTANCE; + } + + /** + * Set progress monitor + * + * @param monitor + * a progress monitor + * @return this instance + */ + public PackRefsCommand setProgressMonitor(ProgressMonitor monitor) { + this.monitor = monitor; + return this; + } + + /** + * Specify whether to pack all the references. + * + * @param all + * if <code>true</code> all the loose refs will be packed + * @return this instance + */ + public PackRefsCommand setAll(boolean all) { + this.all = all; + return this; + } + + /** + * Whether to pack all the references + * + * @return whether to pack all the references + */ + public boolean isAll() { + return all; + } + + @Override + public String call() throws GitAPIException { + checkCallable(); + try { + repo.getRefDatabase().packRefs(monitor, this); + return JGitText.get().packRefsSuccessful; + } catch (IOException e) { + throw new JGitInternalException(JGitText.get().packRefsFailed, e); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java index 76dc87e72b..fdfe533618 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java @@ -360,18 +360,22 @@ public class RawText extends Sequence { length = maxLength; isComplete = false; } - byte last = 'x'; // Just something inconspicuous. - for (int ptr = 0; ptr < length; ptr++) { - byte curr = raw[ptr]; - if (isBinary(curr, last)) { + + int ptr = -1; + byte current; + while (ptr < length - 2) { + current = raw[++ptr]; + if (current == '\0' || (current == '\r' && raw[++ptr] != '\n')) { return true; } - last = curr; } - if (isComplete) { - // Buffer contains everything... - return last == '\r'; // ... so this must be a lone CR + + if (ptr == length - 2) { + // if '\r' be last, then if isComplete then return binary + current = raw[++ptr]; + return current == '\0' || (current == '\r' && isComplete); } + return false; } @@ -467,26 +471,30 @@ public class RawText extends Sequence { */ public static boolean isCrLfText(byte[] raw, int length, boolean complete) { boolean has_crlf = false; - byte last = 'x'; // Just something inconspicuous - for (int ptr = 0; ptr < length; ptr++) { - byte curr = raw[ptr]; - if (isBinary(curr, last)) { + + int ptr = -1; + byte current; + while (ptr < length - 2) { + current = raw[++ptr]; + if (current == '\0') { return false; } - if (curr == '\n' && last == '\r') { + if (current == '\r') { + if (raw[++ptr] != '\n') { + return false; + } has_crlf = true; } - last = curr; } - if (last == '\r') { - if (complete) { - // Lone CR: it's binary after all. + + if (ptr == length - 2) { + // if '\r' be last, then if isComplete then return binary + current = raw[++ptr]; + if (current == '\0' || (current == '\r' && complete)) { return false; } - // Tough call. If the next byte, which we don't have, would be a - // '\n', it'd be a CR-LF text, otherwise it'd be binary. Just decide - // based on what we already scanned; it wasn't binary until now. } + return has_crlf; } @@ -578,4 +586,5 @@ public class RawText extends Sequence { return new RawText(data, RawParseUtils.lineMapOrBinary(data, 0, (int) sz)); } } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java index 5de7bac112..fb98df7c9e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java @@ -80,7 +80,7 @@ class SimilarityRenameDetector { private long[] matrix; /** Score a pair must exceed to be considered a rename. */ - private int renameScore = 60; + private int renameScore = 50; /** * File size threshold (in bytes) for detecting renames. Files larger diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index c3091828ac..c8012d6aa4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -625,6 +625,8 @@ public class JGitText extends TranslationBundle { /***/ public String packingCancelledDuringObjectsWriting; /***/ public String packObjectCountMismatch; /***/ public String packRefs; + /***/ public String packRefsFailed; + /***/ public String packRefsSuccessful; /***/ public String packSizeNotSetYet; /***/ public String packTooLargeForIndexVersion1; /***/ public String packWasDeleted; @@ -668,8 +670,6 @@ public class JGitText extends TranslationBundle { /***/ public String readerIsRequired; /***/ public String readingObjectsFromLocalRepositoryFailed; /***/ public String readLastModifiedFailed; - /***/ public String readPipeIsNotAllowed; - /***/ public String readPipeIsNotAllowedRequiredPermission; /***/ public String readTimedOut; /***/ public String receivePackObjectTooLarge1; /***/ public String receivePackObjectTooLarge2; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java new file mode 100644 index 0000000000..295b702fa7 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + +import java.util.List; + +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * Aggregates values for all given {@link BlockCacheStats}. + */ +class AggregatedBlockCacheStats implements BlockCacheStats { + private final List<BlockCacheStats> blockCacheStats; + + static BlockCacheStats fromStatsList( + List<BlockCacheStats> blockCacheStats) { + if (blockCacheStats.size() == 1) { + return blockCacheStats.get(0); + } + return new AggregatedBlockCacheStats(blockCacheStats); + } + + private AggregatedBlockCacheStats(List<BlockCacheStats> blockCacheStats) { + this.blockCacheStats = blockCacheStats; + } + + @Override + public String getName() { + return AggregatedBlockCacheStats.class.getName(); + } + + @Override + public long[] getCurrentSize() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getCurrentSize()); + } + return sums; + } + + @Override + public long[] getHitCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getHitCount()); + } + return sums; + } + + @Override + public long[] getMissCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getMissCount()); + } + return sums; + } + + @Override + public long[] getTotalRequestCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getTotalRequestCount()); + } + return sums; + } + + @Override + public long[] getHitRatio() { + long[] hit = getHitCount(); + long[] miss = getMissCount(); + long[] ratio = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < ratio.length; i++) { + if (i >= hit.length) { + ratio[i] = 0; + } else if (i >= miss.length) { + ratio[i] = 100; + } else { + long total = hit[i] + miss[i]; + ratio[i] = total == 0 ? 0 : hit[i] * 100 / total; + } + } + return ratio; + } + + @Override + public long[] getEvictions() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getEvictions()); + } + return sums; + } + + private static long[] emptyPackStats() { + return new long[PackExt.values().length]; + } + + private static long[] add(long[] first, long[] second) { + long[] sums = new long[Integer.max(first.length, second.length)]; + int i; + for (i = 0; i < Integer.min(first.length, second.length); i++) { + sums[i] = first[i] + second[i]; + } + for (int j = i; j < first.length; j++) { + sums[j] = first[i]; + } + for (int j = i; j < second.length; j++) { + sums[j] = second[i]; + } + return sums; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java index ce71a71d67..587d482583 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; import java.time.Duration; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -47,6 +48,11 @@ import org.eclipse.jgit.internal.storage.pack.PackExt; * invocations is also fixed in size. */ final class ClockBlockCacheTable implements DfsBlockCacheTable { + /** + * Table name. + */ + private final String name; + /** Number of entries in {@link #table}. */ private final int tableSize; @@ -129,14 +135,20 @@ final class ClockBlockCacheTable implements DfsBlockCacheTable { -1, 0, null); clockHand.next = clockHand; - this.dfsBlockCacheStats = new DfsBlockCacheStats(); + this.name = cfg.getName(); + this.dfsBlockCacheStats = new DfsBlockCacheStats(this.name); this.refLockWaitTime = cfg.getRefLockWaitTimeConsumer(); this.indexEventConsumer = cfg.getIndexEventConsumer(); } @Override - public BlockCacheStats getBlockCacheStats() { - return dfsBlockCacheStats; + public List<BlockCacheStats> getBlockCacheStats() { + return List.of(dfsBlockCacheStats); + } + + @Override + public String getName() { + return name; } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java index 3e1300c867..f8e0831e1f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java @@ -11,7 +11,10 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + import java.io.IOException; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.LongStream; @@ -97,7 +100,12 @@ public final class DfsBlockCache { double streamRatio = cfg.getStreamRatio(); maxStreamThroughCache = (long) (maxBytes * streamRatio); - dfsBlockCacheTable = new ClockBlockCacheTable(cfg); + if (!cfg.getPackExtCacheConfigurations().isEmpty()) { + dfsBlockCacheTable = PackExtBlockCacheTable + .fromBlockCacheConfigs(cfg); + } else { + dfsBlockCacheTable = new ClockBlockCacheTable(cfg); + } for (int i = 0; i < PackExt.values().length; ++i) { Integer limit = cfg.getCacheHotMap().get(PackExt.values()[i]); @@ -119,7 +127,7 @@ public final class DfsBlockCache { * @return total number of bytes in the cache, per pack file extension. */ public long[] getCurrentSize() { - return dfsBlockCacheTable.getBlockCacheStats().getCurrentSize(); + return getAggregatedBlockCacheStats().getCurrentSize(); } /** @@ -138,7 +146,7 @@ public final class DfsBlockCache { * extension. */ public long[] getHitCount() { - return dfsBlockCacheTable.getBlockCacheStats().getHitCount(); + return getAggregatedBlockCacheStats().getHitCount(); } /** @@ -149,7 +157,7 @@ public final class DfsBlockCache { * extension. */ public long[] getMissCount() { - return dfsBlockCacheTable.getBlockCacheStats().getMissCount(); + return getAggregatedBlockCacheStats().getMissCount(); } /** @@ -158,8 +166,7 @@ public final class DfsBlockCache { * @return total number of requests (hit + miss), per pack file extension. */ public long[] getTotalRequestCount() { - return dfsBlockCacheTable.getBlockCacheStats() - .getTotalRequestCount(); + return getAggregatedBlockCacheStats().getTotalRequestCount(); } /** @@ -168,7 +175,7 @@ public final class DfsBlockCache { * @return hit ratios */ public long[] getHitRatio() { - return dfsBlockCacheTable.getBlockCacheStats().getHitRatio(); + return getAggregatedBlockCacheStats().getHitRatio(); } /** @@ -179,7 +186,18 @@ public final class DfsBlockCache { * file extension. */ public long[] getEvictions() { - return dfsBlockCacheTable.getBlockCacheStats().getEvictions(); + return getAggregatedBlockCacheStats().getEvictions(); + } + + /** + * Get the list of {@link BlockCacheStats} for all underlying caches. + * <p> + * Useful in monitoring caches with breakdown. + * + * @return the list of {@link BlockCacheStats} for all underlying caches. + */ + public List<BlockCacheStats> getAllBlockCacheStats() { + return dfsBlockCacheTable.getBlockCacheStats(); } /** @@ -259,6 +277,11 @@ public final class DfsBlockCache { return dfsBlockCacheTable.get(key, position); } + private BlockCacheStats getAggregatedBlockCacheStats() { + return AggregatedBlockCacheStats + .fromStatsList(dfsBlockCacheTable.getBlockCacheStats()); + } + static final class Ref<T> { final DfsStreamKey key; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index fa68b979fb..17bf51876d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -19,6 +19,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO; +import java.io.PrintWriter; import java.text.MessageFormat; import java.time.Duration; import java.util.ArrayList; @@ -29,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import org.eclipse.jgit.internal.JGitText; @@ -49,6 +51,10 @@ public class DfsBlockCacheConfig { /** Default number of max cache hits. */ public static final int DEFAULT_CACHE_HOT_MAX = 1; + static final String DEFAULT_NAME = "<default>"; //$NON-NLS-1$ + + private String name; + private long blockLimit; private int blockSize; @@ -69,6 +75,7 @@ public class DfsBlockCacheConfig { * Create a default configuration. */ public DfsBlockCacheConfig() { + name = DEFAULT_NAME; setBlockLimit(32 * MB); setBlockSize(64 * KB); setStreamRatio(0.30); @@ -78,6 +85,72 @@ public class DfsBlockCacheConfig { } /** + * Print the current cache configuration to the given {@link PrintWriter}. + * + * @param writer + * {@link PrintWriter} to write the cache's configuration to. + */ + public void print(PrintWriter writer) { + print(/* linePrefix= */ "", /* pad= */ " ", writer); //$NON-NLS-1$//$NON-NLS-2$ + } + + /** + * Print the current cache configuration to the given {@link PrintWriter}. + * + * @param linePrefix + * prefix to prepend all writen lines with. Ex a string of 0 or + * more " " entries. + * @param pad + * filler used to extend linePrefix. Ex a multiple of " ". + * @param writer + * {@link PrintWriter} to write the cache's configuration to. + */ + @SuppressWarnings("nls") + private void print(String linePrefix, String pad, PrintWriter writer) { + String currentPrefixLevel = linePrefix; + if (!name.isEmpty() || !packExtCacheConfigurations.isEmpty()) { + writer.println(linePrefix + "Name: " + + (name.isEmpty() ? DEFAULT_NAME : this.name)); + currentPrefixLevel += pad; + } + writer.println(currentPrefixLevel + "BlockLimit: " + blockLimit); + writer.println(currentPrefixLevel + "BlockSize: " + blockSize); + writer.println(currentPrefixLevel + "StreamRatio: " + streamRatio); + writer.println( + currentPrefixLevel + "ConcurrencyLevel: " + concurrencyLevel); + for (Map.Entry<PackExt, Integer> entry : cacheHotMap.entrySet()) { + writer.println(currentPrefixLevel + "CacheHotMapEntry: " + + entry.getKey() + " : " + entry.getValue()); + } + for (DfsBlockCachePackExtConfig extConfig : packExtCacheConfigurations) { + extConfig.print(currentPrefixLevel, pad, writer); + } + } + + /** + * Get the name for the block cache configured by this cache config. + * + * @return the name for the block cache configured by this cache config. + */ + public String getName() { + return name; + } + + /** + * Set the name for the block cache configured by this cache config. + * <p> + * Made visible for testing. + * + * @param name + * the name for the block cache configured by this cache config. + * @return {@code this} + */ + DfsBlockCacheConfig setName(String name) { + this.name = name; + return this; + } + + /** * Get maximum number bytes of heap memory to dedicate to caching pack file * data. * @@ -226,12 +299,24 @@ public class DfsBlockCacheConfig { * map of hot count per pack extension for {@code DfsBlockCache}. * @return {@code this} */ + /* + * TODO The cache HotMap configuration should be set as a config option and + * not passed in through a setter. + */ public DfsBlockCacheConfig setCacheHotMap( Map<PackExt, Integer> cacheHotMap) { this.cacheHotMap = Collections.unmodifiableMap(cacheHotMap); + setCacheHotMapToPackExtConfigs(this.cacheHotMap); return this; } + private void setCacheHotMapToPackExtConfigs( + Map<PackExt, Integer> cacheHotMap) { + for (DfsBlockCachePackExtConfig packExtConfig : packExtCacheConfigurations) { + packExtConfig.setCacheHotMap(cacheHotMap); + } + } + /** * Get the consumer of cache index events. * @@ -308,6 +393,11 @@ public class DfsBlockCacheConfig { Long.valueOf(cfgBlockLimit), Long.valueOf(cfgBlockSize))); } + // Set name only if `core dfs` is configured, otherwise fall back to the + // default. + if (rc.getSubsections(section).contains(subSection)) { + this.name = subSection; + } setBlockLimit(cfgBlockLimit); setBlockSize(cfgBlockSize); @@ -354,6 +444,7 @@ public class DfsBlockCacheConfig { cacheConfigs.add(cacheConfig); } packExtCacheConfigurations = cacheConfigs; + setCacheHotMapToPackExtConfigs(this.cacheHotMap); } private static <T> Set<T> intersection(Set<T> first, Set<T> second) { @@ -472,6 +563,14 @@ public class DfsBlockCacheConfig { return packExtCacheConfiguration; } + void setCacheHotMap(Map<PackExt, Integer> cacheHotMap) { + Map<PackExt, Integer> packExtHotMap = packExts.stream() + .filter(cacheHotMap::containsKey) + .collect(Collectors.toUnmodifiableMap(Function.identity(), + cacheHotMap::get)); + packExtCacheConfiguration.setCacheHotMap(packExtHotMap); + } + private static DfsBlockCachePackExtConfig fromConfig(Config config, String section, String subSection) { String packExtensions = config.getString(section, subSection, @@ -498,5 +597,11 @@ public class DfsBlockCacheConfig { return new DfsBlockCachePackExtConfig(EnumSet.copyOf(packExts), dfsBlockCacheConfig); } + + void print(String linePrefix, String pad, PrintWriter writer) { + packExtCacheConfiguration.print(linePrefix, pad, writer); + writer.println(linePrefix + pad + "PackExts: " //$NON-NLS-1$ + + packExts.stream().sorted().collect(Collectors.toList())); + } } -}
\ No newline at end of file +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java new file mode 100644 index 0000000000..436f57437e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * Keeps track of stats for a Block Cache table. + */ +class DfsBlockCacheStats implements BlockCacheStats { + private final String name; + + /** + * Number of times a block was found in the cache, per pack file extension. + */ + private final AtomicReference<AtomicLong[]> statHit; + + /** + * Number of times a block was not found, and had to be loaded, per pack + * file extension. + */ + private final AtomicReference<AtomicLong[]> statMiss; + + /** + * Number of blocks evicted due to cache being full, per pack file + * extension. + */ + private final AtomicReference<AtomicLong[]> statEvict; + + /** + * Number of bytes currently loaded in the cache, per pack file extension. + */ + private final AtomicReference<AtomicLong[]> liveBytes; + + DfsBlockCacheStats() { + this(""); //$NON-NLS-1$ + } + + DfsBlockCacheStats(String name) { + this.name = name; + statHit = new AtomicReference<>(newCounters()); + statMiss = new AtomicReference<>(newCounters()); + statEvict = new AtomicReference<>(newCounters()); + liveBytes = new AtomicReference<>(newCounters()); + } + + @Override + public String getName() { + return name; + } + + /** + * Increment the {@code statHit} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementHit(DfsStreamKey key) { + getStat(statHit, key).incrementAndGet(); + } + + /** + * Increment the {@code statMiss} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementMiss(DfsStreamKey key) { + getStat(statMiss, key).incrementAndGet(); + } + + /** + * Increment the {@code statEvict} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementEvict(DfsStreamKey key) { + getStat(statEvict, key).incrementAndGet(); + } + + /** + * Add {@code size} to the {@code liveBytes} count. + * + * @param key + * key identifying which liveBytes entry to update. + * @param size + * amount to increment the count by. + */ + void addToLiveBytes(DfsStreamKey key, long size) { + getStat(liveBytes, key).addAndGet(size); + } + + @Override + public long[] getCurrentSize() { + return getStatVals(liveBytes); + } + + @Override + public long[] getHitCount() { + return getStatVals(statHit); + } + + @Override + public long[] getMissCount() { + return getStatVals(statMiss); + } + + @Override + public long[] getTotalRequestCount() { + AtomicLong[] hit = statHit.get(); + AtomicLong[] miss = statMiss.get(); + long[] cnt = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < hit.length; i++) { + cnt[i] += hit[i].get(); + } + for (int i = 0; i < miss.length; i++) { + cnt[i] += miss[i].get(); + } + return cnt; + } + + @Override + public long[] getHitRatio() { + AtomicLong[] hit = statHit.get(); + AtomicLong[] miss = statMiss.get(); + long[] ratio = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < ratio.length; i++) { + if (i >= hit.length) { + ratio[i] = 0; + } else if (i >= miss.length) { + ratio[i] = 100; + } else { + long hitVal = hit[i].get(); + long missVal = miss[i].get(); + long total = hitVal + missVal; + ratio[i] = total == 0 ? 0 : hitVal * 100 / total; + } + } + return ratio; + } + + @Override + public long[] getEvictions() { + return getStatVals(statEvict); + } + + private static AtomicLong[] newCounters() { + AtomicLong[] ret = new AtomicLong[PackExt.values().length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = new AtomicLong(); + } + return ret; + } + + private static long[] getStatVals(AtomicReference<AtomicLong[]> stat) { + AtomicLong[] stats = stat.get(); + long[] cnt = new long[stats.length]; + for (int i = 0; i < stats.length; i++) { + cnt[i] = stats[i].get(); + } + return cnt; + } + + private static AtomicLong getStat(AtomicReference<AtomicLong[]> stats, + DfsStreamKey key) { + int pos = key.packExtPos; + while (true) { + AtomicLong[] vals = stats.get(); + if (pos < vals.length) { + return vals[pos]; + } + AtomicLong[] expect = vals; + vals = new AtomicLong[Math.max(pos + 1, PackExt.values().length)]; + System.arraycopy(expect, 0, vals, 0, expect.length); + for (int i = expect.length; i < vals.length; i++) { + vals[i] = new AtomicLong(); + } + if (stats.compareAndSet(expect, vals)) { + return vals[pos]; + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java index 309f2d1595..c3fd07b7bb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java @@ -11,10 +11,7 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import org.eclipse.jgit.internal.storage.pack.PackExt; +import java.util.List; /** * Block cache table. @@ -129,18 +126,37 @@ public interface DfsBlockCacheTable { <T> T get(DfsStreamKey key, long position); /** - * Get the {@link BlockCacheStats} object for this block cache table's - * statistics. + * Get the list of {@link BlockCacheStats} held by this cache. + * <p> + * The returned list has a {@link BlockCacheStats} per configured cache + * table, with a minimum of 1 {@link BlockCacheStats} object returned. + * + * Use {@link AggregatedBlockCacheStats} to combine the results of the stats + * in the list for an aggregated view of the cache's stats. + * + * @return the list of {@link BlockCacheStats} held by this cache. + */ + List<BlockCacheStats> getBlockCacheStats(); + + /** + * Get the name of the table. * - * @return the {@link BlockCacheStats} tracking this block cache table's - * statistics. + * @return this table's name. */ - BlockCacheStats getBlockCacheStats(); + String getName(); /** * Provides methods used with Block Cache statistics. */ interface BlockCacheStats { + + /** + * Get the name of the block cache generating this instance. + * + * @return this cache's name. + */ + String getName(); + /** * Get total number of bytes in the cache, per pack file extension. * @@ -190,174 +206,4 @@ public interface DfsBlockCacheTable { */ long[] getEvictions(); } - - /** - * Keeps track of stats for a Block Cache table. - */ - class DfsBlockCacheStats implements BlockCacheStats { - /** - * Number of times a block was found in the cache, per pack file - * extension. - */ - private final AtomicReference<AtomicLong[]> statHit; - - /** - * Number of times a block was not found, and had to be loaded, per pack - * file extension. - */ - private final AtomicReference<AtomicLong[]> statMiss; - - /** - * Number of blocks evicted due to cache being full, per pack file - * extension. - */ - private final AtomicReference<AtomicLong[]> statEvict; - - /** - * Number of bytes currently loaded in the cache, per pack file - * extension. - */ - private final AtomicReference<AtomicLong[]> liveBytes; - - DfsBlockCacheStats() { - statHit = new AtomicReference<>(newCounters()); - statMiss = new AtomicReference<>(newCounters()); - statEvict = new AtomicReference<>(newCounters()); - liveBytes = new AtomicReference<>(newCounters()); - } - - /** - * Increment the {@code statHit} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementHit(DfsStreamKey key) { - getStat(statHit, key).incrementAndGet(); - } - - /** - * Increment the {@code statMiss} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementMiss(DfsStreamKey key) { - getStat(statMiss, key).incrementAndGet(); - } - - /** - * Increment the {@code statEvict} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementEvict(DfsStreamKey key) { - getStat(statEvict, key).incrementAndGet(); - } - - /** - * Add {@code size} to the {@code liveBytes} count. - * - * @param key - * key identifying which liveBytes entry to update. - * @param size - * amount to increment the count by. - */ - void addToLiveBytes(DfsStreamKey key, long size) { - getStat(liveBytes, key).addAndGet(size); - } - - @Override - public long[] getCurrentSize() { - return getStatVals(liveBytes); - } - - @Override - public long[] getHitCount() { - return getStatVals(statHit); - } - - @Override - public long[] getMissCount() { - return getStatVals(statMiss); - } - - @Override - public long[] getTotalRequestCount() { - AtomicLong[] hit = statHit.get(); - AtomicLong[] miss = statMiss.get(); - long[] cnt = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < hit.length; i++) { - cnt[i] += hit[i].get(); - } - for (int i = 0; i < miss.length; i++) { - cnt[i] += miss[i].get(); - } - return cnt; - } - - @Override - public long[] getHitRatio() { - AtomicLong[] hit = statHit.get(); - AtomicLong[] miss = statMiss.get(); - long[] ratio = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < ratio.length; i++) { - if (i >= hit.length) { - ratio[i] = 0; - } else if (i >= miss.length) { - ratio[i] = 100; - } else { - long hitVal = hit[i].get(); - long missVal = miss[i].get(); - long total = hitVal + missVal; - ratio[i] = total == 0 ? 0 : hitVal * 100 / total; - } - } - return ratio; - } - - @Override - public long[] getEvictions() { - return getStatVals(statEvict); - } - - private static AtomicLong[] newCounters() { - AtomicLong[] ret = new AtomicLong[PackExt.values().length]; - for (int i = 0; i < ret.length; i++) { - ret[i] = new AtomicLong(); - } - return ret; - } - - private static long[] getStatVals(AtomicReference<AtomicLong[]> stat) { - AtomicLong[] stats = stat.get(); - long[] cnt = new long[stats.length]; - for (int i = 0; i < stats.length; i++) { - cnt[i] = stats[i].get(); - } - return cnt; - } - - private static AtomicLong getStat(AtomicReference<AtomicLong[]> stats, - DfsStreamKey key) { - int pos = key.packExtPos; - while (true) { - AtomicLong[] vals = stats.get(); - if (pos < vals.length) { - return vals[pos]; - } - AtomicLong[] expect = vals; - vals = new AtomicLong[Math.max(pos + 1, - PackExt.values().length)]; - System.arraycopy(expect, 0, vals, 0, expect.length); - for (int i = expect.length; i < vals.length; i++) { - vals[i] = new AtomicLong(); - } - if (stats.compareAndSet(expect, vals)) { - return vals[pos]; - } - } - } - } -}
\ No newline at end of file +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index a177669788..e6068a15ec 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -18,13 +18,13 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RE import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -100,6 +100,7 @@ public class DfsGarbageCollector { private Set<ObjectId> allTags; private Set<ObjectId> nonHeads; private Set<ObjectId> tagTargets; + private Instant refLogExpire; /** * Initialize a garbage collector. @@ -200,6 +201,22 @@ public class DfsGarbageCollector { return this; } + + /** + * Set time limit to the reflog history. + * <p> + * Garbage Collector prunes entries from reflog history older than {@code refLogExpire} + * <p> + * + * @param refLogExpire + * instant in time which defines refLog expiration + * @return {@code this} + */ + public DfsGarbageCollector setRefLogExpire(Instant refLogExpire) { + this.refLogExpire = refLogExpire; + return this; + } + /** * Set maxUpdateIndex for the initial reftable created during conversion. * @@ -687,14 +704,7 @@ public class DfsGarbageCollector { pack.setBlockSize(PACK, out.blockSize()); } - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { - CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); - } + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) { try (DfsOutputStream out = objdb.writeFile(pack, @@ -741,6 +751,10 @@ public class DfsGarbageCollector { compact.addAll(stack.readers()); compact.setIncludeDeletes(includeDeletes); compact.setConfig(configureReftable(reftableConfig, out)); + if(refLogExpire != null ){ + compact.setReflogExpireOldestReflogTimeMillis( + refLogExpire.toEpochMilli()); + } compact.compact(); pack.addFileExt(REFTABLE); pack.setReftableStats(compact.getStats()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index a07d8416c4..16315bf4f2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -41,12 +41,13 @@ import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackIndex; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; @@ -54,7 +55,6 @@ import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ObjectStream; -import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.IO; @@ -71,6 +71,8 @@ public class DfsInserter extends ObjectInserter { private static final int INDEX_VERSION = 2; final DfsObjDatabase db; + + private final int minBytesForObjectSizeIndex; int compression = Deflater.BEST_COMPRESSION; List<PackedObjectInfo> objectList; @@ -83,8 +85,6 @@ public class DfsInserter extends ObjectInserter { private boolean rollback; private boolean checkExisting = true; - private int minBytesForObjectSizeIndex = -1; - /** * Initialize a new inserter. * @@ -93,8 +93,9 @@ public class DfsInserter extends ObjectInserter { */ protected DfsInserter(DfsObjDatabase db) { this.db = db; - PackConfig pc = new PackConfig(db.getRepository().getConfig()); - this.minBytesForObjectSizeIndex = pc.getMinBytesForObjSizeIndex(); + this.minBytesForObjectSizeIndex = db.getRepository().getConfig().getInt( + ConfigConstants.CONFIG_PACK_SECTION, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, -1); } /** @@ -112,21 +113,6 @@ public class DfsInserter extends ObjectInserter { void setCompressionLevel(int compression) { this.compression = compression; } - - /** - * Set minimum size for an object to be included in the object size index. - * - * <p> - * Use 0 for all and -1 for nothing (the pack won't have object size index). - * - * @param minBytes - * only objects with size bigger or equal to this are included in - * the index. - */ - protected void setMinBytesForObjectSizeIndex(int minBytes) { - this.minBytesForObjectSizeIndex = minBytes; - } - @Override public DfsPackParser newPackParser(InputStream in) throws IOException { return new DfsPackParser(db, this, in); @@ -333,7 +319,7 @@ public class DfsInserter extends ObjectInserter { private static void index(OutputStream out, byte[] packHash, List<PackedObjectInfo> list) throws IOException { - PackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); + BasePackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); } void writeObjectSizeIndex(DfsPackDescription pack, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 616563ffdd..efd666ff27 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static java.util.stream.Collectors.joining; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import java.io.FileNotFoundException; import java.io.IOException; @@ -27,7 +28,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackBitmapIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AnyObjectId; @@ -771,4 +774,35 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } }; } + + /** + * Returns a writer to store the pack index in this object database. + * + * @param pack + * Pack file to which the index is associated. + * @param indexVersion + * which version of the index to write + * @return a writer to store the index associated with the pack + * @throws IOException + * when some I/O problem occurs while creating or writing to + * output stream + */ + public PackIndexWriter getPackIndexWriter( + DfsPackDescription pack, int indexVersion) + throws IOException { + return (objectsToStore, packDataChecksum) -> { + try (DfsOutputStream out = writeFile(pack, INDEX); + CountingOutputStream cnt = new CountingOutputStream(out)) { + final PackIndexWriter iw = BasePackIndexWriter + .createVersion(cnt, + indexVersion); + iw.write(objectsToStore, packDataChecksum); + pack.addFileExt(INDEX); + pack.setFileSize(INDEX, cnt.getCount()); + pack.setBlockSize(INDEX, out.blockSize()); + pack.setIndexVersion(indexVersion); + } + }; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index 86144b389c..f9c01b9d6e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -12,7 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA; @@ -249,6 +249,7 @@ public class DfsPackCompactor { try { writePack(objdb, outDesc, pw, pm); writeIndex(objdb, outDesc, pw); + writeObjectSizeIndex(objdb, outDesc, pw); PackStatistics stats = pw.getStatistics(); @@ -458,13 +459,20 @@ public class DfsPackCompactor { private static void writeIndex(DfsObjDatabase objdb, DfsPackDescription pack, PackWriter pw) throws IOException { - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); + } + + private static void writeObjectSizeIndex(DfsObjDatabase objdb, + DfsPackDescription pack, + PackWriter pw) throws IOException { + try (DfsOutputStream out = objdb.writeFile(pack, OBJECT_SIZE_INDEX)) { CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); + pw.writeObjectSizeIndex(cnt); + if (cnt.getCount() > 0) { + pack.addFileExt(OBJECT_SIZE_INDEX); + pack.setFileSize(OBJECT_SIZE_INDEX, cnt.getCount()); + pack.setBlockSize(OBJECT_SIZE_INDEX, out.blockSize()); + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 9cfcbaa5f7..62f6753e5d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -511,18 +511,15 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { throw new MissingObjectException(objectId.copy(), typeHint); } - if (typeHint != Constants.OBJ_BLOB || !pack.hasObjectSizeIndex(this)) { + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { return pack.getObjectSize(this, objectId); } - long sz = pack.getIndexedObjectSize(this, objectId); + Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); + long sz = maybeSz.orElse(-1L); if (sz >= 0) { - stats.objectSizeIndexHit += 1; return sz; } - - // Object wasn't in the index - stats.objectSizeIndexMiss += 1; return pack.getObjectSize(this, objectId); } @@ -541,23 +538,61 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } stats.isNotLargerThanCallCount += 1; - if (typeHint != Constants.OBJ_BLOB || !pack.hasObjectSizeIndex(this)) { + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { + return pack.getObjectSize(this, objectId) <= limit; + } + + Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); + if (maybeSz.isEmpty()) { + // Exception in object size index return pack.getObjectSize(this, objectId) <= limit; } - long sz = pack.getIndexedObjectSize(this, objectId); + long sz = maybeSz.get(); + if (sz >= 0) { + return sz <= limit; + } + + if (isLimitInsideIndexThreshold(pack, limit)) { + // With threshold T, not-found means object < T + // If limit L > T, then object < T < L + return true; + } + + return pack.getObjectSize(this, objectId) <= limit; + } + + private boolean safeHasObjectSizeIndex(DfsPackFile pack) { + try { + return pack.hasObjectSizeIndex(this); + } catch (IOException e) { + return false; + } + } + + private Optional<Long> safeGetIndexedObjectSize(DfsPackFile pack, + AnyObjectId objectId) { + long sz; + try { + sz = pack.getIndexedObjectSize(this, objectId); + } catch (IOException e) { + // Do not count the exception as an index miss + return Optional.empty(); + } if (sz < 0) { stats.objectSizeIndexMiss += 1; } else { stats.objectSizeIndexHit += 1; } + return Optional.of(sz); + } - // Got size from index or we didn't but we are sure it should be there. - if (sz >= 0 || pack.getObjectSizeIndexThreshold(this) <= limit) { - return sz <= limit; + private boolean isLimitInsideIndexThreshold(DfsPackFile pack, long limit) { + try { + return pack.getObjectSizeIndexThreshold(this) <= limit; + } catch (IOException e) { + return false; } - - return pack.getObjectSize(this, objectId) <= limit; } private DfsPackFile findPackWithObject(AnyObjectId objectId) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java index 5f5e81977b..c3974691a1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java @@ -13,6 +13,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_BASE_CACHE_LIMIT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_BUFFER; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_THRESHOLD; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_USE_OBJECT_SIZE_INDEX; @@ -197,6 +198,9 @@ public class DfsReaderOptions { setUseObjectSizeIndex(rc.getBoolean(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, CONFIG_KEY_USE_OBJECT_SIZE_INDEX, false)); + setLoadRevIndexInParallel( + rc.getBoolean(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL, false)); return this; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java index 858f731b70..bb44f9397a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java @@ -37,6 +37,11 @@ import org.eclipse.jgit.internal.storage.pack.PackExt; * type. */ class PackExtBlockCacheTable implements DfsBlockCacheTable { + /** + * Table name. + */ + private final String name; + private final DfsBlockCacheTable defaultBlockCacheTable; // Holds the unique tables backing the extBlockCacheTables values. @@ -120,13 +125,19 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { Set<DfsBlockCacheTable> blockCacheTables = new HashSet<>(); blockCacheTables.add(defaultBlockCacheTable); blockCacheTables.addAll(packExtBlockCacheTables.values()); - return new PackExtBlockCacheTable(defaultBlockCacheTable, + String name = defaultBlockCacheTable.getName() + "," //$NON-NLS-1$ + + packExtBlockCacheTables.values().stream() + .map(DfsBlockCacheTable::getName).sorted() + .collect(Collectors.joining(",")); //$NON-NLS-1$ + return new PackExtBlockCacheTable(name, defaultBlockCacheTable, List.copyOf(blockCacheTables), packExtBlockCacheTables); } - private PackExtBlockCacheTable(DfsBlockCacheTable defaultBlockCacheTable, + private PackExtBlockCacheTable(String name, + DfsBlockCacheTable defaultBlockCacheTable, List<DfsBlockCacheTable> blockCacheTableList, Map<PackExt, DfsBlockCacheTable> extBlockCacheTables) { + this.name = name; this.defaultBlockCacheTable = defaultBlockCacheTable; this.blockCacheTableList = blockCacheTableList; this.extBlockCacheTables = extBlockCacheTables; @@ -177,10 +188,15 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { } @Override - public BlockCacheStats getBlockCacheStats() { - return new CacheStats(blockCacheTableList.stream() - .map(DfsBlockCacheTable::getBlockCacheStats) - .collect(Collectors.toList())); + public List<BlockCacheStats> getBlockCacheStats() { + return blockCacheTableList.stream() + .flatMap(cacheTable -> cacheTable.getBlockCacheStats().stream()) + .collect(Collectors.toList()); + } + + @Override + public String getName() { + return name; } private DfsBlockCacheTable getTable(PackExt packExt) { @@ -196,94 +212,4 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { private static PackExt getPackExt(DfsStreamKey key) { return PackExt.values()[key.packExtPos]; } - - private static class CacheStats implements BlockCacheStats { - private final List<BlockCacheStats> blockCacheStats; - - private CacheStats(List<BlockCacheStats> blockCacheStats) { - this.blockCacheStats = blockCacheStats; - } - - @Override - public long[] getCurrentSize() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getCurrentSize()); - } - return sums; - } - - @Override - public long[] getHitCount() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getHitCount()); - } - return sums; - } - - @Override - public long[] getMissCount() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getMissCount()); - } - return sums; - } - - @Override - public long[] getTotalRequestCount() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getTotalRequestCount()); - } - return sums; - } - - @Override - public long[] getHitRatio() { - long[] hit = getHitCount(); - long[] miss = getMissCount(); - long[] ratio = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < ratio.length; i++) { - if (i >= hit.length) { - ratio[i] = 0; - } else if (i >= miss.length) { - ratio[i] = 100; - } else { - long total = hit[i] + miss[i]; - ratio[i] = total == 0 ? 0 : hit[i] * 100 / total; - } - } - return ratio; - } - - @Override - public long[] getEvictions() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getEvictions()); - } - return sums; - } - - private static long[] emptyPackStats() { - return new long[PackExt.values().length]; - } - - private static long[] add(long[] first, long[] second) { - long[] sums = new long[Integer.max(first.length, second.length)]; - int i; - for (i = 0; i < Integer.min(first.length, second.length); i++) { - sums[i] = first[i] + second[i]; - } - for (int j = i; j < first.length; j++) { - sums[j] = first[i]; - } - for (int j = i; j < second.length; j++) { - sums[j] = second[i]; - } - return sums; - } - } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java index 87e0b44d46..b89cc1ebf4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java @@ -19,6 +19,7 @@ import java.text.MessageFormat; import java.util.List; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.NB; @@ -31,7 +32,7 @@ import org.eclipse.jgit.util.NB; * random access to any object in the pack by associating an ObjectId to the * byte offset within the pack where the object's data can be read. */ -public abstract class PackIndexWriter { +public abstract class BasePackIndexWriter implements PackIndexWriter { /** Magic constant indicating post-version 1 format. */ protected static final byte[] TOC = { -1, 't', 'O', 'c' }; @@ -147,7 +148,7 @@ public abstract class PackIndexWriter { * the stream this instance outputs to. If not already buffered * it will be automatically wrapped in a buffered stream. */ - protected PackIndexWriter(OutputStream dst) { + protected BasePackIndexWriter(OutputStream dst) { out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst : new BufferedOutputStream(dst), Constants.newMessageDigest()); @@ -172,6 +173,7 @@ public abstract class PackIndexWriter { * an error occurred while writing to the output stream, or this * index format cannot store the object data supplied. */ + @Override public void write(final List<? extends PackedObjectInfo> toStore, final byte[] packDataChecksum) throws IOException { entries = toStore; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index 80240e5062..25b7583b95 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -28,8 +28,10 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.PackRefsCommand; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.events.RefsChangedEvent; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.reftable.MergedReftable; import org.eclipse.jgit.internal.storage.reftable.ReftableBatchRefUpdate; import org.eclipse.jgit.internal.storage.reftable.ReftableDatabase; @@ -39,6 +41,7 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefRename; @@ -108,6 +111,22 @@ public class FileReftableDatabase extends RefDatabase { } /** + * {@inheritDoc} + * + * For Reftable, all the data is compacted into a single table. + */ + @Override + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + pm.beginTask(JGitText.get().packRefs, 1); + try { + compactFully(); + } finally { + pm.endTask(); + } + } + + /** * Runs a full compaction for GC purposes. * @throws IOException on I/O errors */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index b5d29a3fc8..84c85659ff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -33,6 +33,7 @@ import java.util.Set; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesNodeProvider; @@ -60,7 +61,6 @@ import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; @@ -595,13 +595,12 @@ public class FileRepository extends Repository { @Override public void autoGC(ProgressMonitor monitor) { GC gc = new GC(this); - gc.setPackConfig(new PackConfig(this)); gc.setProgressMonitor(monitor); gc.setAuto(true); gc.setBackground(shouldAutoDetach()); try { gc.gc(); - } catch (ParseException | IOException e) { + } catch (ParseException | IOException | GitAPIException e) { throw new JGitInternalException(JGitText.get().gcFailed, e); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index e61e2a712f..cc4513dd63 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -63,6 +63,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.PackRefsCommand; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.CancelledException; import org.eclipse.jgit.errors.CorruptObjectException; @@ -233,9 +235,11 @@ public class GC { * @throws java.text.ParseException * If the configuration parameter "gc.pruneexpire" couldn't be * parsed + * @throws GitAPIException + * If packing refs failed */ public CompletableFuture<Collection<Pack>> gc() - throws IOException, ParseException { + throws IOException, ParseException, GitAPIException { if (!background) { return CompletableFuture.completedFuture(doGc()); } @@ -254,7 +258,7 @@ public class GC { gcLog.commit(); } return newPacks; - } catch (IOException | ParseException e) { + } catch (IOException | ParseException | GitAPIException e) { try { gcLog.write(e.getMessage()); StringWriter sw = new StringWriter(); @@ -277,7 +281,8 @@ public class GC { return (executor != null) ? executor : WorkQueue.getExecutor(); } - private Collection<Pack> doGc() throws IOException, ParseException { + private Collection<Pack> doGc() + throws IOException, ParseException, GitAPIException { if (automatic && !needGc()) { return Collections.emptyList(); } @@ -286,7 +291,8 @@ public class GC { return Collections.emptyList(); } pm.start(6 /* tasks */); - packRefs(); + new PackRefsCommand(repo).setProgressMonitor(pm).setAll(true) + .call(); // TODO: implement reflog_expire(pm, repo); Collection<Pack> newPacks = repack(); prune(Collections.emptySet()); @@ -780,43 +786,6 @@ public class GC { } /** - * Pack ref storage. For a RefDirectory database, this packs all - * non-symbolic, loose refs into packed-refs. For Reftable, all of the data - * is compacted into a single table. - * - * @throws java.io.IOException - * if an IO error occurred - */ - public void packRefs() throws IOException { - RefDatabase refDb = repo.getRefDatabase(); - if (refDb instanceof FileReftableDatabase) { - // TODO: abstract this more cleanly. - pm.beginTask(JGitText.get().packRefs, 1); - try { - ((FileReftableDatabase) refDb).compactFully(); - } finally { - pm.endTask(); - } - return; - } - - Collection<Ref> refs = refDb.getRefsByPrefix(Constants.R_REFS); - List<String> refsToBePacked = new ArrayList<>(refs.size()); - pm.beginTask(JGitText.get().packRefs, refs.size()); - try { - for (Ref ref : refs) { - checkCancelled(); - if (!ref.isSymbolic() && ref.getStorage().isLoose()) - refsToBePacked.add(ref.getName()); - pm.update(1); - } - ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked); - } finally { - pm.endTask(); - } - } - - /** * Packs all objects which reachable from any of the heads into one pack * file. Additionally all objects which are not reachable from any head but * which are reachable from any of the other refs (e.g. tags), special refs @@ -1509,7 +1478,7 @@ public class GC { public long numberOfPackFiles; /** - * The number of pack files that were created after the last bitmap + * The number of pack files that were created since the last bitmap * generation. */ public long numberOfPackFilesSinceBitmap; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 9f27f4bd6e..746e124e1f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -28,6 +28,7 @@ import java.util.zip.Deflater; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; @@ -110,7 +111,7 @@ public class ObjectDirectoryPackParser extends PackParser { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; @@ -386,9 +387,9 @@ public class ObjectDirectoryPackParser extends PackParser { try (FileOutputStream os = new FileOutputStream(tmpIdx)) { final PackIndexWriter iw; if (indexVersion <= 0) - iw = PackIndexWriter.createOldestPossible(os, list); + iw = BasePackIndexWriter.createOldestPossible(os, list); else - iw = PackIndexWriter.createVersion(os, indexVersion); + iw = BasePackIndexWriter.createVersion(os, indexVersion); iw.write(list, packHash); os.getChannel().force(true); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index c0540d5a4c..7189ce20a6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -109,7 +109,7 @@ public interface PackIndex } private static boolean isTOC(byte[] h) { - final byte[] toc = PackIndexWriter.TOC; + final byte[] toc = BasePackIndexWriter.TOC; for (int i = 0; i < toc.length; i++) if (h[i] != toc[i]) return false; @@ -302,8 +302,9 @@ public interface PackIndex * */ class MutableEntry { + /** Buffer of the ObjectId visited by the EntriesIterator. */ final MutableObjectId idBuffer = new MutableObjectId(); - + /** Offset into the packfile of the current object. */ long offset; /** @@ -321,7 +322,6 @@ public interface PackIndex * @return hex string describing the object id of this entry. */ public String name() { - ensureId(); return idBuffer.name(); } @@ -331,7 +331,6 @@ public interface PackIndex * @return a copy of the object id. */ public ObjectId toObjectId() { - ensureId(); return idBuffer.toObjectId(); } @@ -342,32 +341,33 @@ public interface PackIndex */ public MutableEntry cloneEntry() { final MutableEntry r = new MutableEntry(); - ensureId(); r.idBuffer.fromObjectId(idBuffer); r.offset = offset; return r; } - - void ensureId() { - // Override in implementations. - } } /** * Base implementation of the iterator over index entries. */ abstract class EntriesIterator implements Iterator<MutableEntry> { - protected final MutableEntry entry = initEntry(); - private final long objectCount; + private final MutableEntry entry = new MutableEntry(); + + /** Counts number of entries accessed so far. */ + private long returnedNumber = 0; + + /** + * Construct an iterator that can move objectCount times forward. + * + * @param objectCount + * the number of objects in the PackFile. + */ protected EntriesIterator(long objectCount) { this.objectCount = objectCount; } - protected long returnedNumber = 0; - - protected abstract MutableEntry initEntry(); @Override public boolean hasNext() { @@ -379,7 +379,55 @@ public interface PackIndex * element. */ @Override - public abstract MutableEntry next(); + public MutableEntry next() { + readNext(); + returnedNumber++; + return entry; + } + + /** + * Used by subclasses to load the next entry into the MutableEntry. + * <p> + * Subclasses are expected to populate the entry with + * {@link #setIdBuffer} and {@link #setOffset}. + */ + protected abstract void readNext(); + + + /** + * Copies to the entry an {@link ObjectId} from the int buffer and + * position idx + * + * @param raw + * the raw data + * @param idx + * the index into {@code raw} + */ + protected void setIdBuffer(int[] raw, int idx) { + entry.idBuffer.fromRaw(raw, idx); + } + + /** + * Copies to the entry an {@link ObjectId} from the byte array at + * position idx. + * + * @param raw + * the raw data + * @param idx + * the index into {@code raw} + */ + protected void setIdBuffer(byte[] raw, int idx) { + entry.idBuffer.fromRaw(raw, idx); + } + + /** + * Sets the {@code offset} to the entry + * + * @param offset the offset in the pack file + */ + protected void setOffset(long offset) { + entry.offset = offset; + } @Override public void remove() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index d7c83785d8..be48358a0d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -203,7 +203,7 @@ class PackIndexV1 implements PackIndex { @Override public Iterator<MutableEntry> iterator() { - return new IndexV1Iterator(objectCnt); + return new EntriesIteratorV1(this); } @Override @@ -246,36 +246,30 @@ class PackIndexV1 implements PackIndex { return packChecksum; } - private class IndexV1Iterator extends EntriesIterator { - int levelOne; + private static class EntriesIteratorV1 extends EntriesIterator { + private int levelOne; - int levelTwo; + private int levelTwo; - IndexV1Iterator(long objectCount) { - super(objectCount); - } + private final PackIndexV1 packIndex; - @Override - protected MutableEntry initEntry() { - return new MutableEntry() { - @Override - protected void ensureId() { - idBuffer.fromRaw(idxdata[levelOne], levelTwo - - Constants.OBJECT_ID_LENGTH); - } - }; + private EntriesIteratorV1(PackIndexV1 packIndex) { + super(packIndex.objectCnt); + this.packIndex = packIndex; } @Override - public MutableEntry next() { - for (; levelOne < idxdata.length; levelOne++) { - if (idxdata[levelOne] == null) + protected void readNext() { + for (; levelOne < packIndex.idxdata.length; levelOne++) { + if (packIndex.idxdata[levelOne] == null) continue; - if (levelTwo < idxdata[levelOne].length) { - entry.offset = NB.decodeUInt32(idxdata[levelOne], levelTwo); - levelTwo += Constants.OBJECT_ID_LENGTH + 4; - returnedNumber++; - return entry; + if (levelTwo < packIndex.idxdata[levelOne].length) { + super.setOffset(NB.decodeUInt32(packIndex.idxdata[levelOne], + levelTwo)); + this.levelTwo += Constants.OBJECT_ID_LENGTH + 4; + super.setIdBuffer(packIndex.idxdata[levelOne], + levelTwo - Constants.OBJECT_ID_LENGTH); + return; } levelTwo = 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index caf8b71180..36e54fcd62 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -224,7 +224,7 @@ class PackIndexV2 implements PackIndex { @Override public Iterator<MutableEntry> iterator() { - return new EntriesIteratorV2(objectCnt); + return new EntriesIteratorV2(this); } @Override @@ -289,41 +289,34 @@ class PackIndexV2 implements PackIndex { return packChecksum; } - private class EntriesIteratorV2 extends EntriesIterator { - int levelOne; + private static class EntriesIteratorV2 extends EntriesIterator { + private int levelOne = 0; - int levelTwo; + private int levelTwo = 0; - EntriesIteratorV2(long objectCount){ - super(objectCount); - } + private final PackIndexV2 packIndex; - @Override - protected MutableEntry initEntry() { - return new MutableEntry() { - @Override - protected void ensureId() { - idBuffer.fromRaw(names[levelOne], levelTwo - - Constants.OBJECT_ID_LENGTH / 4); - } - }; + private EntriesIteratorV2(PackIndexV2 packIndex) { + super(packIndex.objectCnt); + this.packIndex = packIndex; } @Override - public MutableEntry next() { - for (; levelOne < names.length; levelOne++) { - if (levelTwo < names[levelOne].length) { + protected void readNext() { + for (; levelOne < packIndex.names.length; levelOne++) { + if (levelTwo < packIndex.names[levelOne].length) { int idx = levelTwo / (Constants.OBJECT_ID_LENGTH / 4) * 4; - long offset = NB.decodeUInt32(offset32[levelOne], idx); + long offset = NB.decodeUInt32(packIndex.offset32[levelOne], + idx); if ((offset & IS_O64) != 0) { idx = (8 * (int) (offset & ~IS_O64)); - offset = NB.decodeUInt64(offset64, idx); + offset = NB.decodeUInt64(packIndex.offset64, idx); } - entry.offset = offset; - - levelTwo += Constants.OBJECT_ID_LENGTH / 4; - returnedNumber++; - return entry; + super.setOffset(offset); + this.levelTwo += Constants.OBJECT_ID_LENGTH / 4; + super.setIdBuffer(packIndex.names[levelOne], + levelTwo - Constants.OBJECT_ID_LENGTH / 4); + return; } levelTwo = 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java index 7e28b5eb2b..f0b6193066 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java @@ -21,10 +21,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 1 (old style) pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV1 */ -class PackIndexWriterV1 extends PackIndexWriter { +class PackIndexWriterV1 extends BasePackIndexWriter { static boolean canStore(PackedObjectInfo oe) { // We are limited to 4 GB per pack as offset is 32 bit unsigned int. // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java index fc5ef61912..b72b35a464 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java @@ -19,10 +19,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 2 pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV2 */ -class PackIndexWriterV2 extends PackIndexWriter { +class PackIndexWriterV2 extends BasePackIndexWriter { private static final int MAX_OFFSET_32 = 0x7fffffff; private static final int IS_OFFSET_64 = 0x80000000; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 1b092a3332..55e047bd43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -77,6 +77,7 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -320,7 +321,8 @@ public class PackInserter extends ObjectInserter { private static void writePackIndex(File idx, byte[] packHash, List<PackedObjectInfo> list) throws IOException { try (OutputStream os = new FileOutputStream(idx)) { - PackIndexWriter w = PackIndexWriter.createVersion(os, INDEX_VERSION); + PackIndexWriter w = BasePackIndexWriter.createVersion(os, + INDEX_VERSION); w.write(list, packHash); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java index ef9753cd79..720a3bcbff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java @@ -75,20 +75,20 @@ public interface PackReverseIndex { throws CorruptObjectException; /** - * Find the position in the primary index of the object at the given pack + * Find the position in the reverse index of the object at the given pack * offset. * * @param offset * the pack offset of the object - * @return the position in the primary index of the object + * @return the position in the reverse index of the object */ int findPosition(long offset); /** - * Find the object that is in the given position in the primary index. + * Find the object that is in the given position in the reverse index. * * @param nthPosition - * the position of the object in the primary index + * the position of the object in the reverse index * @return the object in that position */ ObjectId findObjectByPosition(int nthPosition); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 604868133e..6aa1157e37 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -57,6 +57,7 @@ import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.PackRefsCommand; import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.errors.MissingObjectException; @@ -69,6 +70,7 @@ import org.eclipse.jgit.lib.CoreConfig.TrustLooseRefStat; import org.eclipse.jgit.lib.CoreConfig.TrustPackedRefsStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefComparator; import org.eclipse.jgit.lib.RefDatabase; @@ -287,6 +289,33 @@ public class RefDirectory extends RefDatabase { clearReferences(); } + /** + * {@inheritDoc} + * + * For a RefDirectory database, by default this packs non-symbolic, loose + * tag refs into packed-refs. If {@code all} flag is set, this packs all the + * non-symbolic, loose refs. + */ + @Override + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + String prefix = packRefs.isAll() ? R_REFS : R_TAGS; + Collection<Ref> refs = getRefsByPrefix(prefix); + List<String> refsToBePacked = new ArrayList<>(refs.size()); + pm.beginTask(JGitText.get().packRefs, refs.size()); + try { + for (Ref ref : refs) { + if (!ref.isSymbolic() && ref.getStorage().isLoose()) { + refsToBePacked.add(ref.getName()); + } + pm.update(1); + } + pack(refsToBePacked); + } finally { + pm.endTask(); + } + } + @Override public boolean isNameConflicting(String name) throws IOException { // Cannot be nested within an existing reference. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 01f514b939..11c45471e4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -205,7 +205,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { * @param cnt * number of bytes to copy. This value may exceed the number of * bytes remaining in the window starting at offset - * <code>pos</code>. + * <code>position</code>. * @return number of bytes actually copied; this may be less than * <code>cnt</code> if <code>cnt</code> exceeded the number of bytes * available. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java new file mode 100644 index 0000000000..f69e68d4ba --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.pack; + +import java.io.IOException; +import java.util.List; + +import org.eclipse.jgit.transport.PackedObjectInfo; + +/** + * Represents a function that accepts a collection of objects to write into a + * primary pack index storage format. + */ +public interface PackIndexWriter { + /** + * Write all object entries to the index stream. + * + * @param toStore + * sorted list of objects to store in the index. The caller must + * have previously sorted the list using + * {@link org.eclipse.jgit.transport.PackedObjectInfo}'s native + * {@link java.lang.Comparable} implementation. + * @param packDataChecksum + * checksum signature of the entire pack data content. This is + * traditionally the last 20 bytes of the pack file's own stream. + * @throws java.io.IOException + * an error occurred while writing to the output stream, or the + * underlying format cannot store the object data supplied. + */ + void write(List<? extends PackedObjectInfo> toStore, + byte[] packDataChecksum) throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 4350f97915..f025d4e3d5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -58,10 +58,10 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.SearchForReuseTimeout; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; +import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.file.PackReverseIndexWriter; -import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AsyncObjectSizeQueue; import org.eclipse.jgit.lib.BatchingProgressMonitor; @@ -118,7 +118,7 @@ import org.eclipse.jgit.util.TemporaryBuffer; * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the * pack is being stored as a file the matching index can be written out after - * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap + * writing the pack by {@link #writeIndex(PackIndexWriter)}. An optional bitmap * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)} * followed by {@link #writeBitmapIndex(PackBitmapIndexWriter)}. * </p> @@ -303,19 +303,6 @@ public class PackWriter implements AutoCloseable { } /** - * Create a writer to load objects from the specified reader. - * <p> - * Objects for packing are specified in {@link #preparePack(Iterator)} or - * {@link #preparePack(ProgressMonitor, Set, Set)}. - * - * @param reader - * reader to read from the repository with. - */ - public PackWriter(ObjectReader reader) { - this(new PackConfig(), reader); - } - - /** * Create writer for specified repository. * <p> * Objects for packing are specified in {@link #preparePack(Iterator)} or @@ -1091,7 +1078,7 @@ public class PackWriter implements AutoCloseable { if (indexVersion <= 0) { for (BlockList<ObjectToPack> objs : objectsLists) indexVersion = Math.max(indexVersion, - PackIndexWriter.oldestPossibleFormat(objs)); + BasePackIndexWriter.oldestPossibleFormat(objs)); } return indexVersion; } @@ -1112,12 +1099,28 @@ public class PackWriter implements AutoCloseable { * the index data could not be written to the supplied stream. */ public void writeIndex(OutputStream indexStream) throws IOException { + writeIndex(BasePackIndexWriter.createVersion(indexStream, + getIndexVersion())); + } + + /** + * Create an index file to match the pack file just written. + * <p> + * Called after + * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. + * <p> + * Writing an index is only required for local pack storage. Packs sent on + * the network do not need to create an index. + * + * @param iw + * an {@link PackIndexWriter} instance to write the index + * @throws java.io.IOException + * the index data could not be written to the supplied stream. + */ + public void writeIndex(PackIndexWriter iw) throws IOException { if (isIndexDisabled()) throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); - long writeStart = System.currentTimeMillis(); - final PackIndexWriter iw = PackIndexWriter.createVersion( - indexStream, getIndexVersion()); iw.write(sortByName(), packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } @@ -2461,7 +2464,7 @@ public class PackWriter implements AutoCloseable { * object graph at selected commits. Writing a bitmap index is an optional * feature that not all pack users may require. * <p> - * Called after {@link #writeIndex(OutputStream)}. + * Called after {@link #writeIndex(PackIndexWriter)}. * <p> * To reduce memory internal state is cleared during this method, rendering * the PackWriter instance useless for anything further than a call to write diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java index 3e75a9dde3..542d6e94f3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java @@ -427,7 +427,35 @@ public class OpenSshConfigFile implements SshConfigStore { return value; } - private static boolean patternMatchesHost(String pattern, String name) { + /** + * Tells whether a given {@code name} matches the given list of patterns, + * accounting for negative matches. + * + * @param patterns + * to test {@code name} against; any pattern starting with an + * exclamation mark is a negative pattern + * @param name + * to test + * @return {@code true} if the {@code name} matches at least one of the + * non-negative patterns and none of the negative patterns, + * {@code false} otherwise + * @since 7.1 + */ + public static boolean patternMatch(Iterable<String> patterns, String name) { + boolean doesMatch = false; + for (String pattern : patterns) { + if (pattern.startsWith("!")) { //$NON-NLS-1$ + if (patternMatches(pattern.substring(1), name)) { + return false; + } + } else if (!doesMatch && patternMatches(pattern, name)) { + doesMatch = true; + } + } + return doesMatch; + } + + private static boolean patternMatches(String pattern, String name) { if (pattern.indexOf('*') >= 0 || pattern.indexOf('?') >= 0) { final FileNameMatcher fn; try { @@ -680,18 +708,7 @@ public class OpenSshConfigFile implements SshConfigStore { } boolean matches(String hostName) { - boolean doesMatch = false; - for (String pattern : patterns) { - if (pattern.startsWith("!")) { //$NON-NLS-1$ - if (patternMatchesHost(pattern.substring(1), hostName)) { - return false; - } - } else if (!doesMatch - && patternMatchesHost(pattern, hostName)) { - doesMatch = true; - } - } - return doesMatch; + return patternMatch(patterns, hostName); } private static String toKey(String key) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java index d232be6276..0c1da83dfb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java @@ -15,6 +15,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BARE; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WORKTREE; import static org.eclipse.jgit.lib.Constants.CONFIG; import static org.eclipse.jgit.lib.Constants.DOT_GIT; +import static org.eclipse.jgit.lib.Constants.GITDIR_FILE; import static org.eclipse.jgit.lib.Constants.GIT_ALTERNATE_OBJECT_DIRECTORIES_KEY; import static org.eclipse.jgit.lib.Constants.GIT_CEILING_DIRECTORIES_KEY; import static org.eclipse.jgit.lib.Constants.GIT_COMMON_DIR_KEY; @@ -23,7 +24,6 @@ import static org.eclipse.jgit.lib.Constants.GIT_INDEX_FILE_KEY; import static org.eclipse.jgit.lib.Constants.GIT_OBJECT_DIRECTORY_KEY; import static org.eclipse.jgit.lib.Constants.GIT_WORK_TREE_KEY; import static org.eclipse.jgit.lib.Constants.OBJECTS; -import static org.eclipse.jgit.lib.Constants.GITDIR_FILE; import java.io.File; import java.io.IOException; @@ -485,7 +485,7 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re if (getAlternateObjectDirectories() == null) { String val = sr.getenv(GIT_ALTERNATE_OBJECT_DIRECTORIES_KEY); if (val != null) { - for (String path : val.split(File.pathSeparator)) + for (String path : val.split(File.pathSeparator, -1)) addAlternateObjectDirectory(new File(path)); } } @@ -505,7 +505,7 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re if (ceilingDirectories == null) { String val = sr.getenv(GIT_CEILING_DIRECTORIES_KEY); if (val != null) { - for (String path : val.split(File.pathSeparator)) + for (String path : val.split(File.pathSeparator, -1)) addCeilingDirectory(new File(path)); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index acb54d7405..a57f1b714a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -206,7 +206,36 @@ public final class ConfigConstants { public static final String CONFIG_KEY_SIGNINGKEY = "signingKey"; /** + * The "ssh" subsection key. + * + * @since 7.1 + */ + public static final String CONFIG_SSH_SUBSECTION = "ssh"; + + /** + * The "defaultKeyCommand" key. + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_DEFAULT_KEY_COMMAND = "defaultKeyCommand"; + + /** + * The "allowedSignersFile" key. + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_ALLOWED_SIGNERS_FILE = "allowedSignersFile"; + + /** + * The "revocationFile" key, + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_REVOCATION_FILE = "revocationFile"; + + /** * The "commit" section + * * @since 5.2 */ public static final String CONFIG_COMMIT_SECTION = "commit"; @@ -1027,4 +1056,11 @@ public final class ConfigConstants { * @since 7.0 */ public static final String CONFIG_KEY_USE_OBJECT_SIZE_INDEX = "useObjectSizeIndex"; + + /** + * The "loadRevIndexInParallel" key + * + * @since 7.1 + */ + public static final String CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL = "loadRevIndexInParallel"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index fb5c904215..76ed36a6e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -61,6 +61,12 @@ public class GpgConfig { private final boolean forceAnnotated; + private final String sshDefaultKeyCommand; + + private final String sshAllowedSignersFile; + + private final String sshRevocationFile; + /** * Create a new GPG config that reads the configuration from config. * @@ -88,6 +94,17 @@ public class GpgConfig { ConfigConstants.CONFIG_KEY_GPGSIGN, false); forceAnnotated = config.getBoolean(ConfigConstants.CONFIG_TAG_SECTION, ConfigConstants.CONFIG_KEY_FORCE_SIGN_ANNOTATED, false); + sshDefaultKeyCommand = config.getString( + ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_DEFAULT_KEY_COMMAND); + sshAllowedSignersFile = config.getString( + ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_ALLOWED_SIGNERS_FILE); + sshRevocationFile = config.getString(ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_REVOCATION_FILE); } /** @@ -151,4 +168,37 @@ public class GpgConfig { public boolean isSignAnnotated() { return forceAnnotated; } + + /** + * Retrieves the value of git config {@code gpg.ssh.defaultKeyCommand}. + * + * @return the value of {@code gpg.ssh.defaultKeyCommand} + * + * @since 7.1 + */ + public String getSshDefaultKeyCommand() { + return sshDefaultKeyCommand; + } + + /** + * Retrieves the value of git config {@code gpg.ssh.allowedSignersFile}. + * + * @return the value of {@code gpg.ssh.allowedSignersFile} + * + * @since 7.1 + */ + public String getSshAllowedSignersFile() { + return sshAllowedSignersFile; + } + + /** + * Retrieves the value of git config {@code gpg.ssh.revocationFile}. + * + * @return the value of {@code gpg.ssh.revocationFile} + * + * @since 7.1 + */ + public String getSshRevocationFile() { + return sshRevocationFile; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index 3ba055aae8..5d3db9e6ee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -13,9 +13,11 @@ package org.eclipse.jgit.lib; import java.io.Serializable; -import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Locale; import java.util.TimeZone; @@ -49,6 +51,17 @@ public class PersonIdent implements Serializable { } /** + * Translate a minutes offset into a ZoneId + * + * @param tzOffset as minutes east of UTC + * @return a ZoneId for this offset + * @since 7.1 + */ + public static ZoneId getZoneId(int tzOffset) { + return ZoneOffset.ofHoursMinutes(tzOffset / 60, tzOffset % 60); + } + + /** * Format a timezone offset. * * @param r @@ -121,13 +134,17 @@ public class PersonIdent implements Serializable { } } + // Write offsets as [+-]HHMM + private static final DateTimeFormatter OFFSET_FORMATTER = DateTimeFormatter + .ofPattern("Z", Locale.US); //$NON-NLS-1$ + private final String name; private final String emailAddress; - private final long when; + private final Instant when; - private final int tzOffset; + private final ZoneId tzOffset; /** * Creates new PersonIdent from config info in repository, with current time. @@ -160,7 +177,7 @@ public class PersonIdent implements Serializable { * a {@link java.lang.String} object. */ public PersonIdent(String aName, String aEmailAddress) { - this(aName, aEmailAddress, SystemReader.getInstance().getCurrentTime()); + this(aName, aEmailAddress, SystemReader.getInstance().now()); } /** @@ -177,7 +194,7 @@ public class PersonIdent implements Serializable { */ public PersonIdent(String aName, String aEmailAddress, ProposedTimestamp when) { - this(aName, aEmailAddress, when.millis()); + this(aName, aEmailAddress, when.instant()); } /** @@ -189,8 +206,25 @@ public class PersonIdent implements Serializable { * local time * @param tz * time zone + * @deprecated Use {@link #PersonIdent(PersonIdent, Instant, ZoneId)} instead. */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, Date when, TimeZone tz) { + this(pi.getName(), pi.getEmailAddress(), when.toInstant(), tz.toZoneId()); + } + + /** + * Copy a PersonIdent, but alter the clone's time stamp + * + * @param pi + * original {@link org.eclipse.jgit.lib.PersonIdent} + * @param when + * local time + * @param tz + * time zone offset + * @since 7.1 + */ + public PersonIdent(PersonIdent pi, Instant when, ZoneId tz) { this(pi.getName(), pi.getEmailAddress(), when, tz); } @@ -202,9 +236,12 @@ public class PersonIdent implements Serializable { * original {@link org.eclipse.jgit.lib.PersonIdent} * @param aWhen * local time + * @deprecated Use the variant with an Instant instead */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, Date aWhen) { - this(pi.getName(), pi.getEmailAddress(), aWhen.getTime(), pi.tzOffset); + this(pi.getName(), pi.getEmailAddress(), aWhen.toInstant(), + pi.tzOffset); } /** @@ -218,7 +255,7 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public PersonIdent(PersonIdent pi, Instant aWhen) { - this(pi.getName(), pi.getEmailAddress(), aWhen.toEpochMilli(), pi.tzOffset); + this(pi.getName(), pi.getEmailAddress(), aWhen, pi.tzOffset); } /** @@ -230,11 +267,12 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(final String aName, final String aEmailAddress, final Date aWhen, final TimeZone aTZ) { - this(aName, aEmailAddress, aWhen.getTime(), aTZ.getOffset(aWhen - .getTime()) / (60 * 1000)); + this(aName, aEmailAddress, aWhen.toInstant(), aTZ.toZoneId()); } /** @@ -252,10 +290,16 @@ public class PersonIdent implements Serializable { */ public PersonIdent(final String aName, String aEmailAddress, Instant aWhen, ZoneId zoneId) { - this(aName, aEmailAddress, aWhen.toEpochMilli(), - TimeZone.getTimeZone(zoneId) - .getOffset(aWhen - .toEpochMilli()) / (60 * 1000)); + if (aName == null) + throw new IllegalArgumentException( + JGitText.get().personIdentNameNonNull); + if (aEmailAddress == null) + throw new IllegalArgumentException( + JGitText.get().personIdentEmailNonNull); + name = aName; + emailAddress = aEmailAddress; + when = aWhen; + tzOffset = zoneId; } /** @@ -267,15 +311,18 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, long aWhen, int aTZ) { - this(pi.getName(), pi.getEmailAddress(), aWhen, aTZ); + this(pi.getName(), pi.getEmailAddress(), Instant.ofEpochMilli(aWhen), + getZoneId(aTZ)); } private PersonIdent(final String aName, final String aEmailAddress, - long when) { + Instant when) { this(aName, aEmailAddress, when, SystemReader.getInstance() - .getTimezone(when)); + .getTimeZoneAt(when)); } private PersonIdent(UserConfig config) { @@ -298,19 +345,12 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(final String aName, final String aEmailAddress, final long aWhen, final int aTZ) { - if (aName == null) - throw new IllegalArgumentException( - JGitText.get().personIdentNameNonNull); - if (aEmailAddress == null) - throw new IllegalArgumentException( - JGitText.get().personIdentEmailNonNull); - name = aName; - emailAddress = aEmailAddress; - when = aWhen; - tzOffset = aTZ; + this(aName, aEmailAddress, Instant.ofEpochMilli(aWhen), getZoneId(aTZ)); } /** @@ -335,9 +375,12 @@ public class PersonIdent implements Serializable { * Get timestamp * * @return timestamp + * + * @deprecated Use getWhenAsInstant instead */ + @Deprecated(since = "7.1") public Date getWhen() { - return new Date(when); + return Date.from(when); } /** @@ -347,16 +390,19 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public Instant getWhenAsInstant() { - return Instant.ofEpochMilli(when); + return when; } /** * Get this person's declared time zone * * @return this person's declared time zone; null if time zone is unknown. + * + * @deprecated Use getZoneId instead */ + @Deprecated(since = "7.1") public TimeZone getTimeZone() { - return getTimeZone(tzOffset); + return TimeZone.getTimeZone(tzOffset); } /** @@ -366,7 +412,17 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public ZoneId getZoneId() { - return getTimeZone().toZoneId(); + return tzOffset; + } + + /** + * Return the offset in this timezone at the specific time + * + * @return the offset + * @since 7.1 + */ + public ZoneOffset getZoneOffset() { + return tzOffset.getRules().getOffset(when); } /** @@ -374,9 +430,11 @@ public class PersonIdent implements Serializable { * * @return this person's declared time zone as minutes east of UTC. If the * timezone is to the west of UTC it is negative. + * @deprecated Use {@link #getZoneOffset()} and read minutes from there */ + @Deprecated(since = "7.1") public int getTimeZoneOffset() { - return tzOffset; + return getZoneOffset().getTotalSeconds() / 60; } /** @@ -388,7 +446,7 @@ public class PersonIdent implements Serializable { public int hashCode() { int hc = getEmailAddress().hashCode(); hc *= 31; - hc += (int) (when / 1000L); + hc += when.hashCode(); return hc; } @@ -398,7 +456,9 @@ public class PersonIdent implements Serializable { final PersonIdent p = (PersonIdent) o; return getName().equals(p.getName()) && getEmailAddress().equals(p.getEmailAddress()) - && when / 1000L == p.when / 1000L; + // commmit timestamps are stored with 1 second precision + && when.truncatedTo(ChronoUnit.SECONDS) + .equals(p.when.truncatedTo(ChronoUnit.SECONDS)); } return false; } @@ -414,9 +474,9 @@ public class PersonIdent implements Serializable { r.append(" <"); //$NON-NLS-1$ appendSanitized(r, getEmailAddress()); r.append("> "); //$NON-NLS-1$ - r.append(when / 1000); + r.append(when.toEpochMilli() / 1000); r.append(' '); - appendTimezone(r, tzOffset); + r.append(OFFSET_FORMATTER.format(getZoneOffset())); return r.toString(); } @@ -424,19 +484,16 @@ public class PersonIdent implements Serializable { @SuppressWarnings("nls") public String toString() { final StringBuilder r = new StringBuilder(); - final SimpleDateFormat dtfmt; - dtfmt = new SimpleDateFormat("EEE MMM d HH:mm:ss yyyy Z", Locale.US); - dtfmt.setTimeZone(getTimeZone()); - + DateTimeFormatter dtfmt = DateTimeFormatter + .ofPattern("EEE MMM d HH:mm:ss yyyy Z", Locale.US) //$NON-NLS-1$ + .withZone(tzOffset); r.append("PersonIdent["); r.append(getName()); r.append(", "); r.append(getEmailAddress()); r.append(", "); - r.append(dtfmt.format(Long.valueOf(when))); + r.append(dtfmt.format(when)); r.append("]"); - return r.toString(); } } - diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 2cf24185c7..09cb5a83dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -26,6 +26,7 @@ import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.PackRefsCommand; /** * Abstraction of name to {@link org.eclipse.jgit.lib.ObjectId} mapping. @@ -160,7 +161,7 @@ public abstract class RefDatabase { if (existing.startsWith(prefix)) conflicting.add(existing); - return conflicting; + return Collections.unmodifiableList(conflicting); } /** @@ -593,4 +594,22 @@ public abstract class RefDatabase { } return null; } + + /** + * Optimize pack ref storage. + * + * @param pm + * a progress monitor + * + * @param packRefs + * {@link PackRefsCommand} to control ref packing behavior + * + * @throws java.io.IOException + * if an IO error occurred + * @since 7.1 + */ + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + // nothing + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java index 1162a615f2..fc5ab62898 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.TimeZone; +import java.util.stream.Collectors; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -185,12 +186,15 @@ public class RecursiveMerger extends ResolveMerger { if (mergeTrees(bcTree, currentBase.getTree(), nextBase.getTree(), true)) currentBase = createCommitForTree(resultTree, parents); - else + else { + String failedPaths = failingPathsMessage(); throw new NoMergeBaseException( NoMergeBaseException.MergeBaseFailureReason.CONFLICTS_DURING_MERGE_BASE_CALCULATION, MessageFormat.format( JGitText.get().mergeRecursiveConflictsWhenMergingCommonAncestors, - currentBase.getName(), nextBase.getName())); + currentBase.getName(), nextBase.getName(), + failedPaths)); + } } } finally { inCore = oldIncore; @@ -236,4 +240,17 @@ public class RecursiveMerger extends ResolveMerger { new Date((time + 1) * 1000L), TimeZone.getTimeZone("GMT+0000")); //$NON-NLS-1$ } + + private String failingPathsMessage() { + int max = 25; + String failedPaths = failingPaths.entrySet().stream().limit(max) + .map(entry -> entry.getKey() + ":" + entry.getValue()) //$NON-NLS-1$ + .collect(Collectors.joining("\n")); //$NON-NLS-1$ + + if (failingPaths.size() > max) { + failedPaths = String.format("%s\n... (%s failing paths omitted)", //$NON-NLS-1$ + failedPaths, Integer.valueOf(failingPaths.size() - max)); + } + return failedPaths; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index a50a6440b9..dc96f65b87 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -1281,6 +1281,13 @@ public class ResolveMerger extends ThreeWayMerger { default: break; } + if (ignoreConflicts) { + // If the path is selected to be treated as binary via attributes, we do not perform + // content merge. When ignoreConflicts = true, we simply keep OURS to allow virtual commit + // to be built. + keep(ourDce); + return true; + } // add the conflicting path to merge result String currentPath = tw.getPathString(); MergeResult<RawText> result = new MergeResult<>( @@ -1320,8 +1327,12 @@ public class ResolveMerger extends ThreeWayMerger { addToCheckout(currentPath, null, attributes); return true; } catch (BinaryBlobException e) { - // if the file is binary in either OURS, THEIRS or BASE - // here, we don't have an option to ignore conflicts + // The file is binary in either OURS, THEIRS or BASE + if (ignoreConflicts) { + // When ignoreConflicts = true, we simply keep OURS to allow virtual commit to be built. + keep(ourDce); + return true; + } } } switch (getContentMergeStrategy()) { @@ -1362,6 +1373,8 @@ public class ResolveMerger extends ThreeWayMerger { } } } else { + // This is reachable if contentMerge() call above threw BinaryBlobException, so we don't + // need to check ignoreConflicts here, since it's already handled above. result.setContainsConflicts(true); addConflict(base, ours, theirs); unmergedPaths.add(currentPath); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 8373d6809a..863b79466a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -50,7 +50,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.zip.Deflater; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Repository; @@ -995,7 +995,7 @@ public class PackConfig { * * @return the index version, the special version 0 designates the oldest * (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public int getIndexVersion() { return indexVersion; @@ -1009,7 +1009,7 @@ public class PackConfig { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java index a0194ea8b1..8120df0698 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java @@ -11,8 +11,6 @@ package org.eclipse.jgit.transport; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.Iterator; import java.util.ServiceLoader; @@ -99,9 +97,8 @@ public abstract class SshSessionFactory { * @since 5.2 */ public static String getLocalUserName() { - return AccessController - .doPrivileged((PrivilegedAction<String>) () -> SystemReader - .getInstance().getProperty(Constants.OS_USER_NAME_KEY)); + return SystemReader.getInstance() + .getProperty(Constants.OS_USER_NAME_KEY); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 5ba8270892..d972067e40 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -2384,7 +2384,8 @@ public class UploadPack implements Closeable { : req.getDepth() - 1; pw.setShallowPack(req.getDepth(), unshallowCommits); - // Ownership is transferred below + // dw borrows the reader from walk which is closed by #close + @SuppressWarnings("resource") DepthWalk.RevWalk dw = new DepthWalk.RevWalk( walk.getObjectReader(), walkDepth); dw.setDeepenSince(req.getDeepenSince()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java index 125ee6cbe9..95b8221a8b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java @@ -36,29 +36,39 @@ import org.eclipse.jgit.annotations.NonNull; */ public interface HttpConnection { /** + * HttpURLConnection#HTTP_OK + * * @see HttpURLConnection#HTTP_OK */ int HTTP_OK = java.net.HttpURLConnection.HTTP_OK; /** + * HttpURLConnection#HTTP_NOT_AUTHORITATIVE + * * @see HttpURLConnection#HTTP_NOT_AUTHORITATIVE * @since 5.8 */ int HTTP_NOT_AUTHORITATIVE = java.net.HttpURLConnection.HTTP_NOT_AUTHORITATIVE; /** + * HttpURLConnection#HTTP_MOVED_PERM + * * @see HttpURLConnection#HTTP_MOVED_PERM * @since 4.7 */ int HTTP_MOVED_PERM = java.net.HttpURLConnection.HTTP_MOVED_PERM; /** + * HttpURLConnection#HTTP_MOVED_TEMP + * * @see HttpURLConnection#HTTP_MOVED_TEMP * @since 4.9 */ int HTTP_MOVED_TEMP = java.net.HttpURLConnection.HTTP_MOVED_TEMP; /** + * HttpURLConnection#HTTP_SEE_OTHER + * * @see HttpURLConnection#HTTP_SEE_OTHER * @since 4.9 */ @@ -85,16 +95,22 @@ public interface HttpConnection { int HTTP_11_MOVED_PERM = 308; /** + * HttpURLConnection#HTTP_NOT_FOUND + * * @see HttpURLConnection#HTTP_NOT_FOUND */ int HTTP_NOT_FOUND = java.net.HttpURLConnection.HTTP_NOT_FOUND; /** + * HttpURLConnection#HTTP_UNAUTHORIZED + * * @see HttpURLConnection#HTTP_UNAUTHORIZED */ int HTTP_UNAUTHORIZED = java.net.HttpURLConnection.HTTP_UNAUTHORIZED; /** + * HttpURLConnection#HTTP_FORBIDDEN + * * @see HttpURLConnection#HTTP_FORBIDDEN */ int HTTP_FORBIDDEN = java.net.HttpURLConnection.HTTP_FORBIDDEN; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java index bcf79a285d..33db6ea661 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java @@ -13,12 +13,12 @@ package org.eclipse.jgit.treewalk.filter; -import org.eclipse.jgit.util.RawParseUtils; - import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; +import org.eclipse.jgit.util.RawParseUtils; + /** * Specialized set for byte arrays, interpreted as strings for use in * {@link PathFilterGroup.Group}. Most methods assume the hash is already know @@ -141,13 +141,19 @@ class ByteArraySet { } /** + * Returns number of arrays in the set + * * @return number of arrays in the set */ int size() { return size; } - /** @return true if {@link #size()} is 0. */ + /** + * Returns true if {@link #size()} is 0 + * + * @return true if {@link #size()} is 0 + */ boolean isEmpty() { return size == 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 860c1c92fd..59bbacfa76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -30,7 +30,6 @@ import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; -import java.security.AccessControlException; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; @@ -262,31 +261,6 @@ public abstract class FS { private static final AtomicInteger threadNumber = new AtomicInteger(1); /** - * Don't use the default thread factory of the ForkJoinPool for the - * CompletableFuture; it runs without any privileges, which causes - * trouble if a SecurityManager is present. - * <p> - * Instead use normal daemon threads. They'll belong to the - * SecurityManager's thread group, or use the one of the calling thread, - * as appropriate. - * </p> - * - * @see java.util.concurrent.Executors#newCachedThreadPool() - */ - private static final ExecutorService FUTURE_RUNNER = new ThreadPoolExecutor( - 5, 5, 30L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - runnable -> { - Thread t = new Thread(runnable, - "JGit-FileStoreAttributeReader-" //$NON-NLS-1$ - + threadNumber.getAndIncrement()); - // Make sure these threads don't prevent application/JVM - // shutdown. - t.setDaemon(true); - return t; - }); - - /** * Use a separate executor with at most one thread to synchronize * writing to the config. We write asynchronously since the config * itself might be on a different file system, which might otherwise @@ -463,7 +437,7 @@ public abstract class FS { locks.remove(s); } return attributes; - }, FUTURE_RUNNER); + }); f = f.exceptionally(e -> { LOG.error(e.getLocalizedMessage(), e); return Optional.empty(); @@ -1391,13 +1365,6 @@ public abstract class FS { } } catch (IOException e) { LOG.error("Caught exception in FS.readPipe()", e); //$NON-NLS-1$ - } catch (AccessControlException e) { - LOG.warn(MessageFormat.format( - JGitText.get().readPipeIsNotAllowedRequiredPermission, - command, dir, e.getPermission())); - } catch (SecurityException e) { - LOG.warn(MessageFormat.format(JGitText.get().readPipeIsNotAllowed, - command, dir)); } if (debug) { LOG.debug("readpipe returns null"); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java index 635351ac84..237879110a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java @@ -14,8 +14,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.OutputStream; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -43,10 +41,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { * @return true if cygwin is found */ public static boolean isCygwin() { - final String path = AccessController - .doPrivileged((PrivilegedAction<String>) () -> System - .getProperty("java.library.path") //$NON-NLS-1$ - ); + final String path = System.getProperty("java.library.path"); //$NON-NLS-1$ if (path == null) return false; File found = FS.searchPath(path, "cygpath.exe"); //$NON-NLS-1$ @@ -99,9 +94,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { @Override protected File userHomeImpl() { - final String home = AccessController.doPrivileged( - (PrivilegedAction<String>) () -> System.getenv("HOME") //$NON-NLS-1$ - ); + final String home = System.getenv("HOME"); //$NON-NLS-1$ if (home == null || home.length() == 0) return super.userHomeImpl(); return resolve(new File("."), home); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java index 6a4b39652a..f080056546 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java @@ -28,7 +28,10 @@ import org.eclipse.jgit.internal.JGitText; * used. One example is the parsing of the config parameter gc.pruneexpire. The * parser can handle only subset of what native gits approxidate parser * understands. + * + * @deprecated Use {@link GitTimeParser} instead. */ +@Deprecated(since = "7.1") public class GitDateParser { /** * The Date representing never. Though this is a concrete value, most diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java new file mode 100644 index 0000000000..7d00fcd5ed --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java @@ -0,0 +1,213 @@ +/* + * Copyright (C) 2024 Christian Halstrick and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import java.text.MessageFormat; +import java.text.ParseException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.EnumMap; +import java.util.Map; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.JGitText; + +/** + * Parses strings with time and date specifications into + * {@link java.time.Instant}. + * + * When git needs to parse strings specified by the user this parser can be + * used. One example is the parsing of the config parameter gc.pruneexpire. The + * parser can handle only subset of what native gits approxidate parser + * understands. + * + * @since 7.1 + */ +public class GitTimeParser { + + private static final Map<ParseableSimpleDateFormat, DateTimeFormatter> formatCache = new EnumMap<>( + ParseableSimpleDateFormat.class); + + // An enum of all those formats which this parser can parse with the help of + // a DateTimeFormatter. There are other formats (e.g. the relative formats + // like "yesterday" or "1 week ago") which this parser can parse but which + // are not listed here because they are parsed without the help of a + // DateTimeFormatter. + enum ParseableSimpleDateFormat { + ISO("yyyy-MM-dd HH:mm:ss Z"), // //$NON-NLS-1$ + RFC("EEE, dd MMM yyyy HH:mm:ss Z"), // //$NON-NLS-1$ + SHORT("yyyy-MM-dd"), // //$NON-NLS-1$ + SHORT_WITH_DOTS_REVERSE("dd.MM.yyyy"), // //$NON-NLS-1$ + SHORT_WITH_DOTS("yyyy.MM.dd"), // //$NON-NLS-1$ + SHORT_WITH_SLASH("MM/dd/yyyy"), // //$NON-NLS-1$ + DEFAULT("EEE MMM dd HH:mm:ss yyyy Z"), // //$NON-NLS-1$ + LOCAL("EEE MMM dd HH:mm:ss yyyy"); //$NON-NLS-1$ + + private final String formatStr; + + ParseableSimpleDateFormat(String formatStr) { + this.formatStr = formatStr; + } + } + + private GitTimeParser() { + // This class is not supposed to be instantiated + } + + /** + * Parses a string into a {@link java.time.LocalDateTime} using the default + * locale. Since this parser also supports relative formats (e.g. + * "yesterday") the caller can specify the reference date. These types of + * strings can be parsed: + * <ul> + * <li>"never"</li> + * <li>"now"</li> + * <li>"yesterday"</li> + * <li>"(x) years|months|weeks|days|hours|minutes|seconds ago"<br> + * Multiple specs can be combined like in "2 weeks 3 days ago". Instead of ' + * ' one can use '.' to separate the words</li> + * <li>"yyyy-MM-dd HH:mm:ss Z" (ISO)</li> + * <li>"EEE, dd MMM yyyy HH:mm:ss Z" (RFC)</li> + * <li>"yyyy-MM-dd"</li> + * <li>"yyyy.MM.dd"</li> + * <li>"MM/dd/yyyy",</li> + * <li>"dd.MM.yyyy"</li> + * <li>"EEE MMM dd HH:mm:ss yyyy Z" (DEFAULT)</li> + * <li>"EEE MMM dd HH:mm:ss yyyy" (LOCAL)</li> + * </ul> + * + * @param dateStr + * the string to be parsed + * @return the parsed {@link java.time.LocalDateTime} + * @throws java.text.ParseException + * if the given dateStr was not recognized + */ + public static LocalDateTime parse(String dateStr) throws ParseException { + return parse(dateStr, SystemReader.getInstance().civilNow()); + } + + // Only tests seem to use this method + static LocalDateTime parse(String dateStr, LocalDateTime now) + throws ParseException { + dateStr = dateStr.trim(); + + if (dateStr.equalsIgnoreCase("never")) { //$NON-NLS-1$ + return LocalDateTime.MAX; + } + LocalDateTime ret = parseRelative(dateStr, now); + if (ret != null) { + return ret; + } + for (ParseableSimpleDateFormat f : ParseableSimpleDateFormat.values()) { + try { + return parseSimple(dateStr, f); + } catch (DateTimeParseException e) { + // simply proceed with the next parser + } + } + ParseableSimpleDateFormat[] values = ParseableSimpleDateFormat.values(); + StringBuilder allFormats = new StringBuilder("\"") //$NON-NLS-1$ + .append(values[0].formatStr); + for (int i = 1; i < values.length; i++) { + allFormats.append("\", \"").append(values[i].formatStr); //$NON-NLS-1$ + } + allFormats.append("\""); //$NON-NLS-1$ + throw new ParseException( + MessageFormat.format(JGitText.get().cannotParseDate, dateStr, + allFormats.toString()), + 0); + } + + // tries to parse a string with the formats supported by DateTimeFormatter + private static LocalDateTime parseSimple(String dateStr, + ParseableSimpleDateFormat f) throws DateTimeParseException { + DateTimeFormatter dateFormat = formatCache.computeIfAbsent(f, + format -> DateTimeFormatter + .ofPattern(f.formatStr) + .withLocale(SystemReader.getInstance().getLocale())); + TemporalAccessor parsed = dateFormat.parse(dateStr); + return parsed.isSupported(ChronoField.HOUR_OF_DAY) + ? LocalDateTime.from(parsed) + : LocalDate.from(parsed).atStartOfDay(); + } + + // tries to parse a string with a relative time specification + @SuppressWarnings("nls") + @Nullable + private static LocalDateTime parseRelative(String dateStr, + LocalDateTime now) { + // check for the static words "yesterday" or "now" + if (dateStr.equals("now")) { + return now; + } + + if (dateStr.equals("yesterday")) { + return now.minusDays(1); + } + + // parse constructs like "3 days ago", "5.week.2.day.ago" + String[] parts = dateStr.split("\\.| ", -1); + int partsLength = parts.length; + // check we have an odd number of parts (at least 3) and that the last + // part is "ago" + if (partsLength < 3 || (partsLength & 1) == 0 + || !parts[parts.length - 1].equals("ago")) { + return null; + } + int number; + for (int i = 0; i < parts.length - 2; i += 2) { + try { + number = Integer.parseInt(parts[i]); + } catch (NumberFormatException e) { + return null; + } + if (parts[i + 1] == null) { + return null; + } + switch (parts[i + 1]) { + case "year": + case "years": + now = now.minusYears(number); + break; + case "month": + case "months": + now = now.minusMonths(number); + break; + case "week": + case "weeks": + now = now.minusWeeks(number); + break; + case "day": + case "days": + now = now.minusDays(number); + break; + case "hour": + case "hours": + now = now.minusHours(number); + break; + case "minute": + case "minutes": + now = now.minusMinutes(number); + break; + case "second": + case "seconds": + now = now.minusSeconds(number); + break; + default: + return null; + } + } + return now; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java index d957deb34c..efa6e7ddc3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java @@ -43,14 +43,18 @@ public class Stats { } /** - * @return number of the added values + * Returns the number of added values + * + * @return the number of added values */ public int count() { return n; } /** - * @return minimum of the added values + * Returns the smallest value added + * + * @return the smallest value added */ public double min() { if (n < 1) { @@ -60,7 +64,9 @@ public class Stats { } /** - * @return maximum of the added values + * Returns the biggest value added + * + * @return the biggest value added */ public double max() { if (n < 1) { @@ -70,9 +76,10 @@ public class Stats { } /** - * @return average of the added values + * Returns the average of the added values + * + * @return the average of the added values */ - public double avg() { if (n < 1) { return Double.NaN; @@ -81,7 +88,9 @@ public class Stats { } /** - * @return variance of the added values + * Returns the variance of the added values + * + * @return the variance of the added values */ public double var() { if (n < 2) { @@ -91,7 +100,9 @@ public class Stats { } /** - * @return standard deviation of the added values + * Returns the standard deviation of the added values + * + * @return the standard deviation of the added values */ public double stddev() { return Math.sqrt(this.var()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index ed62c71371..55cc878e02 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -23,10 +23,12 @@ import java.nio.charset.UnsupportedCharsetException; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Locale; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicReference; @@ -169,6 +171,11 @@ public abstract class SystemReader { } @Override + public Instant now() { + return Instant.now(); + } + + @Override public int getTimezone(long when) { return getTimeZone().getOffset(when) / (60 * 1000); } @@ -230,9 +237,19 @@ public abstract class SystemReader { } @Override + public Instant now() { + return delegate.now(); + } + + @Override public int getTimezone(long when) { return delegate.getTimezone(when); } + + @Override + public ZoneOffset getTimeZoneAt(Instant when) { + return delegate.getTimeZoneAt(when); + } } private static volatile SystemReader INSTANCE = DEFAULT; @@ -503,10 +520,37 @@ public abstract class SystemReader { * Get the current system time * * @return the current system time + * + * @deprecated Use {@link #now()} */ + @Deprecated public abstract long getCurrentTime(); /** + * Get the current system time + * + * @return the current system time + * + * @since 7.1 + */ + public Instant now() { + // Subclasses overriding getCurrentTime should keep working + // TODO(ifrade): Once we remove getCurrentTime, use Instant.now() + return Instant.ofEpochMilli(getCurrentTime()); + } + + /** + * Get "now" as civil time, in the System timezone + * + * @return the current system time + * + * @since 7.1 + */ + public LocalDateTime civilNow() { + return LocalDateTime.ofInstant(now(), getTimeZoneId()); + } + + /** * Get clock instance preferred by this system. * * @return clock instance preferred by this system. @@ -522,20 +566,48 @@ public abstract class SystemReader { * @param when * a system timestamp * @return the local time zone + * + * @deprecated Use {@link #getTimeZoneAt(Instant)} instead. */ + @Deprecated public abstract int getTimezone(long when); /** + * Get the local time zone offset at "when" time + * + * @param when + * a system timestamp + * @return the local time zone + * @since 7.1 + */ + public ZoneOffset getTimeZoneAt(Instant when) { + return getTimeZoneId().getRules().getOffset(when); + } + + /** * Get system time zone, possibly mocked for testing * * @return system time zone, possibly mocked for testing * @since 1.2 + * + * @deprecated Use {@link #getTimeZoneId()} */ + @Deprecated public TimeZone getTimeZone() { return TimeZone.getDefault(); } /** + * Get system time zone, possibly mocked for testing + * + * @return system time zone, possibly mocked for testing + * @since 7.1 + */ + public ZoneId getTimeZoneId() { + return ZoneId.systemDefault(); + } + + /** * Get the locale to use * * @return the locale to use @@ -670,9 +742,7 @@ public abstract class SystemReader { } private String getOsName() { - return AccessController.doPrivileged( - (PrivilegedAction<String>) () -> getProperty("os.name") //$NON-NLS-1$ - ); + return getProperty("os.name"); //$NON-NLS-1$ } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java index 4764676c88..13982b133c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java @@ -11,8 +11,6 @@ package org.eclipse.jgit.util.io; import java.io.IOException; import java.io.Writer; -import java.security.AccessController; -import java.security.PrivilegedAction; import org.eclipse.jgit.util.SystemReader; @@ -35,10 +33,7 @@ public class ThrowingPrintWriter extends Writer { */ public ThrowingPrintWriter(Writer out) { this.out = out; - LF = AccessController - .doPrivileged((PrivilegedAction<String>) () -> SystemReader - .getInstance().getProperty("line.separator") //$NON-NLS-1$ - ); + LF = SystemReader.getInstance().getProperty("line.separator"); //$NON-NLS-1$ } @Override |