diff options
Diffstat (limited to 'org.eclipse.jgit/src')
183 files changed, 8097 insertions, 3586 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index c895dc9aaa..b4d1cab513 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com> - * Copyright (C) 2010, Stefan Lay <stefan.lay@sap.com> and others + * Copyright (C) 2010, 2025 Stefan Lay <stefan.lay@sap.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -17,6 +17,7 @@ import static org.eclipse.jgit.lib.FileMode.TYPE_TREE; import java.io.IOException; import java.io.InputStream; +import java.text.MessageFormat; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -59,8 +60,15 @@ public class AddCommand extends GitCommand<DirCache> { private WorkingTreeIterator workingTreeIterator; + // Update only known index entries, don't add new ones. If there's no file + // for an index entry, remove it: stage deletions. private boolean update = false; + // If TRUE, also stage deletions, otherwise only update and add index + // entries. + // If not set explicitly + private Boolean all; + // This defaults to true because it's what JGit has been doing // traditionally. The C git default would be false. private boolean renormalize = true; @@ -82,6 +90,17 @@ public class AddCommand extends GitCommand<DirCache> { * A directory name (e.g. <code>dir</code> to add <code>dir/file1</code> and * <code>dir/file2</code>) can also be given to add all files in the * directory, recursively. Fileglobs (e.g. *.c) are not yet supported. + * </p> + * <p> + * If a pattern {@code "."} is added, all changes in the git repository's + * working tree will be added. + * </p> + * <p> + * File patterns are required unless {@code isUpdate() == true} or + * {@link #setAll(boolean)} is called. If so and no file patterns are given, + * all changes will be added (i.e., a file pattern of {@code "."} is + * implied). + * </p> * * @param filepattern * repository-relative path of file/directory to add (with @@ -113,15 +132,41 @@ public class AddCommand extends GitCommand<DirCache> { * Executes the {@code Add} command. Each instance of this class should only * be used for one invocation of the command. Don't call this method twice * on an instance. + * </p> + * + * @throws JGitInternalException + * on errors, but also if {@code isUpdate() == true} _and_ + * {@link #setAll(boolean)} had been called + * @throws NoFilepatternException + * if no file patterns are given if {@code isUpdate() == false} + * and {@link #setAll(boolean)} was not called */ @Override public DirCache call() throws GitAPIException, NoFilepatternException { - - if (filepatterns.isEmpty()) - throw new NoFilepatternException(JGitText.get().atLeastOnePatternIsRequired); checkCallable(); + + if (update && all != null) { + throw new JGitInternalException(MessageFormat.format( + JGitText.get().illegalCombinationOfArguments, + "--update", "--all/--no-all")); //$NON-NLS-1$ //$NON-NLS-2$ + } + boolean addAll; + if (filepatterns.isEmpty()) { + if (update || all != null) { + addAll = true; + } else { + throw new NoFilepatternException( + JGitText.get().atLeastOnePatternIsRequired); + } + } else { + addAll = filepatterns.contains("."); //$NON-NLS-1$ + if (all == null && !update) { + all = Boolean.TRUE; + } + } + boolean stageDeletions = update || (all != null && all.booleanValue()); + DirCache dc = null; - boolean addAll = filepatterns.contains("."); //$NON-NLS-1$ try (ObjectInserter inserter = repo.newObjectInserter(); NameConflictTreeWalk tw = new NameConflictTreeWalk(repo)) { @@ -181,7 +226,8 @@ public class AddCommand extends GitCommand<DirCache> { if (f == null) { // working tree file does not exist if (entry != null - && (!update || GITLINK == entry.getFileMode())) { + && (!stageDeletions + || GITLINK == entry.getFileMode())) { builder.add(entry); } continue; @@ -252,7 +298,8 @@ public class AddCommand extends GitCommand<DirCache> { } /** - * Set whether to only match against already tracked files + * Set whether to only match against already tracked files. If + * {@code update == true}, re-sets a previous {@link #setAll(boolean)}. * * @param update * If set to true, the command only matches {@code filepattern} @@ -314,4 +361,32 @@ public class AddCommand extends GitCommand<DirCache> { public boolean isRenormalize() { return renormalize; } + + /** + * Defines whether the command will use '--all' mode: update existing index + * entries, add new entries, and remove index entries for which there is no + * file. (In other words: also stage deletions.) + * <p> + * The setting is independent of {@link #setUpdate(boolean)}. + * </p> + * + * @param all + * whether to enable '--all' mode + * @return {@code this} + * @since 7.2 + */ + public AddCommand setAll(boolean all) { + this.all = Boolean.valueOf(all); + return this; + } + + /** + * Tells whether '--all' has been set for this command. + * + * @return {@code true} if it was set; {@code false}Â otherwise + * @since 7.2 + */ + public boolean isAll() { + return all != null && all.booleanValue(); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java index c133219d4d..32c242f271 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java @@ -644,24 +644,6 @@ public class CheckoutCommand extends GitCommand<Ref> { /** * Specify to force the ref update in case of a branch switch. * - * @param force - * if <code>true</code> and the branch with the given name - * already exists, the start-point of an existing branch will be - * set to a new start-point; if false, the existing branch will - * not be changed - * @return this instance - * @deprecated this method was badly named comparing its semantics to native - * git's checkout --force option, use - * {@link #setForceRefUpdate(boolean)} instead - */ - @Deprecated - public CheckoutCommand setForce(boolean force) { - return setForceRefUpdate(force); - } - - /** - * Specify to force the ref update in case of a branch switch. - * * In releases prior to 5.2 this method was called setForce() but this name * was misunderstood to implement native git's --force option, which is not * true. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java index 3e034f1a6a..4a536b9534 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java @@ -67,6 +67,8 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { private boolean bare; + private boolean relativePaths; + private FS fs; private String remote = Constants.DEFAULT_REMOTE_NAME; @@ -264,6 +266,7 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { private Repository init() throws GitAPIException { InitCommand command = Git.init(); command.setBare(bare); + command.setRelativeDirs(relativePaths); if (fs != null) { command.setFs(fs); } @@ -555,6 +558,20 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { } /** + * Set whether the cloned repository shall use relative paths for GIT_DIR + * and GIT_WORK_TREE + * + * @param relativePaths + * if true, use relative paths for GIT_DIR and GIT_WORK_TREE + * @return this instance + * @since 7.2 + */ + public CloneCommand setRelativePaths(boolean relativePaths) { + this.relativePaths = relativePaths; + return this; + } + + /** * Set the file system abstraction to be used for repositories created by * this command. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index a1a2cc09d2..a7d409c3f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -51,9 +51,6 @@ import org.eclipse.jgit.lib.CommitConfig.CleanupMode; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgConfig.GpgFormat; -import org.eclipse.jgit.lib.GpgObjectSigner; -import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; @@ -62,6 +59,8 @@ import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryState; +import org.eclipse.jgit.lib.Signer; +import org.eclipse.jgit.lib.Signers; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevTag; @@ -129,7 +128,7 @@ public class CommitCommand extends GitCommand<RevCommit> { private String signingKey; - private GpgSigner gpgSigner; + private Signer signer; private GpgConfig gpgConfig; @@ -319,30 +318,22 @@ public class CommitCommand extends GitCommand<RevCommit> { } } - private void sign(CommitBuilder commit) throws ServiceUnavailableException, - CanceledException, UnsupportedSigningFormatException { - if (gpgSigner == null) { - gpgSigner = GpgSigner.getDefault(); - if (gpgSigner == null) { - throw new ServiceUnavailableException( - JGitText.get().signingServiceUnavailable); + private void sign(CommitBuilder commit) + throws CanceledException, IOException, + UnsupportedSigningFormatException { + if (signer == null) { + signer = Signers.get(gpgConfig.getKeyFormat()); + if (signer == null) { + throw new UnsupportedSigningFormatException(MessageFormat + .format(JGitText.get().signatureTypeUnknown, + gpgConfig.getKeyFormat().toConfigValue())); } } if (signingKey == null) { signingKey = gpgConfig.getSigningKey(); } - if (gpgSigner instanceof GpgObjectSigner) { - ((GpgObjectSigner) gpgSigner).signObject(commit, - signingKey, committer, credentialsProvider, - gpgConfig); - } else { - if (gpgConfig.getKeyFormat() != GpgFormat.OPENPGP) { - throw new UnsupportedSigningFormatException(JGitText - .get().onlyOpenPgpSupportedForSigning); - } - gpgSigner.sign(commit, signingKey, committer, - credentialsProvider); - } + signer.signObject(repo, gpgConfig, commit, committer, signingKey, + credentialsProvider); } private void updateRef(RepositoryState state, ObjectId headId, @@ -1097,22 +1088,22 @@ public class CommitCommand extends GitCommand<RevCommit> { } /** - * Sets the {@link GpgSigner} to use if the commit is to be signed. + * Sets the {@link Signer} to use if the commit is to be signed. * * @param signer * to use; if {@code null}, the default signer will be used * @return {@code this} - * @since 5.11 + * @since 7.0 */ - public CommitCommand setGpgSigner(GpgSigner signer) { + public CommitCommand setSigner(Signer signer) { checkCallable(); - this.gpgSigner = signer; + this.signer = signer; return this; } /** * Sets an external {@link GpgConfig} to use. Whether it will be used is at - * the discretion of the {@link #setGpgSigner(GpgSigner)}. + * the discretion of the {@link #setSigner(Signer)}. * * @param config * to set; if {@code null}, the config will be loaded from the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java index 805a886392..d2526287f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java @@ -15,11 +15,11 @@ import static org.eclipse.jgit.lib.TypedConfigGetter.UNSET_INT; import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; @@ -76,6 +76,11 @@ public class DescribeCommand extends GitCommand<String> { private List<FileNameMatcher> matchers = new ArrayList<>(); /** + * Pattern matchers to be applied to tags for exclusion. + */ + private List<FileNameMatcher> excludeMatchers = new ArrayList<>(); + + /** * Whether to use all refs in the refs/ namespace */ private boolean useAll; @@ -263,6 +268,27 @@ public class DescribeCommand extends GitCommand<String> { return this; } + /** + * Sets one or more {@code glob(7)} patterns that tags must not match to be + * considered. If multiple patterns are provided, they will all be applied. + * + * @param patterns + * the {@code glob(7)} pattern or patterns + * @return {@code this} + * @throws org.eclipse.jgit.errors.InvalidPatternException + * if the pattern passed in was invalid. + * @see <a href= + * "https://www.kernel.org/pub/software/scm/git/docs/git-describe.html" + * >Git documentation about describe</a> + * @since 7.2 + */ + public DescribeCommand setExclude(String... patterns) throws InvalidPatternException { + for (String p : patterns) { + excludeMatchers.add(new FileNameMatcher(p, null)); + } + return this; + } + private final Comparator<Ref> TAG_TIE_BREAKER = new Comparator<>() { @Override @@ -274,25 +300,28 @@ public class DescribeCommand extends GitCommand<String> { } } - private Date tagDate(Ref tag) throws IOException { + private Instant tagDate(Ref tag) throws IOException { RevTag t = w.parseTag(tag.getObjectId()); w.parseBody(t); - return t.getTaggerIdent().getWhen(); + return t.getTaggerIdent().getWhenAsInstant(); } }; private Optional<Ref> getBestMatch(List<Ref> tags) { if (tags == null || tags.isEmpty()) { return Optional.empty(); - } else if (matchers.isEmpty()) { + } else if (matchers.isEmpty() && excludeMatchers.isEmpty()) { Collections.sort(tags, TAG_TIE_BREAKER); return Optional.of(tags.get(0)); - } else { + } + + Stream<Ref> matchingTags; + if (!matchers.isEmpty()) { // Find the first tag that matches in the stream of all tags // filtered by matchers ordered by tie break order - Stream<Ref> matchingTags = Stream.empty(); + matchingTags = Stream.empty(); for (FileNameMatcher matcher : matchers) { - Stream<Ref> m = tags.stream().filter( + Stream<Ref> m = tags.stream().filter( // tag -> { matcher.append(formatRefName(tag.getName())); boolean result = matcher.isMatch(); @@ -301,8 +330,22 @@ public class DescribeCommand extends GitCommand<String> { }); matchingTags = Stream.of(matchingTags, m).flatMap(i -> i); } - return matchingTags.sorted(TAG_TIE_BREAKER).findFirst(); + } else { + // If there are no matchers, there are only excluders + // Assume all tags match for now before applying excluders + matchingTags = tags.stream(); + } + + for (FileNameMatcher matcher : excludeMatchers) { + matchingTags = matchingTags.filter( // + tag -> { + matcher.append(formatRefName(tag.getName())); + boolean result = matcher.isMatch(); + matcher.reset(); + return !result; + }); } + return matchingTags.sorted(TAG_TIE_BREAKER).findFirst(); } private ObjectId getObjectIdFromRef(Ref r) throws JGitInternalException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java index 0713c38931..f24127bd51 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java @@ -124,7 +124,7 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> { FetchRecurseSubmodulesMode mode = repo.getConfig().getEnum( FetchRecurseSubmodulesMode.values(), ConfigConstants.CONFIG_SUBMODULE_SECTION, path, - ConfigConstants.CONFIG_KEY_FETCH_RECURSE_SUBMODULES, null); + ConfigConstants.CONFIG_KEY_FETCH_RECURSE_SUBMODULES); if (mode != null) { return mode; } @@ -132,7 +132,7 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> { // Fall back to fetch.recurseSubmodules, if set mode = repo.getConfig().getEnum(FetchRecurseSubmodulesMode.values(), ConfigConstants.CONFIG_FETCH_SECTION, null, - ConfigConstants.CONFIG_KEY_RECURSE_SUBMODULES, null); + ConfigConstants.CONFIG_KEY_RECURSE_SUBMODULES); if (mode != null) { return mode; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java index 88d7e91860..f6935e1c67 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.api; import java.io.IOException; import java.text.MessageFormat; import java.text.ParseException; +import java.time.Instant; import java.util.Date; import java.util.Properties; import java.util.concurrent.ExecutionException; @@ -59,7 +60,7 @@ public class GarbageCollectCommand extends GitCommand<Properties> { private ProgressMonitor monitor; - private Date expire; + private Instant expire; private PackConfig pconfig; @@ -98,8 +99,29 @@ public class GarbageCollectCommand extends GitCommand<Properties> { * @param expire * minimal age of objects to be pruned. * @return this instance + * @deprecated use {@link #setExpire(Instant)} instead */ + @Deprecated(since = "7.2") public GarbageCollectCommand setExpire(Date expire) { + if (expire != null) { + this.expire = expire.toInstant(); + } + return this; + } + + /** + * During gc() or prune() each unreferenced, loose object which has been + * created or modified after <code>expire</code> will not be pruned. Only + * older objects may be pruned. If set to null then every object is a + * candidate for pruning. Use {@link org.eclipse.jgit.util.GitTimeParser} to + * parse time formats used by git gc. + * + * @param expire + * minimal age of objects to be pruned. + * @return this instance + * @since 7.2 + */ + public GarbageCollectCommand setExpire(Instant expire) { this.expire = expire; return this; } @@ -108,8 +130,8 @@ public class GarbageCollectCommand extends GitCommand<Properties> { * Whether to use aggressive mode or not. If set to true JGit behaves more * similar to native git's "git gc --aggressive". If set to * <code>true</code> compressed objects found in old packs are not reused - * but every object is compressed again. Configuration variables - * pack.window and pack.depth are set to 250 for this GC. + * but every object is compressed again. Configuration variables pack.window + * and pack.depth are set to 250 for this GC. * * @since 3.6 * @param aggressive diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java index 3dc53ec248..5bc035a46a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java @@ -714,6 +714,16 @@ public class Git implements AutoCloseable { } /** + * Return a command object to execute a {@code PackRefs} command + * + * @return a {@link org.eclipse.jgit.api.PackRefsCommand} + * @since 7.1 + */ + public PackRefsCommand packRefs() { + return new PackRefsCommand(repo); + } + + /** * Return a command object to find human-readable names of revisions. * * @return a {@link org.eclipse.jgit.api.NameRevCommand}. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java index 240290f4f9..1da71aa6eb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java @@ -19,6 +19,7 @@ import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; @@ -44,6 +45,8 @@ public class InitCommand implements Callable<Git> { private String initialBranch; + private boolean relativePaths; + /** * {@inheritDoc} * <p> @@ -100,7 +103,11 @@ public class InitCommand implements Callable<Git> { : initialBranch); Repository repository = builder.build(); if (!repository.getObjectDatabase().exists()) - repository.create(bare); + if (repository instanceof FileRepository) { + ((FileRepository) repository).create(bare, relativePaths); + } else { + repository.create(bare); + } return new Git(repository, true); } catch (IOException | ConfigInvalidException e) { throw new JGitInternalException(e.getMessage(), e); @@ -214,4 +221,18 @@ public class InitCommand implements Callable<Git> { this.initialBranch = branch; return this; } + + /** + * * Set whether the repository shall use relative paths for GIT_DIR and + * GIT_WORK_TREE + * + * @param relativePaths + * if true, use relative paths for GIT_DIR and GIT_WORK_TREE + * @return {@code this} + * @since 7.2 + */ + public InitCommand setRelativeDirs(boolean relativePaths) { + this.relativePaths = relativePaths; + return this; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java new file mode 100644 index 0000000000..29a69c5ac4 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.api; + +import java.io.IOException; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Repository; + +/** + * Optimize storage of references. + * + * @since 7.1 + */ +public class PackRefsCommand extends GitCommand<String> { + private ProgressMonitor monitor; + + private boolean all; + + /** + * Creates a new {@link PackRefsCommand} instance with default values. + * + * @param repo + * the repository this command will be used on + */ + public PackRefsCommand(Repository repo) { + super(repo); + this.monitor = NullProgressMonitor.INSTANCE; + } + + /** + * Set progress monitor + * + * @param monitor + * a progress monitor + * @return this instance + */ + public PackRefsCommand setProgressMonitor(ProgressMonitor monitor) { + this.monitor = monitor; + return this; + } + + /** + * Specify whether to pack all the references. + * + * @param all + * if <code>true</code> all the loose refs will be packed + * @return this instance + */ + public PackRefsCommand setAll(boolean all) { + this.all = all; + return this; + } + + /** + * Whether to pack all the references + * + * @return whether to pack all the references + */ + public boolean isAll() { + return all; + } + + @Override + public String call() throws GitAPIException { + checkCallable(); + try { + repo.getRefDatabase().packRefs(monitor, this); + return JGitText.get().packRefsSuccessful; + } catch (IOException e) { + throw new JGitInternalException(JGitText.get().packRefsFailed, e); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java index 83ae0fc9d4..4b2cee45c2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java @@ -533,9 +533,9 @@ public class PullCommand extends TransportCommand<PullCommand, PullResult> { Config config) { BranchRebaseMode mode = config.getEnum(BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, - branchName, ConfigConstants.CONFIG_KEY_REBASE, null); + branchName, ConfigConstants.CONFIG_KEY_REBASE); if (mode == null) { - mode = config.getEnum(BranchRebaseMode.values(), + mode = config.getEnum( ConfigConstants.CONFIG_PULL_SECTION, null, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE); } @@ -549,7 +549,7 @@ public class PullCommand extends TransportCommand<PullCommand, PullResult> { Config config = repo.getConfig(); Merge ffMode = config.getEnum(Merge.values(), ConfigConstants.CONFIG_PULL_SECTION, null, - ConfigConstants.CONFIG_KEY_FF, null); + ConfigConstants.CONFIG_KEY_FF); return ffMode != null ? FastForwardMode.valueOf(ffMode) : null; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 858bd961cd..3ae7a6c81e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -18,6 +18,8 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -1835,23 +1837,26 @@ public class RebaseCommand extends GitCommand<RebaseResult> { // the time is saved as <seconds since 1970> <timezone offset> int timeStart = 0; - if (time.startsWith("@")) //$NON-NLS-1$ + if (time.startsWith("@")) { //$NON-NLS-1$ timeStart = 1; - else + } else { timeStart = 0; - long when = Long - .parseLong(time.substring(timeStart, time.indexOf(' '))) * 1000; + } + Instant when = Instant.ofEpochSecond( + Long.parseLong(time.substring(timeStart, time.indexOf(' ')))); String tzOffsetString = time.substring(time.indexOf(' ') + 1); int multiplier = -1; - if (tzOffsetString.charAt(0) == '+') + if (tzOffsetString.charAt(0) == '+') { multiplier = 1; + } int hours = Integer.parseInt(tzOffsetString.substring(1, 3)); int minutes = Integer.parseInt(tzOffsetString.substring(3, 5)); // this is in format (+/-)HHMM (hours and minutes) - // we need to convert into minutes - int tz = (hours * 60 + minutes) * multiplier; - if (name != null && email != null) + ZoneOffset tz = ZoneOffset.ofHoursMinutes(hours * multiplier, + minutes * multiplier); + if (name != null && email != null) { return new PersonIdent(name, email, when, tz); + } return null; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java index dead2749b7..a149649004 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java @@ -68,7 +68,7 @@ public class ReflogCommand extends GitCommand<Collection<ReflogEntry>> { checkCallable(); try { - ReflogReader reader = repo.getReflogReader(ref); + ReflogReader reader = repo.getRefDatabase().getReflogReader(ref); if (reader == null) throw new RefNotFoundException(MessageFormat.format( JGitText.get().refNotResolved, ref)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java index 553fc2e7a0..ad553f07a9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java @@ -49,18 +49,6 @@ public class RemoteRemoveCommand extends GitCommand<RemoteConfig> { /** * The name of the remote to remove. * - * @param name - * a remote name - * @deprecated use {@link #setRemoteName} instead - */ - @Deprecated - public void setName(String name) { - this.remoteName = name; - } - - /** - * The name of the remote to remove. - * * @param remoteName * a remote name * @return {@code this} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java index e3d01861fc..68ddce361f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java @@ -71,18 +71,6 @@ public class RemoteSetUrlCommand extends GitCommand<RemoteConfig> { /** * The name of the remote to change the URL for. * - * @param name - * a remote name - * @deprecated use {@link #setRemoteName} instead - */ - @Deprecated - public void setName(String name) { - this.remoteName = name; - } - - /** - * The name of the remote to change the URL for. - * * @param remoteName * a remote remoteName * @return {@code this} @@ -96,18 +84,6 @@ public class RemoteSetUrlCommand extends GitCommand<RemoteConfig> { /** * The new URL for the remote. * - * @param uri - * an URL for the remote - * @deprecated use {@link #setRemoteUri} instead - */ - @Deprecated - public void setUri(URIish uri) { - this.remoteUri = uri; - } - - /** - * The new URL for the remote. - * * @param remoteUri * an URL for the remote * @return {@code this} @@ -121,23 +97,6 @@ public class RemoteSetUrlCommand extends GitCommand<RemoteConfig> { /** * Whether to change the push URL of the remote instead of the fetch URL. * - * @param push - * <code>true</code> to set the push url, <code>false</code> to - * set the fetch url - * @deprecated use {@link #setUriType} instead - */ - @Deprecated - public void setPush(boolean push) { - if (push) { - setUriType(UriType.PUSH); - } else { - setUriType(UriType.FETCH); - } - } - - /** - * Whether to change the push URL of the remote instead of the fetch URL. - * * @param type * the <code>UriType</code> value to set * @return {@code this} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java index 855c3b1cf3..6643c83662 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com> and others + * Copyright (C) 2010, 2024 Christian Halstrick <christian.halstrick@sap.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -143,8 +143,8 @@ public class RevertCommand extends GitCommand<RevCommit> { merger.setCommitNames(new String[] { "BASE", ourName, revertName }); //$NON-NLS-1$ - String shortMessage = "Revert \"" + srcCommit.getShortMessage() //$NON-NLS-1$ - + "\""; //$NON-NLS-1$ + String shortMessage = "Revert \"" //$NON-NLS-1$ + + srcCommit.getFirstMessageLine() + '"'; String newMessage = shortMessage + "\n\n" //$NON-NLS-1$ + "This reverts commit " + srcCommit.getId().getName() //$NON-NLS-1$ + ".\n"; //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java index e4157286f1..b0b715e06b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java @@ -263,18 +263,6 @@ public class StashApplyCommand extends GitCommand<ObjectId> { /** * Whether to restore the index state * - * @param applyIndex - * true (default) if the command should restore the index state - * @deprecated use {@link #setRestoreIndex} instead - */ - @Deprecated - public void setApplyIndex(boolean applyIndex) { - this.restoreIndex = applyIndex; - } - - /** - * Whether to restore the index state - * * @param restoreIndex * true (default) if the command should restore the index state * @return {@code this} @@ -319,19 +307,6 @@ public class StashApplyCommand extends GitCommand<ObjectId> { /** * Whether the command should restore untracked files * - * @param applyUntracked - * true (default) if the command should restore untracked files - * @since 3.4 - * @deprecated use {@link #setRestoreUntracked} instead - */ - @Deprecated - public void setApplyUntracked(boolean applyUntracked) { - this.restoreUntracked = applyUntracked; - } - - /** - * Whether the command should restore untracked files - * * @param restoreUntracked * true (default) if the command should restore untracked files * @return {@code this} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java index 23fbe0197f..2dba0ef0f2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java @@ -165,7 +165,8 @@ public class StashDropCommand extends GitCommand<ObjectId> { List<ReflogEntry> entries; try { - ReflogReader reader = repo.getReflogReader(R_STASH); + ReflogReader reader = repo.getRefDatabase() + .getReflogReader(R_STASH); if (reader == null) { throw new RefNotFoundException(MessageFormat .format(JGitText.get().refNotResolved, stashRef)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java index 8fb5d60b85..5105dfc2e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java @@ -176,8 +176,9 @@ public class SubmoduleAddCommand extends CloneCommand clone = Git.cloneRepository(); configure(clone); clone.setDirectory(moduleDirectory); - clone.setGitDir(new File(new File(repo.getDirectory(), - Constants.MODULES), path)); + clone.setGitDir(new File( + new File(repo.getCommonDirectory(), Constants.MODULES), path)); + clone.setRelativePaths(true); clone.setURI(resolvedUri); if (monitor != null) clone.setProgressMonitor(monitor); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java index df73164161..5e4b2ee0b7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java @@ -28,6 +28,7 @@ import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.api.errors.WrongRepositoryStateException; import org.eclipse.jgit.dircache.DirCacheCheckout; import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.internal.storage.file.LockFile; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -39,6 +40,7 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.treewalk.filter.PathFilterGroup; +import org.eclipse.jgit.util.FileUtils; /** * A class used to execute a submodule update command. @@ -62,6 +64,8 @@ public class SubmoduleUpdateCommand extends private boolean fetch = false; + private boolean clonedRestored; + /** * <p> * Constructor for SubmoduleUpdateCommand. @@ -116,25 +120,77 @@ public class SubmoduleUpdateCommand extends return this; } + private static boolean submoduleExists(File gitDir) { + if (gitDir != null && gitDir.isDirectory()) { + File[] files = gitDir.listFiles(); + return files != null && files.length != 0; + } + return false; + } + + private static void restoreSubmodule(File gitDir, File workingTree) + throws IOException { + LockFile dotGitLock = new LockFile( + new File(workingTree, Constants.DOT_GIT)); + if (dotGitLock.lock()) { + String content = Constants.GITDIR + + getRelativePath(gitDir, workingTree); + dotGitLock.write(Constants.encode(content)); + dotGitLock.commit(); + } + } + + private static String getRelativePath(File gitDir, File workingTree) { + File relPath; + try { + relPath = workingTree.toPath().relativize(gitDir.toPath()) + .toFile(); + } catch (IllegalArgumentException e) { + relPath = gitDir; + } + return FileUtils.pathToString(relPath); + } + + private String determineUpdateMode(String mode) { + if (clonedRestored) { + return ConfigConstants.CONFIG_KEY_CHECKOUT; + } + return mode; + } + private Repository getOrCloneSubmodule(SubmoduleWalk generator, String url) throws IOException, GitAPIException { Repository repository = generator.getRepository(); + boolean restored = false; + boolean cloned = false; if (repository == null) { - if (callback != null) { - callback.cloningSubmodule(generator.getPath()); - } - CloneCommand clone = Git.cloneRepository(); - configure(clone); - clone.setURI(url); - clone.setDirectory(generator.getDirectory()); - clone.setGitDir( - new File(new File(repo.getDirectory(), Constants.MODULES), - generator.getPath())); - if (monitor != null) { - clone.setProgressMonitor(monitor); + File gitDir = new File( + new File(repo.getCommonDirectory(), Constants.MODULES), + generator.getPath()); + if (submoduleExists(gitDir)) { + restoreSubmodule(gitDir, generator.getDirectory()); + restored = true; + clonedRestored = true; + repository = generator.getRepository(); + } else { + if (callback != null) { + callback.cloningSubmodule(generator.getPath()); + } + CloneCommand clone = Git.cloneRepository(); + configure(clone); + clone.setURI(url); + clone.setDirectory(generator.getDirectory()); + clone.setGitDir(gitDir); + clone.setRelativePaths(true); + if (monitor != null) { + clone.setProgressMonitor(monitor); + } + repository = clone.call().getRepository(); + cloned = true; + clonedRestored = true; } - repository = clone.call().getRepository(); - } else if (this.fetch) { + } + if ((this.fetch || restored) && !cloned) { if (fetchCallback != null) { fetchCallback.fetchingSubmodule(generator.getPath()); } @@ -171,15 +227,17 @@ public class SubmoduleUpdateCommand extends continue; // Skip submodules not registered in parent repository's config String url = generator.getConfigUrl(); - if (url == null) + if (url == null) { continue; - + } + clonedRestored = false; try (Repository submoduleRepo = getOrCloneSubmodule(generator, url); RevWalk walk = new RevWalk(submoduleRepo)) { RevCommit commit = walk .parseCommit(generator.getObjectId()); - String update = generator.getConfigUpdate(); + String update = determineUpdateMode( + generator.getConfigUpdate()); if (ConfigConstants.CONFIG_KEY_MERGE.equals(update)) { MergeCommand merge = new MergeCommand(submoduleRepo); merge.include(commit); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java index 3edaf5e748..cc8589fa1c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java @@ -18,14 +18,11 @@ import org.eclipse.jgit.api.errors.InvalidTagNameException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.api.errors.NoHeadException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; -import org.eclipse.jgit.api.errors.ServiceUnavailableException; import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.GpgConfig; import org.eclipse.jgit.lib.GpgConfig.GpgFormat; -import org.eclipse.jgit.lib.GpgObjectSigner; -import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; @@ -33,6 +30,8 @@ import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.Signer; +import org.eclipse.jgit.lib.Signers; import org.eclipse.jgit.lib.TagBuilder; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; @@ -79,7 +78,7 @@ public class TagCommand extends GitCommand<Ref> { private GpgConfig gpgConfig; - private GpgObjectSigner gpgSigner; + private Signer signer; private CredentialsProvider credentialsProvider; @@ -133,9 +132,9 @@ public class TagCommand extends GitCommand<Ref> { newTag.setTagger(tagger); newTag.setObjectId(id); - if (gpgSigner != null) { - gpgSigner.signObject(newTag, signingKey, tagger, - credentialsProvider, gpgConfig); + if (signer != null) { + signer.signObject(repo, gpgConfig, newTag, tagger, signingKey, + credentialsProvider); } // write the tag object @@ -196,15 +195,12 @@ public class TagCommand extends GitCommand<Ref> { * * @throws InvalidTagNameException * if the tag name is null or invalid - * @throws ServiceUnavailableException - * if the tag should be signed but no signer can be found * @throws UnsupportedSigningFormatException * if the tag should be signed but {@code gpg.format} is not * {@link GpgFormat#OPENPGP} */ private void processOptions() - throws InvalidTagNameException, ServiceUnavailableException, - UnsupportedSigningFormatException { + throws InvalidTagNameException, UnsupportedSigningFormatException { if (name == null || !Repository.isValidRefName(Constants.R_TAGS + name)) { throw new InvalidTagNameException( @@ -230,16 +226,15 @@ public class TagCommand extends GitCommand<Ref> { doSign = gpgConfig.isSignAnnotated(); } if (doSign) { - if (signingKey == null) { - signingKey = gpgConfig.getSigningKey(); - } - if (gpgSigner == null) { - GpgSigner signer = GpgSigner.getDefault(); - if (!(signer instanceof GpgObjectSigner)) { - throw new ServiceUnavailableException( - JGitText.get().signingServiceUnavailable); + if (signer == null) { + signer = Signers.get(gpgConfig.getKeyFormat()); + if (signer == null) { + throw new UnsupportedSigningFormatException( + MessageFormat.format( + JGitText.get().signatureTypeUnknown, + gpgConfig.getKeyFormat() + .toConfigValue())); } - gpgSigner = (GpgObjectSigner) signer; } // The message of a signed tag must end in a newline because // the signature will be appended. @@ -326,22 +321,22 @@ public class TagCommand extends GitCommand<Ref> { } /** - * Sets the {@link GpgSigner} to use if the commit is to be signed. + * Sets the {@link Signer} to use if the commit is to be signed. * * @param signer * to use; if {@code null}, the default signer will be used * @return {@code this} - * @since 5.11 + * @since 7.0 */ - public TagCommand setGpgSigner(GpgObjectSigner signer) { + public TagCommand setSigner(Signer signer) { checkCallable(); - this.gpgSigner = signer; + this.signer = signer; return this; } /** * Sets an external {@link GpgConfig} to use. Whether it will be used is at - * the discretion of the {@link #setGpgSigner(GpgObjectSigner)}. + * the discretion of the {@link #setSigner(Signer)}. * * @param config * to set; if {@code null}, the config will be loaded from the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java index 21cddf75b7..f5f4b06e45 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java @@ -9,7 +9,7 @@ */ package org.eclipse.jgit.api; -import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.SignatureVerifier; import org.eclipse.jgit.revwalk.RevObject; /** @@ -34,8 +34,9 @@ public interface VerificationResult { * Retrieves the signature verification result. * * @return the result, or {@code null}Â if none was computed + * @since 7.0 */ - GpgSignatureVerifier.SignatureVerification getVerification(); + SignatureVerifier.SignatureVerification getVerification(); /** * Retrieves the git object of which the signature was verified. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java index 6a2a44ea2d..487ff04323 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java @@ -25,11 +25,10 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgSignatureVerifier; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; -import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.SignatureVerifiers; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; @@ -65,12 +64,8 @@ public class VerifySignatureCommand extends GitCommand<Map<String, VerificationR private VerifyMode mode = VerifyMode.ANY; - private GpgSignatureVerifier verifier; - private GpgConfig config; - private boolean ownVerifier; - /** * Creates a new {@link VerifySignatureCommand} for the given {@link Repository}. * @@ -140,22 +135,7 @@ public class VerifySignatureCommand extends GitCommand<Map<String, VerificationR } /** - * Sets the {@link GpgSignatureVerifier} to use. - * - * @param verifier - * the {@link GpgSignatureVerifier} to use, or {@code null}Â to - * use the default verifier - * @return {@code this} - */ - public VerifySignatureCommand setVerifier(GpgSignatureVerifier verifier) { - checkCallable(); - this.verifier = verifier; - return this; - } - - /** - * Sets an external {@link GpgConfig} to use. Whether it will be used it at - * the discretion of the {@link #setVerifier(GpgSignatureVerifier)}. + * Sets an external {@link GpgConfig} to use. * * @param config * to set; if {@code null}, the config will be loaded from the @@ -170,16 +150,6 @@ public class VerifySignatureCommand extends GitCommand<Map<String, VerificationR } /** - * Retrieves the currently set {@link GpgSignatureVerifier}. Can be used - * after a successful {@link #call()} to get the verifier that was used. - * - * @return the {@link GpgSignatureVerifier} - */ - public GpgSignatureVerifier getVerifier() { - return verifier; - } - - /** * {@link Repository#resolve(String) Resolves} all names added to the * command to git objects and verifies their signature. Non-existing objects * are ignored. @@ -193,9 +163,6 @@ public class VerifySignatureCommand extends GitCommand<Map<String, VerificationR * * @return a map of the given names to the corresponding * {@link VerificationResult}, excluding ignored or skipped objects. - * @throws ServiceUnavailableException - * if no {@link GpgSignatureVerifier} was set and no - * {@link GpgSignatureVerifierFactory} is available * @throws WrongObjectTypeException * if a name resolves to an object of a type not allowed by the * {@link #setMode(VerifyMode)} mode @@ -207,16 +174,6 @@ public class VerifySignatureCommand extends GitCommand<Map<String, VerificationR checkCallable(); setCallable(false); Map<String, VerificationResult> result = new HashMap<>(); - if (verifier == null) { - GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory - .getDefault(); - if (factory == null) { - throw new ServiceUnavailableException( - JGitText.get().signatureVerificationUnavailable); - } - verifier = factory.getVerifier(); - ownVerifier = true; - } if (config == null) { config = new GpgConfig(repo.getConfig()); } @@ -239,10 +196,6 @@ public class VerifySignatureCommand extends GitCommand<Map<String, VerificationR } catch (IOException e) { throw new JGitInternalException( JGitText.get().signatureVerificationError, e); - } finally { - if (ownVerifier) { - verifier.clear(); - } } return result; } @@ -258,8 +211,8 @@ public class VerifySignatureCommand extends GitCommand<Map<String, VerificationR } if (type == Constants.OBJ_COMMIT || type == Constants.OBJ_TAG) { try { - GpgSignatureVerifier.SignatureVerification verification = verifier - .verifySignature(object, config); + SignatureVerification verification = SignatureVerifiers + .verify(repo, config, object); if (verification == null) { // Not signed return null; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java index fe3e22a21f..9c4d8700a2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java @@ -9,6 +9,8 @@ */ package org.eclipse.jgit.attributes; +import org.eclipse.jgit.annotations.Nullable; + /** * Represents an attribute. * <p> @@ -139,6 +141,7 @@ public final class Attribute { * * @return the attribute value (may be <code>null</code>) */ + @Nullable public String getValue() { return value; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java index 77967df2e5..2d499cafce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java @@ -28,6 +28,8 @@ import org.eclipse.jgit.blame.Candidate.BlobCandidate; import org.eclipse.jgit.blame.Candidate.HeadCandidate; import org.eclipse.jgit.blame.Candidate.ReverseCandidate; import org.eclipse.jgit.blame.ReverseWalk.ReverseCommit; +import org.eclipse.jgit.blame.cache.BlameCache; +import org.eclipse.jgit.blame.cache.CacheRegion; import org.eclipse.jgit.diff.DiffAlgorithm; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffEntry.ChangeType; @@ -129,8 +131,19 @@ public class BlameGenerator implements AutoCloseable { /** Blame is currently assigned to this source. */ private Candidate outCandidate; + private Region outRegion; + private final BlameCache blameCache; + + /** + * Blame in reverse order needs the source lines, but we don't have them in + * the cache. We need to ignore the cache in that case. + */ + private boolean useCache = true; + + private final Stats stats = new Stats(); + /** * Create a blame generator for the repository and path (relative to * repository) @@ -142,6 +155,25 @@ public class BlameGenerator implements AutoCloseable { * repository). */ public BlameGenerator(Repository repository, String path) { + this(repository, path, null); + } + + /** + * Create a blame generator for the repository and path (relative to + * repository) + * + * @param repository + * repository to access revision data from. + * @param path + * initial path of the file to start scanning (relative to the + * repository). + * @param blameCache + * previously calculated blames. This generator will *not* + * populate it, just consume it. + * @since 7.2 + */ + public BlameGenerator(Repository repository, String path, + @Nullable BlameCache blameCache) { this.repository = repository; this.resultPath = PathFilter.create(path); @@ -150,6 +182,7 @@ public class BlameGenerator implements AutoCloseable { initRevPool(false); remaining = -1; + this.blameCache = blameCache; } private void initRevPool(boolean reverse) { @@ -159,10 +192,12 @@ public class BlameGenerator implements AutoCloseable { if (revPool != null) revPool.close(); - if (reverse) + if (reverse) { + useCache = false; revPool = new ReverseWalk(getRepository()); - else + } else { revPool = new RevWalk(getRepository()); + } SEEN = revPool.newFlag("SEEN"); //$NON-NLS-1$ reader = revPool.getObjectReader(); @@ -245,6 +280,31 @@ public class BlameGenerator implements AutoCloseable { } /** + * Stats about this generator + * + * @return the stats of this generator + * @since 7.2 + */ + public Stats getStats() { + return stats; + } + + /** + * Enable/disable the use of cache (if present). Enabled by default. + * <p> + * If caller need source line numbers, the generator cannot use the cache + * (source lines are not there). Use this method to disable the cache in + * that case. + * + * @param useCache + * should this generator use the cache. + * @since 7.2 + */ + public void setUseCache(boolean useCache) { + this.useCache = useCache; + } + + /** * Push a candidate blob onto the generator's traversal stack. * <p> * Candidates should be pushed in history order from oldest-to-newest. @@ -591,6 +651,20 @@ public class BlameGenerator implements AutoCloseable { Candidate n = pop(); if (n == null) return done(); + stats.candidatesVisited += 1; + if (blameCache != null && useCache) { + List<CacheRegion> cachedBlame = blameCache.get(repository, + n.sourceCommit, n.sourcePath.getPath()); + if (cachedBlame != null) { + BlameRegionMerger rb = new BlameRegionMerger(repository, + revPool, cachedBlame); + Candidate fullyBlamed = rb.mergeCandidate(n); + if (fullyBlamed != null) { + stats.cacheHit = true; + return result(fullyBlamed); + } + } + } int pCnt = n.getParentCount(); if (pCnt == 1) { @@ -605,7 +679,7 @@ public class BlameGenerator implements AutoCloseable { // Do not generate a tip of a reverse. The region // survives and should not appear to be deleted. - } else /* if (pCnt == 0) */{ + } else /* if (pCnt == 0) */ { // Root commit, with at least one surviving region. // Assign the remaining blame here. return result(n); @@ -846,8 +920,8 @@ public class BlameGenerator implements AutoCloseable { editList = new EditList(0); } else { p.loadText(reader); - editList = diffAlgorithm.diff(textComparator, - p.sourceText, n.sourceText); + editList = diffAlgorithm.diff(textComparator, p.sourceText, + n.sourceText); } if (editList.isEmpty()) { @@ -981,6 +1055,10 @@ public class BlameGenerator implements AutoCloseable { /** * Get first line of the source data that has been blamed for the current * region + * <p> + * This value is not reliable when the generator is reusing cached values. + * Cache doesn't keep the source lines, the returned value is based on the + * result and can be off if the region moved in previous commits. * * @return first line of the source data that has been blamed for the * current region. This is line number of where the region was added @@ -994,6 +1072,10 @@ public class BlameGenerator implements AutoCloseable { /** * Get one past the range of the source data that has been blamed for the * current region + * <p> + * This value is not reliable when the generator is reusing cached values. + * Cache doesn't keep the source lines, the returned value is based on the + * result and can be off if the region moved in previous commits. * * @return one past the range of the source data that has been blamed for * the current region. This is line number of where the region was @@ -1124,4 +1206,39 @@ public class BlameGenerator implements AutoCloseable { return ent.getChangeType() == ChangeType.RENAME || ent.getChangeType() == ChangeType.COPY; } + + /** + * Stats about the work done by the generator + * + * @since 7.2 + */ + public static class Stats { + + /** Candidates taken from the queue */ + private int candidatesVisited; + + private boolean cacheHit; + + /** + * Number of candidates taken from the queue + * <p> + * The generator could signal it's done without exhausting all + * candidates if there is no more remaining lines or the last visited + * candidate is found in the cache. + * + * @return number of candidates taken from the queue + */ + public int getCandidatesVisited() { + return candidatesVisited; + } + + /** + * The generator found a blamed version in the cache + * + * @return true if we used results from the cache + */ + public boolean isCacheHit() { + return cacheHit; + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java new file mode 100644 index 0000000000..67bc6fb789 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.eclipse.jgit.blame.cache.CacheRegion; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; + +/** + * Translates an unblamed region into one or more blamed regions, using the + * fully blamed data from cache. + * <p> + * Blamed and unblamed regions are not symmetrical: An unblamed region is just a + * range of lines over the file. A blamed region is a Candidate (with the commit + * info) with a region inside (the range blamed). + */ +class BlameRegionMerger { + private final Repository repo; + + private final List<CacheRegion> cachedRegions; + + private final RevWalk rw; + + BlameRegionMerger(Repository repo, RevWalk rw, + List<CacheRegion> cachedRegions) { + this.repo = repo; + List<CacheRegion> sorted = new ArrayList<>(cachedRegions); + Collections.sort(sorted); + this.cachedRegions = sorted; + this.rw = rw; + } + + /** + * Return one or more candidates blaming all the regions of the "unblamed" + * incoming candidate. + * + * @param candidate + * a candidate with a list of unblamed regions + * @return A linked list of Candidates with their blamed regions, null if + * there was any error. + */ + Candidate mergeCandidate(Candidate candidate) { + List<Candidate> newCandidates = new ArrayList<>(); + Region r = candidate.regionList; + while (r != null) { + try { + newCandidates.addAll(mergeOneRegion(r)); + } catch (IOException e) { + return null; + } + r = r.next; + } + return asLinkedCandidate(newCandidates); + } + + // Visible for testing + List<Candidate> mergeOneRegion(Region region) throws IOException { + List<CacheRegion> overlaps = findOverlaps(region); + if (overlaps.isEmpty()) { + throw new IOException( + "Cached blame should cover all lines"); + } + /* + * Cached regions cover the whole file. We find first which ones overlap + * with our unblamed region. Then we take the overlapping portions with + * the corresponding blame. + */ + List<Candidate> candidates = new ArrayList<>(); + for (CacheRegion overlap : overlaps) { + Region blamedRegions = intersectRegions(region, overlap); + Candidate c = new Candidate(repo, parse(overlap.getSourceCommit()), + PathFilter.create(overlap.getSourcePath())); + c.regionList = blamedRegions; + candidates.add(c); + } + return candidates; + } + + // Visible for testing + List<CacheRegion> findOverlaps(Region unblamed) { + int unblamedStart = unblamed.sourceStart; + int unblamedEnd = unblamedStart + unblamed.length; + List<CacheRegion> overlapping = new ArrayList<>(); + for (CacheRegion blamed : cachedRegions) { + // End is not included + if (blamed.getEnd() <= unblamedStart) { + // Blamed region is completely before + continue; + } + + if (blamed.getStart() >= unblamedEnd) { + // Blamed region is completely after + // Blamed regions are sorted by start position, nothing will + // match anymore + break; + } + overlapping.add(blamed); + } + return overlapping; + } + + // Visible for testing + /** + * Calculate the intersection between a Region and a CacheRegion, adjusting + * the start if needed. + * <p> + * This should be called only if there is an overlap (filtering the cached + * regions with {@link #findOverlaps(Region)}), otherwise the result is + * meaningless. + * + * @param unblamed + * a region from the blame generator + * @param cached + * a cached region + * @return a new region with the intersection. + */ + static Region intersectRegions(Region unblamed, CacheRegion cached) { + int blamedStart = Math.max(cached.getStart(), unblamed.sourceStart); + int blamedEnd = Math.min(cached.getEnd(), + unblamed.sourceStart + unblamed.length); + int length = blamedEnd - blamedStart; + + // result start and source start should move together + int blameStartDelta = blamedStart - unblamed.sourceStart; + return new Region(unblamed.resultStart + blameStartDelta, blamedStart, + length); + } + + // Tests can override this, so they don't need a real repo, commit and walk + protected RevCommit parse(ObjectId oid) throws IOException { + return rw.parseCommit(oid); + } + + private static Candidate asLinkedCandidate(List<Candidate> c) { + Candidate head = c.get(0); + Candidate tail = head; + for (int i = 1; i < c.size(); i++) { + tail.queueNext = c.get(i); + tail = tail.queueNext; + } + return head; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java index 5e2746cc7c..48f6b7e250 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java @@ -79,6 +79,7 @@ public class BlameResult { BlameResult(BlameGenerator bg, String path, RawText text) { generator = bg; + generator.setUseCache(false); resultPath = path; resultContents = text; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/BlameCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/BlameCache.java new file mode 100644 index 0000000000..d44fb5f62b --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/BlameCache.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame.cache; + +import java.io.IOException; +import java.util.List; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; + +/** + * Keeps the blame information for a path at certain commit. + * <p> + * If there is a result, it covers the whole file at that revision + * + * @since 7.2 + */ +public interface BlameCache { + /** + * Gets the blame of a path at a given commit if available. + * <p> + * Since this cache is used in blame calculation, this get() method should + * only retrieve the cache value, and not re-trigger blame calculation. In + * other words, this acts as "getIfPresent", and not "computeIfAbsent". + * + * @param repo + * repository containing the commit + * @param commitId + * we are looking at the file in this revision + * @param path + * path a file in the repo + * + * @return the blame of a path at a given commit or null if not in cache + * @throws IOException + * error retrieving/parsing values from storage + */ + List<CacheRegion> get(Repository repo, ObjectId commitId, String path) + throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java new file mode 100644 index 0000000000..cf3f978044 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java @@ -0,0 +1,125 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame.cache; + +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.ObjectId; + +/** + * Region of the blame of a file. + * <p> + * Usually all parameters are non-null, except when the Region was created + * to fill an unblamed gap (to cover for bugs in the calculation). In that + * case, path, commit and author will be null. + * + * @since 7.2 + **/ +public class CacheRegion implements Comparable<CacheRegion> { + private final String sourcePath; + + private final ObjectId sourceCommit; + + private final int end; + + private final int start; + + /** + * A blamed portion of a file + * + * @param path + * location of the file + * @param commit + * commit that is modifying this region + * @param start + * first line of this region (inclusive) + * @param end + * last line of this region (non-inclusive!) + */ + public CacheRegion(String path, ObjectId commit, + int start, int end) { + allOrNoneNull(path, commit); + this.sourcePath = path; + this.sourceCommit = commit; + this.start = start; + this.end = end; + } + + /** + * First line of this region. Starting by 0, inclusive + * + * @return first line of this region. + */ + public int getStart() { + return start; + } + + /** + * One after last line in this region (or: last line non-inclusive) + * + * @return one after last line in this region. + */ + public int getEnd() { + return end; + } + + + /** + * Path of the file this region belongs to + * + * @return path in the repo/commit + */ + public String getSourcePath() { + return sourcePath; + } + + /** + * Commit this region belongs to + * + * @return commit for this region + */ + public ObjectId getSourceCommit() { + return sourceCommit; + } + + @Override + public int compareTo(CacheRegion o) { + return start - o.start; + } + + @SuppressWarnings("nls") + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + if (sourceCommit != null) { + sb.append(sourceCommit.name(), 0, 7).append(' ') + .append(" (") + .append(sourcePath).append(')'); + } else { + sb.append("<unblamed region>"); + } + sb.append(' ').append("start=").append(start).append(", count=") + .append(end - start); + return sb.toString(); + } + + private static void allOrNoneNull(String path, ObjectId commit) { + if (path != null && commit != null) { + return; + } + + if (path == null && commit == null) { + return; + } + throw new IllegalArgumentException(MessageFormat + .format(JGitText.get().cacheRegionAllOrNoneNull, path, commit)); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java new file mode 100644 index 0000000000..b74444400e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.diff; + +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Built-in drivers for various languages, sorted by name. These drivers will be + * used to determine function names for a hunk. + * <p> + * When writing or updating patterns, assume the contents are syntactically + * correct. Patterns can be simple and need not cover all syntactical corner + * cases, as long as they are sufficiently permissive. + * + * @since 6.10.1 + */ +@SuppressWarnings({"ImmutableEnumChecker", "nls"}) +public enum DiffDriver { + /** + * Built-in diff driver for <a href= + * "https://learn.microsoft.com/en-us/cpp/cpp/cpp-language-reference">c++</a> + */ + cpp(List.of( + /* Jump targets or access declarations */ + "^[ \\t]*[A-Za-z_][A-Za-z_0-9]*:\\s*($|/[/*])"), List.of( + /* functions/methods, variables, and compounds at top level */ + "^((::\\s*)?[A-Za-z_].*)$")), + /** + * Built-in diff driver for <a href= + * "https://devicetree-specification.readthedocs.io/en/stable/source-language.html">device + * tree files</a> + */ + dts(List.of(";", "="), List.of( + /* lines beginning with a word optionally preceded by '&' or the root */ + "^[ \\t]*((/[ \\t]*\\{|&?[a-zA-Z_]).*)")), + /** + * Built-in diff driver for <a href= + * "https://docs.oracle.com/javase/specs/jls/se21/html/index.html">java</a> + */ + java(List.of( + "^[ \\t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)"), + List.of( + /* Class, enum, interface, and record declarations */ + "^[ \\t]*(([a-z-]+[ \\t]+)*(class|enum|interface|record)[ \\t]+.*)$", + /* Method definitions; note that constructor signatures are not */ + /* matched because they are indistinguishable from method calls. */ + "^[ \\t]*(([A-Za-z_<>&\\]\\[][?&<>.,A-Za-z_0-9]*[ \\t]+)+[A-Za-z_]" + + "[A-Za-z_0-9]*[ \\t]*\\([^;]*)$")), + /** + * Built-in diff driver for + * <a href="https://docs.python.org/3/reference/index.html">python</a> + */ + python(List.of("^[ \\t]*((class|(async[ \\t]+)?def)[ \\t].*)$")), + /** + * Built-in diff driver for + * <a href="https://doc.rust-lang.org/reference/introduction.html">rust</a> + */ + rust(List.of("^[\\t ]*((pub(\\([^\\)]+\\))?[\\t ]+)?" + + "((async|const|unsafe|extern([\\t ]+\"[^\"]+\"))[\\t ]+)?" + + "(struct|enum|union|mod|trait|fn|impl|macro_rules!)[< \\t]+[^;]*)$")); + + private final List<Pattern> negatePatterns; + + private final List<Pattern> matchPatterns; + + DiffDriver(List<String> negate, List<String> match, int flags) { + if (negate != null) { + this.negatePatterns = negate.stream() + .map(r -> Pattern.compile(r, flags)) + .collect(Collectors.toList()); + } else { + this.negatePatterns = null; + } + this.matchPatterns = match.stream().map(r -> Pattern.compile(r, flags)) + .collect(Collectors.toList()); + } + + DiffDriver(List<String> match) { + this(null, match, 0); + } + + DiffDriver(List<String> negate, List<String> match) { + this(negate, match, 0); + } + + /** + * Returns the list of patterns used to exclude certain lines from being + * considered as function names. + * + * @return the list of negate patterns + */ + public List<Pattern> getNegatePatterns() { + return negatePatterns; + } + + /** + * Returns the list of patterns used to match lines for potential function + * names. + * + * @return the list of match patterns + */ + public List<Pattern> getMatchPatterns() { + return matchPatterns; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java index 2f472b5c0a..cbac3f90b7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java @@ -30,7 +30,9 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.regex.Pattern; import org.eclipse.jgit.api.errors.CanceledException; +import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.dircache.DirCacheIterator; @@ -703,7 +705,7 @@ public class DiffFormatter implements AutoCloseable { */ public void format(DiffEntry ent) throws IOException { FormatResult res = createFormatResult(ent); - format(res.header, res.a, res.b); + format(res.header, res.a, res.b, getDiffDriver(ent)); } private static byte[] writeGitLinkText(AbbreviatedObjectId id) { @@ -749,11 +751,14 @@ public class DiffFormatter implements AutoCloseable { * text source for the post-image version of the content. This * must match the content of * {@link org.eclipse.jgit.patch.FileHeader#getNewId()}. + * @param diffDriver + * the diff driver used to obtain function names in hunk headers * @throws java.io.IOException - * writing to the supplied stream failed. + * writing to the supplied stream failed. + * @since 6.10.1 */ - public void format(FileHeader head, RawText a, RawText b) - throws IOException { + public void format(FileHeader head, RawText a, RawText b, + DiffDriver diffDriver) throws IOException { // Reuse the existing FileHeader as-is by blindly copying its // header lines, but avoiding its hunks. Instead we recreate // the hunks from the text instances we have been supplied. @@ -763,8 +768,49 @@ public class DiffFormatter implements AutoCloseable { if (!head.getHunks().isEmpty()) end = head.getHunks().get(0).getStartOffset(); out.write(head.getBuffer(), start, end - start); - if (head.getPatchType() == PatchType.UNIFIED) - format(head.toEditList(), a, b); + if (head.getPatchType() == PatchType.UNIFIED) { + format(head.toEditList(), a, b, diffDriver); + } + } + + /** + * Format a patch script, reusing a previously parsed FileHeader. + * <p> + * This formatter is primarily useful for editing an existing patch script + * to increase or reduce the number of lines of context within the script. + * All header lines are reused as-is from the supplied FileHeader. + * + * @param head + * existing file header containing the header lines to copy. + * @param a + * text source for the pre-image version of the content. This must match + * the content of {@link org.eclipse.jgit.patch.FileHeader#getOldId()}. + * @param b + * text source for the post-image version of the content. This must match + * the content of {@link org.eclipse.jgit.patch.FileHeader#getNewId()}. + * @throws java.io.IOException + * writing to the supplied stream failed. + */ + public void format(FileHeader head, RawText a, RawText b) + throws IOException { + format(head, a, b, null); + } + + /** + * Formats a list of edits in unified diff format + * + * @param edits + * some differences which have been calculated between A and B + * @param a + * the text A which was compared + * @param b + * the text B which was compared + * @throws java.io.IOException + * if an IO error occurred + */ + public void format(EditList edits, RawText a, RawText b) + throws IOException { + format(edits, a, b, null); } /** @@ -776,11 +822,14 @@ public class DiffFormatter implements AutoCloseable { * the text A which was compared * @param b * the text B which was compared + * @param diffDriver + * the diff driver used to obtain function names in hunk headers * @throws java.io.IOException * if an IO error occurred + * @since 6.10.1 */ - public void format(EditList edits, RawText a, RawText b) - throws IOException { + public void format(EditList edits, RawText a, RawText b, + DiffDriver diffDriver) throws IOException { for (int curIdx = 0; curIdx < edits.size();) { Edit curEdit = edits.get(curIdx); final int endIdx = findCombinedEnd(edits, curIdx); @@ -791,7 +840,8 @@ public class DiffFormatter implements AutoCloseable { final int aEnd = (int) Math.min(a.size(), (long) endEdit.getEndA() + context); final int bEnd = (int) Math.min(b.size(), (long) endEdit.getEndB() + context); - writeHunkHeader(aCur, aEnd, bCur, bEnd); + writeHunkHeader(aCur, aEnd, bCur, bEnd, + getFuncName(a, aCur - 1, diffDriver)); while (aCur < aEnd || bCur < bEnd) { if (aCur < curEdit.getBeginA() || endIdx + 1 < curIdx) { @@ -881,8 +931,30 @@ public class DiffFormatter implements AutoCloseable { * @throws java.io.IOException * if an IO error occurred */ - protected void writeHunkHeader(int aStartLine, int aEndLine, - int bStartLine, int bEndLine) throws IOException { + protected void writeHunkHeader(int aStartLine, int aEndLine, int bStartLine, + int bEndLine) throws IOException { + writeHunkHeader(aStartLine, aEndLine, bStartLine, bEndLine, null); + } + + /** + * Output a hunk header + * + * @param aStartLine + * within first source + * @param aEndLine + * within first source + * @param bStartLine + * within second source + * @param bEndLine + * within second source + * @param funcName + * function name of this hunk + * @throws java.io.IOException + * if an IO error occurred + * @since 6.10.1 + */ + protected void writeHunkHeader(int aStartLine, int aEndLine, int bStartLine, + int bEndLine, String funcName) throws IOException { out.write('@'); out.write('@'); writeRange('-', aStartLine + 1, aEndLine - aStartLine); @@ -890,6 +962,10 @@ public class DiffFormatter implements AutoCloseable { out.write(' '); out.write('@'); out.write('@'); + if (funcName != null) { + out.write(' '); + out.write(funcName.getBytes()); + } out.write('\n'); } @@ -1247,4 +1323,50 @@ public class DiffFormatter implements AutoCloseable { private static boolean end(Edit edit, int a, int b) { return edit.getEndA() <= a && edit.getEndB() <= b; } + + private String getFuncName(RawText text, int startAt, + DiffDriver diffDriver) { + if (diffDriver != null) { + while (startAt > 0) { + String line = text.getString(startAt); + startAt--; + if (matchesAny(diffDriver.getNegatePatterns(), line)) { + continue; + } + if (matchesAny(diffDriver.getMatchPatterns(), line)) { + String funcName = line.replaceAll("^[ \\t]+", ""); //$NON-NLS-1$//$NON-NLS-2$ + return funcName.substring(0, + Math.min(funcName.length(), 80)).trim(); + } + } + } + return null; + } + + private boolean matchesAny(List<Pattern> patterns, String text) { + if (patterns != null) { + for (Pattern p : patterns) { + if (p.matcher(text).find()) { + return true; + } + } + } + return false; + } + + private DiffDriver getDiffDriver(DiffEntry entry) { + Attribute diffAttr = entry.getDiffAttribute(); + if (diffAttr == null) { + return null; + } + String diffAttrValue = diffAttr.getValue(); + if (diffAttrValue == null) { + return null; + } + try { + return DiffDriver.valueOf(diffAttrValue); + } catch (IllegalArgumentException e) { + return null; + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java index 4343642f9a..b401bbe73d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java @@ -44,8 +44,8 @@ public class PatchIdDiffFormatter extends DiffFormatter { } @Override - protected void writeHunkHeader(int aStartLine, int aEndLine, - int bStartLine, int bEndLine) throws IOException { + protected void writeHunkHeader(int aStartLine, int aEndLine, int bStartLine, + int bEndLine, String funcName) throws IOException { // The hunk header is not taken into account for patch id calculation } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java index 76dc87e72b..fdfe533618 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java @@ -360,18 +360,22 @@ public class RawText extends Sequence { length = maxLength; isComplete = false; } - byte last = 'x'; // Just something inconspicuous. - for (int ptr = 0; ptr < length; ptr++) { - byte curr = raw[ptr]; - if (isBinary(curr, last)) { + + int ptr = -1; + byte current; + while (ptr < length - 2) { + current = raw[++ptr]; + if (current == '\0' || (current == '\r' && raw[++ptr] != '\n')) { return true; } - last = curr; } - if (isComplete) { - // Buffer contains everything... - return last == '\r'; // ... so this must be a lone CR + + if (ptr == length - 2) { + // if '\r' be last, then if isComplete then return binary + current = raw[++ptr]; + return current == '\0' || (current == '\r' && isComplete); } + return false; } @@ -467,26 +471,30 @@ public class RawText extends Sequence { */ public static boolean isCrLfText(byte[] raw, int length, boolean complete) { boolean has_crlf = false; - byte last = 'x'; // Just something inconspicuous - for (int ptr = 0; ptr < length; ptr++) { - byte curr = raw[ptr]; - if (isBinary(curr, last)) { + + int ptr = -1; + byte current; + while (ptr < length - 2) { + current = raw[++ptr]; + if (current == '\0') { return false; } - if (curr == '\n' && last == '\r') { + if (current == '\r') { + if (raw[++ptr] != '\n') { + return false; + } has_crlf = true; } - last = curr; } - if (last == '\r') { - if (complete) { - // Lone CR: it's binary after all. + + if (ptr == length - 2) { + // if '\r' be last, then if isComplete then return binary + current = raw[++ptr]; + if (current == '\0' || (current == '\r' && complete)) { return false; } - // Tough call. If the next byte, which we don't have, would be a - // '\n', it'd be a CR-LF text, otherwise it'd be binary. Just decide - // based on what we already scanned; it wasn't binary until now. } + return has_crlf; } @@ -578,4 +586,5 @@ public class RawText extends Sequence { return new RawText(data, RawParseUtils.lineMapOrBinary(data, 0, (int) sz)); } } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java index 5de7bac112..fb98df7c9e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java @@ -80,7 +80,7 @@ class SimilarityRenameDetector { private long[] matrix; /** Score a pair must exceed to be considered a rename. */ - private int renameScore = 60; + private int renameScore = 50; /** * File size threshold (in bytes) for detecting renames. Files larger diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java index accf732dc7..de02aecdb9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java @@ -217,10 +217,18 @@ public class Checkout { } } try { - if (recursiveDelete && Files.isDirectory(f.toPath(), - LinkOption.NOFOLLOW_LINKS)) { + boolean isDir = Files.isDirectory(f.toPath(), + LinkOption.NOFOLLOW_LINKS); + if (recursiveDelete && isDir) { FileUtils.delete(f, FileUtils.RECURSIVE); } + if (cache.getRepository().isWorkTreeCaseInsensitive() && !isDir) { + // We cannot rely on rename via Files.move() to work correctly + // if the target exists in a case variant. For instance with JDK + // 17 on Mac OS, the existing case-variant name is kept. On + // Windows 11 it would work and use the name given in 'f'. + FileUtils.delete(f, FileUtils.SKIP_MISSING); + } FileUtils.rename(tmpFile, f, StandardCopyOption.ATOMIC_MOVE); cachedParent.remove(f.getName()); } catch (IOException e) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java index 34dba0b5be..c650d6e8e7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java @@ -1037,7 +1037,12 @@ public class DirCache { } } - enum DirCacheVersion implements ConfigEnum { + /** + * DirCache versions + * + * @since 7.2 + */ + public enum DirCacheVersion implements ConfigEnum { /** Minimum index version on-disk format that we support. */ DIRC_VERSION_MINIMUM(2), @@ -1060,6 +1065,9 @@ public class DirCache { this.version = versionCode; } + /** + * @return the version code for this version + */ public int getVersionCode() { return version; } @@ -1078,6 +1086,13 @@ public class DirCache { } } + /** + * Create DirCacheVersion from integer value of the version code. + * + * @param val + * integer value of the version code. + * @return the DirCacheVersion instance of the version code. + */ public static DirCacheVersion fromInt(int val) { for (DirCacheVersion v : DirCacheVersion.values()) { if (val == v.getVersionCode()) { @@ -1098,9 +1113,8 @@ public class DirCache { boolean manyFiles = cfg.getBoolean( ConfigConstants.CONFIG_FEATURE_SECTION, ConfigConstants.CONFIG_KEY_MANYFILES, false); - indexVersion = cfg.getEnum(DirCacheVersion.values(), - ConfigConstants.CONFIG_INDEX_SECTION, null, - ConfigConstants.CONFIG_KEY_VERSION, + indexVersion = cfg.getEnum(ConfigConstants.CONFIG_INDEX_SECTION, + null, ConfigConstants.CONFIG_KEY_VERSION, manyFiles ? DirCacheVersion.DIRC_VERSION_PATHCOMPRESS : DirCacheVersion.DIRC_VERSION_EXTENDED); skipHash = cfg.getBoolean(ConfigConstants.CONFIG_INDEX_SECTION, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 6ae5153c12..18d77482e0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -5,7 +5,7 @@ * Copyright (C) 2006, Shawn O. Pearce <spearce@spearce.org> * Copyright (C) 2010, Chrisian Halstrick <christian.halstrick@sap.com> * Copyright (C) 2019, 2020, Andre Bossert <andre.bossert@siemens.com> - * Copyright (C) 2017, 2023, Thomas Wolf <twolf@apache.org> and others + * Copyright (C) 2017, 2025, Thomas Wolf <twolf@apache.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -31,6 +31,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import org.eclipse.jgit.api.errors.CanceledException; import org.eclipse.jgit.api.errors.FilterFailedException; @@ -66,7 +67,6 @@ import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.ExecutionResult; -import org.eclipse.jgit.util.IntList; import org.eclipse.jgit.util.SystemReader; import org.eclipse.jgit.util.io.EolStreamTypeUtil; import org.slf4j.Logger; @@ -113,9 +113,11 @@ public class DirCacheCheckout { private Map<String, CheckoutMetadata> updated = new LinkedHashMap<>(); + private Set<String> existing; + private ArrayList<String> conflicts = new ArrayList<>(); - private ArrayList<String> removed = new ArrayList<>(); + private TreeSet<String> removed; private ArrayList<String> kept = new ArrayList<>(); @@ -185,7 +187,7 @@ public class DirCacheCheckout { * @return a list of all files removed by this checkout */ public List<String> getRemoved() { - return removed; + return new ArrayList<>(removed); } /** @@ -214,6 +216,14 @@ public class DirCacheCheckout { this.mergeCommitTree = mergeCommitTree; this.workingTree = workingTree; this.initialCheckout = !repo.isBare() && !repo.getIndexFile().exists(); + boolean caseInsensitive = !repo.isBare() + && repo.isWorkTreeCaseInsensitive(); + this.removed = caseInsensitive + ? new TreeSet<>(String::compareToIgnoreCase) + : new TreeSet<>(); + this.existing = caseInsensitive + ? new TreeSet<>(String::compareToIgnoreCase) + : null; } /** @@ -400,9 +410,11 @@ public class DirCacheCheckout { // content to be checked out. update(m); } - } else + } else { update(m); - } else if (f == null || !m.idEqual(i)) { + } + } else if (f == null || !m.idEqual(i) + || m.getEntryRawMode() != i.getEntryRawMode()) { // The working tree file is missing or the merge content differs // from index content update(m); @@ -410,11 +422,11 @@ public class DirCacheCheckout { // The index contains a file (and not a folder) if (f.isModified(i.getDirCacheEntry(), true, this.walk.getObjectReader()) - || i.getDirCacheEntry().getStage() != 0) + || i.getDirCacheEntry().getStage() != 0) { // The working tree file is dirty or the index contains a // conflict update(m); - else { + } else { // update the timestamp of the index with the one from the // file if not set, as we are sure to be in sync here. DirCacheEntry entry = i.getDirCacheEntry(); @@ -424,9 +436,10 @@ public class DirCacheCheckout { } keep(i.getEntryPathString(), entry, f); } - } else + } else { // The index contains a folder keep(i.getEntryPathString(), i.getDirCacheEntry(), f); + } } else { // There is no entry in the merge commit. Means: we want to delete // what's currently in the index and working tree @@ -521,6 +534,13 @@ public class DirCacheCheckout { // update our index builder.finish(); + // On case-insensitive file systems we may have a case variant kept + // and another one removed. In that case, don't remove it. + if (existing != null) { + removed.removeAll(existing); + existing.clear(); + } + // init progress reporting int numTotal = removed.size() + updated.size() + conflicts.size(); monitor.beginTask(JGitText.get().checkingOutFiles, numTotal); @@ -531,9 +551,9 @@ public class DirCacheCheckout { // when deleting files process them in the opposite order as they have // been reported. This ensures the files are deleted before we delete // their parent folders - IntList nonDeleted = new IntList(); - for (int i = removed.size() - 1; i >= 0; i--) { - String r = removed.get(i); + Iterator<String> iter = removed.descendingIterator(); + while (iter.hasNext()) { + String r = iter.next(); file = new File(repo.getWorkTree(), r); if (!file.delete() && repo.getFS().exists(file)) { // The list of stuff to delete comes from the index @@ -542,7 +562,7 @@ public class DirCacheCheckout { // to delete it. A submodule is not empty, so it // is safe to check this after a failed delete. if (!repo.getFS().isDirectory(file)) { - nonDeleted.add(i); + iter.remove(); toBeDeleted.add(r); } } else { @@ -560,8 +580,6 @@ public class DirCacheCheckout { if (file != null) { removeEmptyParents(file); } - removed = filterOut(removed, nonDeleted); - nonDeleted = null; Iterator<Map.Entry<String, CheckoutMetadata>> toUpdate = updated .entrySet().iterator(); Map.Entry<String, CheckoutMetadata> e = null; @@ -633,36 +651,6 @@ public class DirCacheCheckout { return toBeDeleted.isEmpty(); } - private static ArrayList<String> filterOut(ArrayList<String> strings, - IntList indicesToRemove) { - int n = indicesToRemove.size(); - if (n == strings.size()) { - return new ArrayList<>(0); - } - switch (n) { - case 0: - return strings; - case 1: - strings.remove(indicesToRemove.get(0)); - return strings; - default: - int length = strings.size(); - ArrayList<String> result = new ArrayList<>(length - n); - // Process indicesToRemove from the back; we know that it - // contains indices in descending order. - int j = n - 1; - int idx = indicesToRemove.get(j); - for (int i = 0; i < length; i++) { - if (i == idx) { - idx = (--j >= 0) ? indicesToRemove.get(j) : -1; - } else { - result.add(strings.get(i)); - } - } - return result; - } - } - private static boolean isSamePrefix(String a, String b) { int as = a.lastIndexOf('/'); int bs = b.lastIndexOf('/'); @@ -1233,6 +1221,9 @@ public class DirCacheCheckout { if (!FileMode.TREE.equals(e.getFileMode())) { builder.add(e); } + if (existing != null) { + existing.add(path); + } if (force) { if (f == null || f.isModified(e, true, walk.getObjectReader())) { kept.add(path); @@ -1401,127 +1392,6 @@ public class DirCacheCheckout { } /** - * Updates the file in the working tree with content and mode from an entry - * in the index. The new content is first written to a new temporary file in - * the same directory as the real file. Then that new file is renamed to the - * final filename. - * - * <p> - * <b>Note:</b> if the entry path on local file system exists as a non-empty - * directory, and the target entry type is a link or file, the checkout will - * fail with {@link java.io.IOException} since existing non-empty directory - * cannot be renamed to file or link without deleting it recursively. - * </p> - * - * @param repo - * repository managing the destination work tree. - * @param entry - * the entry containing new mode and content - * @param or - * object reader to use for checkout - * @throws java.io.IOException - * if an IO error occurred - * @since 3.6 - * @deprecated since 5.1, use - * {@link #checkoutEntry(Repository, DirCacheEntry, ObjectReader, boolean, CheckoutMetadata, WorkingTreeOptions)} - * instead - */ - @Deprecated - public static void checkoutEntry(Repository repo, DirCacheEntry entry, - ObjectReader or) throws IOException { - checkoutEntry(repo, entry, or, false, null, null); - } - - - /** - * Updates the file in the working tree with content and mode from an entry - * in the index. The new content is first written to a new temporary file in - * the same directory as the real file. Then that new file is renamed to the - * final filename. - * - * <p> - * <b>Note:</b> if the entry path on local file system exists as a file, it - * will be deleted and if it exists as a directory, it will be deleted - * recursively, independently if has any content. - * </p> - * - * @param repo - * repository managing the destination work tree. - * @param entry - * the entry containing new mode and content - * @param or - * object reader to use for checkout - * @param deleteRecursive - * true to recursively delete final path if it exists on the file - * system - * @param checkoutMetadata - * containing - * <ul> - * <li>smudgeFilterCommand to be run for smudging the entry to be - * checked out</li> - * <li>eolStreamType used for stream conversion</li> - * </ul> - * @throws java.io.IOException - * if an IO error occurred - * @since 4.2 - * @deprecated since 6.3, use - * {@link #checkoutEntry(Repository, DirCacheEntry, ObjectReader, boolean, CheckoutMetadata, WorkingTreeOptions)} - * instead - */ - @Deprecated - public static void checkoutEntry(Repository repo, DirCacheEntry entry, - ObjectReader or, boolean deleteRecursive, - CheckoutMetadata checkoutMetadata) throws IOException { - checkoutEntry(repo, entry, or, deleteRecursive, checkoutMetadata, null); - } - - /** - * Updates the file in the working tree with content and mode from an entry - * in the index. The new content is first written to a new temporary file in - * the same directory as the real file. Then that new file is renamed to the - * final filename. - * - * <p> - * <b>Note:</b> if the entry path on local file system exists as a file, it - * will be deleted and if it exists as a directory, it will be deleted - * recursively, independently if has any content. - * </p> - * - * @param repo - * repository managing the destination work tree. - * @param entry - * the entry containing new mode and content - * @param or - * object reader to use for checkout - * @param deleteRecursive - * true to recursively delete final path if it exists on the file - * system - * @param checkoutMetadata - * containing - * <ul> - * <li>smudgeFilterCommand to be run for smudging the entry to be - * checked out</li> - * <li>eolStreamType used for stream conversion</li> - * </ul> - * @param options - * {@link WorkingTreeOptions} that are effective; if {@code null} - * they are loaded from the repository config - * @throws java.io.IOException - * if an IO error occurred - * @since 6.3 - * @deprecated since 6.6.1; use {@link Checkout} instead - */ - @Deprecated - public static void checkoutEntry(Repository repo, DirCacheEntry entry, - ObjectReader or, boolean deleteRecursive, - CheckoutMetadata checkoutMetadata, WorkingTreeOptions options) - throws IOException { - Checkout checkout = new Checkout(repo, options) - .setRecursiveDeletion(deleteRecursive); - checkout.checkout(entry, checkoutMetadata, or, null); - } - - /** * Return filtered content for a specific object (blob). EOL handling and * smudge-filter handling are applied in the same way as it would be done * during a checkout. @@ -1647,6 +1517,8 @@ public class DirCacheCheckout { filterProcessBuilder.directory(repo.getWorkTree()); filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, repo.getDirectory().getAbsolutePath()); + filterProcessBuilder.environment().put(Constants.GIT_COMMON_DIR_KEY, + repo.getCommonDirectory().getAbsolutePath()); ExecutionResult result; int rc; try { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java index c5e1e4e765..5a22938694 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java @@ -396,28 +396,6 @@ public class DirCacheEntry { * timestamp. This method tests to see if file was written out at the same * time as the index. * - * @param smudge_s - * seconds component of the index's last modified time. - * @param smudge_ns - * nanoseconds component of the index's last modified time. - * @return true if extra careful checks should be used. - * @deprecated use {@link #mightBeRacilyClean(Instant)} instead - */ - @Deprecated - public final boolean mightBeRacilyClean(int smudge_s, int smudge_ns) { - return mightBeRacilyClean(Instant.ofEpochSecond(smudge_s, smudge_ns)); - } - - /** - * Is it possible for this entry to be accidentally assumed clean? - * <p> - * The "racy git" problem happens when a work file can be updated faster - * than the filesystem records file modification timestamps. It is possible - * for an application to edit a work file, update the index, then edit it - * again before the filesystem will give the work file a new modification - * timestamp. This method tests to see if file was written out at the same - * time as the index. - * * @param smudge * index's last modified time. * @return true if extra careful checks should be used. @@ -653,22 +631,6 @@ public class DirCacheEntry { } /** - * Get the cached last modification date of this file, in milliseconds. - * <p> - * One of the indicators that the file has been modified by an application - * changing the working tree is if the last modification time for the file - * differs from the time stored in this entry. - * - * @return last modification time of this file, in milliseconds since the - * Java epoch (midnight Jan 1, 1970 UTC). - * @deprecated use {@link #getLastModifiedInstant()} instead - */ - @Deprecated - public long getLastModified() { - return decodeTS(P_MTIME); - } - - /** * Get the cached last modification date of this file. * <p> * One of the indicators that the file has been modified by an application @@ -683,18 +645,6 @@ public class DirCacheEntry { } /** - * Set the cached last modification date of this file, using milliseconds. - * - * @param when - * new cached modification date of the file, in milliseconds. - * @deprecated use {@link #setLastModified(Instant)} instead - */ - @Deprecated - public void setLastModified(long when) { - encodeTS(P_MTIME, when); - } - - /** * Set the cached last modification date of this file. * * @param when diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java index 1fd80867b9..38982fdf23 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java @@ -23,18 +23,6 @@ public class PackInvalidException extends IOException { private static final long serialVersionUID = 1L; /** - * Construct a pack invalid error. - * - * @param path - * path of the invalid pack file. - * @deprecated Use {@link #PackInvalidException(File, Throwable)}. - */ - @Deprecated - public PackInvalidException(File path) { - this(path, null); - } - - /** * Construct a pack invalid error with cause. * * @param path @@ -48,18 +36,6 @@ public class PackInvalidException extends IOException { } /** - * Construct a pack invalid error. - * - * @param path - * path of the invalid pack file. - * @deprecated Use {@link #PackInvalidException(String, Throwable)}. - */ - @Deprecated - public PackInvalidException(String path) { - this(path, null); - } - - /** * Construct a pack invalid error with cause. * * @param path diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java index 3ce97a4ff7..e511a68d2e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java @@ -156,6 +156,9 @@ class BareSuperprojectWriter { ObjectId objectId; if (ObjectId.isId(proj.getRevision())) { objectId = ObjectId.fromString(proj.getRevision()); + if (config.recordRemoteBranch && proj.getUpstream() != null) { + cfg.setString("submodule", name, "ref", proj.getUpstream()); //$NON-NLS-1$//$NON-NLS-2$ + } } else { objectId = callback.sha1(url, proj.getRevision()); if (objectId == null && !config.ignoreRemoteFailures) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index 957b3869f2..b033177e05 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -176,6 +176,10 @@ public class ManifestParser extends DefaultHandler { attributes.getValue("groups")); currentProject .setRecommendShallow(attributes.getValue("clone-depth")); + currentProject + .setUpstream(attributes.getValue("upstream")); + currentProject + .setDestBranch(attributes.getValue("dest-branch")); break; case "remote": String alias = attributes.getValue("alias"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 95c1c8b22e..be77fca459 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -111,32 +111,6 @@ public class RepoCommand extends GitCommand<RevCommit> { public ObjectId sha1(String uri, String ref) throws GitAPIException; /** - * Read a file from a remote repository. - * - * @param uri - * The URI of the remote repository - * @param ref - * The ref (branch/tag/etc.) to read - * @param path - * The relative path (inside the repo) to the file to read - * @return the file content. - * @throws GitAPIException - * If the ref have an invalid or ambiguous name, or it does - * not exist in the repository, - * @throws IOException - * If the object does not exist or is too large - * @since 3.5 - * - * @deprecated Use {@link #readFileWithMode(String, String, String)} - * instead - */ - @Deprecated - public default byte[] readFile(String uri, String ref, String path) - throws GitAPIException, IOException { - return readFileWithMode(uri, ref, path).getContents(); - } - - /** * Read contents and mode (i.e. permissions) of the file from a remote * repository. * @@ -255,7 +229,8 @@ public class RepoCommand extends GitCommand<RevCommit> { @SuppressWarnings("serial") static class ManifestErrorException extends GitAPIException { ManifestErrorException(Throwable cause) { - super(RepoText.get().invalidManifest, cause); + super(RepoText.get().invalidManifest + " " + cause.getMessage(), //$NON-NLS-1$ + cause); } } @@ -615,6 +590,7 @@ public class RepoCommand extends GitCommand<RevCommit> { p.setUrl(proj.getUrl()); p.addCopyFiles(proj.getCopyFiles()); p.addLinkFiles(proj.getLinkFiles()); + p.setUpstream(proj.getUpstream()); ret.add(p); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index 8deb7386a6..2630da34e0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -38,6 +38,8 @@ public class RepoProject implements Comparable<RepoProject> { private final Set<String> groups; private final List<CopyFile> copyfiles; private final List<LinkFile> linkfiles; + private String upstream; + private String destBranch; private String recommendShallow; private String url; private String defaultRevision; @@ -389,6 +391,57 @@ public class RepoProject implements Comparable<RepoProject> { this.linkfiles.clear(); } + /** + * Return the upstream attribute of the project + * + * @return the upstream value if present, null otherwise. + * + * @since 6.10 + */ + public String getUpstream() { + return this.upstream; + } + + /** + * Return the dest-branch attribute of the project + * + * @return the dest-branch value if present, null otherwise. + * + * @since 6.10 + */ + public String getDestBranch() { + return this.destBranch; + } + + /** + * Set the upstream attribute of the project + * + * Name of the git ref in which a sha1 can be found, when the revision is a + * sha1. + * + * @param upstream + * value of the attribute in the manifest + * + * @since 6.10 + */ + public void setUpstream(String upstream) { + this.upstream = upstream; + } + + /** + * Set the dest-branch attribute of the project + * + * Name of a Git branch. + * + * @param destBranch + * value of the attribute in the manifest + * + * @since 6.10 + */ + public void setDestBranch(String destBranch) { + this.destBranch = destBranch; + } + private String getPathWithSlash() { if (path.endsWith("/")) { //$NON-NLS-1$ return path; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 700b54a7a6..bf252f9968 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -94,8 +94,6 @@ public class JGitText extends TranslationBundle { /***/ public String binaryHunkInvalidLength; /***/ public String binaryHunkLineTooShort; /***/ public String binaryHunkMissingNewline; - /***/ public String bitmapAccessErrorForPackfile; - /***/ public String bitmapFailedToGet; /***/ public String bitmapMissingObject; /***/ public String bitmapsMustBePrepared; /***/ public String bitmapUseNoopNoListener; @@ -108,6 +106,7 @@ public class JGitText extends TranslationBundle { /***/ public String buildingBitmaps; /***/ public String cachedPacksPreventsIndexCreation; /***/ public String cachedPacksPreventsListingObjects; + /***/ public String cacheRegionAllOrNoneNull; /***/ public String cannotAccessLastModifiedForSafeDeletion; /***/ public String cannotBeCombined; /***/ public String cannotBeRecursiveWhenTreesAreIncluded; @@ -295,6 +294,7 @@ public class JGitText extends TranslationBundle { /***/ public String deleteTagUnexpectedResult; /***/ public String deletingBranches; /***/ public String deletingNotSupported; + /***/ public String deprecatedTrustFolderStat; /***/ public String depthMustBeAt1; /***/ public String depthWithUnshallow; /***/ public String destinationIsNotAWildcard; @@ -315,6 +315,9 @@ public class JGitText extends TranslationBundle { /***/ public String downloadCancelled; /***/ public String downloadCancelledDuringIndexing; /***/ public String duplicateAdvertisementsOf; + /***/ public String duplicateCacheTablesGiven; + /***/ public String duplicatePackExtensionsForCacheTables; + /***/ public String duplicatePackExtensionsSet; /***/ public String duplicateRef; /***/ public String duplicateRefAttribute; /***/ public String duplicateRemoteRefUpdateIsIllegal; @@ -490,6 +493,7 @@ public class JGitText extends TranslationBundle { /***/ public String invalidTimeUnitValue2; /***/ public String invalidTimeUnitValue3; /***/ public String invalidTreeZeroLengthName; + /***/ public String invalidTrustStat; /***/ public String invalidURL; /***/ public String invalidWildcards; /***/ public String invalidRefSpec; @@ -554,6 +558,8 @@ public class JGitText extends TranslationBundle { /***/ public String month; /***/ public String months; /***/ public String monthsAgo; + /***/ public String multiPackIndexUnexpectedSize; + /***/ public String multiPackIndexWritingCancelled; /***/ public String multipleMergeBasesFor; /***/ public String nameMustNotBeNullOrEmpty; /***/ public String need2Arguments; @@ -569,6 +575,8 @@ public class JGitText extends TranslationBundle { /***/ public String noMergeHeadSpecified; /***/ public String nonBareLinkFilesNotSupported; /***/ public String nonCommitToHeads; + /***/ public String noPackExtConfigurationGiven; + /***/ public String noPackExtGivenForConfiguration; /***/ public String noPathAttributesFound; /***/ public String noSuchRef; /***/ public String noSuchRefKnown; @@ -601,7 +609,6 @@ public class JGitText extends TranslationBundle { /***/ public String oldIdMustNotBeNull; /***/ public String onlyOneFetchSupported; /***/ public String onlyOneOperationCallPerConnectionIsSupported; - /***/ public String onlyOpenPgpSupportedForSigning; /***/ public String openFilesMustBeAtLeast1; /***/ public String openingConnection; /***/ public String operationCanceled; @@ -623,6 +630,8 @@ public class JGitText extends TranslationBundle { /***/ public String packingCancelledDuringObjectsWriting; /***/ public String packObjectCountMismatch; /***/ public String packRefs; + /***/ public String packRefsFailed; + /***/ public String packRefsSuccessful; /***/ public String packSizeNotSetYet; /***/ public String packTooLargeForIndexVersion1; /***/ public String packWasDeleted; @@ -639,6 +648,7 @@ public class JGitText extends TranslationBundle { /***/ public String personIdentEmailNonNull; /***/ public String personIdentNameNonNull; /***/ public String postCommitHookFailed; + /***/ public String precedenceTrustConfig; /***/ public String prefixRemote; /***/ public String problemWithResolvingPushRefSpecsLocally; /***/ public String progressMonUploading; @@ -666,8 +676,6 @@ public class JGitText extends TranslationBundle { /***/ public String readerIsRequired; /***/ public String readingObjectsFromLocalRepositoryFailed; /***/ public String readLastModifiedFailed; - /***/ public String readPipeIsNotAllowed; - /***/ public String readPipeIsNotAllowedRequiredPermission; /***/ public String readTimedOut; /***/ public String receivePackObjectTooLarge1; /***/ public String receivePackObjectTooLarge2; @@ -747,6 +755,8 @@ public class JGitText extends TranslationBundle { /***/ public String shutdownCleanup; /***/ public String shutdownCleanupFailed; /***/ public String shutdownCleanupListenerFailed; + /***/ public String signatureServiceConflict; + /***/ public String signatureTypeUnknown; /***/ public String signatureVerificationError; /***/ public String signatureVerificationUnavailable; /***/ public String signedTagMessageNoLf; @@ -833,6 +843,7 @@ public class JGitText extends TranslationBundle { /***/ public String unableToCheckConnectivity; /***/ public String unableToCreateNewObject; /***/ public String unableToReadFullInt; + /***/ public String unableToReadFullArray; /***/ public String unableToReadPackfile; /***/ public String unableToRemovePath; /***/ public String unableToWrite; @@ -858,6 +869,7 @@ public class JGitText extends TranslationBundle { /***/ public String unknownObjectInIndex; /***/ public String unknownObjectType; /***/ public String unknownObjectType2; + /***/ public String unknownPackExtension; /***/ public String unknownPositionEncoding; /***/ public String unknownRefStorageFormat; /***/ public String unknownRepositoryFormat; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java index 0d9815eceb..55539e2a66 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java @@ -52,6 +52,7 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.EmptyTreeIterator; import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.NB; /** @@ -71,6 +72,9 @@ public class CommitGraphWriter { private static final int MAX_CHANGED_PATHS = 512; + private static final PathDiffCalculator PATH_DIFF_CALCULATOR + = new PathDiffCalculator(); + private final int hashsz; private final GraphCommits graphCommits; @@ -374,37 +378,6 @@ public class CommitGraphWriter { return generations; } - private static Optional<HashSet<ByteBuffer>> computeBloomFilterPaths( - ObjectReader or, RevCommit cmit) throws MissingObjectException, - IncorrectObjectTypeException, CorruptObjectException, IOException { - HashSet<ByteBuffer> paths = new HashSet<>(); - try (TreeWalk walk = new TreeWalk(null, or)) { - walk.setRecursive(true); - if (cmit.getParentCount() == 0) { - walk.addTree(new EmptyTreeIterator()); - } else { - walk.addTree(cmit.getParent(0).getTree()); - } - walk.addTree(cmit.getTree()); - while (walk.next()) { - if (walk.idEqual(0, 1)) { - continue; - } - byte[] rawPath = walk.getRawPath(); - paths.add(ByteBuffer.wrap(rawPath)); - for (int i = 0; i < rawPath.length; i++) { - if (rawPath[i] == '/') { - paths.add(ByteBuffer.wrap(rawPath, 0, i)); - } - if (paths.size() > MAX_CHANGED_PATHS) { - return Optional.empty(); - } - } - } - } - return Optional.of(paths); - } - private BloomFilterChunks computeBloomFilterChunks(ProgressMonitor monitor) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { @@ -435,8 +408,8 @@ public class CommitGraphWriter { filtersReused++; } else { filtersComputed++; - Optional<HashSet<ByteBuffer>> paths = computeBloomFilterPaths( - graphCommits.getObjectReader(), cmit); + Optional<HashSet<ByteBuffer>> paths = PATH_DIFF_CALCULATOR + .changedPaths(graphCommits.getObjectReader(), cmit); if (paths.isEmpty()) { cpf = ChangedPathFilter.FULL; } else { @@ -473,6 +446,44 @@ public class CommitGraphWriter { } } + // Visible for testing + static class PathDiffCalculator { + + // Walk steps in the last invocation of changedPaths + int stepCounter; + + Optional<HashSet<ByteBuffer>> changedPaths( + ObjectReader or, RevCommit cmit) throws MissingObjectException, + IncorrectObjectTypeException, CorruptObjectException, IOException { + stepCounter = 0; + HashSet<ByteBuffer> paths = new HashSet<>(); + try (TreeWalk walk = new TreeWalk(null, or)) { + walk.setRecursive(true); + walk.setFilter(TreeFilter.ANY_DIFF); + if (cmit.getParentCount() == 0) { + walk.addTree(new EmptyTreeIterator()); + } else { + walk.addTree(cmit.getParent(0).getTree()); + } + walk.addTree(cmit.getTree()); + while (walk.next()) { + stepCounter += 1; + byte[] rawPath = walk.getRawPath(); + paths.add(ByteBuffer.wrap(rawPath)); + for (int i = 0; i < rawPath.length; i++) { + if (rawPath[i] == '/') { + paths.add(ByteBuffer.wrap(rawPath, 0, i)); + } + if (paths.size() > MAX_CHANGED_PATHS) { + return Optional.empty(); + } + } + } + } + return Optional.of(paths); + } + } + private static class ChunkHeader { final int id; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java new file mode 100644 index 0000000000..295b702fa7 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + +import java.util.List; + +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * Aggregates values for all given {@link BlockCacheStats}. + */ +class AggregatedBlockCacheStats implements BlockCacheStats { + private final List<BlockCacheStats> blockCacheStats; + + static BlockCacheStats fromStatsList( + List<BlockCacheStats> blockCacheStats) { + if (blockCacheStats.size() == 1) { + return blockCacheStats.get(0); + } + return new AggregatedBlockCacheStats(blockCacheStats); + } + + private AggregatedBlockCacheStats(List<BlockCacheStats> blockCacheStats) { + this.blockCacheStats = blockCacheStats; + } + + @Override + public String getName() { + return AggregatedBlockCacheStats.class.getName(); + } + + @Override + public long[] getCurrentSize() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getCurrentSize()); + } + return sums; + } + + @Override + public long[] getHitCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getHitCount()); + } + return sums; + } + + @Override + public long[] getMissCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getMissCount()); + } + return sums; + } + + @Override + public long[] getTotalRequestCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getTotalRequestCount()); + } + return sums; + } + + @Override + public long[] getHitRatio() { + long[] hit = getHitCount(); + long[] miss = getMissCount(); + long[] ratio = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < ratio.length; i++) { + if (i >= hit.length) { + ratio[i] = 0; + } else if (i >= miss.length) { + ratio[i] = 100; + } else { + long total = hit[i] + miss[i]; + ratio[i] = total == 0 ? 0 : hit[i] * 100 / total; + } + } + return ratio; + } + + @Override + public long[] getEvictions() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getEvictions()); + } + return sums; + } + + private static long[] emptyPackStats() { + return new long[PackExt.values().length]; + } + + private static long[] add(long[] first, long[] second) { + long[] sums = new long[Integer.max(first.length, second.length)]; + int i; + for (i = 0; i < Integer.min(first.length, second.length); i++) { + sums[i] = first[i] + second[i]; + } + for (int j = i; j < first.length; j++) { + sums[j] = first[i]; + } + for (int j = i; j < second.length; j++) { + sums[j] = second[i]; + } + return sums; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java index d0907bcc8d..587d482583 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; import java.time.Duration; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -47,6 +48,11 @@ import org.eclipse.jgit.internal.storage.pack.PackExt; * invocations is also fixed in size. */ final class ClockBlockCacheTable implements DfsBlockCacheTable { + /** + * Table name. + */ + private final String name; + /** Number of entries in {@link #table}. */ private final int tableSize; @@ -129,14 +135,20 @@ final class ClockBlockCacheTable implements DfsBlockCacheTable { -1, 0, null); clockHand.next = clockHand; - this.dfsBlockCacheStats = new DfsBlockCacheStats(); + this.name = cfg.getName(); + this.dfsBlockCacheStats = new DfsBlockCacheStats(this.name); this.refLockWaitTime = cfg.getRefLockWaitTimeConsumer(); this.indexEventConsumer = cfg.getIndexEventConsumer(); } @Override - public DfsBlockCacheStats getDfsBlockCacheStats() { - return dfsBlockCacheStats; + public List<BlockCacheStats> getBlockCacheStats() { + return List.of(dfsBlockCacheStats); + } + + @Override + public String getName() { + return name; } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java index 56719cf0f4..f8e0831e1f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java @@ -11,7 +11,10 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + import java.io.IOException; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.LongStream; @@ -97,7 +100,12 @@ public final class DfsBlockCache { double streamRatio = cfg.getStreamRatio(); maxStreamThroughCache = (long) (maxBytes * streamRatio); - dfsBlockCacheTable = new ClockBlockCacheTable(cfg); + if (!cfg.getPackExtCacheConfigurations().isEmpty()) { + dfsBlockCacheTable = PackExtBlockCacheTable + .fromBlockCacheConfigs(cfg); + } else { + dfsBlockCacheTable = new ClockBlockCacheTable(cfg); + } for (int i = 0; i < PackExt.values().length; ++i) { Integer limit = cfg.getCacheHotMap().get(PackExt.values()[i]); @@ -119,7 +127,7 @@ public final class DfsBlockCache { * @return total number of bytes in the cache, per pack file extension. */ public long[] getCurrentSize() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getCurrentSize(); + return getAggregatedBlockCacheStats().getCurrentSize(); } /** @@ -138,7 +146,7 @@ public final class DfsBlockCache { * extension. */ public long[] getHitCount() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getHitCount(); + return getAggregatedBlockCacheStats().getHitCount(); } /** @@ -149,7 +157,7 @@ public final class DfsBlockCache { * extension. */ public long[] getMissCount() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getMissCount(); + return getAggregatedBlockCacheStats().getMissCount(); } /** @@ -158,8 +166,7 @@ public final class DfsBlockCache { * @return total number of requests (hit + miss), per pack file extension. */ public long[] getTotalRequestCount() { - return dfsBlockCacheTable.getDfsBlockCacheStats() - .getTotalRequestCount(); + return getAggregatedBlockCacheStats().getTotalRequestCount(); } /** @@ -168,7 +175,7 @@ public final class DfsBlockCache { * @return hit ratios */ public long[] getHitRatio() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getHitRatio(); + return getAggregatedBlockCacheStats().getHitRatio(); } /** @@ -179,7 +186,18 @@ public final class DfsBlockCache { * file extension. */ public long[] getEvictions() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getEvictions(); + return getAggregatedBlockCacheStats().getEvictions(); + } + + /** + * Get the list of {@link BlockCacheStats} for all underlying caches. + * <p> + * Useful in monitoring caches with breakdown. + * + * @return the list of {@link BlockCacheStats} for all underlying caches. + */ + public List<BlockCacheStats> getAllBlockCacheStats() { + return dfsBlockCacheTable.getBlockCacheStats(); } /** @@ -259,6 +277,11 @@ public final class DfsBlockCache { return dfsBlockCacheTable.get(key, position); } + private BlockCacheStats getAggregatedBlockCacheStats() { + return AggregatedBlockCacheStats + .fromStatsList(dfsBlockCacheTable.getBlockCacheStats()); + } + static final class Ref<T> { final DfsStreamKey key; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index 77273cec61..17bf51876d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -11,17 +11,27 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_CACHE_PREFIX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_LIMIT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_SIZE; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO; +import java.io.PrintWriter; import java.text.MessageFormat; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -41,25 +51,103 @@ public class DfsBlockCacheConfig { /** Default number of max cache hits. */ public static final int DEFAULT_CACHE_HOT_MAX = 1; + static final String DEFAULT_NAME = "<default>"; //$NON-NLS-1$ + + private String name; + private long blockLimit; + private int blockSize; + private double streamRatio; + private int concurrencyLevel; private Consumer<Long> refLock; + private Map<PackExt, Integer> cacheHotMap; private IndexEventConsumer indexEventConsumer; + private List<DfsBlockCachePackExtConfig> packExtCacheConfigurations; + /** * Create a default configuration. */ public DfsBlockCacheConfig() { + name = DEFAULT_NAME; setBlockLimit(32 * MB); setBlockSize(64 * KB); setStreamRatio(0.30); setConcurrencyLevel(32); cacheHotMap = Collections.emptyMap(); + packExtCacheConfigurations = Collections.emptyList(); + } + + /** + * Print the current cache configuration to the given {@link PrintWriter}. + * + * @param writer + * {@link PrintWriter} to write the cache's configuration to. + */ + public void print(PrintWriter writer) { + print(/* linePrefix= */ "", /* pad= */ " ", writer); //$NON-NLS-1$//$NON-NLS-2$ + } + + /** + * Print the current cache configuration to the given {@link PrintWriter}. + * + * @param linePrefix + * prefix to prepend all writen lines with. Ex a string of 0 or + * more " " entries. + * @param pad + * filler used to extend linePrefix. Ex a multiple of " ". + * @param writer + * {@link PrintWriter} to write the cache's configuration to. + */ + @SuppressWarnings("nls") + private void print(String linePrefix, String pad, PrintWriter writer) { + String currentPrefixLevel = linePrefix; + if (!name.isEmpty() || !packExtCacheConfigurations.isEmpty()) { + writer.println(linePrefix + "Name: " + + (name.isEmpty() ? DEFAULT_NAME : this.name)); + currentPrefixLevel += pad; + } + writer.println(currentPrefixLevel + "BlockLimit: " + blockLimit); + writer.println(currentPrefixLevel + "BlockSize: " + blockSize); + writer.println(currentPrefixLevel + "StreamRatio: " + streamRatio); + writer.println( + currentPrefixLevel + "ConcurrencyLevel: " + concurrencyLevel); + for (Map.Entry<PackExt, Integer> entry : cacheHotMap.entrySet()) { + writer.println(currentPrefixLevel + "CacheHotMapEntry: " + + entry.getKey() + " : " + entry.getValue()); + } + for (DfsBlockCachePackExtConfig extConfig : packExtCacheConfigurations) { + extConfig.print(currentPrefixLevel, pad, writer); + } + } + + /** + * Get the name for the block cache configured by this cache config. + * + * @return the name for the block cache configured by this cache config. + */ + public String getName() { + return name; + } + + /** + * Set the name for the block cache configured by this cache config. + * <p> + * Made visible for testing. + * + * @param name + * the name for the block cache configured by this cache config. + * @return {@code this} + */ + DfsBlockCacheConfig setName(String name) { + this.name = name; + return this; } /** @@ -77,10 +165,10 @@ public class DfsBlockCacheConfig { * Set maximum number bytes of heap memory to dedicate to caching pack file * data. * <p> - * It is strongly recommended to set the block limit to be an integer multiple - * of the block size. This constraint is not enforced by this method (since - * it may be called before {@link #setBlockSize(int)}), but it is enforced by - * {@link #fromConfig(Config)}. + * It is strongly recommended to set the block limit to be an integer + * multiple of the block size. This constraint is not enforced by this + * method (since it may be called before {@link #setBlockSize(int)}), but it + * is enforced by {@link #fromConfig(Config)}. * * @param newLimit * maximum number bytes of heap memory to dedicate to caching @@ -89,9 +177,9 @@ public class DfsBlockCacheConfig { */ public DfsBlockCacheConfig setBlockLimit(long newLimit) { if (newLimit <= 0) { - throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().blockLimitNotPositive, - Long.valueOf(newLimit))); + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().blockLimitNotPositive, + Long.valueOf(newLimit))); } blockLimit = newLimit; return this; @@ -211,12 +299,24 @@ public class DfsBlockCacheConfig { * map of hot count per pack extension for {@code DfsBlockCache}. * @return {@code this} */ + /* + * TODO The cache HotMap configuration should be set as a config option and + * not passed in through a setter. + */ public DfsBlockCacheConfig setCacheHotMap( Map<PackExt, Integer> cacheHotMap) { this.cacheHotMap = Collections.unmodifiableMap(cacheHotMap); + setCacheHotMapToPackExtConfigs(this.cacheHotMap); return this; } + private void setCacheHotMapToPackExtConfigs( + Map<PackExt, Integer> cacheHotMap) { + for (DfsBlockCachePackExtConfig packExtConfig : packExtCacheConfigurations) { + packExtConfig.setCacheHotMap(cacheHotMap); + } + } + /** * Get the consumer of cache index events. * @@ -240,61 +340,121 @@ public class DfsBlockCacheConfig { } /** + * Get the list of pack ext cache configs. + * + * @return the list of pack ext cache configs. + */ + List<DfsBlockCachePackExtConfig> getPackExtCacheConfigurations() { + return packExtCacheConfigurations; + } + + /** + * Set the list of pack ext cache configs. + * + * Made visible for testing. + * + * @param packExtCacheConfigurations + * the list of pack ext cache configs to set. + * @return {@code this} + */ + DfsBlockCacheConfig setPackExtCacheConfigurations( + List<DfsBlockCachePackExtConfig> packExtCacheConfigurations) { + this.packExtCacheConfigurations = packExtCacheConfigurations; + return this; + } + + /** * Update properties by setting fields from the configuration. * <p> * If a property is not defined in the configuration, then it is left * unmodified. * <p> - * Enforces certain constraints on the combination of settings in the config, - * for example that the block limit is a multiple of the block size. + * Enforces certain constraints on the combination of settings in the + * config, for example that the block limit is a multiple of the block size. * * @param rc * configuration to read properties from. * @return {@code this} */ public DfsBlockCacheConfig fromConfig(Config rc) { - long cfgBlockLimit = rc.getLong( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_BLOCK_LIMIT, - getBlockLimit()); - int cfgBlockSize = rc.getInt( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_BLOCK_SIZE, + fromConfig(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, rc); + loadPackExtConfigs(rc); + return this; + } + + private void fromConfig(String section, String subSection, Config rc) { + long cfgBlockLimit = rc.getLong(section, subSection, + CONFIG_KEY_BLOCK_LIMIT, getBlockLimit()); + int cfgBlockSize = rc.getInt(section, subSection, CONFIG_KEY_BLOCK_SIZE, getBlockSize()); if (cfgBlockLimit % cfgBlockSize != 0) { throw new IllegalArgumentException(MessageFormat.format( JGitText.get().blockLimitNotMultipleOfBlockSize, - Long.valueOf(cfgBlockLimit), - Long.valueOf(cfgBlockSize))); + Long.valueOf(cfgBlockLimit), Long.valueOf(cfgBlockSize))); } + // Set name only if `core dfs` is configured, otherwise fall back to the + // default. + if (rc.getSubsections(section).contains(subSection)) { + this.name = subSection; + } setBlockLimit(cfgBlockLimit); setBlockSize(cfgBlockSize); - setConcurrencyLevel(rc.getInt( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_CONCURRENCY_LEVEL, - getConcurrencyLevel())); + setConcurrencyLevel(rc.getInt(section, subSection, + CONFIG_KEY_CONCURRENCY_LEVEL, getConcurrencyLevel())); - String v = rc.getString( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_STREAM_RATIO); + String v = rc.getString(section, subSection, CONFIG_KEY_STREAM_RATIO); if (v != null) { try { setStreamRatio(Double.parseDouble(v)); } catch (NumberFormatException e) { throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().enumValueNotSupported3, - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_STREAM_RATIO, v), e); + JGitText.get().enumValueNotSupported3, section, + subSection, CONFIG_KEY_STREAM_RATIO, v), e); } } - return this; + } + + private void loadPackExtConfigs(Config config) { + List<String> subSections = config.getSubsections(CONFIG_CORE_SECTION) + .stream() + .filter(section -> section.startsWith(CONFIG_DFS_CACHE_PREFIX)) + .collect(Collectors.toList()); + if (subSections.size() == 0) { + return; + } + ArrayList<DfsBlockCachePackExtConfig> cacheConfigs = new ArrayList<>(); + Set<PackExt> extensionsSeen = new HashSet<>(); + for (String subSection : subSections) { + var cacheConfig = DfsBlockCachePackExtConfig.fromConfig(config, + CONFIG_CORE_SECTION, subSection); + Set<PackExt> packExtsDuplicates = intersection(extensionsSeen, + cacheConfig.packExts); + if (packExtsDuplicates.size() > 0) { + String duplicatePackExts = packExtsDuplicates.stream() + .map(PackExt::toString) + .collect(Collectors.joining(",")); //$NON-NLS-1$ + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().duplicatePackExtensionsSet, + CONFIG_CORE_SECTION, subSection, + CONFIG_KEY_PACK_EXTENSIONS, duplicatePackExts)); + } + extensionsSeen.addAll(cacheConfig.packExts); + cacheConfigs.add(cacheConfig); + } + packExtCacheConfigurations = cacheConfigs; + setCacheHotMapToPackExtConfigs(this.cacheHotMap); + } + + private static <T> Set<T> intersection(Set<T> first, Set<T> second) { + Set<T> ret = new HashSet<>(); + for (T entry : second) { + if (first.contains(entry)) { + ret.add(entry); + } + } + return ret; } /** Consumer of DfsBlockCache loading and eviction events for indexes. */ @@ -346,4 +506,102 @@ public class DfsBlockCacheConfig { return false; } } -}
\ No newline at end of file + + /** + * A configuration for a single cache table storing 1 or more Pack + * extensions. + * <p> + * The current pack ext cache tables implementation supports the same + * parameters the ClockBlockCacheTable (current default implementation). + * <p> + * Configuration falls back to the defaults coded values defined in the + * {@link DfsBlockCacheConfig} when not set on each cache table + * configuration and NOT the values of the basic dfs section. + * <p> + * <code> + * + * Format: + * [core "dfs.packCache"] + * packExtensions = "PACK" + * blockSize = 512 + * blockLimit = 100 + * concurrencyLevel = 5 + * + * [core "dfs.multipleExtensionCache"] + * packExtensions = "INDEX REFTABLE BITMAP_INDEX" + * blockSize = 512 + * blockLimit = 100 + * concurrencyLevel = 5 + * </code> + */ + static class DfsBlockCachePackExtConfig { + // Set of pack extensions that will map to the cache instance. + private final EnumSet<PackExt> packExts; + + // Configuration for the cache instance. + private final DfsBlockCacheConfig packExtCacheConfiguration; + + /** + * Made visible for testing. + * + * @param packExts + * Set of {@link PackExt}s associated to this cache config. + * @param packExtCacheConfiguration + * {@link DfsBlockCacheConfig} for this cache config. + */ + DfsBlockCachePackExtConfig(EnumSet<PackExt> packExts, + DfsBlockCacheConfig packExtCacheConfiguration) { + this.packExts = packExts; + this.packExtCacheConfiguration = packExtCacheConfiguration; + } + + Set<PackExt> getPackExts() { + return packExts; + } + + DfsBlockCacheConfig getPackExtCacheConfiguration() { + return packExtCacheConfiguration; + } + + void setCacheHotMap(Map<PackExt, Integer> cacheHotMap) { + Map<PackExt, Integer> packExtHotMap = packExts.stream() + .filter(cacheHotMap::containsKey) + .collect(Collectors.toUnmodifiableMap(Function.identity(), + cacheHotMap::get)); + packExtCacheConfiguration.setCacheHotMap(packExtHotMap); + } + + private static DfsBlockCachePackExtConfig fromConfig(Config config, + String section, String subSection) { + String packExtensions = config.getString(section, subSection, + CONFIG_KEY_PACK_EXTENSIONS); + if (packExtensions == null) { + throw new IllegalArgumentException( + JGitText.get().noPackExtGivenForConfiguration); + } + String[] extensions = packExtensions.split(" ", -1); //$NON-NLS-1$ + Set<PackExt> packExts = new HashSet<>(extensions.length); + for (String extension : extensions) { + try { + packExts.add(PackExt.valueOf(extension)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().unknownPackExtension, section, + subSection, CONFIG_KEY_PACK_EXTENSIONS, extension), + e); + } + } + + DfsBlockCacheConfig dfsBlockCacheConfig = new DfsBlockCacheConfig(); + dfsBlockCacheConfig.fromConfig(section, subSection, config); + return new DfsBlockCachePackExtConfig(EnumSet.copyOf(packExts), + dfsBlockCacheConfig); + } + + void print(String linePrefix, String pad, PrintWriter writer) { + packExtCacheConfiguration.print(linePrefix, pad, writer); + writer.println(linePrefix + pad + "PackExts: " //$NON-NLS-1$ + + packExts.stream().sorted().collect(Collectors.toList())); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java new file mode 100644 index 0000000000..436f57437e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * Keeps track of stats for a Block Cache table. + */ +class DfsBlockCacheStats implements BlockCacheStats { + private final String name; + + /** + * Number of times a block was found in the cache, per pack file extension. + */ + private final AtomicReference<AtomicLong[]> statHit; + + /** + * Number of times a block was not found, and had to be loaded, per pack + * file extension. + */ + private final AtomicReference<AtomicLong[]> statMiss; + + /** + * Number of blocks evicted due to cache being full, per pack file + * extension. + */ + private final AtomicReference<AtomicLong[]> statEvict; + + /** + * Number of bytes currently loaded in the cache, per pack file extension. + */ + private final AtomicReference<AtomicLong[]> liveBytes; + + DfsBlockCacheStats() { + this(""); //$NON-NLS-1$ + } + + DfsBlockCacheStats(String name) { + this.name = name; + statHit = new AtomicReference<>(newCounters()); + statMiss = new AtomicReference<>(newCounters()); + statEvict = new AtomicReference<>(newCounters()); + liveBytes = new AtomicReference<>(newCounters()); + } + + @Override + public String getName() { + return name; + } + + /** + * Increment the {@code statHit} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementHit(DfsStreamKey key) { + getStat(statHit, key).incrementAndGet(); + } + + /** + * Increment the {@code statMiss} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementMiss(DfsStreamKey key) { + getStat(statMiss, key).incrementAndGet(); + } + + /** + * Increment the {@code statEvict} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementEvict(DfsStreamKey key) { + getStat(statEvict, key).incrementAndGet(); + } + + /** + * Add {@code size} to the {@code liveBytes} count. + * + * @param key + * key identifying which liveBytes entry to update. + * @param size + * amount to increment the count by. + */ + void addToLiveBytes(DfsStreamKey key, long size) { + getStat(liveBytes, key).addAndGet(size); + } + + @Override + public long[] getCurrentSize() { + return getStatVals(liveBytes); + } + + @Override + public long[] getHitCount() { + return getStatVals(statHit); + } + + @Override + public long[] getMissCount() { + return getStatVals(statMiss); + } + + @Override + public long[] getTotalRequestCount() { + AtomicLong[] hit = statHit.get(); + AtomicLong[] miss = statMiss.get(); + long[] cnt = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < hit.length; i++) { + cnt[i] += hit[i].get(); + } + for (int i = 0; i < miss.length; i++) { + cnt[i] += miss[i].get(); + } + return cnt; + } + + @Override + public long[] getHitRatio() { + AtomicLong[] hit = statHit.get(); + AtomicLong[] miss = statMiss.get(); + long[] ratio = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < ratio.length; i++) { + if (i >= hit.length) { + ratio[i] = 0; + } else if (i >= miss.length) { + ratio[i] = 100; + } else { + long hitVal = hit[i].get(); + long missVal = miss[i].get(); + long total = hitVal + missVal; + ratio[i] = total == 0 ? 0 : hitVal * 100 / total; + } + } + return ratio; + } + + @Override + public long[] getEvictions() { + return getStatVals(statEvict); + } + + private static AtomicLong[] newCounters() { + AtomicLong[] ret = new AtomicLong[PackExt.values().length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = new AtomicLong(); + } + return ret; + } + + private static long[] getStatVals(AtomicReference<AtomicLong[]> stat) { + AtomicLong[] stats = stat.get(); + long[] cnt = new long[stats.length]; + for (int i = 0; i < stats.length; i++) { + cnt[i] = stats[i].get(); + } + return cnt; + } + + private static AtomicLong getStat(AtomicReference<AtomicLong[]> stats, + DfsStreamKey key) { + int pos = key.packExtPos; + while (true) { + AtomicLong[] vals = stats.get(); + if (pos < vals.length) { + return vals[pos]; + } + AtomicLong[] expect = vals; + vals = new AtomicLong[Math.max(pos + 1, PackExt.values().length)]; + System.arraycopy(expect, 0, vals, 0, expect.length); + for (int i = expect.length; i < vals.length; i++) { + vals[i] = new AtomicLong(); + } + if (stats.compareAndSet(expect, vals)) { + return vals[pos]; + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java index 701d1fdce3..c3fd07b7bb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java @@ -11,10 +11,7 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import org.eclipse.jgit.internal.storage.pack.PackExt; +import java.util.List; /** * Block cache table. @@ -129,99 +126,43 @@ public interface DfsBlockCacheTable { <T> T get(DfsStreamKey key, long position); /** - * Get the DfsBlockCacheStats object for this block cache table's - * statistics. + * Get the list of {@link BlockCacheStats} held by this cache. + * <p> + * The returned list has a {@link BlockCacheStats} per configured cache + * table, with a minimum of 1 {@link BlockCacheStats} object returned. + * + * Use {@link AggregatedBlockCacheStats} to combine the results of the stats + * in the list for an aggregated view of the cache's stats. * - * @return the DfsBlockCacheStats tracking this block cache table's - * statistics. + * @return the list of {@link BlockCacheStats} held by this cache. */ - DfsBlockCacheStats getDfsBlockCacheStats(); + List<BlockCacheStats> getBlockCacheStats(); /** - * Keeps track of stats for a Block Cache table. + * Get the name of the table. + * + * @return this table's name. */ - class DfsBlockCacheStats { - /** - * Number of times a block was found in the cache, per pack file - * extension. - */ - private final AtomicReference<AtomicLong[]> statHit; - - /** - * Number of times a block was not found, and had to be loaded, per pack - * file extension. - */ - private final AtomicReference<AtomicLong[]> statMiss; - - /** - * Number of blocks evicted due to cache being full, per pack file - * extension. - */ - private final AtomicReference<AtomicLong[]> statEvict; - - /** - * Number of bytes currently loaded in the cache, per pack file - * extension. - */ - private final AtomicReference<AtomicLong[]> liveBytes; - - DfsBlockCacheStats() { - statHit = new AtomicReference<>(newCounters()); - statMiss = new AtomicReference<>(newCounters()); - statEvict = new AtomicReference<>(newCounters()); - liveBytes = new AtomicReference<>(newCounters()); - } - - /** - * Increment the {@code statHit} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementHit(DfsStreamKey key) { - getStat(statHit, key).incrementAndGet(); - } - - /** - * Increment the {@code statMiss} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementMiss(DfsStreamKey key) { - getStat(statMiss, key).incrementAndGet(); - } + String getName(); - /** - * Increment the {@code statEvict} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementEvict(DfsStreamKey key) { - getStat(statEvict, key).incrementAndGet(); - } + /** + * Provides methods used with Block Cache statistics. + */ + interface BlockCacheStats { /** - * Add {@code size} to the {@code liveBytes} count. + * Get the name of the block cache generating this instance. * - * @param key - * key identifying which liveBytes entry to update. - * @param size - * amount to increment the count by. + * @return this cache's name. */ - void addToLiveBytes(DfsStreamKey key, long size) { - getStat(liveBytes, key).addAndGet(size); - } + String getName(); /** * Get total number of bytes in the cache, per pack file extension. * * @return total number of bytes in the cache, per pack file extension. */ - long[] getCurrentSize() { - return getStatVals(liveBytes); - } + long[] getCurrentSize(); /** * Get number of requests for items in the cache, per pack file @@ -230,9 +171,7 @@ public interface DfsBlockCacheTable { * @return the number of requests for items in the cache, per pack file * extension. */ - long[] getHitCount() { - return getStatVals(statHit); - } + long[] getHitCount(); /** * Get number of requests for items not in the cache, per pack file @@ -241,9 +180,7 @@ public interface DfsBlockCacheTable { * @return the number of requests for items not in the cache, per pack * file extension. */ - long[] getMissCount() { - return getStatVals(statMiss); - } + long[] getMissCount(); /** * Get total number of requests (hit + miss), per pack file extension. @@ -251,42 +188,14 @@ public interface DfsBlockCacheTable { * @return total number of requests (hit + miss), per pack file * extension. */ - long[] getTotalRequestCount() { - AtomicLong[] hit = statHit.get(); - AtomicLong[] miss = statMiss.get(); - long[] cnt = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < hit.length; i++) { - cnt[i] += hit[i].get(); - } - for (int i = 0; i < miss.length; i++) { - cnt[i] += miss[i].get(); - } - return cnt; - } + long[] getTotalRequestCount(); /** * Get hit ratios. * * @return hit ratios. */ - long[] getHitRatio() { - AtomicLong[] hit = statHit.get(); - AtomicLong[] miss = statMiss.get(); - long[] ratio = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < ratio.length; i++) { - if (i >= hit.length) { - ratio[i] = 0; - } else if (i >= miss.length) { - ratio[i] = 100; - } else { - long hitVal = hit[i].get(); - long missVal = miss[i].get(); - long total = hitVal + missVal; - ratio[i] = total == 0 ? 0 : hitVal * 100 / total; - } - } - return ratio; - } + long[] getHitRatio(); /** * Get number of evictions performed due to cache being full, per pack @@ -295,46 +204,6 @@ public interface DfsBlockCacheTable { * @return the number of evictions performed due to cache being full, * per pack file extension. */ - long[] getEvictions() { - return getStatVals(statEvict); - } - - private static AtomicLong[] newCounters() { - AtomicLong[] ret = new AtomicLong[PackExt.values().length]; - for (int i = 0; i < ret.length; i++) { - ret[i] = new AtomicLong(); - } - return ret; - } - - private static long[] getStatVals(AtomicReference<AtomicLong[]> stat) { - AtomicLong[] stats = stat.get(); - long[] cnt = new long[stats.length]; - for (int i = 0; i < stats.length; i++) { - cnt[i] = stats[i].get(); - } - return cnt; - } - - private static AtomicLong getStat(AtomicReference<AtomicLong[]> stats, - DfsStreamKey key) { - int pos = key.packExtPos; - while (true) { - AtomicLong[] vals = stats.get(); - if (pos < vals.length) { - return vals[pos]; - } - AtomicLong[] expect = vals; - vals = new AtomicLong[Math.max(pos + 1, - PackExt.values().length)]; - System.arraycopy(expect, 0, vals, 0, expect.length); - for (int i = expect.length; i < vals.length; i++) { - vals[i] = new AtomicLong(); - } - if (stats.compareAndSet(expect, vals)) { - return vals[pos]; - } - } - } + long[] getEvictions(); } -}
\ No newline at end of file +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index a177669788..199481cf33 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -18,13 +18,13 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RE import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -90,7 +90,7 @@ public class DfsGarbageCollector { private long coalesceGarbageLimit = 50 << 20; private long garbageTtlMillis = TimeUnit.DAYS.toMillis(1); - private long startTimeMillis; + private Instant startTime; private List<DfsPackFile> packsBefore; private List<DfsReftable> reftablesBefore; private List<DfsPackFile> expiredGarbagePacks; @@ -100,6 +100,7 @@ public class DfsGarbageCollector { private Set<ObjectId> allTags; private Set<ObjectId> nonHeads; private Set<ObjectId> tagTargets; + private Instant refLogExpire; /** * Initialize a garbage collector. @@ -200,6 +201,22 @@ public class DfsGarbageCollector { return this; } + + /** + * Set time limit to the reflog history. + * <p> + * Garbage Collector prunes entries from reflog history older than {@code refLogExpire} + * <p> + * + * @param refLogExpire + * instant in time which defines refLog expiration + * @return {@code this} + */ + public DfsGarbageCollector setRefLogExpire(Instant refLogExpire) { + this.refLogExpire = refLogExpire; + return this; + } + /** * Set maxUpdateIndex for the initial reftable created during conversion. * @@ -335,7 +352,7 @@ public class DfsGarbageCollector { throw new IllegalStateException( JGitText.get().supportOnlyPackIndexVersion2); - startTimeMillis = SystemReader.getInstance().getCurrentTime(); + startTime = SystemReader.getInstance().now(); ctx = objdb.newReader(); try { refdb.refresh(); @@ -418,7 +435,7 @@ public class DfsGarbageCollector { packsBefore = new ArrayList<>(packs.length); expiredGarbagePacks = new ArrayList<>(packs.length); - long now = SystemReader.getInstance().getCurrentTime(); + long now = SystemReader.getInstance().now().toEpochMilli(); for (DfsPackFile p : packs) { DfsPackDescription d = p.getPackDescription(); if (d.getPackSource() != UNREACHABLE_GARBAGE) { @@ -687,14 +704,7 @@ public class DfsGarbageCollector { pack.setBlockSize(PACK, out.blockSize()); } - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { - CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); - } + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) { try (DfsOutputStream out = objdb.writeFile(pack, @@ -713,7 +723,7 @@ public class DfsGarbageCollector { PackStatistics stats = pw.getStatistics(); pack.setPackStats(stats); - pack.setLastModified(startTimeMillis); + pack.setLastModified(startTime.toEpochMilli()); newPackDesc.add(pack); newPackStats.add(stats); newPackObj.add(pw.getObjectSet()); @@ -741,6 +751,10 @@ public class DfsGarbageCollector { compact.addAll(stack.readers()); compact.setIncludeDeletes(includeDeletes); compact.setConfig(configureReftable(reftableConfig, out)); + if(refLogExpire != null ){ + compact.setReflogExpireOldestReflogTimeMillis( + refLogExpire.toEpochMilli()); + } compact.compact(); pack.addFileExt(REFTABLE); pack.setReftableStats(compact.getStats()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index a07d8416c4..16315bf4f2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -41,12 +41,13 @@ import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackIndex; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; @@ -54,7 +55,6 @@ import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ObjectStream; -import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.IO; @@ -71,6 +71,8 @@ public class DfsInserter extends ObjectInserter { private static final int INDEX_VERSION = 2; final DfsObjDatabase db; + + private final int minBytesForObjectSizeIndex; int compression = Deflater.BEST_COMPRESSION; List<PackedObjectInfo> objectList; @@ -83,8 +85,6 @@ public class DfsInserter extends ObjectInserter { private boolean rollback; private boolean checkExisting = true; - private int minBytesForObjectSizeIndex = -1; - /** * Initialize a new inserter. * @@ -93,8 +93,9 @@ public class DfsInserter extends ObjectInserter { */ protected DfsInserter(DfsObjDatabase db) { this.db = db; - PackConfig pc = new PackConfig(db.getRepository().getConfig()); - this.minBytesForObjectSizeIndex = pc.getMinBytesForObjSizeIndex(); + this.minBytesForObjectSizeIndex = db.getRepository().getConfig().getInt( + ConfigConstants.CONFIG_PACK_SECTION, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, -1); } /** @@ -112,21 +113,6 @@ public class DfsInserter extends ObjectInserter { void setCompressionLevel(int compression) { this.compression = compression; } - - /** - * Set minimum size for an object to be included in the object size index. - * - * <p> - * Use 0 for all and -1 for nothing (the pack won't have object size index). - * - * @param minBytes - * only objects with size bigger or equal to this are included in - * the index. - */ - protected void setMinBytesForObjectSizeIndex(int minBytes) { - this.minBytesForObjectSizeIndex = minBytes; - } - @Override public DfsPackParser newPackParser(InputStream in) throws IOException { return new DfsPackParser(db, this, in); @@ -333,7 +319,7 @@ public class DfsInserter extends ObjectInserter { private static void index(OutputStream out, byte[] packHash, List<PackedObjectInfo> list) throws IOException { - PackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); + BasePackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); } void writeObjectSizeIndex(DfsPackDescription pack, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 616563ffdd..efd666ff27 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static java.util.stream.Collectors.joining; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import java.io.FileNotFoundException; import java.io.IOException; @@ -27,7 +28,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackBitmapIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AnyObjectId; @@ -771,4 +774,35 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } }; } + + /** + * Returns a writer to store the pack index in this object database. + * + * @param pack + * Pack file to which the index is associated. + * @param indexVersion + * which version of the index to write + * @return a writer to store the index associated with the pack + * @throws IOException + * when some I/O problem occurs while creating or writing to + * output stream + */ + public PackIndexWriter getPackIndexWriter( + DfsPackDescription pack, int indexVersion) + throws IOException { + return (objectsToStore, packDataChecksum) -> { + try (DfsOutputStream out = writeFile(pack, INDEX); + CountingOutputStream cnt = new CountingOutputStream(out)) { + final PackIndexWriter iw = BasePackIndexWriter + .createVersion(cnt, + indexVersion); + iw.write(objectsToStore, packDataChecksum); + pack.addFileExt(INDEX); + pack.setFileSize(INDEX, cnt.getCount()); + pack.setBlockSize(INDEX, out.blockSize()); + pack.setIndexVersion(indexVersion); + } + }; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index 86144b389c..f9c01b9d6e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -12,7 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA; @@ -249,6 +249,7 @@ public class DfsPackCompactor { try { writePack(objdb, outDesc, pw, pm); writeIndex(objdb, outDesc, pw); + writeObjectSizeIndex(objdb, outDesc, pw); PackStatistics stats = pw.getStatistics(); @@ -458,13 +459,20 @@ public class DfsPackCompactor { private static void writeIndex(DfsObjDatabase objdb, DfsPackDescription pack, PackWriter pw) throws IOException { - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); + } + + private static void writeObjectSizeIndex(DfsObjDatabase objdb, + DfsPackDescription pack, + PackWriter pw) throws IOException { + try (DfsOutputStream out = objdb.writeFile(pack, OBJECT_SIZE_INDEX)) { CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); + pw.writeObjectSizeIndex(cnt); + if (cnt.getCount() > 0) { + pack.addFileExt(OBJECT_SIZE_INDEX); + pack.setFileSize(OBJECT_SIZE_INDEX, cnt.getCount()); + pack.setBlockSize(OBJECT_SIZE_INDEX, out.blockSize()); + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 5cc2a57aba..48ed47a77c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -29,6 +29,7 @@ import java.nio.channels.Channels; import java.text.MessageFormat; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.zip.CRC32; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -106,6 +107,53 @@ public final class DfsPackFile extends BlockBasedFile { /** Lock for {@link #corruptObjects}. */ private final Object corruptObjectsLock = new Object(); + private final IndexFactory indexFactory; + + /** + * Returns the indexes for this pack. + * <p> + * We define indexes in different sub interfaces to allow implementing the + * indexes over different combinations of backends. + * <p> + * Implementations decide if/how to cache the indexes. The calling + * DfsPackFile will keep the reference to the index as long as it needs it. + */ + public interface IndexFactory { + /** + * Take care of loading the primary and reverse indexes for this pack. + */ + interface PackIndexes { + /** + * Load the primary index for the pack. + * + * @param ctx + * reader to find the raw bytes + * @return a primary index + * @throws IOException + * a problem finding/parsing the index + */ + PackIndex index(DfsReader ctx) throws IOException; + + /** + * Load the reverse index of the pack + * + * @param ctx + * reader to find the raw bytes + * @return the reverse index of the pack + * @throws IOException + * a problem finding/parsing the reverse index + */ + PackReverseIndex reverseIndex(DfsReader ctx) throws IOException; + } + + /** + * Returns a provider of the primary and reverse indexes of this pack + * + * @return an implementation of the {@link PackIndexes} interface + */ + PackIndexes getPackIndexes(); + } + /** * Construct a reader for an existing, packfile. * @@ -115,7 +163,8 @@ public final class DfsPackFile extends BlockBasedFile { * description of the pack within the DFS. */ DfsPackFile(DfsBlockCache cache, DfsPackDescription desc) { - this(cache, desc, DEFAULT_BITMAP_LOADER); + this(cache, desc, DEFAULT_BITMAP_LOADER, + new CachedStreamIndexFactory(cache, desc)); } /** @@ -127,9 +176,11 @@ public final class DfsPackFile extends BlockBasedFile { * description of the pack within the DFS * @param bitmapLoader * loader to get the bitmaps of this pack (if any) + * @param indexFactory + * an IndexFactory to get references to the indexes of this pack */ public DfsPackFile(DfsBlockCache cache, DfsPackDescription desc, - PackBitmapIndexLoader bitmapLoader) { + PackBitmapIndexLoader bitmapLoader, IndexFactory indexFactory) { super(cache, desc, PACK); int bs = desc.getBlockSize(PACK); @@ -141,6 +192,7 @@ public final class DfsPackFile extends BlockBasedFile { length = sz > 0 ? sz : -1; this.bitmapLoader = bitmapLoader; + this.indexFactory = indexFactory; } /** @@ -195,19 +247,10 @@ public final class DfsPackFile extends BlockBasedFile { Repository.getGlobalListenerList() .dispatch(new BeforeDfsPackIndexLoadedEvent(this)); try { - DfsStreamKey idxKey = desc.getStreamKey(INDEX); - AtomicBoolean cacheHit = new AtomicBoolean(true); - DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(idxKey, - REF_POSITION, () -> { - cacheHit.set(false); - return loadPackIndex(ctx, idxKey); - }); - if (cacheHit.get()) { - ctx.stats.idxCacheHit++; - } - PackIndex idx = idxref.get(); - if (index == null && idx != null) { - index = idx; + index = indexFactory.getPackIndexes().index(ctx); + if (index == null) { + throw new IOException( + "Couldn't get a reference to the primary index"); //$NON-NLS-1$ } ctx.emitIndexLoad(desc, INDEX, index); return index; @@ -321,20 +364,10 @@ public final class DfsPackFile extends BlockBasedFile { return reverseIndex; } - PackIndex idx = idx(ctx); - DfsStreamKey revKey = desc.getStreamKey(REVERSE_INDEX); - AtomicBoolean cacheHit = new AtomicBoolean(true); - DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(revKey, - REF_POSITION, () -> { - cacheHit.set(false); - return loadReverseIdx(ctx, revKey, idx); - }); - if (cacheHit.get()) { - ctx.stats.ridxCacheHit++; - } - PackReverseIndex revidx = revref.get(); - if (reverseIndex == null && revidx != null) { - reverseIndex = revidx; + reverseIndex = indexFactory.getPackIndexes().reverseIndex(ctx); + if (reverseIndex == null) { + throw new IOException( + "Couldn't get a reference to the reverse index"); //$NON-NLS-1$ } ctx.emitIndexLoad(desc, REVERSE_INDEX, reverseIndex); return reverseIndex; @@ -347,6 +380,7 @@ public final class DfsPackFile extends BlockBasedFile { } if (objectSizeIndexLoadAttempted + || !ctx.getOptions().shouldUseObjectSizeIndex() || !desc.hasFileExt(OBJECT_SIZE_INDEX)) { // Pack doesn't have object size index return null; @@ -1210,48 +1244,6 @@ public final class DfsPackFile extends BlockBasedFile { } } - private DfsBlockCache.Ref<PackIndex> loadPackIndex( - DfsReader ctx, DfsStreamKey idxKey) throws IOException { - try { - ctx.stats.readIdx++; - long start = System.nanoTime(); - try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) { - PackIndex idx = PackIndex.read(alignTo8kBlocks(rc)); - ctx.stats.readIdxBytes += rc.position(); - index = idx; - return new DfsBlockCache.Ref<>( - idxKey, - REF_POSITION, - idx.getObjectCount() * REC_SIZE, - idx); - } finally { - ctx.stats.readIdxMicros += elapsedMicros(start); - } - } catch (EOFException e) { - throw new IOException(MessageFormat.format( - DfsText.get().shortReadOfIndex, - desc.getFileName(INDEX)), e); - } catch (IOException e) { - throw new IOException(MessageFormat.format( - DfsText.get().cannotReadIndex, - desc.getFileName(INDEX)), e); - } - } - - private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx( - DfsReader ctx, DfsStreamKey revKey, PackIndex idx) { - ctx.stats.readReverseIdx++; - long start = System.nanoTime(); - PackReverseIndex revidx = PackReverseIndexFactory.computeFromIndex(idx); - reverseIndex = revidx; - ctx.stats.readReverseIdxMicros += elapsedMicros(start); - return new DfsBlockCache.Ref<>( - revKey, - REF_POSITION, - idx.getObjectCount() * 8, - revidx); - } - private DfsBlockCache.Ref<PackObjectSizeIndex> loadObjectSizeIndex( DfsReader ctx, DfsStreamKey objectSizeIndexKey) throws IOException { ctx.stats.readObjectSizeIndex++; @@ -1288,9 +1280,12 @@ public final class DfsPackFile extends BlockBasedFile { private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(DfsReader ctx, DfsStreamKey bitmapKey) throws IOException { ctx.stats.readBitmap++; + long start = System.nanoTime(); PackBitmapIndexLoader.LoadResult result = bitmapLoader .loadPackBitmapIndex(ctx, this); bitmapIndex = result.bitmapIndex; + ctx.stats.readBitmapIdxBytes += result.bytesRead; + ctx.stats.readBitmapIdxMicros += elapsedMicros(start); return new DfsBlockCache.Ref<>(bitmapKey, REF_POSITION, result.bytesRead, result.bitmapIndex); } @@ -1449,4 +1444,141 @@ public final class DfsPackFile extends BlockBasedFile { } } } + + /** + * An index factory backed by Dfs streams and references cached in + * DfsBlockCache + */ + public static final class CachedStreamIndexFactory implements IndexFactory { + private final CachedStreamPackIndexes indexes; + + /** + * An index factory + * + * @param cache + * DFS block cache to use for the references + * @param desc + * This factory loads indexes for this package + */ + public CachedStreamIndexFactory(DfsBlockCache cache, + DfsPackDescription desc) { + this.indexes = new CachedStreamPackIndexes(cache, desc); + } + + @Override + public PackIndexes getPackIndexes() { + return indexes; + } + } + + /** + * Load primary and reverse index from Dfs streams and cache the references + * in DfsBlockCache. + */ + public static final class CachedStreamPackIndexes implements IndexFactory.PackIndexes { + private final DfsBlockCache cache; + + private final DfsPackDescription desc; + + /** + * An index factory + * + * @param cache + * DFS block cache to use for the references + * @param desc This factory loads indexes for this package + */ + public CachedStreamPackIndexes(DfsBlockCache cache, + DfsPackDescription desc) { + this.cache = cache; + this.desc = desc; + } + + @Override + public PackIndex index(DfsReader ctx) throws IOException { + DfsStreamKey idxKey = desc.getStreamKey(INDEX); + // Keep the value parsed in the loader, in case the Ref<> is + // nullified in ClockBlockCacheTable#reserveSpace + // before we read its value. + AtomicReference<PackIndex> loadedRef = new AtomicReference<>(null); + DfsBlockCache.Ref<PackIndex> cachedRef = cache.getOrLoadRef(idxKey, + REF_POSITION, () -> { + RefWithSize<PackIndex> idx = loadPackIndex(ctx, desc); + loadedRef.set(idx.ref); + return new DfsBlockCache.Ref<>(idxKey, REF_POSITION, + idx.size, idx.ref); + }); + if (loadedRef.get() == null) { + ctx.stats.idxCacheHit++; + } + return cachedRef.get() != null ? cachedRef.get() : loadedRef.get(); + } + + private static RefWithSize<PackIndex> loadPackIndex(DfsReader ctx, + DfsPackDescription desc) throws IOException { + try { + ctx.stats.readIdx++; + long start = System.nanoTime(); + try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) { + PackIndex idx = PackIndex.read(alignTo8kBlocks(rc)); + ctx.stats.readIdxBytes += rc.position(); + return new RefWithSize<>(idx, + idx.getObjectCount() * REC_SIZE); + } finally { + ctx.stats.readIdxMicros += elapsedMicros(start); + } + } catch (EOFException e) { + throw new IOException( + MessageFormat.format(DfsText.get().shortReadOfIndex, + desc.getFileName(INDEX)), + e); + } catch (IOException e) { + throw new IOException( + MessageFormat.format(DfsText.get().cannotReadIndex, + desc.getFileName(INDEX)), + e); + } + } + + @Override + public PackReverseIndex reverseIndex(DfsReader ctx) throws IOException { + PackIndex idx = index(ctx); + DfsStreamKey revKey = desc.getStreamKey(REVERSE_INDEX); + // Keep the value parsed in the loader, in case the Ref<> is + // nullified in ClockBlockCacheTable#reserveSpace + // before we read its value. + AtomicReference<PackReverseIndex> loadedRef = new AtomicReference<>( + null); + DfsBlockCache.Ref<PackReverseIndex> cachedRef = cache + .getOrLoadRef(revKey, REF_POSITION, () -> { + RefWithSize<PackReverseIndex> ridx = loadReverseIdx(ctx, + idx); + loadedRef.set(ridx.ref); + return new DfsBlockCache.Ref<>(revKey, REF_POSITION, + ridx.size, ridx.ref); + }); + if (loadedRef.get() == null) { + ctx.stats.ridxCacheHit++; + } + return cachedRef.get() != null ? cachedRef.get() : loadedRef.get(); + } + + private static RefWithSize<PackReverseIndex> loadReverseIdx( + DfsReader ctx, PackIndex idx) { + ctx.stats.readReverseIdx++; + long start = System.nanoTime(); + PackReverseIndex revidx = PackReverseIndexFactory + .computeFromIndex(idx); + ctx.stats.readReverseIdxMicros += elapsedMicros(start); + return new RefWithSize<>(revidx, idx.getObjectCount() * 8); + } + } + + private static final class RefWithSize<V> { + final V ref; + final long size; + RefWithSize(V ref, long size) { + this.ref = ref; + this.size = size; + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index c939114f5f..62f6753e5d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -307,7 +307,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { private <T extends ObjectId> Iterable<FoundObject<T>> findAll( Iterable<T> objectIds) throws IOException { - Collection<T> pending = new ArrayList<>(); + HashSet<T> pending = new HashSet<>(); for (T id : objectIds) { pending.add(id); } @@ -327,22 +327,21 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } private <T extends ObjectId> void findAllImpl(PackList packList, - Collection<T> pending, List<FoundObject<T>> r) { + HashSet<T> pending, List<FoundObject<T>> r) { DfsPackFile[] packs = packList.packs; if (packs.length == 0) { return; } int lastIdx = 0; DfsPackFile lastPack = packs[lastIdx]; - - OBJECT_SCAN: for (Iterator<T> it = pending.iterator(); it.hasNext();) { - T t = it.next(); + HashSet<T> toRemove = new HashSet<>(); + OBJECT_SCAN: for (T t : pending) { if (!skipGarbagePack(lastPack)) { try { long p = lastPack.findOffset(this, t); if (0 < p) { r.add(new FoundObject<>(t, lastIdx, lastPack, p)); - it.remove(); + toRemove.add(t); continue; } } catch (IOException e) { @@ -360,7 +359,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { long p = pack.findOffset(this, t); if (0 < p) { r.add(new FoundObject<>(t, i, pack, p)); - it.remove(); + toRemove.add(t); lastIdx = i; lastPack = pack; continue OBJECT_SCAN; @@ -370,6 +369,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } } } + pending.removeAll(toRemove); last = lastPack; } @@ -511,18 +511,15 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { throw new MissingObjectException(objectId.copy(), typeHint); } - if (typeHint != Constants.OBJ_BLOB || !pack.hasObjectSizeIndex(this)) { + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { return pack.getObjectSize(this, objectId); } - long sz = pack.getIndexedObjectSize(this, objectId); + Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); + long sz = maybeSz.orElse(-1L); if (sz >= 0) { - stats.objectSizeIndexHit += 1; return sz; } - - // Object wasn't in the index - stats.objectSizeIndexMiss += 1; return pack.getObjectSize(this, objectId); } @@ -541,23 +538,61 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } stats.isNotLargerThanCallCount += 1; - if (typeHint != Constants.OBJ_BLOB || !pack.hasObjectSizeIndex(this)) { + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { + return pack.getObjectSize(this, objectId) <= limit; + } + + Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); + if (maybeSz.isEmpty()) { + // Exception in object size index return pack.getObjectSize(this, objectId) <= limit; } - long sz = pack.getIndexedObjectSize(this, objectId); + long sz = maybeSz.get(); + if (sz >= 0) { + return sz <= limit; + } + + if (isLimitInsideIndexThreshold(pack, limit)) { + // With threshold T, not-found means object < T + // If limit L > T, then object < T < L + return true; + } + + return pack.getObjectSize(this, objectId) <= limit; + } + + private boolean safeHasObjectSizeIndex(DfsPackFile pack) { + try { + return pack.hasObjectSizeIndex(this); + } catch (IOException e) { + return false; + } + } + + private Optional<Long> safeGetIndexedObjectSize(DfsPackFile pack, + AnyObjectId objectId) { + long sz; + try { + sz = pack.getIndexedObjectSize(this, objectId); + } catch (IOException e) { + // Do not count the exception as an index miss + return Optional.empty(); + } if (sz < 0) { stats.objectSizeIndexMiss += 1; } else { stats.objectSizeIndexHit += 1; } + return Optional.of(sz); + } - // Got size from index or we didn't but we are sure it should be there. - if (sz >= 0 || pack.getObjectSizeIndexThreshold(this) <= limit) { - return sz <= limit; + private boolean isLimitInsideIndexThreshold(DfsPackFile pack, long limit) { + try { + return pack.getObjectSizeIndexThreshold(this) <= limit; + } catch (IOException e) { + return false; } - - return pack.getObjectSize(this, objectId) <= limit; } private DfsPackFile findPackWithObject(AnyObjectId objectId) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java index adb4673ae4..fcfa3e0dee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java @@ -40,12 +40,12 @@ public class DfsReaderIoStats { /** Total number of complete pack indexes read into memory. */ long readIdx; - /** Total number of complete bitmap indexes read into memory. */ - long readBitmap; - /** Total number of reverse indexes added into memory. */ long readReverseIdx; + /** Total number of complete bitmap indexes read into memory. */ + long readBitmap; + /** Total number of complete commit graphs read into memory. */ long readCommitGraph; @@ -55,6 +55,9 @@ public class DfsReaderIoStats { /** Total number of bytes read from pack indexes. */ long readIdxBytes; + /** Total number of bytes read from bitmap indexes. */ + long readBitmapIdxBytes; + /** Total number of bytes read from commit graphs. */ long readCommitGraphBytes; @@ -67,18 +70,15 @@ public class DfsReaderIoStats { /** Total microseconds spent creating reverse indexes. */ long readReverseIdxMicros; + /** Total microseconds spent reading bitmap indexes. */ + long readBitmapIdxMicros; + /** Total microseconds spent creating commit graphs. */ long readCommitGraphMicros; /** Total microseconds spent creating object size indexes */ long readObjectSizeIndexMicros; - /** Total number of bytes read from bitmap indexes. */ - long readBitmapIdxBytes; - - /** Total microseconds spent reading bitmap indexes. */ - long readBitmapIdxMicros; - /** Total number of block cache hits. */ long blockCacheHit; @@ -195,21 +195,21 @@ public class DfsReaderIoStats { } /** - * Get total number of times the commit graph read into memory. + * Get total number of complete bitmap indexes read into memory. * - * @return total number of commit graph read into memory. + * @return total number of complete bitmap indexes read into memory. */ - public long getReadCommitGraphCount() { - return stats.readCommitGraph; + public long getReadBitmapIndexCount() { + return stats.readBitmap; } /** - * Get total number of complete bitmap indexes read into memory. + * Get total number of times the commit graph read into memory. * - * @return total number of complete bitmap indexes read into memory. + * @return total number of commit graph read into memory. */ - public long getReadBitmapIndexCount() { - return stats.readBitmap; + public long getReadCommitGraphCount() { + return stats.readCommitGraph; } /** @@ -231,6 +231,15 @@ public class DfsReaderIoStats { } /** + * Get total number of bytes read from bitmap indexes. + * + * @return total number of bytes read from bitmap indexes. + */ + public long getReadBitmapIndexBytes() { + return stats.readBitmapIdxBytes; + } + + /** * Get total number of bytes read from commit graphs. * * @return total number of bytes read from commit graphs. @@ -240,6 +249,15 @@ public class DfsReaderIoStats { } /** + * Get total number of bytes read from object size indexes. + * + * @return total number of bytes read from object size indexes. + */ + public long getObjectSizeIndexBytes() { + return stats.readObjectSizeIndexBytes; + } + + /** * Get total microseconds spent reading pack indexes. * * @return total microseconds spent reading pack indexes. @@ -258,30 +276,30 @@ public class DfsReaderIoStats { } /** - * Get total microseconds spent reading commit graphs. + * Get total microseconds spent reading bitmap indexes. * - * @return total microseconds spent reading commit graphs. + * @return total microseconds spent reading bitmap indexes. */ - public long getReadCommitGraphMicros() { - return stats.readCommitGraphMicros; + public long getReadBitmapIndexMicros() { + return stats.readBitmapIdxMicros; } /** - * Get total number of bytes read from bitmap indexes. + * Get total microseconds spent reading commit graphs. * - * @return total number of bytes read from bitmap indexes. + * @return total microseconds spent reading commit graphs. */ - public long getReadBitmapIndexBytes() { - return stats.readBitmapIdxBytes; + public long getReadCommitGraphMicros() { + return stats.readCommitGraphMicros; } /** - * Get total microseconds spent reading bitmap indexes. + * Get total microseconds spent reading object size indexes. * - * @return total microseconds spent reading bitmap indexes. + * @return total microseconds spent reading object size indexes. */ - public long getReadBitmapIndexMicros() { - return stats.readBitmapIdxMicros; + public long getReadObjectSizeIndexMicros() { + return stats.readObjectSizeIndexMicros; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java index f2ac461deb..c3974691a1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java @@ -13,8 +13,10 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_BASE_CACHE_LIMIT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_BUFFER; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_THRESHOLD; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_USE_OBJECT_SIZE_INDEX; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.storage.pack.PackConfig; @@ -36,6 +38,8 @@ public class DfsReaderOptions { private boolean loadRevIndexInParallel; + private boolean useObjectSizeIndex; + /** * Create a default reader configuration. */ @@ -137,6 +141,28 @@ public class DfsReaderOptions { } /** + * Use the object size index if available. + * + * @return true if the reader should try to use the object size index. if + * false, the reader ignores that index. + */ + public boolean shouldUseObjectSizeIndex() { + return useObjectSizeIndex; + } + + /** + * Set if the reader should try to use the object size index + * + * @param useObjectSizeIndex true to use it, false to ignore the object size index + * + * @return {@code this} + */ + public DfsReaderOptions setUseObjectSizeIndex(boolean useObjectSizeIndex) { + this.useObjectSizeIndex = useObjectSizeIndex; + return this; + } + + /** * Update properties by setting fields from the configuration. * <p> * If a property is not defined in the configuration, then it is left @@ -168,6 +194,13 @@ public class DfsReaderOptions { CONFIG_DFS_SECTION, CONFIG_KEY_STREAM_BUFFER, getStreamPackBufferSize())); + + setUseObjectSizeIndex(rc.getBoolean(CONFIG_CORE_SECTION, + CONFIG_DFS_SECTION, CONFIG_KEY_USE_OBJECT_SIZE_INDEX, + false)); + setLoadRevIndexInParallel( + rc.getBoolean(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL, false)); return this; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java index 3ba74b26fc..2751cd2969 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java @@ -28,6 +28,7 @@ import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.util.RefList; @@ -177,6 +178,11 @@ public class DfsReftableDatabase extends DfsRefDatabase { } @Override + public ReflogReader getReflogReader(Ref ref) throws IOException { + return reftableDatabase.getReflogReader(ref.getName()); + } + + @Override public Set<Ref> getTipsWithSha1(ObjectId id) throws IOException { if (!getReftableConfig().isIndexObjects()) { return super.getTipsWithSha1(id); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java new file mode 100644 index 0000000000..bb44f9397a --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.ReadableChannelSupplier; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.Ref; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.RefLoader; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DfsBlockCachePackExtConfig; +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * A table that holds multiple cache tables accessed by {@link PackExt} types. + * + * <p> + * Allows the separation of entries from different {@link PackExt} types to + * limit churn in cache caused by entries of differing sizes. + * <p> + * Separating these tables enables the fine-tuning of cache tables per extension + * type. + */ +class PackExtBlockCacheTable implements DfsBlockCacheTable { + /** + * Table name. + */ + private final String name; + + private final DfsBlockCacheTable defaultBlockCacheTable; + + // Holds the unique tables backing the extBlockCacheTables values. + private final List<DfsBlockCacheTable> blockCacheTableList; + + // Holds the mapping of PackExt to DfsBlockCacheTables. + // The relation between the size of extBlockCacheTables entries and + // blockCacheTableList entries is: + // blockCacheTableList.size() <= extBlockCacheTables.size() + private final Map<PackExt, DfsBlockCacheTable> extBlockCacheTables; + + /** + * Builds the PackExtBlockCacheTable from a list of + * {@link DfsBlockCachePackExtConfig}s. + * + * @param cacheConfig + * {@link DfsBlockCacheConfig} containing + * {@link DfsBlockCachePackExtConfig}s used to configure + * PackExtBlockCacheTable. The {@link DfsBlockCacheConfig} holds + * the configuration for the default cache table. + * @return the cache table built from the given configs. + * @throws IllegalArgumentException + * when no {@link DfsBlockCachePackExtConfig} exists in the + * {@link DfsBlockCacheConfig}. + */ + static PackExtBlockCacheTable fromBlockCacheConfigs( + DfsBlockCacheConfig cacheConfig) { + DfsBlockCacheTable defaultTable = new ClockBlockCacheTable(cacheConfig); + Map<PackExt, DfsBlockCacheTable> packExtBlockCacheTables = new HashMap<>(); + List<DfsBlockCachePackExtConfig> packExtConfigs = cacheConfig + .getPackExtCacheConfigurations(); + if (packExtConfigs == null || packExtConfigs.size() == 0) { + throw new IllegalArgumentException( + JGitText.get().noPackExtConfigurationGiven); + } + for (DfsBlockCachePackExtConfig packExtCacheConfig : packExtConfigs) { + DfsBlockCacheTable table = new ClockBlockCacheTable( + packExtCacheConfig.getPackExtCacheConfiguration()); + for (PackExt packExt : packExtCacheConfig.getPackExts()) { + if (packExtBlockCacheTables.containsKey(packExt)) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().duplicatePackExtensionsForCacheTables, + packExt)); + } + packExtBlockCacheTables.put(packExt, table); + } + } + return fromCacheTables(defaultTable, packExtBlockCacheTables); + } + + /** + * Creates a new PackExtBlockCacheTable from the combination of a default + * {@link DfsBlockCacheTable} and a map of {@link PackExt}s to + * {@link DfsBlockCacheTable}s. + * <p> + * This method allows for the PackExtBlockCacheTable to handle a mapping of + * {@link PackExt}s to arbitrarily defined {@link DfsBlockCacheTable} + * implementations. This is especially useful for users wishing to implement + * custom cache tables. + * <p> + * This is currently made visible for testing. + * + * @param defaultBlockCacheTable + * the default table used when a handling a {@link PackExt} type + * that does not map to a {@link DfsBlockCacheTable} mapped by + * packExtsCacheTablePairs. + * @param packExtBlockCacheTables + * the mapping of {@link PackExt}s to + * {@link DfsBlockCacheTable}s. A single + * {@link DfsBlockCacheTable} can be defined for multiple + * {@link PackExt}s in a many-to-one relationship. + * @return the PackExtBlockCacheTable created from the + * defaultBlockCacheTable and packExtsCacheTablePairs mapping. + * @throws IllegalArgumentException + * when a {@link PackExt} is defined for multiple + * {@link DfsBlockCacheTable}s. + */ + static PackExtBlockCacheTable fromCacheTables( + DfsBlockCacheTable defaultBlockCacheTable, + Map<PackExt, DfsBlockCacheTable> packExtBlockCacheTables) { + Set<DfsBlockCacheTable> blockCacheTables = new HashSet<>(); + blockCacheTables.add(defaultBlockCacheTable); + blockCacheTables.addAll(packExtBlockCacheTables.values()); + String name = defaultBlockCacheTable.getName() + "," //$NON-NLS-1$ + + packExtBlockCacheTables.values().stream() + .map(DfsBlockCacheTable::getName).sorted() + .collect(Collectors.joining(",")); //$NON-NLS-1$ + return new PackExtBlockCacheTable(name, defaultBlockCacheTable, + List.copyOf(blockCacheTables), packExtBlockCacheTables); + } + + private PackExtBlockCacheTable(String name, + DfsBlockCacheTable defaultBlockCacheTable, + List<DfsBlockCacheTable> blockCacheTableList, + Map<PackExt, DfsBlockCacheTable> extBlockCacheTables) { + this.name = name; + this.defaultBlockCacheTable = defaultBlockCacheTable; + this.blockCacheTableList = blockCacheTableList; + this.extBlockCacheTables = extBlockCacheTables; + } + + @Override + public boolean hasBlock0(DfsStreamKey key) { + return getTable(key).hasBlock0(key); + } + + @Override + public DfsBlock getOrLoad(BlockBasedFile file, long position, + DfsReader dfsReader, ReadableChannelSupplier fileChannel) + throws IOException { + return getTable(file.ext).getOrLoad(file, position, dfsReader, + fileChannel); + } + + @Override + public <T> Ref<T> getOrLoadRef(DfsStreamKey key, long position, + RefLoader<T> loader) throws IOException { + return getTable(key).getOrLoadRef(key, position, loader); + } + + @Override + public void put(DfsBlock v) { + getTable(v.stream).put(v); + } + + @Override + public <T> Ref<T> put(DfsStreamKey key, long pos, long size, T v) { + return getTable(key).put(key, pos, size, v); + } + + @Override + public <T> Ref<T> putRef(DfsStreamKey key, long size, T v) { + return getTable(key).putRef(key, size, v); + } + + @Override + public boolean contains(DfsStreamKey key, long position) { + return getTable(key).contains(key, position); + } + + @Override + public <T> T get(DfsStreamKey key, long position) { + return getTable(key).get(key, position); + } + + @Override + public List<BlockCacheStats> getBlockCacheStats() { + return blockCacheTableList.stream() + .flatMap(cacheTable -> cacheTable.getBlockCacheStats().stream()) + .collect(Collectors.toList()); + } + + @Override + public String getName() { + return name; + } + + private DfsBlockCacheTable getTable(PackExt packExt) { + return extBlockCacheTables.getOrDefault(packExt, + defaultBlockCacheTable); + } + + private DfsBlockCacheTable getTable(DfsStreamKey key) { + return extBlockCacheTables.getOrDefault(getPackExt(key), + defaultBlockCacheTable); + } + + private static PackExt getPackExt(DfsStreamKey key) { + return PackExt.values()[key.packExtPos]; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java index 87e0b44d46..b89cc1ebf4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java @@ -19,6 +19,7 @@ import java.text.MessageFormat; import java.util.List; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.NB; @@ -31,7 +32,7 @@ import org.eclipse.jgit.util.NB; * random access to any object in the pack by associating an ObjectId to the * byte offset within the pack where the object's data can be read. */ -public abstract class PackIndexWriter { +public abstract class BasePackIndexWriter implements PackIndexWriter { /** Magic constant indicating post-version 1 format. */ protected static final byte[] TOC = { -1, 't', 'O', 'c' }; @@ -147,7 +148,7 @@ public abstract class PackIndexWriter { * the stream this instance outputs to. If not already buffered * it will be automatically wrapped in a buffered stream. */ - protected PackIndexWriter(OutputStream dst) { + protected BasePackIndexWriter(OutputStream dst) { out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst : new BufferedOutputStream(dst), Constants.newMessageDigest()); @@ -172,6 +173,7 @@ public abstract class PackIndexWriter { * an error occurred while writing to the output stream, or this * index format cannot store the object data supplied. */ + @Override public void write(final List<? extends PackedObjectInfo> toStore, final byte[] packDataChecksum) throws IOException { entries = toStore; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index ed2516ddd0..e9782e2e18 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -16,6 +16,7 @@ import static org.eclipse.jgit.lib.Ref.Storage.PACKED; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -23,22 +24,27 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.PackRefsCommand; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.events.RefsChangedEvent; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.reftable.MergedReftable; import org.eclipse.jgit.internal.storage.reftable.ReftableBatchRefUpdate; import org.eclipse.jgit.internal.storage.reftable.ReftableDatabase; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; import org.eclipse.jgit.lib.BatchRefUpdate; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefRename; @@ -67,15 +73,20 @@ public class FileReftableDatabase extends RefDatabase { private final FileReftableStack reftableStack; + private final AtomicBoolean autoRefresh; + FileReftableDatabase(FileRepository repo) throws IOException { - this(repo, new File(new File(repo.getDirectory(), Constants.REFTABLE), + this(repo, new File(new File(repo.getCommonDirectory(), Constants.REFTABLE), Constants.TABLES_LIST)); } FileReftableDatabase(FileRepository repo, File refstackName) throws IOException { this.fileRepository = repo; + this.autoRefresh = new AtomicBoolean(repo.getConfig().getBoolean( + ConfigConstants.CONFIG_REFTABLE_SECTION, + ConfigConstants.CONFIG_KEY_AUTOREFRESH, false)); this.reftableStack = new FileReftableStack(refstackName, - new File(fileRepository.getDirectory(), Constants.REFTABLE), + new File(fileRepository.getCommonDirectory(), Constants.REFTABLE), () -> fileRepository.fireEvent(new RefsChangedEvent()), () -> fileRepository.getConfig()); this.reftableDatabase = new ReftableDatabase() { @@ -87,7 +98,13 @@ public class FileReftableDatabase extends RefDatabase { }; } - ReflogReader getReflogReader(String refname) throws IOException { + @Override + public ReflogReader getReflogReader(Ref ref) throws IOException { + return reftableDatabase.getReflogReader(ref.getName()); + } + + @Override + public ReflogReader getReflogReader(String refname) throws IOException { return reftableDatabase.getReflogReader(refname); } @@ -108,6 +125,22 @@ public class FileReftableDatabase extends RefDatabase { } /** + * {@inheritDoc} + * + * For Reftable, all the data is compacted into a single table. + */ + @Override + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + pm.beginTask(JGitText.get().packRefs, 1); + try { + compactFully(); + } finally { + pm.endTask(); + } + } + + /** * Runs a full compaction for GC purposes. * @throws IOException on I/O errors */ @@ -158,6 +191,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public Ref exactRef(String name) throws IOException { + autoRefresh(); return reftableDatabase.exactRef(name); } @@ -168,6 +202,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public Map<String, Ref> getRefs(String prefix) throws IOException { + autoRefresh(); List<Ref> refs = reftableDatabase.getRefsByPrefix(prefix); RefList.Builder<Ref> builder = new RefList.Builder<>(refs.size()); for (Ref r : refs) { @@ -180,6 +215,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public List<Ref> getRefsByPrefixWithExclusions(String include, Set<String> excludes) throws IOException { + autoRefresh(); return reftableDatabase.getRefsByPrefixWithExclusions(include, excludes); } @@ -198,6 +234,50 @@ public class FileReftableDatabase extends RefDatabase { } + /** + * Whether to auto-refresh the reftable stack if it is out of date. + * + * @param autoRefresh + * whether to auto-refresh the reftable stack if it is out of + * date. + */ + public void setAutoRefresh(boolean autoRefresh) { + this.autoRefresh.set(autoRefresh); + } + + /** + * Whether the reftable stack is auto-refreshed if it is out of date. + * + * @return whether the reftable stack is auto-refreshed if it is out of + * date. + */ + public boolean isAutoRefresh() { + return autoRefresh.get(); + } + + private void autoRefresh() { + if (autoRefresh.get()) { + refresh(); + } + } + + /** + * Check if the reftable stack is up to date, and if not, reload it. + * <p> + * {@inheritDoc} + */ + @Override + public void refresh() { + try { + if (!reftableStack.isUpToDate()) { + reftableDatabase.clearCache(); + reftableStack.reload(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private Ref doPeel(Ref leaf) throws IOException { try (RevWalk rw = new RevWalk(fileRepository)) { RevObject obj = rw.parseAny(leaf.getObjectId()); @@ -318,7 +398,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public void create() throws IOException { FileUtils.mkdir( - new File(fileRepository.getDirectory(), Constants.REFTABLE), + new File(fileRepository.getCommonDirectory(), Constants.REFTABLE), true); } @@ -538,9 +618,10 @@ public class FileReftableDatabase extends RefDatabase { boolean writeLogs) throws IOException { int size = 0; List<Ref> refs = repo.getRefDatabase().getRefs(); + RefDatabase refDb = repo.getRefDatabase(); if (writeLogs) { for (Ref r : refs) { - ReflogReader rlr = repo.getReflogReader(r.getName()); + ReflogReader rlr = refDb.getReflogReader(r); if (rlr != null) { size = Math.max(rlr.getReverseEntries().size(), size); } @@ -563,10 +644,7 @@ public class FileReftableDatabase extends RefDatabase { if (writeLogs) { for (Ref r : refs) { long idx = size; - ReflogReader reader = repo.getReflogReader(r.getName()); - if (reader == null) { - continue; - } + ReflogReader reader = refDb.getReflogReader(r); for (ReflogEntry e : reader.getReverseEntries()) { w.writeLog(r.getName(), idx, e.getWho(), e.getOldId(), e.getNewId(), e.getComment()); @@ -615,7 +693,7 @@ public class FileReftableDatabase extends RefDatabase { FileReftableDatabase newDb = null; File reftableList = null; try { - File reftableDir = new File(repo.getDirectory(), + File reftableDir = new File(repo.getCommonDirectory(), Constants.REFTABLE); reftableList = new File(reftableDir, Constants.TABLES_LIST); if (!reftableDir.isDirectory()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java index 0f5ff0f9f7..b2c88922b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java @@ -18,8 +18,10 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.StandardCopyOption; import java.security.SecureRandom; import java.util.ArrayList; @@ -27,6 +29,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -39,6 +42,8 @@ import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableReader; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.SystemReader; @@ -59,6 +64,9 @@ public class FileReftableStack implements AutoCloseable { private List<StackEntry> stack; + private AtomicReference<FileSnapshot> snapshot = new AtomicReference<>( + FileSnapshot.DIRTY); + private long lastNextUpdateIndex; private final File stackPath; @@ -98,6 +106,8 @@ public class FileReftableStack implements AutoCloseable { private final CompactionStats stats; + private final TrustStat trustTablesListStat; + /** * Creates a stack corresponding to the list of reftables in the argument * @@ -126,6 +136,8 @@ public class FileReftableStack implements AutoCloseable { reload(); stats = new CompactionStats(); + trustTablesListStat = configSupplier.get().get(CoreConfig.KEY) + .getTrustTablesListStat(); } CompactionStats getStats() { @@ -272,8 +284,9 @@ public class FileReftableStack implements AutoCloseable { } private List<String> readTableNames() throws IOException { + FileSnapshot old; List<String> names = new ArrayList<>(stack.size() + 1); - + old = snapshot.get(); try (BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(stackPath), UTF_8))) { String line; @@ -282,8 +295,10 @@ public class FileReftableStack implements AutoCloseable { names.add(line); } } + snapshot.compareAndSet(old, FileSnapshot.save(stackPath)); } catch (FileNotFoundException e) { // file isn't there: empty repository. + snapshot.compareAndSet(old, FileSnapshot.MISSING_FILE); } return names; } @@ -294,9 +309,28 @@ public class FileReftableStack implements AutoCloseable { * on IO problem */ boolean isUpToDate() throws IOException { - // We could use FileSnapshot to avoid reading the file, but the file is - // small so it's probably a minor optimization. try { + switch (trustTablesListStat) { + case NEVER: + break; + case AFTER_OPEN: + try (InputStream stream = Files + .newInputStream(stackPath.toPath())) { + // open the tables.list file to refresh attributes (on some + // NFS clients) + } catch (FileNotFoundException | NoSuchFileException e) { + // ignore + } + //$FALL-THROUGH$ + case ALWAYS: + if (!snapshot.get().isModified(stackPath)) { + return true; + } + break; + case INHERIT: + // only used in CoreConfig internally + throw new IllegalStateException(); + } List<String> names = readTableNames(); if (names.size() != stack.size()) { return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index e5a00d3925..bcf9f1efdf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -2,7 +2,7 @@ * Copyright (C) 2007, Dave Watson <dwatson@mimvista.com> * Copyright (C) 2008-2010, Google Inc. * Copyright (C) 2006-2010, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2006-2024, Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -31,8 +31,8 @@ import java.util.Locale; import java.util.Objects; import java.util.Set; -import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesNodeProvider; @@ -60,7 +60,6 @@ import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; @@ -165,7 +164,7 @@ public class FileRepository extends Repository { throw new IOException(e.getMessage(), e); } repoConfig = new FileBasedConfig(userConfig, getFS().resolve( - getDirectory(), Constants.CONFIG), + getCommonDirectory(), Constants.CONFIG), getFS()); loadRepoConfig(); @@ -193,7 +192,7 @@ public class FileRepository extends Repository { options.getObjectDirectory(), // options.getAlternateObjectDirectories(), // getFS(), // - new File(getDirectory(), Constants.SHALLOW)); + new File(getCommonDirectory(), Constants.SHALLOW)); if (objectDatabase.exists()) { if (repositoryFormatVersion > 1) @@ -215,6 +214,16 @@ public class FileRepository extends Repository { } } + private String getRelativeDir(File base, File other) { + File relPath; + try { + relPath = base.toPath().relativize(other.toPath()).toFile(); + } catch (IllegalArgumentException e) { + relPath = other; + } + return FileUtils.pathToString(relPath); + } + /** * {@inheritDoc} * <p> @@ -223,6 +232,22 @@ public class FileRepository extends Repository { */ @Override public void create(boolean bare) throws IOException { + create(bare, false); + } + + /** + * Create a new Git repository initializing the necessary files and + * directories. + * + * @param bare + * if true, a bare repository (a repository without a working + * directory) is created. + * @param relativePaths + * if true, relative paths are used for GIT_DIR and GIT_WORK_TREE + * @throws IOException + * in case of IO problem + */ + public void create(boolean bare, boolean relativePaths) throws IOException { final FileBasedConfig cfg = getConfig(); if (cfg.getFile().exists()) { throw new IllegalStateException(MessageFormat.format( @@ -293,15 +318,25 @@ public class FileRepository extends Repository { if (!bare) { File workTree = getWorkTree(); if (!getDirectory().getParentFile().equals(workTree)) { + String workTreePath; + String gitDirPath; + if (relativePaths) { + File canonGitDir = getDirectory().getCanonicalFile(); + File canonWorkTree = getWorkTree().getCanonicalFile(); + workTreePath = getRelativeDir(canonGitDir, canonWorkTree); + gitDirPath = getRelativeDir(canonWorkTree, canonGitDir); + } else { + workTreePath = getWorkTree().getAbsolutePath(); + gitDirPath = getDirectory().getAbsolutePath(); + } cfg.setString(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_WORKTREE, getWorkTree() - .getAbsolutePath()); + ConfigConstants.CONFIG_KEY_WORKTREE, workTreePath); LockFile dotGitLockFile = new LockFile(new File(workTree, Constants.DOT_GIT)); try { if (dotGitLockFile.lock()) { dotGitLockFile.write(Constants.encode(Constants.GITDIR - + getDirectory().getAbsolutePath())); + + gitDirPath)); dotGitLockFile.commit(); } } finally { @@ -507,29 +542,6 @@ public class FileRepository extends Repository { } @Override - public ReflogReader getReflogReader(String refName) throws IOException { - if (refs instanceof FileReftableDatabase) { - // Cannot use findRef: reftable stores log data for deleted or renamed - // branches. - return ((FileReftableDatabase)refs).getReflogReader(refName); - } - - // TODO: use exactRef here, which offers more predictable and therefore preferable - // behavior. - Ref ref = findRef(refName); - if (ref == null) { - return null; - } - return new ReflogReaderImpl(this, ref.getName()); - } - - @Override - public @NonNull ReflogReader getReflogReader(@NonNull Ref ref) - throws IOException { - return new ReflogReaderImpl(this, ref.getName()); - } - - @Override public AttributesNodeProvider createAttributesNodeProvider() { return new AttributesNodeProviderImpl(this); } @@ -595,13 +607,12 @@ public class FileRepository extends Repository { @Override public void autoGC(ProgressMonitor monitor) { GC gc = new GC(this); - gc.setPackConfig(new PackConfig(this)); gc.setProgressMonitor(monitor); gc.setAuto(true); gc.setBackground(shouldAutoDetach()); try { gc.gc(); - } catch (ParseException | IOException e) { + } catch (ParseException | IOException | GitAPIException e) { throw new JGitInternalException(JGitText.get().gcFailed, e); } } @@ -622,16 +633,17 @@ public class FileRepository extends Repository { * on IO problem */ void convertToPackedRefs(boolean writeLogs, boolean backup) throws IOException { + File commonDirectory = getCommonDirectory(); List<Ref> all = refs.getRefs(); - File packedRefs = new File(getDirectory(), Constants.PACKED_REFS); + File packedRefs = new File(commonDirectory, Constants.PACKED_REFS); if (packedRefs.exists()) { throw new IOException(MessageFormat.format(JGitText.get().fileAlreadyExists, packedRefs.getName())); } - File refsFile = new File(getDirectory(), "refs"); //$NON-NLS-1$ + File refsFile = new File(commonDirectory, "refs"); //$NON-NLS-1$ File refsHeadsFile = new File(refsFile, "heads");//$NON-NLS-1$ - File headFile = new File(getDirectory(), Constants.HEAD); + File headFile = new File(commonDirectory, Constants.HEAD); FileReftableDatabase oldDb = (FileReftableDatabase) refs; // Remove the dummy files that ensure compatibility with older git @@ -661,8 +673,8 @@ public class FileRepository extends Repository { } if (writeLogs) { - List<ReflogEntry> logs = oldDb.getReflogReader(r.getName()) - .getReverseEntries(); + ReflogReader reflogReader = oldDb.getReflogReader(r); + List<ReflogEntry> logs = reflogReader.getReverseEntries(); Collections.reverse(logs); for (ReflogEntry e : logs) { logWriter.log(r.getName(), e); @@ -701,12 +713,14 @@ public class FileRepository extends Repository { } if (!backup) { - File reftableDir = new File(getDirectory(), Constants.REFTABLE); + File reftableDir = new File(commonDirectory, Constants.REFTABLE); FileUtils.delete(reftableDir, FileUtils.RECURSIVE | FileUtils.IGNORE_ERRORS); } repoConfig.unset(ConfigConstants.CONFIG_EXTENSIONS_SECTION, null, ConfigConstants.CONFIG_KEY_REF_STORAGE); + repoConfig.setLong(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION, 0); repoConfig.save(); } @@ -730,8 +744,10 @@ public class FileRepository extends Repository { @SuppressWarnings("nls") void convertToReftable(boolean writeLogs, boolean backup) throws IOException { - File reftableDir = new File(getDirectory(), Constants.REFTABLE); - File headFile = new File(getDirectory(), Constants.HEAD); + File commonDirectory = getCommonDirectory(); + File directory = getDirectory(); + File reftableDir = new File(commonDirectory, Constants.REFTABLE); + File headFile = new File(directory, Constants.HEAD); if (reftableDir.exists() && FileUtils.hasFiles(reftableDir.toPath())) { throw new IOException(JGitText.get().reftableDirExists); } @@ -739,28 +755,28 @@ public class FileRepository extends Repository { // Ignore return value, as it is tied to temporary newRefs file. FileReftableDatabase.convertFrom(this, writeLogs); - File refsFile = new File(getDirectory(), "refs"); + File refsFile = new File(commonDirectory, "refs"); // non-atomic: remove old data. - File packedRefs = new File(getDirectory(), Constants.PACKED_REFS); - File logsDir = new File(getDirectory(), Constants.LOGS); + File packedRefs = new File(commonDirectory, Constants.PACKED_REFS); + File logsDir = new File(commonDirectory, Constants.LOGS); List<String> additional = getRefDatabase().getAdditionalRefs().stream() .map(Ref::getName).collect(toList()); additional.add(Constants.HEAD); if (backup) { - FileUtils.rename(refsFile, new File(getDirectory(), "refs.old")); + FileUtils.rename(refsFile, new File(commonDirectory, "refs.old")); if (packedRefs.exists()) { - FileUtils.rename(packedRefs, new File(getDirectory(), + FileUtils.rename(packedRefs, new File(commonDirectory, Constants.PACKED_REFS + ".old")); } if (logsDir.exists()) { FileUtils.rename(logsDir, - new File(getDirectory(), Constants.LOGS + ".old")); + new File(commonDirectory, Constants.LOGS + ".old")); } for (String r : additional) { - FileUtils.rename(new File(getDirectory(), r), - new File(getDirectory(), r + ".old")); + FileUtils.rename(new File(commonDirectory, r), + new File(commonDirectory, r + ".old")); } } else { FileUtils.delete(packedRefs, FileUtils.SKIP_MISSING); @@ -770,7 +786,7 @@ public class FileRepository extends Repository { FileUtils.delete(refsFile, FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); for (String r : additional) { - new File(getDirectory(), r).delete(); + new File(commonDirectory, r).delete(); } } @@ -784,7 +800,7 @@ public class FileRepository extends Repository { // Some tools might write directly into .git/refs/heads/BRANCH. By // putting a file here, this fails spectacularly. - FileUtils.createNewFile(new File(refsFile, "heads")); + FileUtils.createNewFile(new File(refsFile, Constants.HEADS)); repoConfig.setString(ConfigConstants.CONFIG_EXTENSIONS_SECTION, null, ConfigConstants.CONFIG_KEY_REF_STORAGE, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index c88ac984ec..b6bde6eea0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -15,6 +15,7 @@ import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_TIMESTAMP_RE import java.io.File; import java.io.IOException; +import java.nio.file.FileSystemException; import java.nio.file.NoSuchFileException; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; @@ -22,10 +23,12 @@ import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.Locale; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.FileStoreAttributes; import org.slf4j.Logger; @@ -139,29 +142,6 @@ public class FileSnapshot { * @param modified * the last modification time of the file * @return the snapshot. - * @deprecated use {@link #save(Instant)} instead. - */ - @Deprecated - public static FileSnapshot save(long modified) { - final Instant read = Instant.now(); - return new FileSnapshot(read, Instant.ofEpochMilli(modified), - UNKNOWN_SIZE, FALLBACK_TIMESTAMP_RESOLUTION, MISSING_FILEKEY); - } - - /** - * Record a snapshot for a file for which the last modification time is - * already known. - * <p> - * This method should be invoked before the file is accessed. - * <p> - * Note that this method cannot rely on measuring file timestamp resolution - * to avoid racy git issues caused by finite file timestamp resolution since - * it's unknown in which filesystem the file is located. Hence the worst - * case fallback for timestamp resolution is used. - * - * @param modified - * the last modification time of the file - * @return the snapshot. */ public static FileSnapshot save(Instant modified) { final Instant read = Instant.now(); @@ -231,14 +211,8 @@ public class FileSnapshot { this.useConfig = useConfig; BasicFileAttributes fileAttributes = null; try { - fileAttributes = FS.DETECTED.fileAttributes(file); - } catch (NoSuchFileException e) { - this.lastModified = Instant.EPOCH; - this.size = 0L; - this.fileKey = MISSING_FILEKEY; - return; - } catch (IOException e) { - LOG.error(e.getMessage(), e); + fileAttributes = getFileAttributes(file); + } catch (NoSuchElementException e) { this.lastModified = Instant.EPOCH; this.size = 0L; this.fileKey = MISSING_FILEKEY; @@ -282,17 +256,6 @@ public class FileSnapshot { * Get time of last snapshot update * * @return time of last snapshot update - * @deprecated use {@link #lastModifiedInstant()} instead - */ - @Deprecated - public long lastModified() { - return lastModified.toEpochMilli(); - } - - /** - * Get time of last snapshot update - * - * @return time of last snapshot update */ public Instant lastModifiedInstant() { return lastModified; @@ -319,16 +282,11 @@ public class FileSnapshot { long currSize; Object currFileKey; try { - BasicFileAttributes fileAttributes = FS.DETECTED.fileAttributes(path); + BasicFileAttributes fileAttributes = getFileAttributes(path); currLastModified = fileAttributes.lastModifiedTime().toInstant(); currSize = fileAttributes.size(); currFileKey = getFileKey(fileAttributes); - } catch (NoSuchFileException e) { - currLastModified = Instant.EPOCH; - currSize = 0L; - currFileKey = MISSING_FILEKEY; - } catch (IOException e) { - LOG.error(e.getMessage(), e); + } catch (NoSuchElementException e) { currLastModified = Instant.EPOCH; currSize = 0L; currFileKey = MISSING_FILEKEY; @@ -586,4 +544,27 @@ public class FileSnapshot { } return fileStoreAttributeCache; } + + private static BasicFileAttributes getFileAttributes(File path) + throws NoSuchElementException { + try { + try { + return FS.DETECTED.fileAttributes(path); + } catch (IOException e) { + if (!FileUtils.isStaleFileHandle(e)) { + throw e; + } + } + } catch (NoSuchFileException e) { + // ignore + } catch (FileSystemException e) { + String msg = e.getMessage(); + if (!msg.endsWith("Not a directory")) { //$NON-NLS-1$ + LOG.error(msg, e); + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + throw new NoSuchElementException(path.toString()); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index cf26f8d284..05bd970789 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -63,6 +63,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.PackRefsCommand; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.CancelledException; import org.eclipse.jgit.errors.CorruptObjectException; @@ -100,9 +102,8 @@ import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.LockToken; import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.GitDateParser; +import org.eclipse.jgit.util.GitTimeParser; import org.eclipse.jgit.util.StringUtils; -import org.eclipse.jgit.util.SystemReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -158,11 +159,11 @@ public class GC { private long expireAgeMillis = -1; - private Date expire; + private Instant expire; private long packExpireAgeMillis = -1; - private Date packExpire; + private Instant packExpire; private Boolean packKeptObjects; @@ -233,9 +234,11 @@ public class GC { * @throws java.text.ParseException * If the configuration parameter "gc.pruneexpire" couldn't be * parsed + * @throws GitAPIException + * If packing refs failed */ public CompletableFuture<Collection<Pack>> gc() - throws IOException, ParseException { + throws IOException, ParseException, GitAPIException { if (!background) { return CompletableFuture.completedFuture(doGc()); } @@ -254,7 +257,7 @@ public class GC { gcLog.commit(); } return newPacks; - } catch (IOException | ParseException e) { + } catch (IOException | ParseException | GitAPIException e) { try { gcLog.write(e.getMessage()); StringWriter sw = new StringWriter(); @@ -277,7 +280,8 @@ public class GC { return (executor != null) ? executor : WorkQueue.getExecutor(); } - private Collection<Pack> doGc() throws IOException, ParseException { + private Collection<Pack> doGc() + throws IOException, ParseException, GitAPIException { if (automatic && !needGc()) { return Collections.emptyList(); } @@ -286,7 +290,8 @@ public class GC { return Collections.emptyList(); } pm.start(6 /* tasks */); - packRefs(); + new PackRefsCommand(repo).setProgressMonitor(pm).setAll(true) + .call(); // TODO: implement reflog_expire(pm, repo); Collection<Pack> newPacks = repack(); prune(Collections.emptySet()); @@ -692,16 +697,18 @@ public class GC { if (expire == null && expireAgeMillis == -1) { String pruneExpireStr = getPruneExpireStr(); - if (pruneExpireStr == null) + if (pruneExpireStr == null) { pruneExpireStr = PRUNE_EXPIRE_DEFAULT; - expire = GitDateParser.parse(pruneExpireStr, null, SystemReader - .getInstance().getLocale()); + } + expire = GitTimeParser.parseInstant(pruneExpireStr); expireAgeMillis = -1; } - if (expire != null) - expireDate = expire.getTime(); - if (expireAgeMillis != -1) + if (expire != null) { + expireDate = expire.toEpochMilli(); + } + if (expireAgeMillis != -1) { expireDate = System.currentTimeMillis() - expireAgeMillis; + } return expireDate; } @@ -718,16 +725,18 @@ public class GC { String prunePackExpireStr = repo.getConfig().getString( ConfigConstants.CONFIG_GC_SECTION, null, ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE); - if (prunePackExpireStr == null) + if (prunePackExpireStr == null) { prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT; - packExpire = GitDateParser.parse(prunePackExpireStr, null, - SystemReader.getInstance().getLocale()); + } + packExpire = GitTimeParser.parseInstant(prunePackExpireStr); packExpireAgeMillis = -1; } - if (packExpire != null) - packExpireDate = packExpire.getTime(); - if (packExpireAgeMillis != -1) + if (packExpire != null) { + packExpireDate = packExpire.toEpochMilli(); + } + if (packExpireAgeMillis != -1) { packExpireDate = System.currentTimeMillis() - packExpireAgeMillis; + } return packExpireDate; } @@ -780,43 +789,6 @@ public class GC { } /** - * Pack ref storage. For a RefDirectory database, this packs all - * non-symbolic, loose refs into packed-refs. For Reftable, all of the data - * is compacted into a single table. - * - * @throws java.io.IOException - * if an IO error occurred - */ - public void packRefs() throws IOException { - RefDatabase refDb = repo.getRefDatabase(); - if (refDb instanceof FileReftableDatabase) { - // TODO: abstract this more cleanly. - pm.beginTask(JGitText.get().packRefs, 1); - try { - ((FileReftableDatabase) refDb).compactFully(); - } finally { - pm.endTask(); - } - return; - } - - Collection<Ref> refs = refDb.getRefsByPrefix(Constants.R_REFS); - List<String> refsToBePacked = new ArrayList<>(refs.size()); - pm.beginTask(JGitText.get().packRefs, refs.size()); - try { - for (Ref ref : refs) { - checkCancelled(); - if (!ref.isSymbolic() && ref.getStorage().isLoose()) - refsToBePacked.add(ref.getName()); - pm.update(1); - } - ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked); - } finally { - pm.endTask(); - } - } - - /** * Packs all objects which reachable from any of the heads into one pack * file. Additionally all objects which are not reachable from any head but * which are reachable from any of the other refs (e.g. tags), special refs @@ -1047,7 +1019,7 @@ public class GC { } private void deleteEmptyRefsFolders() throws IOException { - Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS); + Path refs = repo.getCommonDirectory().toPath().resolve(Constants.R_REFS); // Avoid deleting a folder that was created after the threshold so that concurrent // operations trying to create a reference are not impacted Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS); @@ -1185,7 +1157,7 @@ public class GC { * if an IO error occurred */ private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException { - ReflogReader reflogReader = repo.getReflogReader(ref); + ReflogReader reflogReader = repo.getRefDatabase().getReflogReader(ref); List<ReflogEntry> rlEntries = reflogReader .getReverseEntries(); if (rlEntries == null || rlEntries.isEmpty()) @@ -1509,6 +1481,18 @@ public class GC { public long numberOfPackFiles; /** + * The number of pack files that were created since the last bitmap + * generation. + */ + public long numberOfPackFilesSinceBitmap; + + /** + * The number of objects stored in pack files and as loose object + * created after the last bitmap generation. + */ + public long numberOfObjectsSinceBitmap; + + /** * The number of objects stored as loose objects. */ public long numberOfLooseObjects; @@ -1543,6 +1527,10 @@ public class GC { final StringBuilder b = new StringBuilder(); b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$ + b.append(", numberOfPackFilesSinceBitmap=") //$NON-NLS-1$ + .append(numberOfPackFilesSinceBitmap); + b.append(", numberOfObjectsSinceBitmap=") //$NON-NLS-1$ + .append(numberOfObjectsSinceBitmap); b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$ b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$ b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$ @@ -1563,12 +1551,22 @@ public class GC { public RepoStatistics getStatistics() throws IOException { RepoStatistics ret = new RepoStatistics(); Collection<Pack> packs = repo.getObjectDatabase().getPacks(); + long latestBitmapTime = 0L; for (Pack p : packs) { - ret.numberOfPackedObjects += p.getIndex().getObjectCount(); + long packedObjects = p.getIndex().getObjectCount(); + ret.numberOfPackedObjects += packedObjects; ret.numberOfPackFiles++; ret.sizeOfPackedObjects += p.getPackFile().length(); - if (p.getBitmapIndex() != null) + if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); + if (latestBitmapTime == 0L) { + latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli(); + } + } + else if (latestBitmapTime == 0L) { + ret.numberOfPackFilesSinceBitmap++; + ret.numberOfObjectsSinceBitmap += packedObjects; + } } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); @@ -1584,6 +1582,9 @@ public class GC { continue; ret.numberOfLooseObjects++; ret.sizeOfLooseObjects += f.length(); + if (f.lastModified() > latestBitmapTime) { + ret.numberOfObjectsSinceBitmap ++; + } } } } @@ -1659,12 +1660,31 @@ public class GC { * candidate for pruning. * * @param expire - * instant in time which defines object expiration - * objects with modification time before this instant are expired - * objects with modification time newer or equal to this instant - * are not expired + * instant in time which defines object expiration objects with + * modification time before this instant are expired objects with + * modification time newer or equal to this instant are not + * expired + * @deprecated use {@link #setExpire(Instant)} instead */ + @Deprecated(since = "7.2") public void setExpire(Date expire) { + this.expire = expire.toInstant(); + expireAgeMillis = -1; + } + + /** + * During gc() or prune() each unreferenced, loose object which has been + * created or modified after or at <code>expire</code> will not be pruned. + * Only older objects may be pruned. If set to null then every object is a + * candidate for pruning. + * + * @param expire + * instant in time which defines object expiration objects with + * modification time before this instant are expired objects with + * modification time newer or equal to this instant are not + * expired + */ + public void setExpire(Instant expire) { this.expire = expire; expireAgeMillis = -1; } @@ -1677,8 +1697,24 @@ public class GC { * * @param packExpire * instant in time which defines packfile expiration + * @deprecated use {@link #setPackExpire(Instant)} instead */ + @Deprecated(since = "7.2") public void setPackExpire(Date packExpire) { + this.packExpire = packExpire.toInstant(); + packExpireAgeMillis = -1; + } + + /** + * During gc() or prune() packfiles which are created or modified after or + * at <code>packExpire</code> will not be deleted. Only older packfiles may + * be deleted. If set to null then every packfile is a candidate for + * deletion. + * + * @param packExpire + * instant in time which defines packfile expiration + */ + public void setPackExpire(Instant packExpire) { this.packExpire = packExpire; packExpireAgeMillis = -1; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java index 628bf5db0c..862aaab0ee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java @@ -23,8 +23,7 @@ import java.time.Instant; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.GitDateParser; -import org.eclipse.jgit.util.SystemReader; +import org.eclipse.jgit.util.GitTimeParser; /** * This class manages the gc.log file for a {@link FileRepository}. @@ -50,7 +49,7 @@ class GcLog { */ GcLog(FileRepository repo) { this.repo = repo; - logFile = new File(repo.getDirectory(), "gc.log"); //$NON-NLS-1$ + logFile = new File(repo.getCommonDirectory(), "gc.log"); //$NON-NLS-1$ lock = new LockFile(logFile); } @@ -62,8 +61,7 @@ class GcLog { if (logExpiryStr == null) { logExpiryStr = LOG_EXPIRY_DEFAULT; } - gcLogExpire = GitDateParser.parse(logExpiryStr, null, - SystemReader.getInstance().getLocale()).toInstant(); + gcLogExpire = GitTimeParser.parseInstant(logExpiryStr); } return gcLogExpire; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java index 11d842b246..e8d442b8fb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java @@ -46,7 +46,7 @@ public class InfoAttributesNode extends AttributesNode { FS fs = repository.getFS(); - File attributes = fs.resolve(repository.getDirectory(), + File attributes = fs.resolve(repository.getCommonDirectory(), Constants.INFO_ATTRIBUTES); FileRepository.AttributesNodeProviderImpl.loadRulesFromFile(r, attributes); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java index a2d8bd0140..9e12ee8a0c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java @@ -24,6 +24,7 @@ import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.FileTime; import java.text.MessageFormat; @@ -141,9 +142,8 @@ public class LockFile { throw new IllegalStateException( MessageFormat.format(JGitText.get().lockAlreadyHeld, ref)); } - FileUtils.mkdirs(lck.getParentFile(), true); try { - token = FS.DETECTED.createNewFileAtomic(lck); + token = createLockFileWithRetry(); } catch (IOException e) { LOG.error(JGitText.get().failedCreateLockFile, lck, e); throw e; @@ -160,6 +160,19 @@ public class LockFile { return obtainedLock; } + private FS.LockToken createLockFileWithRetry() throws IOException { + try { + return createLockFile(); + } catch (NoSuchFileException e) { + return createLockFile(); + } + } + + private FS.LockToken createLockFile() throws IOException { + FileUtils.mkdirs(lck.getParentFile(), true); + return FS.DETECTED.createNewFileAtomic(lck); + } + /** * Try to establish the lock for appending. * @@ -515,17 +528,6 @@ public class LockFile { * Get the modification time of the output file when it was committed. * * @return modification time of the lock file right before we committed it. - * @deprecated use {@link #getCommitLastModifiedInstant()} instead - */ - @Deprecated - public long getCommitLastModified() { - return commitSnapshot.lastModified(); - } - - /** - * Get the modification time of the output file when it was committed. - * - * @return modification time of the lock file right before we committed it. */ public Instant getCommitLastModifiedInstant() { return commitSnapshot.lastModifiedInstant(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java index b4bb2a9293..909b3e3082 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -26,8 +26,9 @@ import org.eclipse.jgit.internal.storage.file.FileObjectDatabase.InsertLooseObje import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.util.FileUtils; @@ -49,13 +50,13 @@ class LooseObjects { * Maximum number of attempts to read a loose object for which a stale file * handle exception is thrown */ - private final static int MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS = 5; + private final static int MAX_STALE_READ_RETRIES = 5; private final File directory; private final UnpackedObjectCache unpackedObjectCache; - private final boolean trustFolderStat; + private final TrustStat trustLooseObjectStat; /** * Initialize a reference to an on-disk object directory. @@ -68,9 +69,8 @@ class LooseObjects { LooseObjects(Config config, File dir) { directory = dir; unpackedObjectCache = new UnpackedObjectCache(); - trustFolderStat = config.getBoolean( - ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + trustLooseObjectStat = config.get(CoreConfig.KEY) + .getTrustLooseObjectStat(); } /** @@ -108,7 +108,8 @@ class LooseObjects { */ boolean has(AnyObjectId objectId) { boolean exists = hasWithoutRefresh(objectId); - if (trustFolderStat || exists) { + if (trustLooseObjectStat == TrustStat.ALWAYS + || exists) { return exists; } try (InputStream stream = Files.newInputStream(directory.toPath())) { @@ -163,13 +164,31 @@ class LooseObjects { } ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException { - int readAttempts = 0; - while (readAttempts < MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS) { - readAttempts++; - File path = fileFor(id); - if (trustFolderStat && !path.exists()) { + File path = fileFor(id); + for (int retries = 0; retries < MAX_STALE_READ_RETRIES; retries++) { + boolean reload = true; + switch (trustLooseObjectStat) { + case NEVER: break; + case AFTER_OPEN: + try (InputStream stream = Files + .newInputStream(path.getParentFile().toPath())) { + // open the loose object's fanout directory to refresh + // attributes (on some NFS clients) + } catch (FileNotFoundException | NoSuchFileException e) { + // ignore + } + //$FALL-THROUGH$ + case ALWAYS: + if (!path.exists()) { + reload = false; + } + break; + case INHERIT: + // only used in CoreConfig internally + throw new IllegalStateException(); } + if (reload) { try { return getObjectLoader(curs, path, id); } catch (FileNotFoundException noFile) { @@ -183,9 +202,10 @@ class LooseObjects { } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( - JGitText.get().looseObjectHandleIsStale, id.name(), - Integer.valueOf(readAttempts), Integer.valueOf( - MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS))); + JGitText.get().looseObjectHandleIsStale, + id.name(), Integer.valueOf(retries), + Integer.valueOf(MAX_STALE_READ_RETRIES))); + } } } } @@ -211,7 +231,7 @@ class LooseObjects { try { return getObjectLoaderWithoutRefresh(curs, path, id); } catch (FileNotFoundException e) { - if (trustFolderStat) { + if (trustLooseObjectStat == TrustStat.ALWAYS) { throw e; } try (InputStream stream = Files @@ -248,7 +268,7 @@ class LooseObjects { return getSizeWithoutRefresh(curs, id); } catch (FileNotFoundException noFile) { try { - if (trustFolderStat) { + if (trustLooseObjectStat == TrustStat.ALWAYS) { throw noFile; } try (InputStream stream = Files diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 9f27f4bd6e..746e124e1f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -28,6 +28,7 @@ import java.util.zip.Deflater; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; @@ -110,7 +111,7 @@ public class ObjectDirectoryPackParser extends PackParser { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; @@ -386,9 +387,9 @@ public class ObjectDirectoryPackParser extends PackParser { try (FileOutputStream os = new FileOutputStream(tmpIdx)) { final PackIndexWriter iw; if (indexVersion <= 0) - iw = PackIndexWriter.createOldestPossible(os, list); + iw = BasePackIndexWriter.createOldestPossible(os, list); else - iw = PackIndexWriter.createVersion(os, indexVersion); + iw = BasePackIndexWriter.createVersion(os, indexVersion); iw.write(list, packHash); os.getChannel().force(true); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index f87329ccc2..5813d39e9a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -95,6 +95,9 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { private RandomAccessFile fd; + /** For managing open/close accounting of {@link #fd}. */ + private final Object activeLock = new Object(); + /** Serializes reads performed against {@link #fd}. */ private final Object readLock = new Object(); @@ -113,13 +116,13 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { private volatile Exception invalidatingCause; @Nullable - private PackFile bitmapIdxFile; + private volatile PackFile bitmapIdxFile; private AtomicInteger transientErrorCount = new AtomicInteger(); private byte[] packChecksum; - private volatile Optionally<PackIndex> loadedIdx = Optionally.empty(); + private Optionally<PackIndex> loadedIdx = Optionally.empty(); private Optionally<PackReverseIndex> reverseIdx = Optionally.empty(); @@ -159,60 +162,54 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { length = Long.MAX_VALUE; } - private PackIndex idx() throws IOException { + private synchronized PackIndex idx() throws IOException { Optional<PackIndex> optional = loadedIdx.getOptional(); if (optional.isPresent()) { return optional.get(); } - synchronized (this) { - optional = loadedIdx.getOptional(); - if (optional.isPresent()) { - return optional.get(); - } - if (invalid) { - throw new PackInvalidException(packFile, invalidatingCause); + if (invalid) { + throw new PackInvalidException(packFile, invalidatingCause); + } + try { + long start = System.currentTimeMillis(); + PackFile idxFile = packFile.create(INDEX); + PackIndex idx = PackIndex.open(idxFile); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ + idxFile.getAbsolutePath(), + Float.valueOf(idxFile.length() + / (1024f * 1024)), + Long.valueOf(System.currentTimeMillis() + - start))); } - try { - long start = System.currentTimeMillis(); - PackFile idxFile = packFile.create(INDEX); - PackIndex idx = PackIndex.open(idxFile); - if (LOG.isDebugEnabled()) { - LOG.debug(String.format( - "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ - idxFile.getAbsolutePath(), - Float.valueOf(idxFile.length() - / (1024f * 1024)), - Long.valueOf(System.currentTimeMillis() - - start))); - } - if (packChecksum == null) { - packChecksum = idx.packChecksum; - fileSnapshot.setChecksum( - ObjectId.fromRaw(packChecksum)); - } else if (!Arrays.equals(packChecksum, - idx.packChecksum)) { - throw new PackMismatchException(MessageFormat - .format(JGitText.get().packChecksumMismatch, - packFile.getPath(), - PackExt.PACK.getExtension(), - Hex.toHexString(packChecksum), - PackExt.INDEX.getExtension(), - Hex.toHexString(idx.packChecksum))); - } - loadedIdx = optionally(idx); - return idx; - } catch (InterruptedIOException e) { - // don't invalidate the pack, we are interrupted from - // another thread - throw e; - } catch (IOException e) { - invalid = true; - invalidatingCause = e; - throw e; + packChecksum = idx.getChecksum(); + fileSnapshot.setChecksum( + ObjectId.fromRaw(packChecksum)); + } else if (!Arrays.equals(packChecksum, + idx.getChecksum())) { + throw new PackMismatchException(MessageFormat + .format(JGitText.get().packChecksumMismatch, + packFile.getPath(), + PackExt.PACK.getExtension(), + Hex.toHexString(packChecksum), + PackExt.INDEX.getExtension(), + Hex.toHexString(idx.getChecksum()))); } + loadedIdx = optionally(idx); + return idx; + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from + // another thread + throw e; + } catch (IOException e) { + invalid = true; + invalidatingCause = e; + throw e; } } + /** * Get the File object which locates this pack on disk. * @@ -296,15 +293,28 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } /** - * Close the resources utilized by this repository + * Close the resources utilized by these pack files + * + * @param packs + * packs to close + */ + public static void close(Set<Pack> packs) { + WindowCache.purge(packs); + packs.forEach(p -> p.closeIndices()); + } + + /** + * Close the resources utilized by this pack file */ public void close() { WindowCache.purge(this); - synchronized (this) { - loadedIdx.clear(); - reverseIdx.clear(); - bitmapIdx.clear(); - } + closeIndices(); + } + + private synchronized void closeIndices() { + loadedIdx.clear(); + reverseIdx.clear(); + bitmapIdx.clear(); } /** @@ -416,185 +426,202 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { final CRC32 crc2 = validate ? new CRC32() : null; final byte[] buf = out.getCopyBuffer(); + boolean isHeaderWritten = false; // Rip apart the header so we can discover the size. // - readFully(src.offset, buf, 0, 20, curs); - int c = buf[0] & 0xff; - final int typeCode = (c >> 4) & 7; - long inflatedLength = c & 15; - int shift = 4; - int headerCnt = 1; - while ((c & 0x80) != 0) { - c = buf[headerCnt++] & 0xff; - inflatedLength += ((long) (c & 0x7f)) << shift; - shift += 7; - } - - if (typeCode == Constants.OBJ_OFS_DELTA) { - do { + try { + readFully(src.offset, buf, 0, 20, curs); + + int c = buf[0] & 0xff; + final int typeCode = (c >> 4) & 7; + long inflatedLength = c & 15; + int shift = 4; + int headerCnt = 1; + while ((c & 0x80) != 0) { c = buf[headerCnt++] & 0xff; - } while ((c & 128) != 0); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); + inflatedLength += ((long) (c & 0x7f)) << shift; + shift += 7; } - } else if (typeCode == Constants.OBJ_REF_DELTA) { - if (validate) { + + if (typeCode == Constants.OBJ_OFS_DELTA) { + do { + c = buf[headerCnt++] & 0xff; + } while ((c & 128) != 0); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + } else if (typeCode == Constants.OBJ_REF_DELTA) { + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + + readFully(src.offset + headerCnt, buf, 0, 20, curs); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, 20); + crc2.update(buf, 0, 20); + } + headerCnt += 20; + } else if (validate) { assert(crc1 != null && crc2 != null); crc1.update(buf, 0, headerCnt); crc2.update(buf, 0, headerCnt); } - readFully(src.offset + headerCnt, buf, 0, 20, curs); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, 20); - crc2.update(buf, 0, 20); - } - headerCnt += 20; - } else if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } - - final long dataOffset = src.offset + headerCnt; - final long dataLength = src.length; - final long expectedCRC; - final ByteArrayWindow quickCopy; + final long dataOffset = src.offset + headerCnt; + final long dataLength = src.length; + final long expectedCRC; + final ByteArrayWindow quickCopy; - // Verify the object isn't corrupt before sending. If it is, - // we report it missing instead. - // - try { - quickCopy = curs.quickCopy(this, dataOffset, dataLength); + // Verify the object isn't corrupt before sending. If it is, + // we report it missing instead. + // + try { + quickCopy = curs.quickCopy(this, dataOffset, dataLength); - if (validate && idx().hasCRC32Support()) { - assert(crc1 != null); - // Index has the CRC32 code cached, validate the object. - // - expectedCRC = idx().findCRC32(src); - if (quickCopy != null) { - quickCopy.crc32(crc1, dataOffset, (int) dataLength); - } else { - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - pos += n; - cnt -= n; + if (validate && idx().hasCRC32Support()) { + assert(crc1 != null); + // Index has the CRC32 code cached, validate the object. + // + expectedCRC = idx().findCRC32(src); + if (quickCopy != null) { + quickCopy.crc32(crc1, dataOffset, (int) dataLength); + } else { + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + pos += n; + cnt -= n; + } } + if (crc1.getValue() != expectedCRC) { + setCorrupt(src.offset); + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } + } else if (validate) { + // We don't have a CRC32 code in the index, so compute it + // now while inflating the raw data to get zlib to tell us + // whether or not the data is safe. + // + Inflater inf = curs.inflater(); + byte[] tmp = new byte[1024]; + if (quickCopy != null) { + quickCopy.check(inf, tmp, dataOffset, (int) dataLength); + } else { + assert(crc1 != null); + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + inf.setInput(buf, 0, n); + while (inf.inflate(tmp, 0, tmp.length) > 0) + continue; + pos += n; + cnt -= n; + } + } + if (!inf.finished() || inf.getBytesRead() != dataLength) { + setCorrupt(src.offset); + throw new EOFException(MessageFormat.format( + JGitText.get().shortCompressedStreamAt, + Long.valueOf(src.offset))); + } + assert(crc1 != null); + expectedCRC = crc1.getValue(); + } else { + expectedCRC = -1; } - if (crc1.getValue() != expectedCRC) { - setCorrupt(src.offset); - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); - } - } else if (validate) { - // We don't have a CRC32 code in the index, so compute it - // now while inflating the raw data to get zlib to tell us - // whether or not the data is safe. + } catch (DataFormatException dataFormat) { + setCorrupt(src.offset); + + CorruptObjectException corruptObject = new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile()), + dataFormat); + + throw new StoredObjectRepresentationNotAvailableException( + corruptObject); + } + + if (quickCopy != null) { + // The entire object fits into a single byte array window slice, + // and we have it pinned. Write this out without copying. // - Inflater inf = curs.inflater(); - byte[] tmp = new byte[1024]; - if (quickCopy != null) { - quickCopy.check(inf, tmp, dataOffset, (int) dataLength); - } else { - assert(crc1 != null); + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + quickCopy.write(out, dataOffset, (int) dataLength); + + } else if (dataLength <= buf.length) { + // Tiny optimization: Lots of objects are very small deltas or + // deflated commits that are likely to fit in the copy buffer. + // + if (!validate) { long pos = dataOffset; long cnt = dataLength; while (cnt > 0) { final int n = (int) Math.min(cnt, buf.length); readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - inf.setInput(buf, 0, n); - while (inf.inflate(tmp, 0, tmp.length) > 0) - continue; pos += n; cnt -= n; } } - if (!inf.finished() || inf.getBytesRead() != dataLength) { - setCorrupt(src.offset); - throw new EOFException(MessageFormat.format( - JGitText.get().shortCompressedStreamAt, - Long.valueOf(src.offset))); - } - assert(crc1 != null); - expectedCRC = crc1.getValue(); + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + out.write(buf, 0, (int) dataLength); } else { - expectedCRC = -1; - } - } catch (DataFormatException dataFormat) { - setCorrupt(src.offset); - - CorruptObjectException corruptObject = new CorruptObjectException( - MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile()), - dataFormat); - - throw new StoredObjectRepresentationNotAvailableException( - corruptObject); - - } catch (IOException ioError) { - throw new StoredObjectRepresentationNotAvailableException(ioError); - } - - if (quickCopy != null) { - // The entire object fits into a single byte array window slice, - // and we have it pinned. Write this out without copying. - // - out.writeHeader(src, inflatedLength); - quickCopy.write(out, dataOffset, (int) dataLength); - - } else if (dataLength <= buf.length) { - // Tiny optimization: Lots of objects are very small deltas or - // deflated commits that are likely to fit in the copy buffer. - // - if (!validate) { + // Now we are committed to sending the object. As we spool it out, + // check its CRC32 code to make sure there wasn't corruption between + // the verification we did above, and us actually outputting it. + // long pos = dataOffset; long cnt = dataLength; while (cnt > 0) { final int n = (int) Math.min(cnt, buf.length); readFully(pos, buf, 0, n, curs); - pos += n; + if (validate) { + assert(crc2 != null); + crc2.update(buf, 0, n); + } cnt -= n; + if (!isHeaderWritten) { + if (invalid && cnt > 0) { + // Since this is not the last iteration and the packfile is invalid, + // better to assume the iterations will not all complete here while + // it is still likely recoverable. + throw new StoredObjectRepresentationNotAvailableException(invalidatingCause); + } + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + } + out.write(buf, 0, n); + pos += n; } - } - out.writeHeader(src, inflatedLength); - out.write(buf, 0, (int) dataLength); - } else { - // Now we are committed to sending the object. As we spool it out, - // check its CRC32 code to make sure there wasn't corruption between - // the verification we did above, and us actually outputting it. - // - out.writeHeader(src, inflatedLength); - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); if (validate) { assert(crc2 != null); - crc2.update(buf, 0, n); + if (crc2.getValue() != expectedCRC) { + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } } - out.write(buf, 0, n); - pos += n; - cnt -= n; } - if (validate) { - assert(crc2 != null); - if (crc2.getValue() != expectedCRC) { - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); - } + } catch (IOException ioError) { + if (!isHeaderWritten) { + throw new StoredObjectRepresentationNotAvailableException(ioError); } + throw ioError; } } @@ -621,42 +648,53 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { throw new EOFException(); } - private synchronized void beginCopyAsIs() + private void beginCopyAsIs() throws StoredObjectRepresentationNotAvailableException { - if (++activeCopyRawData == 1 && activeWindows == 0) { - try { - doOpen(); - } catch (IOException thisPackNotValid) { - throw new StoredObjectRepresentationNotAvailableException( - thisPackNotValid); + synchronized (activeLock) { + if (++activeCopyRawData == 1 && activeWindows == 0) { + try { + doOpen(); + } catch (IOException thisPackNotValid) { + throw new StoredObjectRepresentationNotAvailableException( + thisPackNotValid); + } } } } - private synchronized void endCopyAsIs() { - if (--activeCopyRawData == 0 && activeWindows == 0) - doClose(); + private void endCopyAsIs() { + synchronized (activeLock) { + if (--activeCopyRawData == 0 && activeWindows == 0) { + doClose(); + } + } } - synchronized boolean beginWindowCache() throws IOException { - if (++activeWindows == 1) { - if (activeCopyRawData == 0) - doOpen(); - return true; + boolean beginWindowCache() throws IOException { + synchronized (activeLock) { + if (++activeWindows == 1) { + if (activeCopyRawData == 0) { + doOpen(); + } + return true; + } + return false; } - return false; } - synchronized boolean endWindowCache() { - final boolean r = --activeWindows == 0; - if (r && activeCopyRawData == 0) - doClose(); - return r; + boolean endWindowCache() { + synchronized (activeLock) { + boolean r = --activeWindows == 0; + if (r && activeCopyRawData == 0) { + doClose(); + } + return r; + } } private void doOpen() throws IOException { if (invalid) { - openFail(true, invalidatingCause); + openFail(invalidatingCause); throw new PackInvalidException(packFile, invalidatingCause); } try { @@ -667,39 +705,41 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } } catch (InterruptedIOException e) { // don't invalidate the pack, we are interrupted from another thread - openFail(false, e); + openFail(e); throw e; } catch (FileNotFoundException fn) { - // don't invalidate the pack if opening an existing file failed - // since it may be related to a temporary lack of resources (e.g. - // max open files) - openFail(!packFile.exists(), fn); + if (!packFile.exists()) { + // Failure to open an existing file may be related to a temporary lack of resources + // (e.g. max open files) + invalid = true; + } + openFail(fn); throw fn; } catch (EOFException | AccessDeniedException | NoSuchFileException | CorruptObjectException | NoPackSignatureException | PackMismatchException | UnpackException | UnsupportedPackIndexVersionException | UnsupportedPackVersionException pe) { - // exceptions signaling permanent problems with a pack - openFail(true, pe); + invalid = true; // exceptions signaling permanent problems with a pack + openFail(pe); throw pe; } catch (IOException ioe) { - // mark this packfile as invalid when NFS stale file handle error - // occur - openFail(FileUtils.isStaleFileHandleInCausalChain(ioe), ioe); + if (FileUtils.isStaleFileHandleInCausalChain(ioe)) { + invalid = true; + } + openFail(ioe); throw ioe; } catch (RuntimeException ge) { // generic exceptions could be transient so we should not mark the // pack invalid to avoid false MissingObjectExceptions - openFail(false, ge); + openFail(ge); throw ge; } } - private void openFail(boolean invalidate, Exception cause) { + private void openFail(Exception cause) { activeWindows = 0; activeCopyRawData = 0; - invalid = invalidate; invalidatingCause = cause; doClose(); } @@ -791,7 +831,7 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { MessageFormat.format(JGitText.get().packChecksumMismatch, getPackFile(), PackExt.PACK.getExtension(), Hex.toHexString(buf), PackExt.INDEX.getExtension(), - Hex.toHexString(idx.packChecksum))); + Hex.toHexString(idx.getChecksum()))); } } @@ -1173,17 +1213,8 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { return null; } - synchronized void refreshBitmapIndex(PackFile bitmapIndexFile) { - this.bitmapIdx = Optionally.empty(); - this.invalid = false; + void setBitmapIndexFile(PackFile bitmapIndexFile) { this.bitmapIdxFile = bitmapIndexFile; - try { - getBitmapIndex(); - } catch (IOException e) { - LOG.warn(JGitText.get().bitmapFailedToGet, bitmapIdxFile, e); - this.bitmapIdx = Optionally.empty(); - this.bitmapIdxFile = null; - } } private synchronized PackReverseIndex getReverseIdx() throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 8221cff442..f50c17eafa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -17,6 +17,8 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; @@ -24,6 +26,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -41,7 +44,8 @@ import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.util.FileUtils; @@ -71,7 +75,7 @@ class PackDirectory { private final AtomicReference<PackList> packList; - private final boolean trustFolderStat; + private final TrustStat trustPackStat; /** * Initialize a reference to an on-disk 'pack' directory. @@ -85,14 +89,7 @@ class PackDirectory { this.config = config; this.directory = directory; packList = new AtomicReference<>(NO_PACKS); - - // Whether to trust the pack folder's modification time. If set to false - // we will always scan the .git/objects/pack folder to check for new - // pack files. If set to true (default) we use the folder's size, - // modification time, and key (inode) and assume that no new pack files - // can be in this folder if these attributes have not changed. - trustFolderStat = config.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + trustPackStat = config.get(CoreConfig.KEY).getTrustPackStat(); } /** @@ -111,9 +108,7 @@ class PackDirectory { void close() { PackList packs = packList.get(); if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { - for (Pack p : packs.packs) { - p.close(); - } + Pack.close(Set.of(packs.packs)); } } @@ -314,38 +309,42 @@ class PackDirectory { } private void handlePackError(IOException e, Pack p) { - String warnTmpl = null; + String warnTemplate = null; + String debugTemplate = null; int transientErrorCount = 0; - String errTmpl = JGitText.get().exceptionWhileReadingPack; + String errorTemplate = JGitText.get().exceptionWhileReadingPack; if ((e instanceof CorruptObjectException) || (e instanceof PackInvalidException)) { - warnTmpl = JGitText.get().corruptPack; - LOG.warn(MessageFormat.format(warnTmpl, + warnTemplate = JGitText.get().corruptPack; + LOG.warn(MessageFormat.format(warnTemplate, p.getPackFile().getAbsolutePath()), e); // Assume the pack is corrupted, and remove it from the list. remove(p); } else if (e instanceof FileNotFoundException) { if (p.getPackFile().exists()) { - errTmpl = JGitText.get().packInaccessible; + errorTemplate = JGitText.get().packInaccessible; transientErrorCount = p.incrementTransientErrorCount(); } else { - warnTmpl = JGitText.get().packWasDeleted; + debugTemplate = JGitText.get().packWasDeleted; remove(p); } } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { - warnTmpl = JGitText.get().packHandleIsStale; + warnTemplate = JGitText.get().packHandleIsStale; remove(p); } else { transientErrorCount = p.incrementTransientErrorCount(); } - if (warnTmpl != null) { - LOG.warn(MessageFormat.format(warnTmpl, + if (warnTemplate != null) { + LOG.warn(MessageFormat.format(warnTemplate, p.getPackFile().getAbsolutePath()), e); + } else if (debugTemplate != null) { + LOG.debug(MessageFormat.format(debugTemplate, + p.getPackFile().getAbsolutePath()), e); } else { if (doLogExponentialBackoff(transientErrorCount)) { // Don't remove the pack from the list, as the error may be // transient. - LOG.error(MessageFormat.format(errTmpl, + LOG.error(MessageFormat.format(errorTemplate, p.getPackFile().getAbsolutePath(), Integer.valueOf(transientErrorCount)), e); } @@ -362,8 +361,26 @@ class PackDirectory { } boolean searchPacksAgain(PackList old) { - return (!trustFolderStat || old.snapshot.isModified(directory)) - && old != scanPacks(old); + switch (trustPackStat) { + case NEVER: + break; + case AFTER_OPEN: + try (InputStream stream = Files + .newInputStream(directory.toPath())) { + // open the pack directory to refresh attributes (on some NFS clients) + } catch (IOException e) { + // ignore + } + //$FALL-THROUGH$ + case ALWAYS: + if (!old.snapshot.isModified(directory)) { + return false; + } + break; + case INHERIT: + // only used in CoreConfig internally + } + return old != scanPacks(old); } void insert(Pack pack) { @@ -460,12 +477,9 @@ class PackDirectory { && !oldPack.getFileSnapshot().isModified(packFile)) { forReuse.remove(packFile.getName()); list.add(oldPack); - try { - if(oldPack.getBitmapIndex() == null) { - oldPack.refreshBitmapIndex(packFilesByExt.get(BITMAP_INDEX)); - } - } catch (IOException e) { - LOG.warn(JGitText.get().bitmapAccessErrorForPackfile, oldPack.getPackName(), e); + PackFile bitMaps = packFilesByExt.get(BITMAP_INDEX); + if (bitMaps != null) { + oldPack.setBitmapIndexFile(bitMaps); } continue; } @@ -484,9 +498,7 @@ class PackDirectory { return old; } - for (Pack p : forReuse.values()) { - p.close(); - } + Pack.close(new HashSet<>(forReuse.values())); if (list.isEmpty()) { return new PackList(snapshot, NO_PACKS.packs); @@ -545,7 +557,7 @@ class PackDirectory { for (String name : nameList) { try { PackFile pack = new PackFile(directory, name); - if (pack.getPackExt() != null) { + if (pack.getPackExt() != null && !pack.isTmpGCFile()) { Map<PackExt, PackFile> packByExt = packFilesByExtById .get(pack.getId()); if (packByExt == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index 19979d0ed5..c9b05ad025 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -27,6 +27,7 @@ public class PackFile extends File { private static final long serialVersionUID = 1L; private static final String PREFIX = "pack-"; //$NON-NLS-1$ + private static final String TMP_GC_PREFIX = ".tmp-"; //$NON-NLS-1$ private final String base; // PREFIX + id i.e. // pack-0123456789012345678901234567890123456789 @@ -126,6 +127,13 @@ public class PackFile extends File { } /** + * @return whether the file is a temporary GC file + */ + public boolean isTmpGCFile() { + return id.startsWith(TMP_GC_PREFIX); + } + + /** * Create a new similar PackFile with the given extension instead. * * @param ext diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index c2c3775d67..b3e4efb4fc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -42,8 +42,8 @@ import org.eclipse.jgit.util.io.SilentFileInputStream; * by ObjectId. * </p> */ -public abstract class PackIndex - implements Iterable<PackIndex.MutableEntry>, ObjectIdSet { +public interface PackIndex + extends Iterable<PackIndex.MutableEntry>, ObjectIdSet { /** * Open an existing pack <code>.idx</code> file for reading. * <p> @@ -61,7 +61,7 @@ public abstract class PackIndex * the file exists but could not be read due to security errors, * unrecognized data version, or unexpected data corruption. */ - public static PackIndex open(File idxFile) throws IOException { + static PackIndex open(File idxFile) throws IOException { try (SilentFileInputStream fd = new SilentFileInputStream( idxFile)) { return read(fd); @@ -92,7 +92,7 @@ public abstract class PackIndex * @throws org.eclipse.jgit.errors.CorruptObjectException * the stream does not contain a valid pack index. */ - public static PackIndex read(InputStream fd) throws IOException, + static PackIndex read(InputStream fd) throws IOException, CorruptObjectException { final byte[] hdr = new byte[8]; IO.readFully(fd, hdr, 0, hdr.length); @@ -109,16 +109,13 @@ public abstract class PackIndex } private static boolean isTOC(byte[] h) { - final byte[] toc = PackIndexWriter.TOC; + final byte[] toc = BasePackIndexWriter.TOC; for (int i = 0; i < toc.length; i++) if (h[i] != toc[i]) return false; return true; } - /** Footer checksum applied on the bottom of the pack file. */ - protected byte[] packChecksum; - /** * Determine if an object is contained within the pack file. * @@ -126,12 +123,12 @@ public abstract class PackIndex * the object to look for. Must not be null. * @return true if the object is listed in this index; false otherwise. */ - public boolean hasObject(AnyObjectId id) { + default boolean hasObject(AnyObjectId id) { return findOffset(id) != -1; } @Override - public boolean contains(AnyObjectId id) { + default boolean contains(AnyObjectId id) { return findOffset(id) != -1; } @@ -147,7 +144,7 @@ public abstract class PackIndex * </p> */ @Override - public abstract Iterator<MutableEntry> iterator(); + Iterator<MutableEntry> iterator(); /** * Obtain the total number of objects described by this index. @@ -155,7 +152,7 @@ public abstract class PackIndex * @return number of objects in this index, and likewise in the associated * pack that this index was generated from. */ - public abstract long getObjectCount(); + long getObjectCount(); /** * Obtain the total number of objects needing 64 bit offsets. @@ -163,7 +160,7 @@ public abstract class PackIndex * @return number of objects in this index using a 64 bit offset; that is an * object positioned after the 2 GB position within the file. */ - public abstract long getOffset64Count(); + long getOffset64Count(); /** * Get ObjectId for the n-th object entry returned by {@link #iterator()}. @@ -185,7 +182,7 @@ public abstract class PackIndex * is 0, the second is 1, etc. * @return the ObjectId for the corresponding entry. */ - public abstract ObjectId getObjectId(long nthPosition); + ObjectId getObjectId(long nthPosition); /** * Get ObjectId for the n-th object entry returned by {@link #iterator()}. @@ -209,7 +206,7 @@ public abstract class PackIndex * negative, but still valid. * @return the ObjectId for the corresponding entry. */ - public final ObjectId getObjectId(int nthPosition) { + default ObjectId getObjectId(int nthPosition) { if (nthPosition >= 0) return getObjectId((long) nthPosition); final int u31 = nthPosition >>> 1; @@ -228,7 +225,7 @@ public abstract class PackIndex * etc. Positions past 2**31-1 are negative, but still valid. * @return the offset in a pack for the corresponding entry. */ - protected abstract long getOffset(long nthPosition); + long getOffset(long nthPosition); /** * Locate the file offset position for the requested object. @@ -239,7 +236,7 @@ public abstract class PackIndex * object does not exist in this index and is thus not stored in the * associated pack. */ - public abstract long findOffset(AnyObjectId objId); + long findOffset(AnyObjectId objId); /** * Locate the position of this id in the list of object-ids in the index @@ -250,7 +247,7 @@ public abstract class PackIndex * of ids stored in this index; -1 if the object does not exist in * this index and is thus not stored in the associated pack. */ - public abstract int findPosition(AnyObjectId objId); + int findPosition(AnyObjectId objId); /** * Retrieve stored CRC32 checksum of the requested object raw-data @@ -264,7 +261,7 @@ public abstract class PackIndex * @throws java.lang.UnsupportedOperationException * when this index doesn't support CRC32 checksum */ - public abstract long findCRC32(AnyObjectId objId) + long findCRC32(AnyObjectId objId) throws MissingObjectException, UnsupportedOperationException; /** @@ -272,7 +269,7 @@ public abstract class PackIndex * * @return true if CRC32 is stored, false otherwise */ - public abstract boolean hasCRC32Support(); + boolean hasCRC32Support(); /** * Find objects matching the prefix abbreviation. @@ -288,8 +285,8 @@ public abstract class PackIndex * @throws java.io.IOException * the index cannot be read. */ - public abstract void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, - int matchLimit) throws IOException; + void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) throws IOException; /** * Get pack checksum @@ -297,18 +294,18 @@ public abstract class PackIndex * @return the checksum of the pack; caller must not modify it * @since 5.5 */ - public byte[] getChecksum() { - return packChecksum; - } + byte[] getChecksum(); /** * Represent mutable entry of pack index consisting of object id and offset * in pack (both mutable). * */ - public static class MutableEntry { + class MutableEntry { + /** Buffer of the ObjectId visited by the EntriesIterator. */ final MutableObjectId idBuffer = new MutableObjectId(); + /** Offset into the packfile of the current object. */ long offset; /** @@ -326,7 +323,6 @@ public abstract class PackIndex * @return hex string describing the object id of this entry. */ public String name() { - ensureId(); return idBuffer.name(); } @@ -336,7 +332,6 @@ public abstract class PackIndex * @return a copy of the object id. */ public ObjectId toObjectId() { - ensureId(); return idBuffer.toObjectId(); } @@ -347,27 +342,64 @@ public abstract class PackIndex */ public MutableEntry cloneEntry() { final MutableEntry r = new MutableEntry(); - ensureId(); r.idBuffer.fromObjectId(idBuffer); r.offset = offset; return r; } - void ensureId() { - // Override in implementations. + /** + * Similar to {@link Comparable#compareTo(Object)}, using only the + * object id in the entry. + * + * @param other + * Another mutable entry (probably from another index) + * + * @return a negative integer, zero, or a positive integer as this + * object is less than, equal to, or greater than the specified + * object. + */ + public int compareBySha1To(MutableEntry other) { + return idBuffer.compareTo(other.idBuffer); + } + + /** + * Copy the current ObjectId to dest + * <p> + * Like {@link #toObjectId()}, but reusing the destination instead of + * creating a new ObjectId instance. + * + * @param dest + * destination for the object id + */ + public void copyOidTo(MutableObjectId dest) { + dest.fromObjectId(idBuffer); } } + /** + * Base implementation of the iterator over index entries. + */ abstract class EntriesIterator implements Iterator<MutableEntry> { - protected final MutableEntry entry = initEntry(); + private final long objectCount; - protected long returnedNumber = 0; + private final MutableEntry entry = new MutableEntry(); - protected abstract MutableEntry initEntry(); + /** Counts number of entries accessed so far. */ + private long returnedNumber = 0; + + /** + * Construct an iterator that can move objectCount times forward. + * + * @param objectCount + * the number of objects in the PackFile. + */ + protected EntriesIterator(long objectCount) { + this.objectCount = objectCount; + } @Override public boolean hasNext() { - return returnedNumber < getObjectCount(); + return returnedNumber < objectCount; } /** @@ -375,7 +407,55 @@ public abstract class PackIndex * element. */ @Override - public abstract MutableEntry next(); + public MutableEntry next() { + readNext(); + returnedNumber++; + return entry; + } + + /** + * Used by subclasses to load the next entry into the MutableEntry. + * <p> + * Subclasses are expected to populate the entry with + * {@link #setIdBuffer} and {@link #setOffset}. + */ + protected abstract void readNext(); + + /** + * Copies to the entry an {@link ObjectId} from the int buffer and + * position idx + * + * @param raw + * the raw data + * @param idx + * the index into {@code raw} + */ + protected void setIdBuffer(int[] raw, int idx) { + entry.idBuffer.fromRaw(raw, idx); + } + + /** + * Copies to the entry an {@link ObjectId} from the byte array at + * position idx. + * + * @param raw + * the raw data + * @param idx + * the index into {@code raw} + */ + protected void setIdBuffer(byte[] raw, int idx) { + entry.idBuffer.fromRaw(raw, idx); + } + + /** + * Sets the {@code offset} to the entry + * + * @param offset + * the offset in the pack file + */ + protected void setOffset(long offset) { + entry.offset = offset; + } @Override public void remove() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index 5180df46bf..be48358a0d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -29,13 +29,16 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.NB; -class PackIndexV1 extends PackIndex { +class PackIndexV1 implements PackIndex { private static final int IDX_HDR_LEN = 256 * 4; private static final int RECORD_SIZE = 4 + Constants.OBJECT_ID_LENGTH; private final long[] idxHeader; + /** Footer checksum applied on the bottom of the pack file. */ + protected byte[] packChecksum; + byte[][] idxdata; private long objectCnt; @@ -118,7 +121,7 @@ class PackIndexV1 extends PackIndex { } @Override - protected long getOffset(long nthPosition) { + public long getOffset(long nthPosition) { final int levelOne = findLevelOne(nthPosition); final int levelTwo = getLevelTwo(nthPosition, levelOne); final int p = (4 + Constants.OBJECT_ID_LENGTH) * levelTwo; @@ -200,7 +203,7 @@ class PackIndexV1 extends PackIndex { @Override public Iterator<MutableEntry> iterator() { - return new IndexV1Iterator(); + return new EntriesIteratorV1(this); } @Override @@ -238,32 +241,35 @@ class PackIndexV1 extends PackIndex { return (RECORD_SIZE * mid) + 4; } - private class IndexV1Iterator extends EntriesIterator { - int levelOne; + @Override + public byte[] getChecksum() { + return packChecksum; + } - int levelTwo; + private static class EntriesIteratorV1 extends EntriesIterator { + private int levelOne; - @Override - protected MutableEntry initEntry() { - return new MutableEntry() { - @Override - protected void ensureId() { - idBuffer.fromRaw(idxdata[levelOne], levelTwo - - Constants.OBJECT_ID_LENGTH); - } - }; + private int levelTwo; + + private final PackIndexV1 packIndex; + + private EntriesIteratorV1(PackIndexV1 packIndex) { + super(packIndex.objectCnt); + this.packIndex = packIndex; } @Override - public MutableEntry next() { - for (; levelOne < idxdata.length; levelOne++) { - if (idxdata[levelOne] == null) + protected void readNext() { + for (; levelOne < packIndex.idxdata.length; levelOne++) { + if (packIndex.idxdata[levelOne] == null) continue; - if (levelTwo < idxdata[levelOne].length) { - entry.offset = NB.decodeUInt32(idxdata[levelOne], levelTwo); - levelTwo += Constants.OBJECT_ID_LENGTH + 4; - returnedNumber++; - return entry; + if (levelTwo < packIndex.idxdata[levelOne].length) { + super.setOffset(NB.decodeUInt32(packIndex.idxdata[levelOne], + levelTwo)); + this.levelTwo += Constants.OBJECT_ID_LENGTH + 4; + super.setIdBuffer(packIndex.idxdata[levelOne], + levelTwo - Constants.OBJECT_ID_LENGTH); + return; } levelTwo = 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index 751b62dc40..36e54fcd62 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -28,7 +28,7 @@ import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.NB; /** Support for the pack index v2 format. */ -class PackIndexV2 extends PackIndex { +class PackIndexV2 implements PackIndex { private static final long IS_O64 = 1L << 31; private static final int FANOUT = 256; @@ -37,6 +37,9 @@ class PackIndexV2 extends PackIndex { private static final byte[] NO_BYTES = {}; + /** Footer checksum applied on the bottom of the pack file. */ + protected byte[] packChecksum; + private long objectCnt; private final long[] fanoutTable; @@ -221,7 +224,7 @@ class PackIndexV2 extends PackIndex { @Override public Iterator<MutableEntry> iterator() { - return new EntriesIteratorV2(); + return new EntriesIteratorV2(this); } @Override @@ -281,37 +284,39 @@ class PackIndexV2 extends PackIndex { return -1; } - private class EntriesIteratorV2 extends EntriesIterator { - int levelOne; + @Override + public byte[] getChecksum() { + return packChecksum; + } - int levelTwo; + private static class EntriesIteratorV2 extends EntriesIterator { + private int levelOne = 0; - @Override - protected MutableEntry initEntry() { - return new MutableEntry() { - @Override - protected void ensureId() { - idBuffer.fromRaw(names[levelOne], levelTwo - - Constants.OBJECT_ID_LENGTH / 4); - } - }; + private int levelTwo = 0; + + private final PackIndexV2 packIndex; + + private EntriesIteratorV2(PackIndexV2 packIndex) { + super(packIndex.objectCnt); + this.packIndex = packIndex; } @Override - public MutableEntry next() { - for (; levelOne < names.length; levelOne++) { - if (levelTwo < names[levelOne].length) { + protected void readNext() { + for (; levelOne < packIndex.names.length; levelOne++) { + if (levelTwo < packIndex.names[levelOne].length) { int idx = levelTwo / (Constants.OBJECT_ID_LENGTH / 4) * 4; - long offset = NB.decodeUInt32(offset32[levelOne], idx); + long offset = NB.decodeUInt32(packIndex.offset32[levelOne], + idx); if ((offset & IS_O64) != 0) { idx = (8 * (int) (offset & ~IS_O64)); - offset = NB.decodeUInt64(offset64, idx); + offset = NB.decodeUInt64(packIndex.offset64, idx); } - entry.offset = offset; - - levelTwo += Constants.OBJECT_ID_LENGTH / 4; - returnedNumber++; - return entry; + super.setOffset(offset); + this.levelTwo += Constants.OBJECT_ID_LENGTH / 4; + super.setIdBuffer(packIndex.names[levelOne], + levelTwo - Constants.OBJECT_ID_LENGTH / 4); + return; } levelTwo = 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java index 7e28b5eb2b..f0b6193066 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java @@ -21,10 +21,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 1 (old style) pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV1 */ -class PackIndexWriterV1 extends PackIndexWriter { +class PackIndexWriterV1 extends BasePackIndexWriter { static boolean canStore(PackedObjectInfo oe) { // We are limited to 4 GB per pack as offset is 32 bit unsigned int. // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java index fc5ef61912..b72b35a464 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java @@ -19,10 +19,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 2 pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV2 */ -class PackIndexWriterV2 extends PackIndexWriter { +class PackIndexWriterV2 extends BasePackIndexWriter { private static final int MAX_OFFSET_32 = 0x7fffffff; private static final int IS_OFFSET_64 = 0x80000000; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 1b092a3332..55e047bd43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -77,6 +77,7 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -320,7 +321,8 @@ public class PackInserter extends ObjectInserter { private static void writePackIndex(File idx, byte[] packHash, List<PackedObjectInfo> list) throws IOException { try (OutputStream os = new FileOutputStream(idx)) { - PackIndexWriter w = PackIndexWriter.createVersion(os, INDEX_VERSION); + PackIndexWriter w = BasePackIndexWriter.createVersion(os, + INDEX_VERSION); w.write(list, packHash); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java index a3d74be040..9957f54fbf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java @@ -12,7 +12,7 @@ package org.eclipse.jgit.internal.storage.file; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.util.Arrays; +import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.util.NB; @@ -35,7 +35,7 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { private final UInt24Array positions24; - private final int[] positions32; + private final IntArray positions32; /** * Parallel array to concat(positions24, positions32) with the size of the @@ -45,35 +45,37 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { * doesn't fit in an int and |value|-1 is the position for the size in the * size64 array e.g. a value of -1 is sizes64[0], -2 = sizes64[1], ... */ - private final int[] sizes32; + private final IntArray sizes32; - private final long[] sizes64; + private final LongArray sizes64; static PackObjectSizeIndex parse(InputStream in) throws IOException { /** Header and version already out of the input */ - IndexInputStreamReader stream = new IndexInputStreamReader(in); - int threshold = stream.readInt(); // minSize - int objCount = stream.readInt(); + byte[] buffer = new byte[8]; + in.readNBytes(buffer, 0, 8); + int threshold = NB.decodeInt32(buffer, 0); // minSize + int objCount = NB.decodeInt32(buffer, 4); if (objCount == 0) { return new EmptyPackObjectSizeIndex(threshold); } - return new PackObjectSizeIndexV1(stream, threshold, objCount); + return new PackObjectSizeIndexV1(in, threshold, objCount); } - private PackObjectSizeIndexV1(IndexInputStreamReader stream, int threshold, + private PackObjectSizeIndexV1(InputStream stream, int threshold, int objCount) throws IOException { this.threshold = threshold; UInt24Array pos24 = null; - int[] pos32 = null; + IntArray pos32 = null; + StreamHelper helper = new StreamHelper(); byte positionEncoding; - while ((positionEncoding = stream.readByte()) != 0) { + while ((positionEncoding = helper.readByte(stream)) != 0) { if (Byte.compareUnsigned(positionEncoding, BITS_24) == 0) { - int sz = stream.readInt(); + int sz = helper.readInt(stream); pos24 = new UInt24Array(stream.readNBytes(sz * 3)); } else if (Byte.compareUnsigned(positionEncoding, BITS_32) == 0) { - int sz = stream.readInt(); - pos32 = stream.readIntArray(sz); + int sz = helper.readInt(stream); + pos32 = IntArray.from(stream, sz); } else { throw new UnsupportedEncodingException( String.format(JGitText.get().unknownPositionEncoding, @@ -81,16 +83,16 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { } } positions24 = pos24 != null ? pos24 : UInt24Array.EMPTY; - positions32 = pos32 != null ? pos32 : new int[0]; + positions32 = pos32 != null ? pos32 : IntArray.EMPTY; - sizes32 = stream.readIntArray(objCount); - int c64sizes = stream.readInt(); + sizes32 = IntArray.from(stream, objCount); + int c64sizes = helper.readInt(stream); if (c64sizes == 0) { - sizes64 = new long[0]; + sizes64 = LongArray.EMPTY; return; } - sizes64 = stream.readLongArray(c64sizes); - int c128sizes = stream.readInt(); + sizes64 = LongArray.from(stream, c64sizes); + int c128sizes = helper.readInt(stream); if (c128sizes != 0) { // this MUST be 0 (we don't support 128 bits sizes yet) throw new IOException(JGitText.get().unsupportedSizesObjSizeIndex); @@ -102,8 +104,8 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { int pos = -1; if (!positions24.isEmpty() && idxOffset <= positions24.getLastValue()) { pos = positions24.binarySearch(idxOffset); - } else if (positions32.length > 0 && idxOffset >= positions32[0]) { - int pos32 = Arrays.binarySearch(positions32, idxOffset); + } else if (!positions32.empty() && idxOffset >= positions32.get(0)) { + int pos32 = positions32.binarySearch(idxOffset); if (pos32 >= 0) { pos = pos32 + positions24.size(); } @@ -112,17 +114,17 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { return -1; } - int objSize = sizes32[pos]; + int objSize = sizes32.get(pos); if (objSize < 0) { int secondPos = Math.abs(objSize) - 1; - return sizes64[secondPos]; + return sizes64.get(secondPos); } return objSize; } @Override public long getObjectCount() { - return (long) positions24.size() + positions32.length; + return (long) positions24.size() + positions32.size(); } @Override @@ -131,69 +133,128 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { } /** - * Wrapper to read parsed content from the byte stream + * A byte[] that should be interpreted as an int[] */ - private static class IndexInputStreamReader { + private static class IntArray { + private static final IntArray EMPTY = new IntArray(new byte[0]); - private final byte[] buffer = new byte[8]; + private static final int INT_SIZE = 4; - private final InputStream in; + private final byte[] data; - IndexInputStreamReader(InputStream in) { - this.in = in; - } + private final int size; - int readInt() throws IOException { - int n = in.readNBytes(buffer, 0, 4); - if (n < 4) { - throw new IOException(JGitText.get().unableToReadFullInt); + static IntArray from(InputStream in, int ints) throws IOException { + int expectedBytes = ints * INT_SIZE; + byte[] data = in.readNBytes(expectedBytes); + if (data.length < expectedBytes) { + throw new IOException(MessageFormat + .format(JGitText.get().unableToReadFullArray, + Integer.valueOf(ints))); } - return NB.decodeInt32(buffer, 0); + return new IntArray(data); + } + + private IntArray(byte[] data) { + this.data = data; + size = data.length / INT_SIZE; } - int[] readIntArray(int intsCount) throws IOException { - if (intsCount == 0) { - return new int[0]; + /** + * Returns position of element in array, -1 if not there + * + * @param needle + * element to look for + * @return position of the element in the array or -1 if not found + */ + int binarySearch(int needle) { + if (size == 0) { + return -1; } + int high = size; + int low = 0; + do { + int mid = (low + high) >>> 1; + int cmp = Integer.compare(needle, get(mid)); + if (cmp < 0) + high = mid; + else if (cmp == 0) { + return mid; + } else + low = mid + 1; + } while (low < high); + return -1; + } - int[] dest = new int[intsCount]; - for (int i = 0; i < intsCount; i++) { - dest[i] = readInt(); + int get(int position) { + if (position < 0 || position >= size) { + throw new IndexOutOfBoundsException(position); } - return dest; + return NB.decodeInt32(data, position * INT_SIZE); } - long readLong() throws IOException { - int n = in.readNBytes(buffer, 0, 8); - if (n < 8) { - throw new IOException(JGitText.get().unableToReadFullInt); + boolean empty() { + return size == 0; + } + + int size() { + return size; + } + } + + /** + * A byte[] that should be interpreted as an long[] + */ + private static class LongArray { + private static final LongArray EMPTY = new LongArray(new byte[0]); + + private static final int LONG_SIZE = 8; // bytes + + private final byte[] data; + + private final int size; + + static LongArray from(InputStream in, int longs) throws IOException { + byte[] data = in.readNBytes(longs * LONG_SIZE); + if (data.length < longs * LONG_SIZE) { + throw new IOException(MessageFormat + .format(JGitText.get().unableToReadFullArray, + Integer.valueOf(longs))); } - return NB.decodeInt64(buffer, 0); + return new LongArray(data); } - long[] readLongArray(int longsCount) throws IOException { - if (longsCount == 0) { - return new long[0]; + private LongArray(byte[] data) { + this.data = data; + size = data.length / LONG_SIZE; + } + + long get(int position) { + if (position < 0 || position >= size) { + throw new IndexOutOfBoundsException(position); } + return NB.decodeInt64(data, position * LONG_SIZE); + } + } - long[] dest = new long[longsCount]; - for (int i = 0; i < longsCount; i++) { - dest[i] = readLong(); + private static class StreamHelper { + private final byte[] buffer = new byte[8]; + + int readInt(InputStream in) throws IOException { + int n = in.readNBytes(buffer, 0, 4); + if (n < 4) { + throw new IOException(JGitText.get().unableToReadFullInt); } - return dest; + return NB.decodeInt32(buffer, 0); } - byte readByte() throws IOException { + byte readByte(InputStream in) throws IOException { int n = in.readNBytes(buffer, 0, 1); if (n != 1) { throw new IOException(JGitText.get().cannotReadByte); } return buffer[0]; } - - byte[] readNBytes(int sz) throws IOException { - return in.readNBytes(sz); - } } private static class EmptyPackObjectSizeIndex diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java index ef9753cd79..720a3bcbff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java @@ -75,20 +75,20 @@ public interface PackReverseIndex { throws CorruptObjectException; /** - * Find the position in the primary index of the object at the given pack + * Find the position in the reverse index of the object at the given pack * offset. * * @param offset * the pack offset of the object - * @return the position in the primary index of the object + * @return the position in the reverse index of the object */ int findPosition(long offset); /** - * Find the object that is in the given position in the primary index. + * Find the object that is in the given position in the reverse index. * * @param nthPosition - * the position of the object in the primary index + * the position of the object in the reverse index * @return the object in that position */ ObjectId findObjectByPosition(int nthPosition); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 8e57bf9f2f..05f1ef53a1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -16,6 +16,7 @@ package org.eclipse.jgit.internal.storage.file; import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.HEAD; import static org.eclipse.jgit.lib.Constants.LOGS; +import static org.eclipse.jgit.lib.Constants.L_LOGS; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; import static org.eclipse.jgit.lib.Constants.PACKED_REFS; import static org.eclipse.jgit.lib.Constants.R_HEADS; @@ -56,23 +57,25 @@ import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.PackRefsCommand; import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.ObjectWritingException; import org.eclipse.jgit.events.RefsChangedEvent; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.CoreConfig.TrustLooseRefStat; -import org.eclipse.jgit.lib.CoreConfig.TrustPackedRefsStat; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefComparator; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefWriter; +import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.SymbolicRef; import org.eclipse.jgit.revwalk.RevObject; @@ -124,6 +127,8 @@ public class RefDirectory extends RefDatabase { private final File gitDir; + private final File gitCommonDir; + final File refsDir; final File packedRefsFile; @@ -179,24 +184,19 @@ public class RefDirectory extends RefDatabase { private List<Integer> retrySleepMs = RETRY_SLEEP_MS; - private final boolean trustFolderStat; - - private final TrustPackedRefsStat trustPackedRefsStat; - - private final TrustLooseRefStat trustLooseRefStat; + private final CoreConfig coreConfig; RefDirectory(RefDirectory refDb) { parent = refDb.parent; gitDir = refDb.gitDir; + gitCommonDir = refDb.gitCommonDir; refsDir = refDb.refsDir; logsDir = refDb.logsDir; logsRefsDir = refDb.logsRefsDir; packedRefsFile = refDb.packedRefsFile; looseRefs.set(refDb.looseRefs.get()); packedRefs.set(refDb.packedRefs.get()); - trustFolderStat = refDb.trustFolderStat; - trustPackedRefsStat = refDb.trustPackedRefsStat; - trustLooseRefStat = refDb.trustLooseRefStat; + coreConfig = refDb.coreConfig; inProcessPackedRefsLock = refDb.inProcessPackedRefsLock; } @@ -204,24 +204,15 @@ public class RefDirectory extends RefDatabase { final FS fs = db.getFS(); parent = db; gitDir = db.getDirectory(); - refsDir = fs.resolve(gitDir, R_REFS); - logsDir = fs.resolve(gitDir, LOGS); - logsRefsDir = fs.resolve(gitDir, LOGS + '/' + R_REFS); - packedRefsFile = fs.resolve(gitDir, PACKED_REFS); + gitCommonDir = db.getCommonDirectory(); + refsDir = fs.resolve(gitCommonDir, R_REFS); + logsDir = fs.resolve(gitCommonDir, LOGS); + logsRefsDir = fs.resolve(gitCommonDir, L_LOGS + R_REFS); + packedRefsFile = fs.resolve(gitCommonDir, PACKED_REFS); looseRefs.set(RefList.<LooseRef> emptyList()); packedRefs.set(NO_PACKED_REFS); - trustFolderStat = db.getConfig() - .getBoolean(ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); - trustPackedRefsStat = db.getConfig() - .getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_TRUST_PACKED_REFS_STAT, - TrustPackedRefsStat.UNSET); - trustLooseRefStat = db.getConfig() - .getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_TRUST_LOOSE_REF_STAT, - TrustLooseRefStat.ALWAYS); + coreConfig = db.getConfig().get(CoreConfig.KEY); inProcessPackedRefsLock = new ReentrantLock(true); } @@ -282,6 +273,33 @@ public class RefDirectory extends RefDatabase { clearReferences(); } + /** + * {@inheritDoc} + * + * For a RefDirectory database, by default this packs non-symbolic, loose + * tag refs into packed-refs. If {@code all} flag is set, this packs all the + * non-symbolic, loose refs. + */ + @Override + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + String prefix = packRefs.isAll() ? R_REFS : R_TAGS; + Collection<Ref> refs = getRefsByPrefix(prefix); + List<String> refsToBePacked = new ArrayList<>(refs.size()); + pm.beginTask(JGitText.get().packRefs, refs.size()); + try { + for (Ref ref : refs) { + if (!ref.isSymbolic() && ref.getStorage().isLoose()) { + refsToBePacked.add(ref.getName()); + } + pm.update(1); + } + pack(refsToBePacked); + } finally { + pm.endTask(); + } + } + @Override public boolean isNameConflicting(String name) throws IOException { // Cannot be nested within an existing reference. @@ -422,6 +440,11 @@ public class RefDirectory extends RefDatabase { return ret; } + @Override + public ReflogReader getReflogReader(Ref ref) throws IOException { + return new ReflogReaderImpl(getRepository(), ref.getName()); + } + @SuppressWarnings("unchecked") private RefList<Ref> upcast(RefList<? extends Ref> loose) { return (RefList<Ref>) loose; @@ -939,7 +962,7 @@ public class RefDirectory extends RefDatabase { PackedRefList getPackedRefs() throws IOException { final PackedRefList curList = packedRefs.get(); - switch (trustPackedRefsStat) { + switch (coreConfig.getTrustPackedRefsStat()) { case NEVER: break; case AFTER_OPEN: @@ -955,12 +978,8 @@ public class RefDirectory extends RefDatabase { return curList; } break; - case UNSET: - if (trustFolderStat - && !curList.snapshot.isModified(packedRefsFile)) { - return curList; - } - break; + case INHERIT: + // only used in CoreConfig internally } return refreshPackedRefs(curList); @@ -1146,7 +1165,7 @@ public class RefDirectory extends RefDatabase { LooseRef scanRef(LooseRef ref, String name) throws IOException { final File path = fileFor(name); - if (trustLooseRefStat.equals(TrustLooseRefStat.AFTER_OPEN)) { + if (coreConfig.getTrustLooseRefStat() == TrustStat.AFTER_OPEN) { refreshPathToLooseRef(Paths.get(name)); } @@ -1329,7 +1348,12 @@ public class RefDirectory extends RefDatabase { name = name.substring(R_REFS.length()); return new File(refsDir, name); } - return new File(gitDir, name); + // HEAD needs to get resolved from git dir as resolving it from common dir + // would always lead back to current default branch + if (name.equals(HEAD)) { + return new File(gitDir, name); + } + return new File(gitCommonDir, name); } static int levelsIn(String name) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java index 21b5a54eb7..f1888eb90f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.lib.Constants.HEAD; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -37,7 +39,9 @@ class ReflogReaderImpl implements ReflogReader { * {@code Ref} name */ ReflogReaderImpl(Repository db, String refname) { - logName = new File(db.getDirectory(), Constants.LOGS + '/' + refname); + File logBaseDir = refname.equals(HEAD) ? db.getDirectory() + : db.getCommonDirectory(); + logName = new File(logBaseDir, Constants.L_LOGS + refname); } /* (non-Javadoc) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 30f8240aa9..15c125c684 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -15,8 +15,10 @@ import java.io.IOException; import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; import java.util.Collections; +import java.util.HashSet; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -397,7 +399,11 @@ public class WindowCache { } static final void purge(Pack pack) { - cache.removeAll(pack); + purge(Collections.singleton(pack)); + } + + static final void purge(Set<Pack> packs) { + cache.queueRemoveAll(packs); } /** cleanup released and/or garbage collected windows. */ @@ -441,6 +447,29 @@ public class WindowCache { private final boolean useStrongIndexRefs; + /** Removers are purely CPU/mem bound (no I/O), so likely should not go above # CPUs */ + private final int idealNumRemovers; + + /** Number of blocks to split the Pack removal into. + * + * Consolidation is better with more blocks since it increases + * the wait before moving to the next set by allowing more work + * to accumulate in the next set. On the flip side, the more + * blocks, the more synchronization overhead increasing each + * removers latency. + */ + private final int numRemovalBlocks; + + private final int removalBlockSize; + + private Set<Pack> packsToRemove = new HashSet<>(); + + private Set<Pack> packsBeingRemoved; + + private int numRemovers; + + private int blockBeingRemoved; + private WindowCache(WindowCacheConfig cfg) { tableSize = tableSize(cfg); final int lockCount = lockCount(cfg); @@ -479,6 +508,23 @@ public class WindowCache { statsRecorder = mbean; publishMBean.set(cfg.getExposeStatsViaJmx()); + /* Since each worker will only process up to one full set of blocks, at least 2 + * workers are needed anytime there are queued removals to ensure that all the + * blocks will get processed. However, if workers are maxed out at only 2, then + * enough newer workers will never start in order to make it safe for older + * workers to quit early. At least 3 workers are needed to make older worker + * relief transitions possible. + */ + idealNumRemovers = Math.max(3, Runtime.getRuntime().availableProcessors()); + + int bs = 1024; + if (tableSize < 2 * bs) { + bs = tableSize / 2; + } + removalBlockSize = bs; + numRemovalBlocks = tableSize / removalBlockSize; + blockBeingRemoved = numRemovalBlocks - 1; + if (maxFiles < 1) throw new IllegalArgumentException(JGitText.get().openFilesMustBeAtLeast1); if (maxBytes < windowSize) @@ -708,22 +754,105 @@ public class WindowCache { } /** - * Clear all entries related to a single file. + * Asynchronously clear all entries related to files. * <p> - * Typically this method is invoked during {@link Pack#close()}, when we - * know the pack is never going to be useful to us again (for example, it no - * longer exists on disk). A concurrent reader loading an entry from this - * same pack may cause the pack to become stuck in the cache anyway. + * Typically this method is invoked during {@link Pack#close()}, or + * {@link Pack#close(Set)}, when we know the packs are never going to be + * useful to us again (for example, they no longer exist on disk after gc). + * A concurrent reader loading an entry from these same packs may cause a + * pack to become stuck in the cache anyway. * - * @param pack - * the file to purge all entries of. + * Work on clearing files will be split up into blocks so that removing + * can be shared by more than one thread. This potential work sharing + * can provide 2 optimizations for removals: + * <ol> + * <li> It provides an opportunity for separate removal requests to be + * consolidated into one removal pass.</li> + * <li> It can reduce removing thread latencies by sharing the removal work + * with other removing threads which otherwise might not have any work to do + * due to their removal request being consolidated. This makes the system + * more efficient and can actually reduce latencies as system load increases + * due to pack removals!</li> + * </ol> + * The optimizations above are all achieved without blockng threads to wait + * for other threads to complete (although naturally there are some + * synchronization points), and while ensuring that no threads do more work + * than if they were the only thread available to perform a removal. + * + * @param packs + * the files to purge all entries of */ - private void removeAll(Pack pack) { - for (int s = 0; s < tableSize; s++) { + private void queueRemoveAll(Set<Pack> packs) { + synchronized (this) { + packsToRemove.addAll(packs); + if (numRemovers >= idealNumRemovers) { + return; + } + numRemovers++; + } + for (int numRemoved = 0; removeNextBlock(numRemoved); numRemoved++) { + // empty + } + synchronized (this) { + if (numRemovers > 0) { + numRemovers--; + } + } + } + + /** Determine which block to remove next, if any, and do so. + * + * @param numRemoved + * the number of already processed block removals by the current thread + * @return whether more processing should be done by the current thread + */ + private boolean removeNextBlock(int numRemoved) { + Set<Pack> toRemove; + int block; + synchronized (this) { + if (packsBeingRemoved == null || blockBeingRemoved >= numRemovalBlocks - 1) { + if (packsToRemove.isEmpty()) { + return false; + } + + blockBeingRemoved = 0; + packsBeingRemoved = packsToRemove; + packsToRemove = new HashSet<>(); + } else { + blockBeingRemoved++; + } + + toRemove = packsBeingRemoved; + block = blockBeingRemoved; + } + + removeBlock(toRemove, block); + numRemoved++; + + /* Ensure threads never work on more than a full set of blocks (the equivalent + * of removing one Pack) */ + boolean isLast = numRemoved >= numRemovalBlocks; + synchronized (this) { + if (numRemovers >= idealNumRemovers) { + isLast = true; + } + } + return !isLast; + } + + /** Remove a block of entries for a Set of files + * @param packs + * the files to purge all entries of + * @param block + * the specific block to process removals for + */ + private void removeBlock(Set<Pack> packs, int block) { + int starting = block * removalBlockSize; + for (int s = starting; s < starting + removalBlockSize && s < tableSize; s++) { final Entry e1 = table.get(s); boolean hasDead = false; for (Entry e = e1; e != null; e = e.next) { - if (e.ref.getPack() == pack) { + if (packs.contains(e.ref.getPack())) { e.kill(); hasDead = true; } else if (e.dead) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 01f514b939..11c45471e4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -205,7 +205,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { * @param cnt * number of bytes to copy. This value may exceed the number of * bytes remaining in the window starting at offset - * <code>pos</code>. + * <code>position</code>. * @return number of bytes actually copied; this may be less than * <code>cnt</code> if <code>cnt</code> exceeded the number of bytes * available. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java new file mode 100644 index 0000000000..6122a9a143 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +class MultiPackIndexConstants { + static final int MIDX_SIGNATURE = 0x4d494458; /* MIDX */ + + static final byte MIDX_VERSION = 1; + + /** + * We infer the length of object IDs (OIDs) from this value: + * + * <pre> + * 1 => SHA-1 + * 2 => SHA-256 + * </pre> + */ + static final byte OID_HASH_VERSION = 1; + + static final int MULTIPACK_INDEX_FANOUT_SIZE = 4 * 256; + + /** + * First 4 bytes describe the chunk id. Value 0 is a terminating label. + * Other 8 bytes provide the byte-offset in current file for chunk to start. + */ + static final int CHUNK_LOOKUP_WIDTH = 12; + + /** "PNAM" chunk */ + static final int MIDX_CHUNKID_PACKNAMES = 0x504e414d; + + /** "OIDF" chunk */ + static final int MIDX_CHUNKID_OIDFANOUT = 0x4f494446; + + /** "OIDL" chunk */ + static final int MIDX_CHUNKID_OIDLOOKUP = 0x4f49444c; + + /** "OOFF" chunk */ + static final int MIDX_CHUNKID_OBJECTOFFSETS = 0x4f4f4646; + + /** "LOFF" chunk */ + static final int MIDX_CHUNKID_LARGEOFFSETS = 0x4c4f4646; + + /** "RIDX" chunk */ + static final int MIDX_CHUNKID_REVINDEX = 0x52494458; + + private MultiPackIndexConstants() { + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java new file mode 100644 index 0000000000..795d39e375 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.CHUNK_LOOKUP_WIDTH; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.util.NB; + +/** + * Prints a multipack index file in a human-readable format. + * + * @since 7.2 + */ +@SuppressWarnings({ "boxing", "nls" }) +public class MultiPackIndexPrettyPrinter { + + /** + * Writes to out, in human-readable format, the multipack index in rawMidx + * + * @param rawMidx the bytes of a multipack index + * @param out a writer + */ + public static void prettyPrint(byte[] rawMidx, PrintWriter out) { + // Header (12 bytes) + out.println("[ 0] Magic: " + new String(rawMidx, 0, 4, UTF_8)); + out.println("[ 4] Version number: " + (int) rawMidx[4]); + out.println("[ 5] OID version: " + (int) rawMidx[5]); + int chunkCount = rawMidx[6]; + out.println("[ 6] # of chunks: " + chunkCount); + out.println("[ 7] # of bases: " + (int) rawMidx[7]); + int numberOfPacks = NB.decodeInt32(rawMidx, 8); + out.println("[ 8] # of packs: " + numberOfPacks); + + // Chunk lookup table + List<ChunkSegment> chunkSegments = new ArrayList<>(); + int current = printChunkLookup(out, rawMidx, chunkCount, chunkSegments); + + for (int i = 0; i < chunkSegments.size() - 1; i++) { + ChunkSegment segment = chunkSegments.get(i); + if (current != segment.startOffset()) { + throw new IllegalStateException(String.format( + "We are at byte %d, but segment should start at %d", + current, segment.startOffset())); + } + out.printf("Starting chunk: %s @ %d%n", segment.chunkName(), + segment.startOffset()); + switch (segment.chunkName()) { + case "OIDF" -> current = printOIDF(out, rawMidx, current); + case "OIDL" -> current = printOIDL(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + case "OOFF" -> current = printOOFF(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + case "PNAM" -> current = printPNAM(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + case "RIDX" -> current = printRIDX(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + default -> { + out.printf( + "Skipping %s (don't know how to print it yet)%n", + segment.chunkName()); + current = (int) chunkSegments.get(i + 1).startOffset(); + } + } + } + // Checksum is a SHA-1, use ObjectId to parse it + out.printf("[ %d] Checksum %s%n", current, + ObjectId.fromRaw(rawMidx, current).name()); + out.printf("Total size: " + (current + 20)); + } + + private static int printChunkLookup(PrintWriter out, byte[] rawMidx, int chunkCount, + List<ChunkSegment> chunkSegments) { + out.println("Starting chunk lookup @ 12"); + int current = 12; + for (int i = 0; i < chunkCount; i++) { + String chunkName = new String(rawMidx, current, 4, UTF_8); + long offset = NB.decodeInt64(rawMidx, current + 4); + out.printf("[ %d] |%8s|%8d|%n", current, chunkName, offset); + current += CHUNK_LOOKUP_WIDTH; + chunkSegments.add(new ChunkSegment(chunkName, offset)); + } + String chunkName = "0000"; + long offset = NB.decodeInt64(rawMidx, current + 4); + out.printf("[ %d] |%8s|%8d|%n", current, chunkName, offset); + current += CHUNK_LOOKUP_WIDTH; + chunkSegments.add(new ChunkSegment(chunkName, offset)); + return current; + } + + private static int printOIDF(PrintWriter out, byte[] rawMidx, int start) { + int current = start; + for (short i = 0; i < 256; i++) { + out.printf("[ %d] (%02X) %d%n", current, i, + NB.decodeInt32(rawMidx, current)); + current += 4; + } + return current; + } + + private static int printOIDL(PrintWriter out, byte[] rawMidx, int start, long end) { + int i = start; + while (i < end) { + out.printf("[ %d] %s%n", i, + ObjectId.fromRaw(rawMidx, i).name()); + i += 20; + } + return i; + } + + private static int printOOFF(PrintWriter out, byte[] rawMidx, int start, long end) { + int i = start; + while (i < end) { + out.printf("[ %d] %d %d%n", i, NB.decodeInt32(rawMidx, i), + NB.decodeInt32(rawMidx, i + 4)); + i += 8; + } + return i; + } + + private static int printRIDX(PrintWriter out, byte[] rawMidx, int start, long end) { + int i = start; + while (i < end) { + out.printf("[ %d] %d%n", i, NB.decodeInt32(rawMidx, i)); + i += 4; + } + return (int) end; + } + + private static int printPNAM(PrintWriter out, byte[] rawMidx, int start, long end) { + int nameStart = start; + for (int i = start; i < end; i++) { + if (rawMidx[i] == 0) { + out + .println(new String(rawMidx, nameStart, i - nameStart)); + nameStart = i + 1; + } + } + return (int) end; + } + + private record ChunkSegment(String chunkName, long startOffset) { + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java new file mode 100644 index 0000000000..bddf3ac4ad --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java @@ -0,0 +1,426 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.CHUNK_LOOKUP_WIDTH; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_LARGEOFFSETS; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OBJECTOFFSETS; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDFANOUT; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDLOOKUP; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_PACKNAMES; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_REVINDEX; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_SIGNATURE; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_VERSION; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MULTIPACK_INDEX_FANOUT_SIZE; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.OID_HASH_VERSION; +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.internal.storage.io.CancellableDigestOutputStream; +import org.eclipse.jgit.internal.storage.midx.PackIndexMerger.MidxMutableEntry; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.util.NB; + +/** + * Writes a collection of indexes as a multipack index. + * <p> + * See <a href= + * "https://git-scm.com/docs/pack-format#_multi_pack_index_midx_files_have_the_following_format">multipack + * index format spec</a> + * + * @since 7.2 + */ +public class MultiPackIndexWriter { + + private static final int LIMIT_31_BITS = (1 << 31) - 1; + + private static final int MIDX_HEADER_SIZE = 12; + + /** + * Writes the inputs in the multipack index format in the outputStream. + * + * @param monitor + * progress monitor + * @param outputStream + * stream to write the multipack index file + * @param inputs + * pairs of name and index for each pack to include in the + * multipack index. + * @throws IOException + * Error writing to the stream + */ + public void write(ProgressMonitor monitor, OutputStream outputStream, + Map<String, PackIndex> inputs) throws IOException { + PackIndexMerger data = new PackIndexMerger(inputs); + + // List of chunks in the order they need to be written + List<ChunkHeader> chunkHeaders = createChunkHeaders(data); + long expectedSize = calculateExpectedSize(chunkHeaders); + try (CancellableDigestOutputStream out = new CancellableDigestOutputStream( + monitor, outputStream)) { + writeHeader(out, chunkHeaders.size(), data.getPackCount()); + writeChunkLookup(out, chunkHeaders); + + WriteContext ctx = new WriteContext(out, data); + for (ChunkHeader chunk : chunkHeaders) { + chunk.writerFn.write(ctx); + } + writeCheckSum(out); + if (expectedSize != out.length()) { + throw new IllegalStateException(String.format( + JGitText.get().multiPackIndexUnexpectedSize, + Long.valueOf(expectedSize), + Long.valueOf(out.length()))); + } + } catch (InterruptedIOException e) { + throw new IOException(JGitText.get().multiPackIndexWritingCancelled, + e); + } + } + + private static long calculateExpectedSize(List<ChunkHeader> chunks) { + int chunkLookup = (chunks.size() + 1) * CHUNK_LOOKUP_WIDTH; + long chunkContent = chunks.stream().mapToLong(c -> c.size).sum(); + return /* header */ 12 + chunkLookup + chunkContent + /* CRC */ 20; + } + + private List<ChunkHeader> createChunkHeaders(PackIndexMerger data) { + List<ChunkHeader> chunkHeaders = new ArrayList<>(); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_OIDFANOUT, + MULTIPACK_INDEX_FANOUT_SIZE, this::writeFanoutTable)); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_OIDLOOKUP, + (long) data.getUniqueObjectCount() * OBJECT_ID_LENGTH, + this::writeOidLookUp)); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_OBJECTOFFSETS, + 8L * data.getUniqueObjectCount(), this::writeObjectOffsets)); + if (data.needsLargeOffsetsChunk()) { + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_LARGEOFFSETS, + 8L * data.getOffsetsOver31BitsCount(), + this::writeObjectLargeOffsets)); + } + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_REVINDEX, + 4L * data.getUniqueObjectCount(), this::writeRidx)); + + int packNamesSize = data.getPackNames().stream() + .mapToInt(String::length).map(i -> i + 1 /* null at the end */) + .sum(); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_PACKNAMES, packNamesSize, + this::writePackfileNames)); + return chunkHeaders; + } + + /** + * Write the first 12 bytes of the multipack index. + * <p> + * These bytes include things like magic number, version, number of + * chunks... + * + * @param out + * output stream to write + * @param numChunks + * number of chunks this multipack index is going to have + * @param packCount + * number of packs covered by this multipack index + * @throws IOException + * error writing to the output stream + */ + private void writeHeader(CancellableDigestOutputStream out, int numChunks, + int packCount) throws IOException { + byte[] headerBuffer = new byte[MIDX_HEADER_SIZE]; + NB.encodeInt32(headerBuffer, 0, MIDX_SIGNATURE); + byte[] buff = { MIDX_VERSION, OID_HASH_VERSION, (byte) numChunks, + (byte) 0 }; + System.arraycopy(buff, 0, headerBuffer, 4, 4); + NB.encodeInt32(headerBuffer, 8, packCount); + out.write(headerBuffer, 0, headerBuffer.length); + out.flush(); + } + + /** + * Write a table of "chunkId, start-offset", with a special value "0, + * end-of-previous_chunk", to mark the end. + * + * @param out + * output stream to write + * @param chunkHeaders + * list of chunks in the order they are expected to be written + * @throws IOException + * error writing to the output stream + */ + private void writeChunkLookup(CancellableDigestOutputStream out, + List<ChunkHeader> chunkHeaders) throws IOException { + + // first chunk will start at header + this lookup block + long chunkStart = MIDX_HEADER_SIZE + + (long) (chunkHeaders.size() + 1) * CHUNK_LOOKUP_WIDTH; + byte[] chunkEntry = new byte[CHUNK_LOOKUP_WIDTH]; + for (ChunkHeader chunkHeader : chunkHeaders) { + NB.encodeInt32(chunkEntry, 0, chunkHeader.chunkId); + NB.encodeInt64(chunkEntry, 4, chunkStart); + out.write(chunkEntry); + chunkStart += chunkHeader.size; + } + // Terminating label for the block + // (chunkid 0, offset where the next block would start) + NB.encodeInt32(chunkEntry, 0, 0); + NB.encodeInt64(chunkEntry, 4, chunkStart); + out.write(chunkEntry); + } + + /** + * Write the fanout table for the object ids + * <p> + * Table with 256 entries (one byte), where the ith entry, F[i], stores the + * number of OIDs with first byte at most i. Thus, F[255] stores the total + * number of objects. + * + * @param ctx + * write context + * @throws IOException + * error writing to the output stream + */ + + private void writeFanoutTable(WriteContext ctx) throws IOException { + byte[] tmp = new byte[4]; + int[] fanout = new int[256]; + Iterator<MidxMutableEntry> iterator = ctx.data.bySha1Iterator(); + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + fanout[e.getObjectId().getFirstByte() & 0xff]++; + } + for (int i = 1; i < fanout.length; i++) { + fanout[i] += fanout[i - 1]; + } + for (int n : fanout) { + NB.encodeInt32(tmp, 0, n); + ctx.out.write(tmp, 0, 4); + } + } + + /** + * Write the OID lookup chunk + * <p> + * A list of OIDs in sha1 order. + * + * @param ctx + * write context + * @throws IOException + * error writing to the output stream + */ + private void writeOidLookUp(WriteContext ctx) throws IOException { + byte[] tmp = new byte[OBJECT_ID_LENGTH]; + + Iterator<MidxMutableEntry> iterator = ctx.data.bySha1Iterator(); + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + e.getObjectId().copyRawTo(tmp, 0); + ctx.out.write(tmp, 0, OBJECT_ID_LENGTH); + } + } + + /** + * Write the object offsets chunk + * <p> + * A list of offsets, parallel to the list of OIDs. If the offset is too + * large (see {@link #fitsIn31bits(long)}), this contains the position in + * the large offsets list (marked with a 1 in the most significant bit). + * + * @param ctx + * write context + * @throws IOException + * error writing to the output stream + */ + private void writeObjectOffsets(WriteContext ctx) throws IOException { + byte[] entry = new byte[8]; + Iterator<MidxMutableEntry> iterator = ctx.data.bySha1Iterator(); + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + NB.encodeInt32(entry, 0, e.getPackId()); + if (!ctx.data.needsLargeOffsetsChunk() + || fitsIn31bits(e.getOffset())) { + NB.encodeInt32(entry, 4, (int) e.getOffset()); + } else { + int offloadedPosition = ctx.largeOffsets.append(e.getOffset()); + NB.encodeInt32(entry, 4, offloadedPosition | (1 << 31)); + } + ctx.out.write(entry); + } + } + + /** + * Writes the reverse index chunk + * <p> + * This stores the position of the objects in the main index, ordered first + * by pack and then by offset + * + * @param ctx + * write context + * @throws IOException + * erorr writing to the output stream + */ + private void writeRidx(WriteContext ctx) throws IOException { + Map<Integer, List<OffsetPosition>> packOffsets = new HashMap<>( + ctx.data.getPackCount()); + // TODO(ifrade): Brute force solution loading all offsets/packs in + // memory. We could also iterate reverse indexes looking up + // their position in the midx (and discarding if the pack doesn't + // match). + Iterator<MidxMutableEntry> iterator = ctx.data.bySha1Iterator(); + int midxPosition = 0; + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + OffsetPosition op = new OffsetPosition(e.getOffset(), midxPosition); + midxPosition++; + packOffsets.computeIfAbsent(Integer.valueOf(e.getPackId()), + k -> new ArrayList<>()).add(op); + } + + for (int i = 0; i < ctx.data.getPackCount(); i++) { + List<OffsetPosition> offsetsForPack = packOffsets + .get(Integer.valueOf(i)); + if (offsetsForPack.isEmpty()) { + continue; + } + offsetsForPack.sort(Comparator.comparing(OffsetPosition::offset)); + byte[] ridxForPack = new byte[4 * offsetsForPack.size()]; + for (int j = 0; j < offsetsForPack.size(); j++) { + NB.encodeInt32(ridxForPack, j * 4, + offsetsForPack.get(j).position); + } + ctx.out.write(ridxForPack); + } + } + + /** + * Write the large offset chunk + * <p> + * A list of large offsets (long). The regular offset chunk will point to a + * position here. + * + * @param ctx + * writer context + * @throws IOException + * error writing to the output stream + */ + private void writeObjectLargeOffsets(WriteContext ctx) throws IOException { + ctx.out.write(ctx.largeOffsets.offsets, 0, + ctx.largeOffsets.bytePosition); + } + + /** + * Write the list of packfiles chunk + * <p> + * List of packfiles (in lexicographical order) with an \0 at the end + * + * @param ctx + * writer context + * @throws IOException + * error writing to the output stream + */ + private void writePackfileNames(WriteContext ctx) throws IOException { + for (String packName : ctx.data.getPackNames()) { + // Spec doesn't talk about encoding. + ctx.out.write(packName.getBytes(StandardCharsets.UTF_8)); + ctx.out.write(0); + } + } + + /** + * Write final checksum of the data written to the stream + * + * @param out + * output stream used to write + * @throws IOException + * error writing to the output stream + */ + private void writeCheckSum(CancellableDigestOutputStream out) + throws IOException { + out.write(out.getDigest()); + out.flush(); + } + + + private record OffsetPosition(long offset, int position) { + } + + /** + * If there is at least one offset value larger than 2^32-1, then the large + * offset chunk must exist, and offsets larger than 2^31-1 must be stored in + * it instead + * + * @param offset object offset + * + * @return true if the offset fits in 31 bits + */ + private static boolean fitsIn31bits(long offset) { + return offset <= LIMIT_31_BITS; + } + + private static class LargeOffsets { + private final byte[] offsets; + + private int bytePosition; + + LargeOffsets(int largeOffsetsCount) { + offsets = new byte[largeOffsetsCount * 8]; + bytePosition = 0; + } + + /** + * Add an offset to the large offset chunk + * + * @param largeOffset + * a large offset + * @return the position of the just inserted offset (as in number of + * offsets, NOT in bytes) + */ + int append(long largeOffset) { + int at = bytePosition; + NB.encodeInt64(offsets, at, largeOffset); + bytePosition += 8; + return at / 8; + } + } + + private record ChunkHeader(int chunkId, long size, ChunkWriter writerFn) { + } + + @FunctionalInterface + private interface ChunkWriter { + void write(WriteContext ctx) throws IOException; + } + + private static class WriteContext { + final CancellableDigestOutputStream out; + + final PackIndexMerger data; + + final LargeOffsets largeOffsets; + + WriteContext(CancellableDigestOutputStream out, PackIndexMerger data) { + this.out = out; + this.data = data; + this.largeOffsets = new LargeOffsets( + data.getOffsetsOver31BitsCount()); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java new file mode 100644 index 0000000000..89814af107 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.MutableObjectId; + +/** + * Collect the stats and offers an iterator over the union of n-pack indexes. + * <p> + * The multipack index is a list of (sha1, packid, offset) ordered by sha1. We + * can build it from the individual pack indexes (sha1, offset) ordered by sha1, + * with a simple merge ignoring duplicates. + * <p> + * This class encapsulates the merging logic and precalculates the stats that + * the index needs (like total count of objects). To limit memory consumption, + * it does the merge as it goes during the iteration and iterators use mutable + * entries. The stats of the combined index are calculated in an iteration at + * construction time. + */ +class PackIndexMerger { + + private static final int LIMIT_31_BITS = (1 << 31) - 1; + + private static final long LIMIT_32_BITS = (1L << 32) - 1; + + /** + * Object returned by the iterator. + * <p> + * The iterator returns (on each next()) the same instance with different + * values, to avoid allocating many short-lived objects. Callers should not + * keep a reference to that returned value. + */ + static class MidxMutableEntry { + // The object id + private final MutableObjectId oid = new MutableObjectId(); + + // Position of the pack in the ordered list of pack in this merger + private int packId; + + // Offset in its pack + private long offset; + + public AnyObjectId getObjectId() { + return oid; + } + + public int getPackId() { + return packId; + } + + public long getOffset() { + return offset; + } + + /** + * Copy values from another mutable entry + * + * @param packId + * packId + * @param other + * another mutable entry + */ + private void fill(int packId, PackIndex.MutableEntry other) { + other.copyOidTo(oid); + this.packId = packId; + this.offset = other.getOffset(); + } + } + + private final List<String> packNames; + + private final List<PackIndex> indexes; + + private final boolean needsLargeOffsetsChunk; + + private final int offsetsOver31BitsCount; + + private final int uniqueObjectCount; + + PackIndexMerger(Map<String, PackIndex> packs) { + this.packNames = packs.keySet().stream().sorted() + .collect(Collectors.toUnmodifiableList()); + + this.indexes = packNames.stream().map(packs::get) + .collect(Collectors.toUnmodifiableList()); + + // Iterate for duplicates + int objectCount = 0; + boolean hasLargeOffsets = false; + int over31bits = 0; + MutableObjectId lastSeen = new MutableObjectId(); + MultiIndexIterator it = new MultiIndexIterator(indexes); + while (it.hasNext()) { + MidxMutableEntry entry = it.next(); + if (lastSeen.equals(entry.oid)) { + continue; + } + // If there is at least one offset value larger than 2^32-1, then + // the large offset chunk must exist, and offsets larger than + // 2^31-1 must be stored in it instead + if (entry.offset > LIMIT_32_BITS) { + hasLargeOffsets = true; + } + if (entry.offset > LIMIT_31_BITS) { + over31bits++; + } + + lastSeen.fromObjectId(entry.oid); + objectCount++; + } + uniqueObjectCount = objectCount; + offsetsOver31BitsCount = over31bits; + needsLargeOffsetsChunk = hasLargeOffsets; + } + + /** + * Object count of the merged index (i.e. without duplicates) + * + * @return object count of the merged index + */ + int getUniqueObjectCount() { + return uniqueObjectCount; + } + + /** + * If any object in any of the indexes has an offset over 2^32-1 + * + * @return true if there is any object with offset > 2^32 -1 + */ + boolean needsLargeOffsetsChunk() { + return needsLargeOffsetsChunk; + } + + /** + * How many object have offsets over 2^31-1 + * <p> + * Per multipack index spec, IF there is large offset chunk, all this + * offsets should be there. + * + * @return number of objects with offsets over 2^31-1 + */ + int getOffsetsOver31BitsCount() { + return offsetsOver31BitsCount; + } + + /** + * List of pack names in alphabetical order. + * <p> + * Order matters: In case of duplicates, the multipack index prefers the + * first package with it. This is in the same order we are using to + * prioritize duplicates. + * + * @return List of pack names, in the order used by the merge. + */ + List<String> getPackNames() { + return packNames; + } + + /** + * How many packs are being merged + * + * @return count of packs merged + */ + int getPackCount() { + return packNames.size(); + } + + /** + * Iterator over the merged indexes in sha1 order without duplicates + * <p> + * The returned entry in the iterator is mutable, callers should NOT keep a + * reference to it. + * + * @return an iterator in sha1 order without duplicates. + */ + Iterator<MidxMutableEntry> bySha1Iterator() { + return new DedupMultiIndexIterator(new MultiIndexIterator(indexes), + getUniqueObjectCount()); + } + + /** + * For testing. Iterate all entries, not skipping duplicates (stable order) + * + * @return an iterator of all objects in sha1 order, including duplicates. + */ + Iterator<MidxMutableEntry> rawIterator() { + return new MultiIndexIterator(indexes); + } + + /** + * Iterator over n-indexes in ObjectId order. + * <p> + * It returns duplicates if the same object id is in different indexes. Wrap + * it with {@link DedupMultiIndexIterator (Iterator, int)} to avoid + * duplicates. + */ + private static final class MultiIndexIterator + implements Iterator<MidxMutableEntry> { + + private final List<PackIndexPeekIterator> indexIterators; + + private final MidxMutableEntry mutableEntry = new MidxMutableEntry(); + + MultiIndexIterator(List<PackIndex> indexes) { + this.indexIterators = new ArrayList<>(indexes.size()); + for (int i = 0; i < indexes.size(); i++) { + PackIndexPeekIterator it = new PackIndexPeekIterator(i, + indexes.get(i)); + // Position in the first element + if (it.next() != null) { + indexIterators.add(it); + } + } + } + + @Override + public boolean hasNext() { + return !indexIterators.isEmpty(); + } + + @Override + public MidxMutableEntry next() { + PackIndexPeekIterator winner = null; + for (int index = 0; index < indexIterators.size(); index++) { + PackIndexPeekIterator current = indexIterators.get(index); + if (winner == null + || current.peek().compareBySha1To(winner.peek()) < 0) { + winner = current; + } + } + + if (winner == null) { + throw new NoSuchElementException(); + } + + mutableEntry.fill(winner.getPackId(), winner.peek()); + if (winner.next() == null) { + indexIterators.remove(winner); + }; + return mutableEntry; + } + } + + private static class DedupMultiIndexIterator + implements Iterator<MidxMutableEntry> { + private final MultiIndexIterator src; + + private int remaining; + + private final MutableObjectId lastOid = new MutableObjectId(); + + DedupMultiIndexIterator(MultiIndexIterator src, int totalCount) { + this.src = src; + this.remaining = totalCount; + } + + @Override + public boolean hasNext() { + return remaining > 0; + } + + @Override + public MidxMutableEntry next() { + MidxMutableEntry next = src.next(); + while (next != null && lastOid.equals(next.oid)) { + next = src.next(); + } + + if (next == null) { + throw new NoSuchElementException(); + } + + lastOid.fromObjectId(next.oid); + remaining--; + return next; + } + } + + /** + * Convenience around the PackIndex iterator to read the current value + * multiple times without consuming it. + * <p> + * This is used to merge indexes in the multipack index, where we need to + * compare the current value between indexes multiple times to find the + * next. + * <p> + * We could also implement this keeping the position (int) and + * MutableEntry#getObjectId, but that would create an ObjectId per entry. + * This implementation reuses the MutableEntry and avoid instantiations. + */ + // Visible for testing + static class PackIndexPeekIterator { + private final Iterator<PackIndex.MutableEntry> it; + + private final int packId; + + PackIndex.MutableEntry current; + + PackIndexPeekIterator(int packId, PackIndex index) { + it = index.iterator(); + this.packId = packId; + } + + PackIndex.MutableEntry next() { + if (it.hasNext()) { + current = it.next(); + } else { + current = null; + } + return current; + } + + PackIndex.MutableEntry peek() { + return current; + } + + int getPackId() { + return packId; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java new file mode 100644 index 0000000000..f69e68d4ba --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.pack; + +import java.io.IOException; +import java.util.List; + +import org.eclipse.jgit.transport.PackedObjectInfo; + +/** + * Represents a function that accepts a collection of objects to write into a + * primary pack index storage format. + */ +public interface PackIndexWriter { + /** + * Write all object entries to the index stream. + * + * @param toStore + * sorted list of objects to store in the index. The caller must + * have previously sorted the list using + * {@link org.eclipse.jgit.transport.PackedObjectInfo}'s native + * {@link java.lang.Comparable} implementation. + * @param packDataChecksum + * checksum signature of the entire pack data content. This is + * traditionally the last 20 bytes of the pack file's own stream. + * @throws java.io.IOException + * an error occurred while writing to the output stream, or the + * underlying format cannot store the object data supplied. + */ + void write(List<? extends PackedObjectInfo> toStore, + byte[] packDataChecksum) throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 4350f97915..f025d4e3d5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -58,10 +58,10 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.SearchForReuseTimeout; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; +import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.file.PackReverseIndexWriter; -import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AsyncObjectSizeQueue; import org.eclipse.jgit.lib.BatchingProgressMonitor; @@ -118,7 +118,7 @@ import org.eclipse.jgit.util.TemporaryBuffer; * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the * pack is being stored as a file the matching index can be written out after - * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap + * writing the pack by {@link #writeIndex(PackIndexWriter)}. An optional bitmap * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)} * followed by {@link #writeBitmapIndex(PackBitmapIndexWriter)}. * </p> @@ -303,19 +303,6 @@ public class PackWriter implements AutoCloseable { } /** - * Create a writer to load objects from the specified reader. - * <p> - * Objects for packing are specified in {@link #preparePack(Iterator)} or - * {@link #preparePack(ProgressMonitor, Set, Set)}. - * - * @param reader - * reader to read from the repository with. - */ - public PackWriter(ObjectReader reader) { - this(new PackConfig(), reader); - } - - /** * Create writer for specified repository. * <p> * Objects for packing are specified in {@link #preparePack(Iterator)} or @@ -1091,7 +1078,7 @@ public class PackWriter implements AutoCloseable { if (indexVersion <= 0) { for (BlockList<ObjectToPack> objs : objectsLists) indexVersion = Math.max(indexVersion, - PackIndexWriter.oldestPossibleFormat(objs)); + BasePackIndexWriter.oldestPossibleFormat(objs)); } return indexVersion; } @@ -1112,12 +1099,28 @@ public class PackWriter implements AutoCloseable { * the index data could not be written to the supplied stream. */ public void writeIndex(OutputStream indexStream) throws IOException { + writeIndex(BasePackIndexWriter.createVersion(indexStream, + getIndexVersion())); + } + + /** + * Create an index file to match the pack file just written. + * <p> + * Called after + * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. + * <p> + * Writing an index is only required for local pack storage. Packs sent on + * the network do not need to create an index. + * + * @param iw + * an {@link PackIndexWriter} instance to write the index + * @throws java.io.IOException + * the index data could not be written to the supplied stream. + */ + public void writeIndex(PackIndexWriter iw) throws IOException { if (isIndexDisabled()) throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); - long writeStart = System.currentTimeMillis(); - final PackIndexWriter iw = PackIndexWriter.createVersion( - indexStream, getIndexVersion()); iw.write(sortByName(), packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } @@ -2461,7 +2464,7 @@ public class PackWriter implements AutoCloseable { * object graph at selected commits. Writing a bitmap index is an optional * feature that not all pack users may require. * <p> - * Called after {@link #writeIndex(OutputStream)}. + * Called after {@link #writeIndex(PackIndexWriter)}. * <p> * To reduce memory internal state is cleared during this method, rendering * the PackWriter instance useless for anything further than a call to write diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java index dabc1f0c5f..bf87c4c9d6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java @@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.file.PackBitmapIndex.FLAG_REUSE; import static org.eclipse.jgit.revwalk.RevFlag.SEEN; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -28,16 +29,16 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.revwalk.AddUnseenToBitmapFilter; import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl; +import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexRemapper; -import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.revwalk.BitmapWalker; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevCommit; @@ -99,10 +100,10 @@ class PackWriterBitmapPreparer { this.excessiveBranchCount = config.getBitmapExcessiveBranchCount(); this.excessiveBranchTipCount = Math.max(excessiveBranchCount, config.getBitmapExcessiveBranchTipCount()); - long now = SystemReader.getInstance().getCurrentTime(); + Instant now = SystemReader.getInstance().now(); long ageInSeconds = (long) config.getBitmapInactiveBranchAgeInDays() * DAY_IN_SECONDS; - this.inactiveBranchTimestamp = (now / 1000) - ageInSeconds; + this.inactiveBranchTimestamp = now.getEpochSecond() - ageInSeconds; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java index d07713db8e..e9ff02700d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java @@ -32,6 +32,8 @@ import static org.eclipse.jgit.lib.Ref.Storage.PACKED; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -245,9 +247,9 @@ class BlockReader { private PersonIdent readPersonIdent() { String name = readValueString(); String email = readValueString(); - long ms = readVarint64() * 1000; - int tz = readInt16(); - return new PersonIdent(name, email, ms, tz); + long epochSeconds = readVarint64(); + ZoneOffset tz = ZoneOffset.ofTotalSeconds(readInt16() * 60); + return new PersonIdent(name, email, Instant.ofEpochSecond(epochSeconds), tz); } void readBlock(BlockSource src, long pos, int fileBlockSize) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java index 3e75a9dde3..542d6e94f3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java @@ -427,7 +427,35 @@ public class OpenSshConfigFile implements SshConfigStore { return value; } - private static boolean patternMatchesHost(String pattern, String name) { + /** + * Tells whether a given {@code name} matches the given list of patterns, + * accounting for negative matches. + * + * @param patterns + * to test {@code name} against; any pattern starting with an + * exclamation mark is a negative pattern + * @param name + * to test + * @return {@code true} if the {@code name} matches at least one of the + * non-negative patterns and none of the negative patterns, + * {@code false} otherwise + * @since 7.1 + */ + public static boolean patternMatch(Iterable<String> patterns, String name) { + boolean doesMatch = false; + for (String pattern : patterns) { + if (pattern.startsWith("!")) { //$NON-NLS-1$ + if (patternMatches(pattern.substring(1), name)) { + return false; + } + } else if (!doesMatch && patternMatches(pattern, name)) { + doesMatch = true; + } + } + return doesMatch; + } + + private static boolean patternMatches(String pattern, String name) { if (pattern.indexOf('*') >= 0 || pattern.indexOf('?') >= 0) { final FileNameMatcher fn; try { @@ -680,18 +708,7 @@ public class OpenSshConfigFile implements SshConfigStore { } boolean matches(String hostName) { - boolean doesMatch = false; - for (String pattern : patterns) { - if (pattern.startsWith("!")) { //$NON-NLS-1$ - if (patternMatchesHost(pattern.substring(1), hostName)) { - return false; - } - } else if (!doesMatch - && patternMatchesHost(pattern, hostName)) { - doesMatch = true; - } - } - return doesMatch; + return patternMatch(patterns, hostName); } private static String toKey(String key) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java index 3447f669ab..270b760562 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java @@ -53,24 +53,24 @@ public interface Optionally<T> { /** * The mutable optional object */ - protected T element; + protected Optional<T> optional; /** * @param element * the mutable optional object */ public Hard(T element) { - this.element = element; + optional = Optional.ofNullable(element); } @Override public void clear() { - element = null; + optional = Optional.empty(); } @Override public Optional<T> getOptional() { - return Optional.ofNullable(element); + return optional; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/AbstractGpgSignatureVerifier.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/AbstractGpgSignatureVerifier.java deleted file mode 100644 index 06a89dc535..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/AbstractGpgSignatureVerifier.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2024, Thomas Wolf <twolf@apache.org> and others -* -* This program and the accompanying materials are made available under the -* terms of the Eclipse Distribution License v. 1.0 which is available at -* https://www.eclipse.org/org/documents/edl-v10.php. -* -* SPDX-License-Identifier: BSD-3-Clause -*/ -package org.eclipse.jgit.lib; - -import java.io.IOException; -import java.util.Arrays; - -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.util.RawParseUtils; - -/** - * Provides a base implementation of - * {@link GpgSignatureVerifier#verifySignature(RevObject, GpgConfig)}. - * - * @since 6.9 - */ -public abstract class AbstractGpgSignatureVerifier - implements GpgSignatureVerifier { - - @Override - public SignatureVerification verifySignature(RevObject object, - GpgConfig config) throws IOException { - if (object instanceof RevCommit) { - RevCommit commit = (RevCommit) object; - byte[] signatureData = commit.getRawGpgSignature(); - if (signatureData == null) { - return null; - } - byte[] raw = commit.getRawBuffer(); - // Now remove the GPG signature - byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; - int start = RawParseUtils.headerStart(header, raw, 0); - if (start < 0) { - return null; - } - int end = RawParseUtils.nextLfSkippingSplitLines(raw, start); - // start is at the beginning of the header's content - start -= header.length + 1; - // end is on the terminating LF; we need to skip that, too - if (end < raw.length) { - end++; - } - byte[] data = new byte[raw.length - (end - start)]; - System.arraycopy(raw, 0, data, 0, start); - System.arraycopy(raw, end, data, start, raw.length - end); - return verify(config, data, signatureData); - } else if (object instanceof RevTag) { - RevTag tag = (RevTag) object; - byte[] signatureData = tag.getRawGpgSignature(); - if (signatureData == null) { - return null; - } - byte[] raw = tag.getRawBuffer(); - // The signature is just tacked onto the end of the message, which - // is last in the buffer. - byte[] data = Arrays.copyOfRange(raw, 0, - raw.length - signatureData.length); - return verify(config, data, signatureData); - } - return null; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java index c58133adab..f742e993a0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java @@ -35,23 +35,6 @@ public abstract class AnyObjectId implements Comparable<AnyObjectId> { * @param secondObjectId * the second identifier to compare. Must not be null. * @return true if the two identifiers are the same. - * @deprecated use {@link #isEqual(AnyObjectId, AnyObjectId)} instead - */ - @Deprecated - @SuppressWarnings("AmbiguousMethodReference") - public static boolean equals(final AnyObjectId firstObjectId, - final AnyObjectId secondObjectId) { - return isEqual(firstObjectId, secondObjectId); - } - - /** - * Compare two object identifier byte sequences for equality. - * - * @param firstObjectId - * the first identifier to compare. Must not be null. - * @param secondObjectId - * the second identifier to compare. Must not be null. - * @return true if the two identifiers are the same. * @since 5.4 */ public static boolean isEqual(final AnyObjectId firstObjectId, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java index 5dfb648faa..0c1da83dfb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java @@ -13,13 +13,17 @@ package org.eclipse.jgit.lib; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BARE; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WORKTREE; +import static org.eclipse.jgit.lib.Constants.CONFIG; import static org.eclipse.jgit.lib.Constants.DOT_GIT; +import static org.eclipse.jgit.lib.Constants.GITDIR_FILE; import static org.eclipse.jgit.lib.Constants.GIT_ALTERNATE_OBJECT_DIRECTORIES_KEY; import static org.eclipse.jgit.lib.Constants.GIT_CEILING_DIRECTORIES_KEY; +import static org.eclipse.jgit.lib.Constants.GIT_COMMON_DIR_KEY; import static org.eclipse.jgit.lib.Constants.GIT_DIR_KEY; import static org.eclipse.jgit.lib.Constants.GIT_INDEX_FILE_KEY; import static org.eclipse.jgit.lib.Constants.GIT_OBJECT_DIRECTORY_KEY; import static org.eclipse.jgit.lib.Constants.GIT_WORK_TREE_KEY; +import static org.eclipse.jgit.lib.Constants.OBJECTS; import java.io.File; import java.io.IOException; @@ -70,7 +74,21 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re && ref[7] == ' '; } - private static File getSymRef(File workTree, File dotGit, FS fs) + /** + * Read symbolic reference file + * + * @param workTree + * the work tree path + * @param dotGit + * the .git file + * @param fs + * th FS util + * @return the file read from symbolic reference file + * @throws java.io.IOException + * the dotGit file is invalid reference + * @since 7.0 + */ + static File getSymRef(File workTree, File dotGit, FS fs) throws IOException { byte[] content = IO.readFully(dotGit); if (!isSymRef(content)) { @@ -102,6 +120,8 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re private File gitDir; + private File gitCommonDir; + private File objectDirectory; private List<File> alternateObjectDirectories; @@ -172,6 +192,30 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re } /** + * Set common dir. + * + * @param gitCommonDir + * {@code GIT_COMMON_DIR}, the common repository meta directory. + * @return {@code this} (for chaining calls). + * @since 7.0 + */ + public B setGitCommonDir(File gitCommonDir) { + this.gitCommonDir = gitCommonDir; + this.config = null; + return self(); + } + + /** + * Get common dir. + * + * @return common dir; null if not set. + * @since 7.0 + */ + public File getGitCommonDir() { + return gitCommonDir; + } + + /** * Set the directory storing the repository's objects. * * @param objectDirectory @@ -396,9 +440,9 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re * Read standard Git environment variables and configure from those. * <p> * This method tries to read the standard Git environment variables, such as - * {@code GIT_DIR} and {@code GIT_WORK_TREE} to configure this builder - * instance. If an environment variable is set, it overrides the value - * already set in this builder. + * {@code GIT_DIR}, {@code GIT_COMMON_DIR}, {@code GIT_WORK_TREE} etc. to + * configure this builder instance. If an environment variable is set, it + * overrides the value already set in this builder. * * @return {@code this} (for chaining calls). */ @@ -410,9 +454,9 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re * Read standard Git environment variables and configure from those. * <p> * This method tries to read the standard Git environment variables, such as - * {@code GIT_DIR} and {@code GIT_WORK_TREE} to configure this builder - * instance. If a property is already set in the builder, the environment - * variable is not used. + * {@code GIT_DIR}, {@code GIT_COMMON_DIR}, {@code GIT_WORK_TREE} etc. to + * configure this builder instance. If a property is already set in the + * builder, the environment variable is not used. * * @param sr * the SystemReader abstraction to access the environment. @@ -425,6 +469,13 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re setGitDir(new File(val)); } + if (getGitCommonDir() == null) { + String val = sr.getenv(GIT_COMMON_DIR_KEY); + if (val != null) { + setGitCommonDir(new File(val)); + } + } + if (getObjectDirectory() == null) { String val = sr.getenv(GIT_OBJECT_DIRECTORY_KEY); if (val != null) @@ -434,7 +485,7 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re if (getAlternateObjectDirectories() == null) { String val = sr.getenv(GIT_ALTERNATE_OBJECT_DIRECTORIES_KEY); if (val != null) { - for (String path : val.split(File.pathSeparator)) + for (String path : val.split(File.pathSeparator, -1)) addAlternateObjectDirectory(new File(path)); } } @@ -454,7 +505,7 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re if (ceilingDirectories == null) { String val = sr.getenv(GIT_CEILING_DIRECTORIES_KEY); if (val != null) { - for (String path : val.split(File.pathSeparator)) + for (String path : val.split(File.pathSeparator, -1)) addCeilingDirectory(new File(path)); } } @@ -601,6 +652,7 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re public B setup() throws IllegalArgumentException, IOException { requireGitDirOrWorkTree(); setupGitDir(); + setupCommonDir(); setupWorkTree(); setupInternals(); return self(); @@ -658,6 +710,20 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re } /** + * Perform standard common dir initialization. + * + * @throws java.io.IOException + * the repository could not be accessed + * @since 7.0 + */ + protected void setupCommonDir() throws IOException { + // no gitCommonDir? Try to get it from gitDir + if (getGitCommonDir() == null) { + setGitCommonDir(safeFS().getCommonDir(getGitDir())); + } + } + + /** * Perform standard work-tree initialization. * <p> * This is a method typically invoked inside of {@link #setup()}, near the @@ -695,8 +761,12 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re * the repository could not be accessed */ protected void setupInternals() throws IOException { - if (getObjectDirectory() == null && getGitDir() != null) - setObjectDirectory(safeFS().resolve(getGitDir(), Constants.OBJECTS)); + if (getObjectDirectory() == null) { + File commonDir = getGitCommonDir(); + if (commonDir != null) { + setObjectDirectory(safeFS().resolve(commonDir, OBJECTS)); + } + } } /** @@ -723,12 +793,13 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re * the configuration is not available. */ protected Config loadConfig() throws IOException { - if (getGitDir() != null) { + File commonDir = getGitCommonDir(); + if (commonDir != null) { // We only want the repository's configuration file, and not // the user file, as these parameters must be unique to this // repository and not inherited from other files. // - File path = safeFS().resolve(getGitDir(), Constants.CONFIG); + File path = safeFS().resolve(commonDir, CONFIG); FileBasedConfig cfg = new FileBasedConfig(path, safeFS()); try { cfg.load(); @@ -749,8 +820,29 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re // String path = cfg.getString(CONFIG_CORE_SECTION, null, CONFIG_KEY_WORKTREE); - if (path != null) + if (path != null) { return safeFS().resolve(getGitDir(), path).getCanonicalFile(); + } + + /* + * We are in worktree's $GIT_DIR folder + * ".git/worktrees/<worktree-name>" and want to get the working + * tree (checkout) path; so here we have an opposite link in file + * "gitdir" showing to the ".git" file located in the working tree read + * it and convert it to absolute path if it's relative + */ + File gitDirFile = new File(getGitDir(), GITDIR_FILE); + if (gitDirFile.isFile()) { + String workDirPath = new String(IO.readFully(gitDirFile)).trim(); + File workTreeDotGitFile = new File(workDirPath); + if (!workTreeDotGitFile.isAbsolute()) { + workTreeDotGitFile = new File(getGitDir(), workDirPath) + .getCanonicalFile(); + } + if (workTreeDotGitFile != null) { + return workTreeDotGitFile.getParentFile(); + } + } // If core.bare is set, honor its value. Assume workTree is // the parent directory of the repository. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java index e15c7af932..7921052aaa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java @@ -187,8 +187,7 @@ public class BranchConfig { * @since 4.5 */ public BranchRebaseMode getRebaseMode() { - return config.getEnum(BranchRebaseMode.values(), - ConfigConstants.CONFIG_BRANCH_SECTION, branchName, + return config.getEnum(ConfigConstants.CONFIG_BRANCH_SECTION, branchName, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java index ea33082229..ad3c2c091d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java @@ -194,19 +194,6 @@ public class CommitBuilder extends ObjectBuilder { } } - /** - * Set the encoding for the commit information. - * - * @param encodingName - * the encoding name. See - * {@link java.nio.charset.Charset#forName(String)}. - * @deprecated use {@link #setEncoding(Charset)} instead. - */ - @Deprecated - public void setEncoding(String encodingName) { - setEncoding(Charset.forName(encodingName)); - } - @Override public byte[] build() throws UnsupportedEncodingException { ByteArrayOutputStream os = new ByteArrayOutputStream(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java index f701a41d67..b1ba5dfa28 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java @@ -119,7 +119,7 @@ public class CommitConfig { if (!StringUtils.isEmptyOrNull(comment)) { if ("auto".equalsIgnoreCase(comment)) { //$NON-NLS-1$ autoCommentChar = true; - } else { + } else if (comment != null) { char first = comment.charAt(0); if (first > ' ' && first < 127) { commentCharacter = first; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java index 07c5fa4500..345cb22f80 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.events.ConfigChangedEvent; import org.eclipse.jgit.events.ConfigChangedListener; @@ -254,9 +255,8 @@ public class Config { * default value to return if no value was present. * @return an integer value from the configuration, or defaultValue. */ - public int getInt(final String section, final String name, - final int defaultValue) { - return typedGetter.getInt(this, section, null, name, defaultValue); + public int getInt(String section, String name, int defaultValue) { + return getInt(section, null, name, defaultValue); } /** @@ -264,6 +264,23 @@ public class Config { * * @param section * section the key is grouped within. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getInt(String section, String name) { + return getInt(section, null, name); + } + + + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. * @param subsection * subsection name, such a remote or branch name. * @param name @@ -272,10 +289,30 @@ public class Config { * default value to return if no value was present. * @return an integer value from the configuration, or defaultValue. */ - public int getInt(final String section, String subsection, - final String name, final int defaultValue) { + public int getInt(String section, String subsection, String name, + int defaultValue) { + Integer v = typedGetter.getInt(this, section, subsection, name, + Integer.valueOf(defaultValue)); + return v == null ? defaultValue : v.intValue(); + } + + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getInt(String section, String subsection, String name) { return typedGetter.getInt(this, section, subsection, name, - defaultValue); + null); } /** @@ -297,8 +334,30 @@ public class Config { */ public int getIntInRange(String section, String name, int minValue, int maxValue, int defaultValue) { - return typedGetter.getIntInRange(this, section, null, name, minValue, - maxValue, defaultValue); + return getIntInRange(section, null, name, + minValue, maxValue, defaultValue); + } + + /** + * Obtain an integer value from the configuration which must be inside given + * range. + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @param minValue + * minimum value + * @param maxValue + * maximum value + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getIntInRange(String section, String name, int minValue, + int maxValue) { + return getIntInRange(section, null, name, minValue, maxValue); } /** @@ -322,8 +381,34 @@ public class Config { */ public int getIntInRange(String section, String subsection, String name, int minValue, int maxValue, int defaultValue) { + Integer v = typedGetter.getIntInRange(this, section, subsection, name, + minValue, maxValue, Integer.valueOf(defaultValue)); + return v == null ? defaultValue : v.intValue(); + } + + /** + * Obtain an integer value from the configuration which must be inside given + * range. + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param minValue + * minimum value + * @param maxValue + * maximum value + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getIntInRange(String section, String subsection, String name, + int minValue, int maxValue) { return typedGetter.getIntInRange(this, section, subsection, name, - minValue, maxValue, defaultValue); + minValue, maxValue, null); } /** @@ -338,7 +423,23 @@ public class Config { * @return an integer value from the configuration, or defaultValue. */ public long getLong(String section, String name, long defaultValue) { - return typedGetter.getLong(this, section, null, name, defaultValue); + return getLong(section, null, name, defaultValue); + } + + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Long getLong(String section, String name) { + return getLong(section, null, name); } /** @@ -355,9 +456,28 @@ public class Config { * @return an integer value from the configuration, or defaultValue. */ public long getLong(final String section, String subsection, - final String name, final long defaultValue) { - return typedGetter.getLong(this, section, subsection, name, - defaultValue); + String name, long defaultValue) { + Long v = typedGetter.getLong(this, section, subsection, name, + Long.valueOf(defaultValue)); + return v == null ? defaultValue : v.longValue(); + } + + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Long getLong(String section, String subsection, String name) { + return typedGetter.getLong(this, section, subsection, name, null); } /** @@ -372,9 +492,26 @@ public class Config { * @return true if any value or defaultValue is true, false for missing or * explicit false */ - public boolean getBoolean(final String section, final String name, - final boolean defaultValue) { - return typedGetter.getBoolean(this, section, null, name, defaultValue); + public boolean getBoolean(String section, String name, + boolean defaultValue) { + Boolean v = typedGetter.getBoolean(this, section, null, name, + Boolean.valueOf(defaultValue)); + return v == null ? defaultValue : v.booleanValue(); + } + + /** + * Get a boolean value from the git config + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @return configured boolean value, or {@code null} if not set. + * @since 7.2 + */ + @Nullable + public Boolean getBoolean(String section, String name) { + return getBoolean(section, null, name); } /** @@ -391,10 +528,28 @@ public class Config { * @return true if any value or defaultValue is true, false for missing or * explicit false */ - public boolean getBoolean(final String section, String subsection, - final String name, final boolean defaultValue) { - return typedGetter.getBoolean(this, section, subsection, name, - defaultValue); + public boolean getBoolean(String section, String subsection, String name, + boolean defaultValue) { + Boolean v = typedGetter.getBoolean(this, section, subsection, name, + Boolean.valueOf(defaultValue)); + return v == null ? defaultValue : v.booleanValue(); + } + + /** + * Get a boolean value from the git config + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return configured boolean value, or {@code null} if not set. + * @since 7.2 + */ + @Nullable + public Boolean getBoolean(String section, String subsection, String name) { + return typedGetter.getBoolean(this, section, subsection, name, null); } /** @@ -412,8 +567,8 @@ public class Config { * default value to return if no value was present. * @return the selected enumeration value, or {@code defaultValue}. */ - public <T extends Enum<?>> T getEnum(final String section, - final String subsection, final String name, final T defaultValue) { + public <T extends Enum<?>> T getEnum(String section, String subsection, + String name, @NonNull T defaultValue) { final T[] all = allValuesOf(defaultValue); return typedGetter.getEnum(this, all, section, subsection, name, defaultValue); @@ -448,14 +603,41 @@ public class Config { * @param defaultValue * default value to return if no value was present. * @return the selected enumeration value, or {@code defaultValue}. + * @deprecated use {@link #getEnum(String, String, String, Enum)} or + * {{@link #getEnum(Enum[], String, String, String)}} instead. */ - public <T extends Enum<?>> T getEnum(final T[] all, final String section, - final String subsection, final String name, final T defaultValue) { + @Nullable + @Deprecated + public <T extends Enum<?>> T getEnum(T[] all, String section, + String subsection, String name, @Nullable T defaultValue) { return typedGetter.getEnum(this, all, section, subsection, name, defaultValue); } /** + * Parse an enumeration from the configuration. + * + * @param <T> + * type of the returned enum + * @param all + * all possible values in the enumeration which should be + * recognized. Typically {@code EnumType.values()}. + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return the selected enumeration value, or {@code null} if not set. + * @since 7.2 + */ + @Nullable + public <T extends Enum<?>> T getEnum(T[] all, String section, + String subsection, String name) { + return typedGetter.getEnum(this, all, section, subsection, name, null); + } + + /** * Get string value or null if not found. * * @param section @@ -466,8 +648,8 @@ public class Config { * the key name * @return a String value from the config, <code>null</code> if not found */ - public String getString(final String section, String subsection, - final String name) { + @Nullable + public String getString(String section, String subsection, String name) { return getRawString(section, subsection, name); } @@ -526,8 +708,34 @@ public class Config { */ public long getTimeUnit(String section, String subsection, String name, long defaultValue, TimeUnit wantUnit) { + Long v = typedGetter.getTimeUnit(this, section, subsection, name, + Long.valueOf(defaultValue), wantUnit); + return v == null ? defaultValue : v.longValue(); + + } + + /** + * Parse a numerical time unit, such as "1 minute", from the configuration. + * + * @param section + * section the key is in. + * @param subsection + * subsection the key is in, or null if not in a subsection. + * @param name + * the key name. + * @param wantUnit + * the units of {@code defaultValue} and the return value, as + * well as the units to assume if the value does not contain an + * indication of the units. + * @return the value, or {@code null} if not set, expressed in + * {@code units}. + * @since 7.2 + */ + @Nullable + public Long getTimeUnit(String section, String subsection, String name, + TimeUnit wantUnit) { return typedGetter.getTimeUnit(this, section, subsection, name, - defaultValue, wantUnit); + null, wantUnit); } /** @@ -555,8 +763,9 @@ public class Config { * @return the {@link Path}, or {@code defaultValue} if not set * @since 5.10 */ + @Nullable public Path getPath(String section, String subsection, String name, - @NonNull FS fs, File resolveAgainst, Path defaultValue) { + @NonNull FS fs, File resolveAgainst, @Nullable Path defaultValue) { return typedGetter.getPath(this, section, subsection, name, fs, resolveAgainst, defaultValue); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 0edf3c5ad0..c4550329d3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -78,6 +78,13 @@ public final class ConfigConstants { public static final String CONFIG_DFS_SECTION = "dfs"; /** + * The dfs cache subsection prefix. + * + * @since 7.0 + */ + public static final String CONFIG_DFS_CACHE_PREFIX = "dfs."; + + /** * The "receive" section * @since 4.6 */ @@ -199,7 +206,36 @@ public final class ConfigConstants { public static final String CONFIG_KEY_SIGNINGKEY = "signingKey"; /** + * The "ssh" subsection key. + * + * @since 7.1 + */ + public static final String CONFIG_SSH_SUBSECTION = "ssh"; + + /** + * The "defaultKeyCommand" key. + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_DEFAULT_KEY_COMMAND = "defaultKeyCommand"; + + /** + * The "allowedSignersFile" key. + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_ALLOWED_SIGNERS_FILE = "allowedSignersFile"; + + /** + * The "revocationFile" key, + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_REVOCATION_FILE = "revocationFile"; + + /** * The "commit" section + * * @since 5.2 */ public static final String CONFIG_COMMIT_SECTION = "commit"; @@ -332,6 +368,13 @@ public final class ConfigConstants { public static final String CONFIG_KEY_DELTA_BASE_CACHE_LIMIT = "deltaBaseCacheLimit"; /** + * The "packExtensions" key + * + * @since 7.0 + **/ + public static final String CONFIG_KEY_PACK_EXTENSIONS = "packExtensions"; + + /** * The "symlinks" key * @since 3.3 */ @@ -345,12 +388,6 @@ public final class ConfigConstants { public static final String CONFIG_KEY_STREAM_FILE_THRESHOLD = "streamFileThreshold"; /** - * @deprecated typo, use CONFIG_KEY_STREAM_FILE_THRESHOLD instead - */ - @Deprecated(since = "6.8") - public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = CONFIG_KEY_STREAM_FILE_THRESHOLD; - - /** * The "packedGitMmap" key * @since 5.1.13 */ @@ -409,6 +446,13 @@ public final class ConfigConstants { /** The "rebase" key */ public static final String CONFIG_KEY_REBASE = "rebase"; + /** + * The "checkout" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_CHECKOUT = "checkout"; + /** The "url" key */ public static final String CONFIG_KEY_URL = "url"; @@ -556,11 +600,21 @@ public final class ConfigConstants { /** * The "trustfolderstat" key in the "core" section + * * @since 3.6 + * @deprecated use {CONFIG_KEY_TRUST_STAT} instead */ + @Deprecated(since = "7.2", forRemoval = true) public static final String CONFIG_KEY_TRUSTFOLDERSTAT = "trustfolderstat"; /** + * The "trustfilestat" key in the "core"section + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_STAT = "truststat"; + + /** * The "supportsAtomicFileCreation" key in the "core" section * * @since 4.5 @@ -979,6 +1033,27 @@ public final class ConfigConstants { public static final String CONFIG_KEY_TRUST_LOOSE_REF_STAT = "trustLooseRefStat"; /** + * The "trustLooseRefStat" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_PACK_STAT = "trustPackStat"; + + /** + * The "trustLooseObjectFileStat" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_LOOSE_OBJECT_STAT = "trustLooseObjectStat"; + + /** + * The "trustTablesListStat" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_TABLESLIST_STAT = "trustTablesListStat"; + + /** * The "pack.preserveOldPacks" key * * @since 5.13.2 @@ -1012,4 +1087,32 @@ public final class ConfigConstants { * @since 6.7 */ public static final String CONFIG_KEY_READ_CHANGED_PATHS = "readChangedPaths"; + + /** + * The "useObjectSizeIndex" key + * + * @since 7.0 + */ + public static final String CONFIG_KEY_USE_OBJECT_SIZE_INDEX = "useObjectSizeIndex"; + + /** + * The "loadRevIndexInParallel" key + * + * @since 7.1 + */ + public static final String CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL = "loadRevIndexInParallel"; + + /** + * The "reftable" section + * + * @since 7.2 + */ + public static final String CONFIG_REFTABLE_SECTION = "reftable"; + + /** + * The "autorefresh" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_AUTOREFRESH = "autorefresh"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 60a23dde05..997f4ed314 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -12,10 +12,13 @@ package org.eclipse.jgit.lib; +import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; @@ -203,24 +206,6 @@ public final class Constants { */ public static final byte[] PACK_SIGNATURE = { 'P', 'A', 'C', 'K' }; - /** - * Native character encoding for commit messages, file names... - * - * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} directly - * instead. - */ - @Deprecated - public static final Charset CHARSET; - - /** - * Native character encoding for commit messages, file names... - * - * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} directly - * instead. - */ - @Deprecated - public static final String CHARACTER_ENCODING; - /** Default main branch name */ public static final String MASTER = "master"; @@ -273,6 +258,20 @@ public final class Constants { public static final String INFO_REFS = "info/refs"; /** + * Name of heads folder or file in refs. + * + * @since 7.0 + */ + public static final String HEADS = "heads"; + + /** + * Prefix for any log. + * + * @since 7.0 + */ + public static final String L_LOGS = LOGS + "/"; + + /** * Info alternates file (goes under OBJECTS) * @since 5.5 */ @@ -358,6 +357,14 @@ public final class Constants { public static final String GIT_DIR_KEY = "GIT_DIR"; /** + * The environment variable that tells us which directory is the common + * ".git" directory. + * + * @since 7.0 + */ + public static final String GIT_COMMON_DIR_KEY = "GIT_COMMON_DIR"; + + /** * The environment variable that tells us which directory is the working * directory. */ @@ -459,6 +466,36 @@ public final class Constants { public static final String GITDIR = "gitdir: "; /** + * Name of the file (inside gitDir) that references the worktree's .git + * file (opposite link). + * + * .git/worktrees/<worktree-name>/gitdir + * + * A text file containing the absolute path back to the .git file that + * points here. This file is used to verify if the linked repository has been + * manually removed in which case this directory is no longer needed. + * The modification time (mtime) of this file should be updated each time + * the linked repository is accessed. + * + * @since 7.0 + */ + public static final String GITDIR_FILE = "gitdir"; + + /** + * Name of the file (inside gitDir) that has reference to $GIT_COMMON_DIR. + * + * .git/worktrees/<worktree-name>/commondir + * + * If this file exists, $GIT_COMMON_DIR will be set to the path specified in + * this file unless it is explicitly set. If the specified path is relative, + * it is relative to $GIT_DIR. The repository with commondir is incomplete + * without the repository pointed by "commondir". + * + * @since 7.0 + */ + public static final String COMMONDIR_FILE = "commondir"; + + /** * Name of the folder (inside gitDir) where submodules are stored * * @since 3.6 @@ -494,6 +531,34 @@ public final class Constants { public static final String ATTR_BUILTIN_BINARY_MERGER = "binary"; //$NON-NLS-1$ /** + * Prefix of a GPG signature. + * + * @since 7.0 + */ + public static final String GPG_SIGNATURE_PREFIX = "-----BEGIN PGP SIGNATURE-----"; //$NON-NLS-1$ + + /** + * Prefix of a CMS signature (X.509, S/MIME). + * + * @since 7.0 + */ + public static final String CMS_SIGNATURE_PREFIX = "-----BEGIN SIGNED MESSAGE-----"; //$NON-NLS-1$ + + /** + * Prefix of an SSH signature. + * + * @since 7.0 + */ + public static final String SSH_SIGNATURE_PREFIX = "-----BEGIN SSH SIGNATURE-----"; //$NON-NLS-1$ + + /** + * Union built-in merge driver + * + * @since 6.10.1 + */ + public static final String ATTR_BUILTIN_UNION_MERGE_DRIVER = "union"; //$NON-NLS-1$ + + /** * Create a new digest function for objects. * * @return a new digest object. @@ -661,44 +726,32 @@ public final class Constants { * the 7-bit ASCII character space. */ public static byte[] encodeASCII(String s) { - final byte[] r = new byte[s.length()]; - for (int k = r.length - 1; k >= 0; k--) { - final char c = s.charAt(k); - if (c > 127) - throw new IllegalArgumentException(MessageFormat.format(JGitText.get().notASCIIString, s)); - r[k] = (byte) c; + try { + CharsetEncoder encoder = US_ASCII.newEncoder() + .onUnmappableCharacter(CodingErrorAction.REPORT) + .onMalformedInput(CodingErrorAction.REPORT); + return encoder.encode(CharBuffer.wrap(s)).array(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().notASCIIString, s), e); } - return r; } /** - * Convert a string to a byte array in the standard character encoding. + * Convert a string to a byte array in the standard character encoding UTF8. * * @param str * the string to convert. May contain any Unicode characters. * @return a byte array representing the requested string, encoded using the * default character encoding (UTF-8). - * @see #CHARACTER_ENCODING */ public static byte[] encode(String str) { - final ByteBuffer bb = UTF_8.encode(str); - final int len = bb.limit(); - if (bb.hasArray() && bb.arrayOffset() == 0) { - final byte[] arr = bb.array(); - if (arr.length == len) - return arr; - } - - final byte[] arr = new byte[len]; - bb.get(arr); - return arr; + return str.getBytes(UTF_8); } static { if (OBJECT_ID_LENGTH != newMessageDigest().getDigestLength()) throw new LinkageError(JGitText.get().incorrectOBJECT_ID_LENGTH); - CHARSET = UTF_8; - CHARACTER_ENCODING = UTF_8.name(); } /** name of the file containing the commit msg for a merge commit */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java index 9fa5d75a3f..0e27b2743c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java @@ -17,12 +17,16 @@ package org.eclipse.jgit.lib; import static java.util.zip.Deflater.DEFAULT_COMPRESSION; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Config.SectionParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class keeps git repository core parameters. */ public class CoreConfig { + private static final Logger LOG = LoggerFactory.getLogger(CoreConfig.class); /** Key for {@link Config#get(SectionParser)}. */ public static final Config.SectionParser<CoreConfig> KEY = CoreConfig::new; @@ -127,7 +131,9 @@ public class CoreConfig { * Permissible values for {@code core.trustPackedRefsStat}. * * @since 6.1.1 + * @deprecated use {@link TrustStat} instead */ + @Deprecated(since = "7.2", forRemoval = true) public enum TrustPackedRefsStat { /** Do not trust file attributes of the packed-refs file. */ NEVER, @@ -135,12 +141,15 @@ public class CoreConfig { /** Trust file attributes of the packed-refs file. */ ALWAYS, - /** Open and close the packed-refs file to refresh its file attributes - * and then trust it. */ + /** + * Open and close the packed-refs file to refresh its file attributes + * and then trust it. + */ AFTER_OPEN, - /** {@code core.trustPackedRefsStat} defaults to this when it is - * not set */ + /** + * {@code core.trustPackedRefsStat} defaults to this when it is not set + */ UNSET } @@ -148,29 +157,66 @@ public class CoreConfig { * Permissible values for {@code core.trustLooseRefStat}. * * @since 6.9 + * @deprecated use {@link TrustStat} instead */ + @Deprecated(since = "7.2", forRemoval = true) public enum TrustLooseRefStat { /** Trust file attributes of the loose ref. */ ALWAYS, - /** Open and close parent directories of the loose ref file until the - * repository root to refresh its file attributes and then trust it. */ + /** + * Open and close parent directories of the loose ref file until the + * repository root to refresh its file attributes and then trust it. + */ AFTER_OPEN, } + /** + * Values for {@code core.trustXXX} options. + * + * @since 7.2 + */ + public enum TrustStat { + /** Do not trust file attributes of a File. */ + NEVER, + + /** Always trust file attributes of a File. */ + ALWAYS, + + /** Open and close the File to refresh its file attributes + * and then trust it. */ + AFTER_OPEN, + + /** + * Used for specific options to inherit value from value set for + * core.trustStat. + */ + INHERIT + } + private final int compression; private final int packIndexVersion; - private final LogRefUpdates logAllRefUpdates; - private final String excludesfile; private final String attributesfile; private final boolean commitGraph; + private final TrustStat trustStat; + + private final TrustStat trustPackedRefsStat; + + private final TrustStat trustLooseRefStat; + + private final TrustStat trustPackStat; + + private final TrustStat trustLooseObjectStat; + + private final TrustStat trustTablesListStat; + /** * Options for symlink handling * @@ -200,14 +246,17 @@ public class CoreConfig { DOTGITONLY } - private CoreConfig(Config rc) { + /** + * Create a new core configuration from the passed configuration. + * + * @param rc + * git configuration + */ + CoreConfig(Config rc) { compression = rc.getInt(ConfigConstants.CONFIG_CORE_SECTION, ConfigConstants.CONFIG_KEY_COMPRESSION, DEFAULT_COMPRESSION); packIndexVersion = rc.getInt(ConfigConstants.CONFIG_PACK_SECTION, ConfigConstants.CONFIG_KEY_INDEXVERSION, 2); - logAllRefUpdates = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, - LogRefUpdates.TRUE); excludesfile = rc.getString(ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_EXCLUDESFILE); attributesfile = rc.getString(ConfigConstants.CONFIG_CORE_SECTION, @@ -215,6 +264,68 @@ public class CoreConfig { commitGraph = rc.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, ConfigConstants.CONFIG_COMMIT_GRAPH, DEFAULT_COMMIT_GRAPH_ENABLE); + + trustStat = parseTrustStat(rc); + trustPackedRefsStat = parseTrustPackedRefsStat(rc); + trustLooseRefStat = parseTrustLooseRefStat(rc); + trustPackStat = parseTrustPackFileStat(rc); + trustLooseObjectStat = parseTrustLooseObjectFileStat(rc); + trustTablesListStat = parseTablesListStat(rc); + } + + private static TrustStat parseTrustStat(Config rc) { + Boolean tfs = rc.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT); + TrustStat ts = rc.getEnum(TrustStat.values(), + ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_TRUST_STAT); + if (tfs != null) { + if (ts == null) { + LOG.warn(JGitText.get().deprecatedTrustFolderStat); + return tfs.booleanValue() ? TrustStat.ALWAYS : TrustStat.NEVER; + } + LOG.warn(JGitText.get().precedenceTrustConfig); + } + if (ts == null) { + ts = TrustStat.ALWAYS; + } else if (ts == TrustStat.INHERIT) { + LOG.warn(JGitText.get().invalidTrustStat); + ts = TrustStat.ALWAYS; + } + return ts; + } + + private TrustStat parseTrustPackedRefsStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_PACKED_REFS_STAT); + } + + private TrustStat parseTrustLooseRefStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_LOOSE_REF_STAT); + } + + private TrustStat parseTrustPackFileStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_PACK_STAT); + } + + private TrustStat parseTrustLooseObjectFileStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_LOOSE_OBJECT_STAT); + } + + private TrustStat inheritParseTrustStat(Config rc, String key) { + TrustStat t = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, key, + TrustStat.INHERIT); + return t == TrustStat.INHERIT ? trustStat : t; + } + + private TrustStat parseTablesListStat(Config rc) { + TrustStat t = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_TRUST_TABLESLIST_STAT, + TrustStat.INHERIT); + return t == TrustStat.INHERIT ? trustStat : t; } /** @@ -236,20 +347,6 @@ public class CoreConfig { } /** - * Whether to log all refUpdates - * - * @return whether to log all refUpdates - * @deprecated since 5.6; default value depends on whether the repository is - * bare. Use - * {@link Config#getEnum(String, String, String, Enum)} - * directly. - */ - @Deprecated - public boolean isLogAllRefUpdates() { - return !LogRefUpdates.FALSE.equals(logAllRefUpdates); - } - - /** * Get path of excludesfile * * @return path of excludesfile @@ -279,4 +376,70 @@ public class CoreConfig { public boolean enableCommitGraph() { return commitGraph; } + + /** + * Get how far we can trust file attributes of packed-refs file which is + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of packed-refs file. + * + * @since 7.2 + */ + public TrustStat getTrustPackedRefsStat() { + return trustPackedRefsStat; + } + + /** + * Get how far we can trust file attributes of loose ref files which are + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of loose ref files. + * + * @since 7.2 + */ + public TrustStat getTrustLooseRefStat() { + return trustLooseRefStat; + } + + /** + * Get how far we can trust file attributes of packed-refs file which is + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of packed-refs file. + * + * @since 7.2 + */ + public TrustStat getTrustPackStat() { + return trustPackStat; + } + + /** + * Get how far we can trust file attributes of loose ref files which are + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of loose ref files. + * + * @since 7.2 + */ + public TrustStat getTrustLooseObjectStat() { + return trustLooseObjectStat; + } + + /** + * Get how far we can trust file attributes of the "tables.list" file which + * is used to store the list of filenames of the files storing + * {@link org.eclipse.jgit.internal.storage.reftable.Reftable}s in + * {@link org.eclipse.jgit.internal.storage.file.FileReftableDatabase}. + * + * @return how far we can trust file attributes of the "tables.list" file. + * + * @since 7.2 + */ + public TrustStat getTrustTablesListStat() { + return trustTablesListStat; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java index a71549c92e..3059f283fe 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java @@ -18,6 +18,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Config.ConfigEnum; import org.eclipse.jgit.transport.RefSpec; @@ -31,27 +32,37 @@ import org.eclipse.jgit.util.StringUtils; */ public class DefaultTypedConfigGetter implements TypedConfigGetter { + @SuppressWarnings("boxed") @Override public boolean getBoolean(Config config, String section, String subsection, String name, boolean defaultValue) { + return neverNull(getBoolean(config, section, subsection, name, + Boolean.valueOf(defaultValue))); + } + + @Nullable + @Override + public Boolean getBoolean(Config config, String section, String subsection, + String name, @Nullable Boolean defaultValue) { String n = config.getString(section, subsection, name); if (n == null) { return defaultValue; } if (Config.isMissing(n)) { - return true; + return Boolean.TRUE; } try { - return StringUtils.toBoolean(n); + return Boolean.valueOf(StringUtils.toBoolean(n)); } catch (IllegalArgumentException err) { throw new IllegalArgumentException(MessageFormat.format( JGitText.get().invalidBooleanValue, section, name, n), err); } } + @Nullable @Override public <T extends Enum<?>> T getEnum(Config config, T[] all, String section, - String subsection, String name, T defaultValue) { + String subsection, String name, @Nullable T defaultValue) { String value = config.getString(section, subsection, name); if (value == null) { return defaultValue; @@ -107,9 +118,27 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public int getInt(Config config, String section, String subsection, String name, int defaultValue) { - long val = config.getLong(section, subsection, name, defaultValue); + return neverNull(getInt(config, section, subsection, name, + Integer.valueOf(defaultValue))); + } + + @Nullable + @Override + @SuppressWarnings("boxing") + public Integer getInt(Config config, String section, String subsection, + String name, @Nullable Integer defaultValue) { + Long longDefault = defaultValue != null + ? Long.valueOf(defaultValue.longValue()) + : null; + Long val = config.getLong(section, subsection, name); + if (val == null) { + val = longDefault; + } + if (val == null) { + return null; + } if (Integer.MIN_VALUE <= val && val <= Integer.MAX_VALUE) { - return (int) val; + return Integer.valueOf(Math.toIntExact(val)); } throw new IllegalArgumentException(MessageFormat .format(JGitText.get().integerValueOutOfRange, section, name)); @@ -118,37 +147,56 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public int getIntInRange(Config config, String section, String subsection, String name, int minValue, int maxValue, int defaultValue) { - int val = getInt(config, section, subsection, name, defaultValue); + return neverNull(getIntInRange(config, section, subsection, name, + minValue, maxValue, Integer.valueOf(defaultValue))); + } + + @Override + @SuppressWarnings("boxing") + public Integer getIntInRange(Config config, String section, + String subsection, String name, int minValue, int maxValue, + Integer defaultValue) { + Integer val = getInt(config, section, subsection, name, defaultValue); + if (val == null) { + return null; + } if ((val >= minValue && val <= maxValue) || val == UNSET_INT) { return val; } if (subsection == null) { - throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().integerValueNotInRange, section, name, - Integer.valueOf(val), Integer.valueOf(minValue), - Integer.valueOf(maxValue))); + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().integerValueNotInRange, + section, name, val, minValue, maxValue)); } throw new IllegalArgumentException(MessageFormat.format( JGitText.get().integerValueNotInRangeSubSection, section, - subsection, name, Integer.valueOf(val), - Integer.valueOf(minValue), Integer.valueOf(maxValue))); + subsection, name, val, minValue, maxValue)); } @Override public long getLong(Config config, String section, String subsection, String name, long defaultValue) { - final String str = config.getString(section, subsection, name); + return neverNull(getLong(config, section, subsection, name, + Long.valueOf(defaultValue))); + } + + @Nullable + @Override + public Long getLong(Config config, String section, String subsection, + String name, @Nullable Long defaultValue) { + String str = config.getString(section, subsection, name); if (str == null) { return defaultValue; } try { - return StringUtils.parseLongWithSuffix(str, false); + return Long.valueOf(StringUtils.parseLongWithSuffix(str, false)); } catch (StringIndexOutOfBoundsException e) { // Empty return defaultValue; } catch (NumberFormatException nfe) { - throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().invalidIntegerValue, section, name, str), + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().invalidIntegerValue, + section, name, str), nfe); } } @@ -156,6 +204,13 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public long getTimeUnit(Config config, String section, String subsection, String name, long defaultValue, TimeUnit wantUnit) { + return neverNull(getTimeUnit(config, section, subsection, name, + Long.valueOf(defaultValue), wantUnit)); + } + + @Override + public Long getTimeUnit(Config config, String section, String subsection, + String name, @Nullable Long defaultValue, TimeUnit wantUnit) { String valueString = config.getString(section, subsection, name); if (valueString == null) { @@ -232,8 +287,8 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { } try { - return wantUnit.convert(Long.parseLong(digits) * inputMul, - inputUnit); + return Long.valueOf(wantUnit + .convert(Long.parseLong(digits) * inputMul, inputUnit)); } catch (NumberFormatException nfe) { IllegalArgumentException iae = notTimeUnit(section, subsection, unitName, valueString); @@ -274,4 +329,14 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { } return result; } + + // Trick for the checkers. When we use this, one is never null, but + // they don't know. + @NonNull + private static <T> T neverNull(T one) { + if (one == null) { + throw new IllegalArgumentException(); + } + return one; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index 427a235f3b..23d16db39f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -24,7 +24,13 @@ public class GpgConfig { /** Value for openpgp */ OPENPGP("openpgp"), //$NON-NLS-1$ /** Value for x509 */ - X509("x509"); //$NON-NLS-1$ + X509("x509"), //$NON-NLS-1$ + /** + * Value for ssh. + * + * @since 7.0 + */ + SSH("ssh"); //$NON-NLS-1$ private final String configValue; @@ -55,26 +61,11 @@ public class GpgConfig { private final boolean forceAnnotated; - /** - * Create a {@link GpgConfig} with the given parameters and default - * {@code true} for signing commits and {@code false} for tags. - * - * @param keySpec - * to use - * @param format - * to use - * @param gpgProgram - * to use - * @since 5.11 - */ - public GpgConfig(String keySpec, GpgFormat format, String gpgProgram) { - keyFormat = format; - signingKey = keySpec; - program = gpgProgram; - signCommits = true; - signAllTags = false; - forceAnnotated = false; - } + private final String sshDefaultKeyCommand; + + private final String sshAllowedSignersFile; + + private final String sshRevocationFile; /** * Create a new GPG config that reads the configuration from config. @@ -83,18 +74,18 @@ public class GpgConfig { * the config to read from */ public GpgConfig(Config config) { - keyFormat = config.getEnum(GpgFormat.values(), - ConfigConstants.CONFIG_GPG_SECTION, null, + keyFormat = config.getEnum(ConfigConstants.CONFIG_GPG_SECTION, null, ConfigConstants.CONFIG_KEY_FORMAT, GpgFormat.OPENPGP); signingKey = config.getString(ConfigConstants.CONFIG_USER_SECTION, null, ConfigConstants.CONFIG_KEY_SIGNINGKEY); String exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, keyFormat.toConfigValue(), ConfigConstants.CONFIG_KEY_PROGRAM); - if (exe == null) { + if (exe == null && GpgFormat.OPENPGP.equals(keyFormat)) { exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, null, ConfigConstants.CONFIG_KEY_PROGRAM); } + program = exe; signCommits = config.getBoolean(ConfigConstants.CONFIG_COMMIT_SECTION, ConfigConstants.CONFIG_KEY_GPGSIGN, false); @@ -102,6 +93,17 @@ public class GpgConfig { ConfigConstants.CONFIG_KEY_GPGSIGN, false); forceAnnotated = config.getBoolean(ConfigConstants.CONFIG_TAG_SECTION, ConfigConstants.CONFIG_KEY_FORCE_SIGN_ANNOTATED, false); + sshDefaultKeyCommand = config.getString( + ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_DEFAULT_KEY_COMMAND); + sshAllowedSignersFile = config.getString( + ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_ALLOWED_SIGNERS_FILE); + sshRevocationFile = config.getString(ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_REVOCATION_FILE); } /** @@ -165,4 +167,37 @@ public class GpgConfig { public boolean isSignAnnotated() { return forceAnnotated; } + + /** + * Retrieves the value of git config {@code gpg.ssh.defaultKeyCommand}. + * + * @return the value of {@code gpg.ssh.defaultKeyCommand} + * + * @since 7.1 + */ + public String getSshDefaultKeyCommand() { + return sshDefaultKeyCommand; + } + + /** + * Retrieves the value of git config {@code gpg.ssh.allowedSignersFile}. + * + * @return the value of {@code gpg.ssh.allowedSignersFile} + * + * @since 7.1 + */ + public String getSshAllowedSignersFile() { + return sshAllowedSignersFile; + } + + /** + * Retrieves the value of git config {@code gpg.ssh.revocationFile}. + * + * @return the value of {@code gpg.ssh.revocationFile} + * + * @since 7.1 + */ + public String getSshRevocationFile() { + return sshRevocationFile; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java deleted file mode 100644 index 074f46567b..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2020 Thomas Wolf <thomas.wolf@paranor.ch> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; -import org.eclipse.jgit.transport.CredentialsProvider; - -/** - * Creates GPG signatures for Git objects. - * - * @since 5.11 - */ -public interface GpgObjectSigner { - - /** - * Signs the specified object. - * - * <p> - * Implementors should obtain the payload for signing from the specified - * object via {@link ObjectBuilder#build()} and create a proper - * {@link GpgSignature}. The generated signature must be set on the - * specified {@code object} (see - * {@link ObjectBuilder#setGpgSignature(GpgSignature)}). - * </p> - * <p> - * Any existing signature on the object must be discarded prior obtaining - * the payload via {@link ObjectBuilder#build()}. - * </p> - * - * @param object - * the object to sign (must not be {@code null} and must be - * complete to allow proper calculation of payload) - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of <code>user.signingkey</code>) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @param config - * GPG settings from the git config - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - * @throws UnsupportedSigningFormatException - * if a config is given and the wanted key format is not - * supported - */ - void signObject(@NonNull ObjectBuilder object, - @Nullable String gpgSigningKey, @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider, GpgConfig config) - throws CanceledException, UnsupportedSigningFormatException; - - /** - * Indicates if a signing key is available for the specified committer - * and/or signing key. - * - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of <code>user.signingkey</code>) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @param config - * GPG settings from the git config - * @return <code>true</code> if a signing key is available, - * <code>false</code> otherwise - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - * @throws UnsupportedSigningFormatException - * if a config is given and the wanted key format is not - * supported - */ - public abstract boolean canLocateSigningKey(@Nullable String gpgSigningKey, - @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider, GpgConfig config) - throws CanceledException, UnsupportedSigningFormatException; - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java deleted file mode 100644 index 91c9bab5a4..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (C) 2021, 2024 Thomas Wolf <twolf@apache.org> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import java.io.IOException; -import java.util.Date; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.revwalk.RevObject; - -/** - * A {@code GpgSignatureVerifier} can verify GPG signatures on git commits and - * tags. - * - * @since 5.11 - */ -public interface GpgSignatureVerifier { - - /** - * Verifies the signature on a signed commit or tag. - * - * @param object - * to verify - * @param config - * the {@link GpgConfig} to use - * @return a {@link SignatureVerification} describing the outcome of the - * verification, or {@code null}Â if the object was not signed - * @throws IOException - * if an error occurs getting a public key - * @throws org.eclipse.jgit.api.errors.JGitInternalException - * if signature verification fails - */ - @Nullable - SignatureVerification verifySignature(@NonNull RevObject object, - @NonNull GpgConfig config) throws IOException; - - /** - * Verifies a given signature for given data. - * - * @param config - * the {@link GpgConfig} - * @param data - * the signature is for - * @param signatureData - * the ASCII-armored signature - * @return a {@link SignatureVerification} describing the outcome - * @throws IOException - * if the signature cannot be parsed - * @throws JGitInternalException - * if signature verification fails - * @since 6.9 - */ - default SignatureVerification verify(@NonNull GpgConfig config, byte[] data, - byte[] signatureData) throws IOException { - // Default implementation for backwards compatibility; override as - // appropriate - return verify(data, signatureData); - } - - /** - * Verifies a given signature for given data. - * - * @param data - * the signature is for - * @param signatureData - * the ASCII-armored signature - * @return a {@link SignatureVerification} describing the outcome - * @throws IOException - * if the signature cannot be parsed - * @throws JGitInternalException - * if signature verification fails - * @deprecated since 6.9, use {@link #verify(GpgConfig, byte[], byte[])} - * instead - */ - @Deprecated - public SignatureVerification verify(byte[] data, byte[] signatureData) - throws IOException; - - /** - * Retrieves the name of this verifier. This should be a short string - * identifying the engine that verified the signature, like "gpg" if GPG is - * used, or "bc" for a BouncyCastle implementation. - * - * @return the name - */ - @NonNull - String getName(); - - /** - * A {@link GpgSignatureVerifier} may cache public keys to speed up - * verifying signatures on multiple objects. This clears this cache, if any. - */ - void clear(); - - /** - * A {@code SignatureVerification} returns data about a (positively or - * negatively) verified signature. - */ - interface SignatureVerification { - - // Data about the signature. - - @NonNull - Date getCreationDate(); - - // Data from the signature used to find a public key. - - /** - * Obtains the signer as stored in the signature, if known. - * - * @return the signer, or {@code null} if unknown - */ - String getSigner(); - - /** - * Obtains the short or long fingerprint of the public key as stored in - * the signature, if known. - * - * @return the fingerprint, or {@code null} if unknown - */ - String getKeyFingerprint(); - - // Some information about the found public key. - - /** - * Obtains the OpenPGP user ID associated with the key. - * - * @return the user id, or {@code null} if unknown - */ - String getKeyUser(); - - /** - * Tells whether the public key used for this signature verification was - * expired when the signature was created. - * - * @return {@code true} if the key was expired already, {@code false} - * otherwise - */ - boolean isExpired(); - - /** - * Obtains the trust level of the public key used to verify the - * signature. - * - * @return the trust level - */ - @NonNull - TrustLevel getTrustLevel(); - - // The verification result. - - /** - * Tells whether the signature verification was successful. - * - * @return {@code true} if the signature was verified successfully; - * {@code false} if not. - */ - boolean getVerified(); - - /** - * Obtains a human-readable message giving additional information about - * the outcome of the verification. - * - * @return the message, or {@code null} if none set. - */ - String getMessage(); - } - - /** - * The owner's trust in a public key. - */ - enum TrustLevel { - UNKNOWN, NEVER, MARGINAL, FULL, ULTIMATE - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java deleted file mode 100644 index 59775c475b..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2021, 2022 Thomas Wolf <thomas.wolf@paranor.ch> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import java.util.Iterator; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A {@code GpgSignatureVerifierFactory} creates {@link GpgSignatureVerifier} instances. - * - * @since 5.11 - */ -public abstract class GpgSignatureVerifierFactory { - - private static final Logger LOG = LoggerFactory - .getLogger(GpgSignatureVerifierFactory.class); - - private static class DefaultFactory { - - private static volatile GpgSignatureVerifierFactory defaultFactory = loadDefault(); - - private static GpgSignatureVerifierFactory loadDefault() { - try { - ServiceLoader<GpgSignatureVerifierFactory> loader = ServiceLoader - .load(GpgSignatureVerifierFactory.class); - Iterator<GpgSignatureVerifierFactory> iter = loader.iterator(); - if (iter.hasNext()) { - return iter.next(); - } - } catch (ServiceConfigurationError e) { - LOG.error(e.getMessage(), e); - } - return null; - } - - private DefaultFactory() { - // No instantiation - } - - public static GpgSignatureVerifierFactory getDefault() { - return defaultFactory; - } - - /** - * Sets the default factory. - * - * @param factory - * the new default factory - */ - public static void setDefault(GpgSignatureVerifierFactory factory) { - defaultFactory = factory; - } - } - - /** - * Retrieves the default factory. - * - * @return the default factory or {@code null} if none set - */ - public static GpgSignatureVerifierFactory getDefault() { - return DefaultFactory.getDefault(); - } - - /** - * Sets the default factory. - * - * @param factory - * the new default factory - */ - public static void setDefault(GpgSignatureVerifierFactory factory) { - DefaultFactory.setDefault(factory); - } - - /** - * Creates a new {@link GpgSignatureVerifier}. - * - * @return the new {@link GpgSignatureVerifier} - */ - public abstract GpgSignatureVerifier getVerifier(); - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java deleted file mode 100644 index b25a61b506..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (C) 2018, 2022 Salesforce and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import java.util.Iterator; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.transport.CredentialsProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Creates GPG signatures for Git objects. - * - * @since 5.3 - */ -public abstract class GpgSigner { - - private static final Logger LOG = LoggerFactory.getLogger(GpgSigner.class); - - private static class DefaultSigner { - - private static volatile GpgSigner defaultSigner = loadGpgSigner(); - - private static GpgSigner loadGpgSigner() { - try { - ServiceLoader<GpgSigner> loader = ServiceLoader - .load(GpgSigner.class); - Iterator<GpgSigner> iter = loader.iterator(); - if (iter.hasNext()) { - return iter.next(); - } - } catch (ServiceConfigurationError e) { - LOG.error(e.getMessage(), e); - } - return null; - } - - private DefaultSigner() { - // No instantiation - } - - public static GpgSigner getDefault() { - return defaultSigner; - } - - public static void setDefault(GpgSigner signer) { - defaultSigner = signer; - } - } - - /** - * Get the default signer, or <code>null</code>. - * - * @return the default signer, or <code>null</code>. - */ - public static GpgSigner getDefault() { - return DefaultSigner.getDefault(); - } - - /** - * Set the default signer. - * - * @param signer - * the new default signer, may be <code>null</code> to select no - * default. - */ - public static void setDefault(GpgSigner signer) { - DefaultSigner.setDefault(signer); - } - - /** - * Signs the specified commit. - * - * <p> - * Implementors should obtain the payload for signing from the specified - * commit via {@link CommitBuilder#build()} and create a proper - * {@link GpgSignature}. The generated signature must be set on the - * specified {@code commit} (see - * {@link CommitBuilder#setGpgSignature(GpgSignature)}). - * </p> - * <p> - * Any existing signature on the commit must be discarded prior obtaining - * the payload via {@link CommitBuilder#build()}. - * </p> - * - * @param commit - * the commit to sign (must not be <code>null</code> and must be - * complete to allow proper calculation of payload) - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of <code>user.signingkey</code>) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - */ - public abstract void sign(@NonNull CommitBuilder commit, - @Nullable String gpgSigningKey, @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider) throws CanceledException; - - /** - * Indicates if a signing key is available for the specified committer - * and/or signing key. - * - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of <code>user.signingkey</code>) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @return <code>true</code> if a signing key is available, - * <code>false</code> otherwise - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - */ - public abstract boolean canLocateSigningKey(@Nullable String gpgSigningKey, - @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider) throws CanceledException; - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java index 8e965c5e9d..a99c64701f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java @@ -639,7 +639,7 @@ public class IndexDiff { // submodule repository in .git/modules doesn't // exist yet it isn't "missing". File gitDir = new File( - new File(repository.getDirectory(), + new File(repository.getCommonDirectory(), Constants.MODULES), subRepoPath); if (!gitDir.isDirectory()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java index 1c31263e33..1b455b974d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.nio.ByteBuffer; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.InvalidObjectIdException; @@ -152,6 +153,22 @@ public class ObjectId extends AnyObjectId implements Serializable { } /** + * Convert an ObjectId from raw binary representation + * + * @param bb + * a bytebuffer with the objectid encoded as 5 consecutive ints. + * This is the reverse of {@link ObjectId#copyRawTo(ByteBuffer)} + * + * @return the converted object id. + * + * @since 7.0 + */ + public static final ObjectId fromRaw(ByteBuffer bb) { + return new ObjectId(bb.getInt(), bb.getInt(), bb.getInt(), bb.getInt(), + bb.getInt()); + } + + /** * Convert an ObjectId from raw binary representation. * * @param is diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index 3ba055aae8..50f4a83b93 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -12,10 +12,15 @@ package org.eclipse.jgit.lib; +import static java.time.ZoneOffset.UTC; + import java.io.Serializable; -import java.text.SimpleDateFormat; +import java.time.DateTimeException; import java.time.Instant; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Locale; import java.util.TimeZone; @@ -40,7 +45,9 @@ public class PersonIdent implements Serializable { * timezone offset as in {@link #getTimeZoneOffset()}. * @return time zone object for the given offset. * @since 4.1 + * @deprecated use {@link #getZoneId(int)} instead */ + @Deprecated(since = "7.2") public static TimeZone getTimeZone(int tzOffset) { StringBuilder tzId = new StringBuilder(8); tzId.append("GMT"); //$NON-NLS-1$ @@ -49,6 +56,21 @@ public class PersonIdent implements Serializable { } /** + * Translate a minutes offset into a ZoneId + * + * @param tzOffset as minutes east of UTC + * @return a ZoneId for this offset (UTC if invalid) + * @since 7.1 + */ + public static ZoneId getZoneId(int tzOffset) { + try { + return ZoneOffset.ofHoursMinutes(tzOffset / 60, tzOffset % 60); + } catch (DateTimeException e) { + return UTC; + } + } + + /** * Format a timezone offset. * * @param r @@ -121,13 +143,17 @@ public class PersonIdent implements Serializable { } } + // Write offsets as [+-]HHMM + private static final DateTimeFormatter OFFSET_FORMATTER = DateTimeFormatter + .ofPattern("Z", Locale.US); //$NON-NLS-1$ + private final String name; private final String emailAddress; - private final long when; + private final Instant when; - private final int tzOffset; + private final ZoneId tzOffset; /** * Creates new PersonIdent from config info in repository, with current time. @@ -160,7 +186,7 @@ public class PersonIdent implements Serializable { * a {@link java.lang.String} object. */ public PersonIdent(String aName, String aEmailAddress) { - this(aName, aEmailAddress, SystemReader.getInstance().getCurrentTime()); + this(aName, aEmailAddress, SystemReader.getInstance().now()); } /** @@ -177,7 +203,7 @@ public class PersonIdent implements Serializable { */ public PersonIdent(String aName, String aEmailAddress, ProposedTimestamp when) { - this(aName, aEmailAddress, when.millis()); + this(aName, aEmailAddress, when.instant()); } /** @@ -189,8 +215,25 @@ public class PersonIdent implements Serializable { * local time * @param tz * time zone + * @deprecated Use {@link #PersonIdent(PersonIdent, Instant, ZoneId)} instead. */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, Date when, TimeZone tz) { + this(pi.getName(), pi.getEmailAddress(), when.toInstant(), tz.toZoneId()); + } + + /** + * Copy a PersonIdent, but alter the clone's time stamp + * + * @param pi + * original {@link org.eclipse.jgit.lib.PersonIdent} + * @param when + * local time + * @param tz + * time zone offset + * @since 7.1 + */ + public PersonIdent(PersonIdent pi, Instant when, ZoneId tz) { this(pi.getName(), pi.getEmailAddress(), when, tz); } @@ -202,9 +245,12 @@ public class PersonIdent implements Serializable { * original {@link org.eclipse.jgit.lib.PersonIdent} * @param aWhen * local time + * @deprecated Use the variant with an Instant instead */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, Date aWhen) { - this(pi.getName(), pi.getEmailAddress(), aWhen.getTime(), pi.tzOffset); + this(pi.getName(), pi.getEmailAddress(), aWhen.toInstant(), + pi.tzOffset); } /** @@ -218,7 +264,7 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public PersonIdent(PersonIdent pi, Instant aWhen) { - this(pi.getName(), pi.getEmailAddress(), aWhen.toEpochMilli(), pi.tzOffset); + this(pi.getName(), pi.getEmailAddress(), aWhen, pi.tzOffset); } /** @@ -230,11 +276,12 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(final String aName, final String aEmailAddress, final Date aWhen, final TimeZone aTZ) { - this(aName, aEmailAddress, aWhen.getTime(), aTZ.getOffset(aWhen - .getTime()) / (60 * 1000)); + this(aName, aEmailAddress, aWhen.toInstant(), aTZ.toZoneId()); } /** @@ -252,10 +299,16 @@ public class PersonIdent implements Serializable { */ public PersonIdent(final String aName, String aEmailAddress, Instant aWhen, ZoneId zoneId) { - this(aName, aEmailAddress, aWhen.toEpochMilli(), - TimeZone.getTimeZone(zoneId) - .getOffset(aWhen - .toEpochMilli()) / (60 * 1000)); + if (aName == null) + throw new IllegalArgumentException( + JGitText.get().personIdentNameNonNull); + if (aEmailAddress == null) + throw new IllegalArgumentException( + JGitText.get().personIdentEmailNonNull); + name = aName; + emailAddress = aEmailAddress; + when = aWhen; + tzOffset = zoneId; } /** @@ -267,15 +320,18 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, long aWhen, int aTZ) { - this(pi.getName(), pi.getEmailAddress(), aWhen, aTZ); + this(pi.getName(), pi.getEmailAddress(), Instant.ofEpochMilli(aWhen), + getZoneId(aTZ)); } private PersonIdent(final String aName, final String aEmailAddress, - long when) { + Instant when) { this(aName, aEmailAddress, when, SystemReader.getInstance() - .getTimezone(when)); + .getTimeZoneAt(when)); } private PersonIdent(UserConfig config) { @@ -298,19 +354,12 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(final String aName, final String aEmailAddress, final long aWhen, final int aTZ) { - if (aName == null) - throw new IllegalArgumentException( - JGitText.get().personIdentNameNonNull); - if (aEmailAddress == null) - throw new IllegalArgumentException( - JGitText.get().personIdentEmailNonNull); - name = aName; - emailAddress = aEmailAddress; - when = aWhen; - tzOffset = aTZ; + this(aName, aEmailAddress, Instant.ofEpochMilli(aWhen), getZoneId(aTZ)); } /** @@ -335,9 +384,12 @@ public class PersonIdent implements Serializable { * Get timestamp * * @return timestamp + * + * @deprecated Use getWhenAsInstant instead */ + @Deprecated(since = "7.1") public Date getWhen() { - return new Date(when); + return Date.from(when); } /** @@ -347,16 +399,19 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public Instant getWhenAsInstant() { - return Instant.ofEpochMilli(when); + return when; } /** * Get this person's declared time zone * * @return this person's declared time zone; null if time zone is unknown. + * + * @deprecated Use getZoneId instead */ + @Deprecated(since = "7.1") public TimeZone getTimeZone() { - return getTimeZone(tzOffset); + return TimeZone.getTimeZone(tzOffset); } /** @@ -366,7 +421,17 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public ZoneId getZoneId() { - return getTimeZone().toZoneId(); + return tzOffset; + } + + /** + * Return the offset in this timezone at the specific time + * + * @return the offset + * @since 7.1 + */ + public ZoneOffset getZoneOffset() { + return tzOffset.getRules().getOffset(when); } /** @@ -374,9 +439,11 @@ public class PersonIdent implements Serializable { * * @return this person's declared time zone as minutes east of UTC. If the * timezone is to the west of UTC it is negative. + * @deprecated Use {@link #getZoneOffset()} and read minutes from there */ + @Deprecated(since = "7.1") public int getTimeZoneOffset() { - return tzOffset; + return getZoneOffset().getTotalSeconds() / 60; } /** @@ -388,7 +455,7 @@ public class PersonIdent implements Serializable { public int hashCode() { int hc = getEmailAddress().hashCode(); hc *= 31; - hc += (int) (when / 1000L); + hc += when.hashCode(); return hc; } @@ -398,7 +465,9 @@ public class PersonIdent implements Serializable { final PersonIdent p = (PersonIdent) o; return getName().equals(p.getName()) && getEmailAddress().equals(p.getEmailAddress()) - && when / 1000L == p.when / 1000L; + // commmit timestamps are stored with 1 second precision + && when.truncatedTo(ChronoUnit.SECONDS) + .equals(p.when.truncatedTo(ChronoUnit.SECONDS)); } return false; } @@ -414,9 +483,9 @@ public class PersonIdent implements Serializable { r.append(" <"); //$NON-NLS-1$ appendSanitized(r, getEmailAddress()); r.append("> "); //$NON-NLS-1$ - r.append(when / 1000); + r.append(when.toEpochMilli() / 1000); r.append(' '); - appendTimezone(r, tzOffset); + r.append(OFFSET_FORMATTER.format(getZoneOffset())); return r.toString(); } @@ -424,19 +493,16 @@ public class PersonIdent implements Serializable { @SuppressWarnings("nls") public String toString() { final StringBuilder r = new StringBuilder(); - final SimpleDateFormat dtfmt; - dtfmt = new SimpleDateFormat("EEE MMM d HH:mm:ss yyyy Z", Locale.US); - dtfmt.setTimeZone(getTimeZone()); - + DateTimeFormatter dtfmt = DateTimeFormatter + .ofPattern("EEE MMM d HH:mm:ss yyyy Z", Locale.US) //$NON-NLS-1$ + .withZone(tzOffset); r.append("PersonIdent["); r.append(getName()); r.append(", "); r.append(getEmailAddress()); r.append(", "); - r.append(dtfmt.format(Long.valueOf(when))); + r.append(dtfmt.format(when)); r.append("]"); - return r.toString(); } } - diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 9e05a39731..49d5224325 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -26,6 +26,7 @@ import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.PackRefsCommand; /** * Abstraction of name to {@link org.eclipse.jgit.lib.ObjectId} mapping. @@ -160,7 +161,7 @@ public abstract class RefDatabase { if (existing.startsWith(prefix)) conflicting.add(existing); - return conflicting; + return Collections.unmodifiableList(conflicting); } /** @@ -238,23 +239,6 @@ public abstract class RefDatabase { } /** - * Compatibility synonym for {@link #findRef(String)}. - * - * @param name - * the name of the reference. May be a short name which must be - * searched for using the standard {@link #SEARCH_PATH}. - * @return the reference (if it exists); else {@code null}. - * @throws IOException - * the reference space cannot be accessed. - * @deprecated Use {@link #findRef(String)} instead. - */ - @Deprecated - @Nullable - public final Ref getRef(String name) throws IOException { - return findRef(name); - } - - /** * Read a single reference. * <p> * Aside from taking advantage of {@link #SEARCH_PATH}, this method may be @@ -372,6 +356,40 @@ public abstract class RefDatabase { } /** + * Get the reflog reader + * + * @param refName + * a {@link java.lang.String} object. + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied + * refname, or {@code null} if the named ref does not exist. + * @throws java.io.IOException + * the ref could not be accessed. + * @since 7.2 + */ + @Nullable + public ReflogReader getReflogReader(String refName) throws IOException { + Ref ref = exactRef(refName); + if (ref == null) { + return null; + } + return getReflogReader(ref); + } + + /** + * Get the reflog reader. + * + * @param ref + * a Ref + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref. + * @throws IOException + * if an IO error occurred + * @since 7.2 + */ + @NonNull + public abstract ReflogReader getReflogReader(@NonNull Ref ref) + throws IOException; + + /** * Get a section of the reference namespace. * * @param prefix @@ -610,4 +628,22 @@ public abstract class RefDatabase { } return null; } + + /** + * Optimize pack ref storage. + * + * @param pm + * a progress monitor + * + * @param packRefs + * {@link PackRefsCommand} to control ref packing behavior + * + * @throws java.io.IOException + * if an IO error occurred + * @since 7.1 + */ + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + // nothing + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index 4722e29b4c..c9dc6da4ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.LinkOption; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; @@ -33,10 +35,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import org.eclipse.jgit.annotations.NonNull; @@ -113,9 +117,12 @@ public abstract class Repository implements AutoCloseable { final AtomicLong closedAt = new AtomicLong(); - /** Metadata directory holding the repository's critical files. */ + /** $GIT_DIR: metadata directory holding the repository's critical files. */ private final File gitDir; + /** $GIT_COMMON_DIR: metadata directory holding the common repository's critical files. */ + private final File gitCommonDir; + /** File abstraction used to resolve paths. */ private final FS fs; @@ -129,6 +136,8 @@ public abstract class Repository implements AutoCloseable { private final String initialBranch; + private final AtomicReference<Boolean> caseInsensitiveWorktree = new AtomicReference<>(); + /** * Initialize a new repository instance. * @@ -137,6 +146,7 @@ public abstract class Repository implements AutoCloseable { */ protected Repository(BaseRepositoryBuilder options) { gitDir = options.getGitDir(); + gitCommonDir = options.getGitCommonDir(); fs = options.getFS(); workTree = options.getWorkTree(); indexFile = options.getIndexFile(); @@ -220,6 +230,16 @@ public abstract class Repository implements AutoCloseable { public abstract String getIdentifier(); /** + * Get common dir. + * + * @return $GIT_COMMON_DIR: local common metadata directory; + * @since 7.0 + */ + public File getCommonDirectory() { + return gitCommonDir; + } + + /** * Get the object database which stores this repository's data. * * @return the object database which stores this repository's data. @@ -293,25 +313,6 @@ public abstract class Repository implements AutoCloseable { } /** - * Whether the specified object is stored in this repo or any of the known - * shared repositories. - * - * @param objectId - * a {@link org.eclipse.jgit.lib.AnyObjectId} object. - * @return true if the specified object is stored in this repo or any of the - * known shared repositories. - * @deprecated use {@code getObjectDatabase().has(objectId)} - */ - @Deprecated - public boolean hasObject(AnyObjectId objectId) { - try { - return getObjectDatabase().has(objectId); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - /** * Open an object from this repository. * <p> * This is a one-shot call interface which may be faster than allocating a @@ -1150,11 +1151,9 @@ public abstract class Repository implements AutoCloseable { * new Ref object representing the same data as Ref, but isPeeled() * will be true and getPeeledObjectId will contain the peeled object * (or null). - * @deprecated use {@code getRefDatabase().peel(ref)} instead. */ - @Deprecated @NonNull - public Ref peel(Ref ref) { + private Ref peel(Ref ref) { try { return getRefDatabase().peel(ref); } catch (IOException e) { @@ -1584,6 +1583,40 @@ public abstract class Repository implements AutoCloseable { } /** + * Tells whether the work tree is on a case-insensitive file system. + * + * @return {@code true} if the work tree is case-insensitive; {@code false} + * otherwise + * @throws NoWorkTreeException + * if the repository is bare + * @since 7.2 + */ + public boolean isWorkTreeCaseInsensitive() throws NoWorkTreeException { + Boolean flag = caseInsensitiveWorktree.get(); + if (flag == null) { + File directory = getWorkTree(); + // See if we can find ".git" also as ".GIT". + File dotGit = new File(directory, Constants.DOT_GIT); + if (Files.exists(dotGit.toPath(), LinkOption.NOFOLLOW_LINKS)) { + dotGit = new File(directory, + Constants.DOT_GIT.toUpperCase(Locale.ROOT)); + flag = Boolean.valueOf(Files.exists(dotGit.toPath(), + LinkOption.NOFOLLOW_LINKS)); + } else { + // Fall back to a mostly sane default. On Mac, HFS+ and APFS + // partitions are case-insensitive by default but can be + // configured to be case-sensitive. + SystemReader system = SystemReader.getInstance(); + flag = Boolean.valueOf(system.isWindows() || system.isMacOS()); + } + if (!caseInsensitiveWorktree.compareAndSet(null, flag)) { + flag = caseInsensitiveWorktree.get(); + } + } + return flag.booleanValue(); + } + + /** * Force a scan for changed refs. Fires an IndexChangedEvent(false) if * changes are detected. * @@ -1699,10 +1732,13 @@ public abstract class Repository implements AutoCloseable { * @throws java.io.IOException * the ref could not be accessed. * @since 3.0 + * @deprecated use {@code #getRefDatabase().getReflogReader(String)} instead */ + @Deprecated(since = "7.2") @Nullable - public abstract ReflogReader getReflogReader(String refName) - throws IOException; + public ReflogReader getReflogReader(String refName) throws IOException { + return getRefDatabase().getReflogReader(refName); + } /** * Get the reflog reader. Subclasses should override this method and provide @@ -1710,15 +1746,17 @@ public abstract class Repository implements AutoCloseable { * * @param ref * a Ref - * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref, - * or {@code null} if the ref does not exist. + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref. * @throws IOException * if an IO error occurred * @since 5.13.2 + * @deprecated use {@code #getRefDatabase().getReflogReader(Ref)} instead */ - public @Nullable ReflogReader getReflogReader(@NonNull Ref ref) + @Deprecated(since = "7.2") + @NonNull + public ReflogReader getReflogReader(@NonNull Ref ref) throws IOException { - return getReflogReader(ref.getName()); + return getRefDatabase().getReflogReader(ref); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java index 6288447a8d..18366541da 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java @@ -450,10 +450,21 @@ public class RepositoryCache { * Git directory. */ public static boolean isGitRepository(File dir, FS fs) { - return fs.resolve(dir, Constants.OBJECTS).exists() - && fs.resolve(dir, "refs").exists() //$NON-NLS-1$ - && (fs.resolve(dir, Constants.REFTABLE).exists() - || isValidHead(new File(dir, Constants.HEAD))); + // check if common-dir available or fallback to git-dir + File commonDir; + try { + commonDir = fs.getCommonDir(dir); + } catch (IOException e) { + commonDir = null; + } + if (commonDir == null) { + commonDir = dir; + } + return fs.resolve(commonDir, Constants.OBJECTS).exists() + && fs.resolve(commonDir, "refs").exists() //$NON-NLS-1$ + && (fs.resolve(commonDir, Constants.REFTABLE).exists() + || isValidHead( + new File(commonDir, Constants.HEAD))); } private static boolean isValidHead(File head) { @@ -496,15 +507,31 @@ public class RepositoryCache { * null if there is no suitable match. */ public static File resolve(File directory, FS fs) { - if (isGitRepository(directory, fs)) + // the folder itself + if (isGitRepository(directory, fs)) { return directory; - if (isGitRepository(new File(directory, Constants.DOT_GIT), fs)) - return new File(directory, Constants.DOT_GIT); - - final String name = directory.getName(); - final File parent = directory.getParentFile(); - if (isGitRepository(new File(parent, name + Constants.DOT_GIT_EXT), fs)) - return new File(parent, name + Constants.DOT_GIT_EXT); + } + // the .git subfolder or file (reference) + File dotDir = new File(directory, Constants.DOT_GIT); + if (dotDir.isFile()) { + try { + File refDir = BaseRepositoryBuilder.getSymRef(directory, + dotDir, fs); + if (refDir != null && isGitRepository(refDir, fs)) { + return refDir; + } + } catch (IOException ignored) { + // Continue searching if gitdir ref isn't found + } + } else if (isGitRepository(dotDir, fs)) { + return dotDir; + } + // the folder extended with .git (bare) + File bareDir = new File(directory.getParentFile(), + directory.getName() + Constants.DOT_GIT_EXT); + if (isGitRepository(bareDir, fs)) { + return bareDir; + } return null; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifier.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifier.java new file mode 100644 index 0000000000..2ce2708cb9 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifier.java @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2021, 2024 Thomas Wolf <twolf@apache.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.util.Date; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.JGitInternalException; + +/** + * A {@code SignatureVerifier} can verify signatures on git commits and tags. + * + * @since 7.0 + */ +public interface SignatureVerifier { + + /** + * Verifies a given signature for given data. + * + * @param repository + * the {@link Repository} the data comes from. + * @param config + * the {@link GpgConfig} + * @param data + * the signature is for + * @param signatureData + * the ASCII-armored signature + * @return a {@link SignatureVerification} describing the outcome + * @throws IOException + * if the signature cannot be parsed + * @throws JGitInternalException + * if signature verification fails + */ + SignatureVerification verify(@NonNull Repository repository, + @NonNull GpgConfig config, byte[] data, byte[] signatureData) + throws IOException; + + /** + * Retrieves the name of this verifier. This should be a short string + * identifying the engine that verified the signature, like "gpg" if GPG is + * used, or "bc" for a BouncyCastle implementation. + * + * @return the name + */ + @NonNull + String getName(); + + /** + * A {@link SignatureVerifier} may cache public keys to speed up + * verifying signatures on multiple objects. This clears this cache, if any. + */ + void clear(); + + /** + * A {@code SignatureVerification} returns data about a (positively or + * negatively) verified signature. + * + * @param verifierName + * the name of the verifier that created this verification result + * @param creationDate + * date and time the signature was created + * @param signer + * the signer as stored in the signature, or {@code null} if + * unknown + * @param keyFingerprint + * fingerprint of the public key, or {@code null} if unknown + * @param keyUser + * user associated with the key, or {@code null} if unknown + * @param verified + * whether the signature verification was successful + * @param expired + * whether the public key used for this signature verification + * was expired when the signature was created + * @param trustLevel + * the trust level of the public key used to verify the signature + * @param message + * human-readable message giving additional information about the + * outcome of the verification, possibly {@code null} + */ + record SignatureVerification( + String verifierName, + Date creationDate, + String signer, + String keyFingerprint, + String keyUser, + boolean verified, + boolean expired, + @NonNull TrustLevel trustLevel, + String message) { + } + + /** + * The owner's trust in a public key. + */ + enum TrustLevel { + UNKNOWN, NEVER, MARGINAL, FULL, ULTIMATE + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifierFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifierFactory.java new file mode 100644 index 0000000000..7844aba3bd --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifierFactory.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Thomas Wolf <twolf@apache.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import org.eclipse.jgit.annotations.NonNull; + +/** + * A factory for {@link SignatureVerifier}s. + * + * @since 7.0 + */ +public interface SignatureVerifierFactory { + + /** + * Tells what kind of {@link SignatureVerifier} this factory creates. + * + * @return the {@link GpgConfig.GpgFormat} of the signer + */ + @NonNull + GpgConfig.GpgFormat getType(); + + /** + * Creates a new instance of a {@link SignatureVerifier} that can produce + * signatures of type {@link #getType()}. + * + * @return a new {@link SignatureVerifier} + */ + @NonNull + SignatureVerifier create(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifiers.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifiers.java new file mode 100644 index 0000000000..01c8422b66 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifiers.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2024 Thomas Wolf <twolf@apache.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.Map; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTag; +import org.eclipse.jgit.util.RawParseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the available signers. + * + * @since 7.0 + */ +public final class SignatureVerifiers { + + private static final Logger LOG = LoggerFactory.getLogger(SignatureVerifiers.class); + + private static final byte[] PGP_PREFIX = Constants.GPG_SIGNATURE_PREFIX + .getBytes(StandardCharsets.US_ASCII); + + private static final byte[] X509_PREFIX = Constants.CMS_SIGNATURE_PREFIX + .getBytes(StandardCharsets.US_ASCII); + + private static final byte[] SSH_PREFIX = Constants.SSH_SIGNATURE_PREFIX + .getBytes(StandardCharsets.US_ASCII); + + private static final Map<GpgConfig.GpgFormat, SignatureVerifierFactory> FACTORIES = loadSignatureVerifiers(); + + private static final Map<GpgConfig.GpgFormat, SignatureVerifier> VERIFIERS = new ConcurrentHashMap<>(); + + private static Map<GpgConfig.GpgFormat, SignatureVerifierFactory> loadSignatureVerifiers() { + Map<GpgConfig.GpgFormat, SignatureVerifierFactory> result = new EnumMap<>( + GpgConfig.GpgFormat.class); + try { + for (SignatureVerifierFactory factory : ServiceLoader + .load(SignatureVerifierFactory.class)) { + GpgConfig.GpgFormat format = factory.getType(); + SignatureVerifierFactory existing = result.get(format); + if (existing != null) { + LOG.warn("{}", //$NON-NLS-1$ + MessageFormat.format( + JGitText.get().signatureServiceConflict, + "SignatureVerifierFactory", format, //$NON-NLS-1$ + existing.getClass().getCanonicalName(), + factory.getClass().getCanonicalName())); + } else { + result.put(format, factory); + } + } + } catch (ServiceConfigurationError e) { + LOG.error(e.getMessage(), e); + } + return result; + } + + private SignatureVerifiers() { + // No instantiation + } + + /** + * Retrieves a {@link Signer} that can produce signatures of the given type + * {@code format}. + * + * @param format + * {@link GpgConfig.GpgFormat} the signer must support + * @return a {@link Signer}, or {@code null} if none is available + */ + public static SignatureVerifier get(@NonNull GpgConfig.GpgFormat format) { + return VERIFIERS.computeIfAbsent(format, f -> { + SignatureVerifierFactory factory = FACTORIES.get(format); + if (factory == null) { + return null; + } + return factory.create(); + }); + } + + /** + * Sets a specific signature verifier to use for a specific signature type. + * + * @param format + * signature type to set the {@code verifier} for + * @param verifier + * the {@link SignatureVerifier} to use for signatures of type + * {@code format}; if {@code null}, a default implementation, if + * available, may be used. + */ + public static void set(@NonNull GpgConfig.GpgFormat format, + SignatureVerifier verifier) { + SignatureVerifier previous; + if (verifier == null) { + previous = VERIFIERS.remove(format); + } else { + previous = VERIFIERS.put(format, verifier); + } + if (previous != null) { + previous.clear(); + } + } + + /** + * Verifies the signature on a signed commit or tag. + * + * @param repository + * the {@link Repository} the object is from + * @param config + * the {@link GpgConfig} to use + * @param object + * to verify + * @return a {@link SignatureVerifier.SignatureVerification} describing the + * outcome of the verification, or {@code null}Â if the object does + * not have a signature of a known type + * @throws IOException + * if an error occurs getting a public key + * @throws org.eclipse.jgit.api.errors.JGitInternalException + * if signature verification fails + */ + @Nullable + public static SignatureVerifier.SignatureVerification verify( + @NonNull Repository repository, @NonNull GpgConfig config, + @NonNull RevObject object) throws IOException { + if (object instanceof RevCommit) { + RevCommit commit = (RevCommit) object; + byte[] signatureData = commit.getRawGpgSignature(); + if (signatureData == null) { + return null; + } + byte[] raw = commit.getRawBuffer(); + // Now remove the GPG signature + byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; + int start = RawParseUtils.headerStart(header, raw, 0); + if (start < 0) { + return null; + } + int end = RawParseUtils.nextLfSkippingSplitLines(raw, start); + // start is at the beginning of the header's content + start -= header.length + 1; + // end is on the terminating LF; we need to skip that, too + if (end < raw.length) { + end++; + } + byte[] data = new byte[raw.length - (end - start)]; + System.arraycopy(raw, 0, data, 0, start); + System.arraycopy(raw, end, data, start, raw.length - end); + return verify(repository, config, data, signatureData); + } else if (object instanceof RevTag) { + RevTag tag = (RevTag) object; + byte[] signatureData = tag.getRawGpgSignature(); + if (signatureData == null) { + return null; + } + byte[] raw = tag.getRawBuffer(); + // The signature is just tacked onto the end of the message, which + // is last in the buffer. + byte[] data = Arrays.copyOfRange(raw, 0, + raw.length - signatureData.length); + return verify(repository, config, data, signatureData); + } + return null; + } + + /** + * Verifies a given signature for some give data. + * + * @param repository + * the {@link Repository} the object is from + * @param config + * the {@link GpgConfig} to use + * @param data + * to verify the signature of + * @param signature + * the given signature of the {@code data} + * @return a {@link SignatureVerifier.SignatureVerification} describing the + * outcome of the verification, or {@code null}Â if the signature + * type is unknown + * @throws IOException + * if an error occurs getting a public key + * @throws org.eclipse.jgit.api.errors.JGitInternalException + * if signature verification fails + */ + @Nullable + public static SignatureVerifier.SignatureVerification verify( + @NonNull Repository repository, @NonNull GpgConfig config, + byte[] data, byte[] signature) throws IOException { + GpgConfig.GpgFormat format = getFormat(signature); + if (format == null) { + return null; + } + SignatureVerifier verifier = get(format); + if (verifier == null) { + return null; + } + return verifier.verify(repository, config, data, signature); + } + + /** + * Determines the type of a given signature. + * + * @param signature + * to get the type of + * @return the signature type, or {@code null} if unknown + */ + @Nullable + public static GpgConfig.GpgFormat getFormat(byte[] signature) { + if (RawParseUtils.match(signature, 0, PGP_PREFIX) > 0) { + return GpgConfig.GpgFormat.OPENPGP; + } + if (RawParseUtils.match(signature, 0, X509_PREFIX) > 0) { + return GpgConfig.GpgFormat.X509; + } + if (RawParseUtils.match(signature, 0, SSH_PREFIX) > 0) { + return GpgConfig.GpgFormat.SSH; + } + return null; + } +}
\ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signer.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signer.java new file mode 100644 index 0000000000..3bb7464d08 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signer.java @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2024 Thomas Wolf <twolf@apache.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.CanceledException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; +import org.eclipse.jgit.transport.CredentialsProvider; + +/** + * Creates signatures for Git objects. + * + * @since 7.0 + */ +public interface Signer { + + /** + * Signs the specified object. + * + * <p> + * Implementors should obtain the payload for signing from the specified + * object via {@link ObjectBuilder#build()} and create a proper + * {@link GpgSignature}. The generated signature is set on the specified + * {@code object} (see {@link ObjectBuilder#setGpgSignature(GpgSignature)}). + * </p> + * <p> + * Any existing signature on the object must be discarded prior obtaining + * the payload via {@link ObjectBuilder#build()}. + * </p> + * + * @param repository + * {@link Repository} the object belongs to + * @param config + * GPG settings from the git config + * @param object + * the object to sign (must not be {@code null} and must be + * complete to allow proper calculation of payload) + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param signingKey + * if non-{@code null} overrides the signing key from the config + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws IOException + * if an I/O error occurs + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + default void signObject(@NonNull Repository repository, + @NonNull GpgConfig config, @NonNull ObjectBuilder object, + @NonNull PersonIdent committer, String signingKey, + CredentialsProvider credentialsProvider) + throws CanceledException, IOException, + UnsupportedSigningFormatException { + try { + object.setGpgSignature(sign(repository, config, object.build(), + committer, signingKey, credentialsProvider)); + } catch (UnsupportedEncodingException e) { + throw new JGitInternalException(e.getMessage(), e); + } + } + + /** + * Signs arbitrary data. + * + * @param repository + * {@link Repository} the signature is created in + * @param config + * GPG settings from the git config + * @param data + * the data to sign + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param signingKey + * if non-{@code null} overrides the signing key from the config + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @return the signature for {@code data} + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws IOException + * if an I/O error occurs + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + GpgSignature sign(@NonNull Repository repository, @NonNull GpgConfig config, + byte[] data, @NonNull PersonIdent committer, String signingKey, + CredentialsProvider credentialsProvider) throws CanceledException, + IOException, UnsupportedSigningFormatException; + + /** + * Indicates if a signing key is available for the specified committer + * and/or signing key. + * + * @param repository + * the current {@link Repository} + * @param config + * GPG settings from the git config + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param signingKey + * if non-{@code null} overrides the signing key from the config + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @return {@code true} if a signing key is available, {@code false} + * otherwise + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + */ + boolean canLocateSigningKey(@NonNull Repository repository, + @NonNull GpgConfig config, @NonNull PersonIdent committer, + String signingKey, CredentialsProvider credentialsProvider) + throws CanceledException; + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignerFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignerFactory.java new file mode 100644 index 0000000000..125d25e3b7 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignerFactory.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Thomas Wolf <twolf@apache.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import org.eclipse.jgit.annotations.NonNull; + +/** + * A factory for {@link Signer}s. + * + * @since 7.0 + */ +public interface SignerFactory { + + /** + * Tells what kind of {@link Signer} this factory creates. + * + * @return the {@link GpgConfig.GpgFormat} of the signer + */ + @NonNull + GpgConfig.GpgFormat getType(); + + /** + * Creates a new instance of a {@link Signer} that can produce signatures of + * type {@link #getType()}. + * + * @return a new {@link Signer} + */ + @NonNull + Signer create(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signers.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signers.java new file mode 100644 index 0000000000..7771b07841 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signers.java @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2024 Thomas Wolf <twolf@apache.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.text.MessageFormat; +import java.util.EnumMap; +import java.util.Map; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.internal.JGitText; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the available signers. + * + * @since 7.0 + */ +public final class Signers { + + private static final Logger LOG = LoggerFactory.getLogger(Signers.class); + + private static final Map<GpgConfig.GpgFormat, SignerFactory> SIGNER_FACTORIES = loadSigners(); + + private static final Map<GpgConfig.GpgFormat, Signer> SIGNERS = new ConcurrentHashMap<>(); + + private static Map<GpgConfig.GpgFormat, SignerFactory> loadSigners() { + Map<GpgConfig.GpgFormat, SignerFactory> result = new EnumMap<>( + GpgConfig.GpgFormat.class); + try { + for (SignerFactory factory : ServiceLoader + .load(SignerFactory.class)) { + GpgConfig.GpgFormat format = factory.getType(); + SignerFactory existing = result.get(format); + if (existing != null) { + LOG.warn("{}", //$NON-NLS-1$ + MessageFormat.format( + JGitText.get().signatureServiceConflict, + "SignerFactory", format, //$NON-NLS-1$ + existing.getClass().getCanonicalName(), + factory.getClass().getCanonicalName())); + } else { + result.put(format, factory); + } + } + } catch (ServiceConfigurationError e) { + LOG.error(e.getMessage(), e); + } + return result; + } + + private Signers() { + // No instantiation + } + + /** + * Retrieves a {@link Signer} that can produce signatures of the given type + * {@code format}. + * + * @param format + * {@link GpgConfig.GpgFormat} the signer must support + * @return a {@link Signer}, or {@code null} if none is available + */ + public static Signer get(@NonNull GpgConfig.GpgFormat format) { + return SIGNERS.computeIfAbsent(format, f -> { + SignerFactory factory = SIGNER_FACTORIES.get(format); + if (factory == null) { + return null; + } + return factory.create(); + }); + } + + /** + * Sets a specific signer to use for a specific signature type. + * + * @param format + * signature type to set the {@code signer} for + * @param signer + * the {@link Signer} to use for signatures of type + * {@code format}; if {@code null}, a default implementation, if + * available, may be used. + */ + public static void set(@NonNull GpgConfig.GpgFormat format, Signer signer) { + if (signer == null) { + SIGNERS.remove(format); + } else { + SIGNERS.put(format, signer); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java index bbc614448f..ea73d95102 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java @@ -206,23 +206,6 @@ public class TagBuilder extends ObjectBuilder { return os.toByteArray(); } - /** - * Format this builder's state as an annotated tag object. - * - * @return this object in the canonical annotated tag format, suitable for - * storage in a repository, or {@code null} if the tag cannot be - * encoded - * @deprecated since 5.11; use {@link #build()} instead - */ - @Deprecated - public byte[] toByteArray() { - try { - return build(); - } catch (UnsupportedEncodingException e) { - return null; - } - } - @SuppressWarnings("nls") @Override public String toString() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java index 0c03adcab8..3d4e0d1f3c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.util.FS; @@ -50,11 +51,36 @@ public interface TypedConfigGetter { * default value to return if no value was present. * @return true if any value or defaultValue is true, false for missing or * explicit false + * @deprecated use + * {@link #getBoolean(Config, String, String, String, Boolean)} + * instead */ + @Deprecated boolean getBoolean(Config config, String section, String subsection, String name, boolean defaultValue); /** + * Get a boolean value from a git {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param defaultValue + * default value to return if no value was present. + * @return true if any value or defaultValue is true, false for missing or + * explicit false + * @since 7.2 + */ + @Nullable + Boolean getBoolean(Config config, String section, String subsection, + String name, @Nullable Boolean defaultValue); + + /** * Parse an enumeration from a git {@link Config}. * * @param <T> @@ -74,8 +100,9 @@ public interface TypedConfigGetter { * default value to return if no value was present. * @return the selected enumeration value, or {@code defaultValue}. */ + @Nullable <T extends Enum<?>> T getEnum(Config config, T[] all, String section, - String subsection, String name, T defaultValue); + String subsection, String name, @Nullable T defaultValue); /** * Obtain an integer value from a git {@link Config}. @@ -91,11 +118,34 @@ public interface TypedConfigGetter { * @param defaultValue * default value to return if no value was present. * @return an integer value from the configuration, or defaultValue. + * @deprecated use {@link #getInt(Config, String, String, String, Integer)} + * instead */ + @Deprecated int getInt(Config config, String section, String subsection, String name, int defaultValue); /** + * Obtain an integer value from a git {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param defaultValue + * default value to return if no value was present. + * @return an integer value from the configuration, or defaultValue. + * @since 7.2 + */ + @Nullable + Integer getInt(Config config, String section, String subsection, + String name, @Nullable Integer defaultValue); + + /** * Obtain an integer value from a git {@link Config} which must be in given * range. * @@ -117,11 +167,43 @@ public interface TypedConfigGetter { * @return an integer value from the configuration, or defaultValue. * {@code #UNSET_INT} if unset. * @since 6.1 + * @deprecated use + * {@link #getIntInRange(Config, String, String, String, int, int, Integer)} + * instead */ + @Deprecated int getIntInRange(Config config, String section, String subsection, String name, int minValue, int maxValue, int defaultValue); /** + * Obtain an integer value from a git {@link Config} which must be in given + * range. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param minValue + * minimal value + * @param maxValue + * maximum value + * @param defaultValue + * default value to return if no value was present. Use + * {@code #UNSET_INT} to set the default to unset. + * @return an integer value from the configuration, or defaultValue. + * {@code #UNSET_INT} if unset. + * @since 7.2 + */ + @Nullable + Integer getIntInRange(Config config, String section, String subsection, + String name, int minValue, int maxValue, + @Nullable Integer defaultValue); + + /** * Obtain a long value from a git {@link Config}. * * @param config @@ -135,11 +217,34 @@ public interface TypedConfigGetter { * @param defaultValue * default value to return if no value was present. * @return a long value from the configuration, or defaultValue. + * @deprecated use {@link #getLong(Config, String, String, String, Long)} + * instead */ + @Deprecated long getLong(Config config, String section, String subsection, String name, long defaultValue); /** + * Obtain a long value from a git {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param defaultValue + * default value to return if no value was present. + * @return a long value from the configuration, or defaultValue. + * @since 7.2 + */ + @Nullable + Long getLong(Config config, String section, String subsection, String name, + @Nullable Long defaultValue); + + /** * Parse a numerical time unit, such as "1 minute", from a git * {@link Config}. * @@ -159,11 +264,41 @@ public interface TypedConfigGetter { * indication of the units. * @return the value, or {@code defaultValue} if not set, expressed in * {@code units}. + * @deprecated use + * {@link #getTimeUnit(Config, String, String, String, Long, TimeUnit)} + * instead */ + @Deprecated long getTimeUnit(Config config, String section, String subsection, String name, long defaultValue, TimeUnit wantUnit); /** + * Parse a numerical time unit, such as "1 minute", from a git + * {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is in. + * @param subsection + * subsection the key is in, or null if not in a subsection. + * @param name + * the key name. + * @param defaultValue + * default value to return if no value was present. + * @param wantUnit + * the units of {@code defaultValue} and the return value, as + * well as the units to assume if the value does not contain an + * indication of the units. + * @return the value, or {@code defaultValue} if not set, expressed in + * {@code units}. + * @since 7.2 + */ + @Nullable + Long getTimeUnit(Config config, String section, String subsection, + String name, @Nullable Long defaultValue, TimeUnit wantUnit); + + /** * Parse a string value from a git {@link Config} and treat it as a file * path, replacing a ~/ prefix by the user's home directory. * <p> @@ -189,9 +324,10 @@ public interface TypedConfigGetter { * @return the {@link Path}, or {@code defaultValue} if not set * @since 5.10 */ + @Nullable default Path getPath(Config config, String section, String subsection, String name, @NonNull FS fs, File resolveAgainst, - Path defaultValue) { + @Nullable Path defaultValue) { String value = config.getString(section, subsection, name); if (value == null) { return defaultValue; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java index 6d568643d5..a835a1dfc5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java @@ -23,5 +23,12 @@ public enum ContentMergeStrategy { OURS, /** Resolve the conflict hunk using the theirs version. */ - THEIRS -}
\ No newline at end of file + THEIRS, + + /** + * Resolve the conflict hunk using a union of both ours and theirs versions. + * + * @since 6.10.1 + */ + UNION +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java index 5734a25276..d0d4d367b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java @@ -120,6 +120,7 @@ public final class MergeAlgorithm { result.add(1, 0, 0, ConflictState.NO_CONFLICT); break; case THEIRS: + case UNION: result.add(2, 0, theirs.size(), ConflictState.NO_CONFLICT); break; @@ -148,6 +149,7 @@ public final class MergeAlgorithm { // we modified, they deleted switch (strategy) { case OURS: + case UNION: result.add(1, 0, ours.size(), ConflictState.NO_CONFLICT); break; case THEIRS: @@ -158,7 +160,7 @@ public final class MergeAlgorithm { result.add(1, 0, ours.size(), ConflictState.FIRST_CONFLICTING_RANGE); result.add(0, 0, base.size(), - ConflictState.BASE_CONFLICTING_RANGE); + ConflictState.BASE_CONFLICTING_RANGE); result.add(2, 0, 0, ConflictState.NEXT_CONFLICTING_RANGE); break; } @@ -333,6 +335,15 @@ public final class MergeAlgorithm { theirsEndB - commonSuffix, ConflictState.NO_CONFLICT); break; + case UNION: + result.add(1, oursBeginB + commonPrefix, + oursEndB - commonSuffix, + ConflictState.NO_CONFLICT); + + result.add(2, theirsBeginB + commonPrefix, + theirsEndB - commonSuffix, + ConflictState.NO_CONFLICT); + break; default: result.add(1, oursBeginB + commonPrefix, oursEndB - commonSuffix, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java index a35b30eb01..079db4a07f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java @@ -22,37 +22,6 @@ import org.eclipse.jgit.diff.RawText; * A class to convert merge results into a Git conformant textual presentation */ public class MergeFormatter { - /** - * Formats the results of a merge of {@link org.eclipse.jgit.diff.RawText} - * objects in a Git conformant way. This method also assumes that the - * {@link org.eclipse.jgit.diff.RawText} objects being merged are line - * oriented files which use LF as delimiter. This method will also use LF to - * separate chunks and conflict metadata, therefore it fits only to texts - * that are LF-separated lines. - * - * @param out - * the output stream where to write the textual presentation - * @param res - * the merge result which should be presented - * @param seqName - * When a conflict is reported each conflicting range will get a - * name. This name is following the "<<<<<<< - * " or ">>>>>>> " conflict markers. The - * names for the sequences are given in this list - * @param charsetName - * the name of the character set used when writing conflict - * metadata - * @throws java.io.IOException - * if an IO error occurred - * @deprecated Use - * {@link #formatMerge(OutputStream, MergeResult, List, Charset)} - * instead. - */ - @Deprecated - public void formatMerge(OutputStream out, MergeResult<RawText> res, - List<String> seqName, String charsetName) throws IOException { - formatMerge(out, res, seqName, Charset.forName(charsetName)); - } /** * Formats the results of a merge of {@link org.eclipse.jgit.diff.RawText} @@ -129,40 +98,6 @@ public class MergeFormatter { * the name ranges from ours should get * @param theirsName * the name ranges from theirs should get - * @param charsetName - * the name of the character set used when writing conflict - * metadata - * @throws java.io.IOException - * if an IO error occurred - * @deprecated use - * {@link #formatMerge(OutputStream, MergeResult, String, String, String, Charset)} - * instead. - */ - @Deprecated - public void formatMerge(OutputStream out, MergeResult res, String baseName, - String oursName, String theirsName, String charsetName) throws IOException { - formatMerge(out, res, baseName, oursName, theirsName, - Charset.forName(charsetName)); - } - - /** - * Formats the results of a merge of exactly two - * {@link org.eclipse.jgit.diff.RawText} objects in a Git conformant way. - * This convenience method accepts the names for the three sequences (base - * and the two merged sequences) as explicit parameters and doesn't require - * the caller to specify a List - * - * @param out - * the {@link java.io.OutputStream} where to write the textual - * presentation - * @param res - * the merge result which should be presented - * @param baseName - * the name ranges from the base should get - * @param oursName - * the name ranges from ours should get - * @param theirsName - * the name ranges from theirs should get * @param charset * the character set used when writing conflict metadata * @throws java.io.IOException diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java index e0c083f55c..039d7d844c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java @@ -92,24 +92,6 @@ public class MergeMessageFormatter { } /** - * Add section with conflicting paths to merge message. Lines are prefixed - * with a hash. - * - * @param message - * the original merge message - * @param conflictingPaths - * the paths with conflicts - * @return merge message with conflicting paths added - * @deprecated since 6.1; use - * {@link #formatWithConflicts(String, Iterable, char)} instead - */ - @Deprecated - public String formatWithConflicts(String message, - List<String> conflictingPaths) { - return formatWithConflicts(message, conflictingPaths, '#'); - } - - /** * Add section with conflicting paths to merge message. * * @param message diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java index 1162a615f2..f58ef4faba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java @@ -18,10 +18,11 @@ package org.eclipse.jgit.merge; import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.ArrayList; -import java.util.Date; import java.util.List; -import java.util.TimeZone; +import java.util.stream.Collectors; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -185,12 +186,15 @@ public class RecursiveMerger extends ResolveMerger { if (mergeTrees(bcTree, currentBase.getTree(), nextBase.getTree(), true)) currentBase = createCommitForTree(resultTree, parents); - else + else { + String failedPaths = failingPathsMessage(); throw new NoMergeBaseException( NoMergeBaseException.MergeBaseFailureReason.CONFLICTS_DURING_MERGE_BASE_CALCULATION, MessageFormat.format( JGitText.get().mergeRecursiveConflictsWhenMergingCommonAncestors, - currentBase.getName(), nextBase.getName())); + currentBase.getName(), nextBase.getName(), + failedPaths)); + } } } finally { inCore = oldIncore; @@ -229,11 +233,23 @@ public class RecursiveMerger extends ResolveMerger { private static PersonIdent mockAuthor(List<RevCommit> parents) { String name = RecursiveMerger.class.getSimpleName(); int time = 0; - for (RevCommit p : parents) + for (RevCommit p : parents) { time = Math.max(time, p.getCommitTime()); - return new PersonIdent( - name, name + "@JGit", //$NON-NLS-1$ - new Date((time + 1) * 1000L), - TimeZone.getTimeZone("GMT+0000")); //$NON-NLS-1$ + } + return new PersonIdent(name, name + "@JGit", //$NON-NLS-1$ + Instant.ofEpochSecond(time+1), ZoneOffset.UTC); + } + + private String failingPathsMessage() { + int max = 25; + String failedPaths = failingPaths.entrySet().stream().limit(max) + .map(entry -> entry.getKey() + ":" + entry.getValue()) //$NON-NLS-1$ + .collect(Collectors.joining("\n")); //$NON-NLS-1$ + + if (failingPaths.size() > max) { + failedPaths = String.format("%s\n... (%s failing paths omitted)", //$NON-NLS-1$ + failedPaths, Integer.valueOf(failingPaths.size() - max)); + } + return failedPaths; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index 1ad41be423..dc96f65b87 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -41,6 +41,7 @@ import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.attributes.Attributes; +import org.eclipse.jgit.attributes.AttributesNodeProvider; import org.eclipse.jgit.diff.DiffAlgorithm; import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; import org.eclipse.jgit.diff.RawText; @@ -837,6 +838,13 @@ public class ResolveMerger extends ThreeWayMerger { @NonNull private ContentMergeStrategy contentStrategy = ContentMergeStrategy.CONFLICT; + /** + * The {@link AttributesNodeProvider} to use while merging trees. + * + * @since 6.10.1 + */ + protected AttributesNodeProvider attributesNodeProvider; + private static MergeAlgorithm getMergeAlgorithm(Config config) { SupportedAlgorithm diffAlg = config.getEnum( CONFIG_DIFF_SECTION, null, CONFIG_KEY_ALGORITHM, @@ -1273,6 +1281,13 @@ public class ResolveMerger extends ThreeWayMerger { default: break; } + if (ignoreConflicts) { + // If the path is selected to be treated as binary via attributes, we do not perform + // content merge. When ignoreConflicts = true, we simply keep OURS to allow virtual commit + // to be built. + keep(ourDce); + return true; + } // add the conflicting path to merge result String currentPath = tw.getPathString(); MergeResult<RawText> result = new MergeResult<>( @@ -1312,8 +1327,12 @@ public class ResolveMerger extends ThreeWayMerger { addToCheckout(currentPath, null, attributes); return true; } catch (BinaryBlobException e) { - // if the file is binary in either OURS, THEIRS or BASE - // here, we don't have an option to ignore conflicts + // The file is binary in either OURS, THEIRS or BASE + if (ignoreConflicts) { + // When ignoreConflicts = true, we simply keep OURS to allow virtual commit to be built. + keep(ourDce); + return true; + } } } switch (getContentMergeStrategy()) { @@ -1354,6 +1373,8 @@ public class ResolveMerger extends ThreeWayMerger { } } } else { + // This is reachable if contentMerge() call above threw BinaryBlobException, so we don't + // need to check ignoreConflicts here, since it's already handled above. result.setContainsConflicts(true); addConflict(base, ours, theirs); unmergedPaths.add(currentPath); @@ -1489,11 +1510,26 @@ public class ResolveMerger extends ThreeWayMerger { : getRawText(ours.getEntryObjectId(), attributes[T_OURS]); RawText theirsText = theirs == null ? RawText.EMPTY_TEXT : getRawText(theirs.getEntryObjectId(), attributes[T_THEIRS]); - mergeAlgorithm.setContentMergeStrategy(strategy); + mergeAlgorithm.setContentMergeStrategy( + getAttributesContentMergeStrategy(attributes[T_OURS], + strategy)); return mergeAlgorithm.merge(RawTextComparator.DEFAULT, baseText, ourText, theirsText); } + private ContentMergeStrategy getAttributesContentMergeStrategy( + Attributes attributes, ContentMergeStrategy strategy) { + Attribute attr = attributes.get(Constants.ATTR_MERGE); + if (attr != null) { + String attrValue = attr.getValue(); + if (attrValue != null && attrValue + .equals(Constants.ATTR_BUILTIN_UNION_MERGE_DRIVER)) { + return ContentMergeStrategy.UNION; + } + } + return strategy; + } + private boolean isIndexDirty() { if (inCore) { return false; @@ -1824,6 +1860,18 @@ public class ResolveMerger extends ThreeWayMerger { this.workingTreeIterator = workingTreeIterator; } + /** + * Sets the {@link AttributesNodeProvider} to be used by this merger. + * + * @param attributesNodeProvider + * the attributeNodeProvider to set + * @since 6.10.1 + */ + public void setAttributesNodeProvider( + AttributesNodeProvider attributesNodeProvider) { + this.attributesNodeProvider = attributesNodeProvider; + } + /** * The resolve conflict way of three way merging @@ -1868,6 +1916,9 @@ public class ResolveMerger extends ThreeWayMerger { WorkTreeUpdater.createWorkTreeUpdater(db, dircache); dircache = workTreeUpdater.getLockedDirCache(); tw = new NameConflictTreeWalk(db, reader); + if (attributesNodeProvider != null) { + tw.setAttributesNodeProvider(attributesNodeProvider); + } tw.addTree(baseTree); tw.setHead(tw.addTree(headTree)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java index 79ceb1316a..30512c17b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java @@ -199,7 +199,7 @@ public class NoteMapMerger { if (child == null) return; if (child instanceof InMemoryNoteBucket) - b.setBucket(cell, ((InMemoryNoteBucket) child).writeTree(inserter)); + b.setBucket(cell, child.writeTree(inserter)); else b.setBucket(cell, child.getTreeId()); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java index a327095c81..23e09b9479 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.StandardCopyOption; @@ -33,12 +34,13 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.zip.InflaterInputStream; + import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.FilterFailedException; -import org.eclipse.jgit.api.errors.PatchFormatException; import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.attributes.Attributes; import org.eclipse.jgit.attributes.FilterCommand; @@ -101,11 +103,12 @@ import org.eclipse.jgit.util.sha1.SHA1; * @since 6.4 */ public class PatchApplier { - private static final byte[] NO_EOL = "\\ No newline at end of file" //$NON-NLS-1$ .getBytes(StandardCharsets.US_ASCII); - /** The tree before applying the patch. Only non-null for inCore operation. */ + /** + * The tree before applying the patch. Only non-null for inCore operation. + */ @Nullable private final RevTree beforeTree; @@ -115,10 +118,14 @@ public class PatchApplier { private final ObjectReader reader; + private final Charset charset; + private WorkingTreeOptions workingTreeOptions; private int inCoreSizeLimit; + private boolean allowConflicts; + /** * @param repo * repository to apply the patch in @@ -128,7 +135,8 @@ public class PatchApplier { inserter = repo.newObjectInserter(); reader = inserter.newReader(); beforeTree = null; - + allowConflicts = false; + charset = StandardCharsets.UTF_8; Config config = repo.getConfig(); workingTreeOptions = config.get(WorkingTreeOptions.KEY); inCoreSizeLimit = config.getInt(ConfigConstants.CONFIG_MERGE_SECTION, @@ -143,11 +151,14 @@ public class PatchApplier { * @param oi * to be used for modifying objects */ - public PatchApplier(Repository repo, RevTree beforeTree, ObjectInserter oi) { + public PatchApplier(Repository repo, RevTree beforeTree, + ObjectInserter oi) { this.repo = repo; this.beforeTree = beforeTree; inserter = oi; reader = oi.newReader(); + allowConflicts = false; + charset = StandardCharsets.UTF_8; } /** @@ -157,7 +168,6 @@ public class PatchApplier { * @since 6.3 */ public static class Result { - /** * A wrapper for a patch applying error that affects a given file. * @@ -166,28 +176,68 @@ public class PatchApplier { // TODO(ms): rename this class in next major release @SuppressWarnings("JavaLangClash") public static class Error { + final String msg; + + final String oldFileName; - private String msg; - private String oldFileName; - private @Nullable HunkHeader hh; + @Nullable + final HunkHeader hh; - private Error(String msg, String oldFileName, - @Nullable HunkHeader hh) { + final boolean isGitConflict; + + Error(String msg, String oldFileName, @Nullable HunkHeader hh, + boolean isGitConflict) { this.msg = msg; this.oldFileName = oldFileName; this.hh = hh; + this.isGitConflict = isGitConflict; + } + + /** + * Signals if as part of encountering this error, conflict markers + * were added to the file. + * + * @return {@code true} if conflict markers were added for this + * error. + * + * @since 6.10 + */ + public boolean isGitConflict() { + return isGitConflict; } @Override public String toString() { if (hh != null) { - return MessageFormat.format(JGitText.get().patchApplyErrorWithHunk, - oldFileName, hh, msg); + return MessageFormat.format( + JGitText.get().patchApplyErrorWithHunk, oldFileName, + hh, msg); } - return MessageFormat.format(JGitText.get().patchApplyErrorWithoutHunk, - oldFileName, msg); + return MessageFormat.format( + JGitText.get().patchApplyErrorWithoutHunk, oldFileName, + msg); } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof Error)) { + return false; + } + Error error = (Error) o; + return Objects.equals(msg, error.msg) + && Objects.equals(oldFileName, error.oldFileName) + && Objects.equals(hh, error.hh) + && isGitConflict == error.isGitConflict; + } + + @Override + public int hashCode() { + return Objects.hash(msg, oldFileName, hh, + Boolean.valueOf(isGitConflict)); + } } private ObjectId treeId; @@ -225,35 +275,15 @@ public class PatchApplier { return errors; } - private void addError(String msg,String oldFileName, @Nullable HunkHeader hh) { - errors.add(new Error(msg, oldFileName, hh)); + private void addError(String msg, String oldFileName, + @Nullable HunkHeader hh) { + errors.add(new Error(msg, oldFileName, hh, false)); } - } - /** - * Applies the given patch - * - * @param patchInput - * the patch to apply. - * @return the result of the patch - * @throws PatchFormatException - * if the patch cannot be parsed - * @throws IOException - * if the patch read fails - * @deprecated use {@link #applyPatch(Patch)} instead - */ - @Deprecated - public Result applyPatch(InputStream patchInput) - throws PatchFormatException, IOException { - Patch p = new Patch(); - try (InputStream inStream = patchInput) { - p.parse(inStream); - - if (!p.getErrors().isEmpty()) { - throw new PatchFormatException(p.getErrors()); - } + private void addErrorWithGitConflict(String msg, String oldFileName, + @Nullable HunkHeader hh) { + errors.add(new Error(msg, oldFileName, hh, true)); } - return applyPatch(p); } /** @@ -357,6 +387,17 @@ public class PatchApplier { return result; } + /** + * Sets up the {@link PatchApplier} to apply patches even if they conflict. + * + * @return the {@link PatchApplier} to apply any patches + * @since 6.10 + */ + public PatchApplier allowConflicts() { + allowConflicts = true; + return this; + } + private File getFile(String path) { return inCore() ? null : new File(repo.getWorkTree(), path); } @@ -439,6 +480,7 @@ public class PatchApplier { return false; } } + private static final int FILE_TREE_INDEX = 1; /** @@ -539,7 +581,9 @@ public class PatchApplier { convertCrLf); resultStreamLoader = applyText(raw, fh, result); } - if (resultStreamLoader == null || !result.getErrors().isEmpty()) { + if (resultStreamLoader == null + || (!result.getErrors().isEmpty() && result.getErrors().stream() + .anyMatch(e -> !e.msg.equals("cannot apply hunk")))) { //$NON-NLS-1$ return; } @@ -961,9 +1005,51 @@ public class PatchApplier { } } if (!applies) { - result.addError(JGitText.get().applyTextPatchCannotApplyHunk, - fh.getOldPath(), hh); - return null; + if (!allowConflicts) { + result.addError( + JGitText.get().applyTextPatchCannotApplyHunk, + fh.getOldPath(), hh); + return null; + } + // Insert conflict markers. This is best-guess because the + // file might have changed completely. But at least we give + // the user a graceful state that they can resolve manually. + // An alternative to this is using the 3-way merger. This + // only works if the pre-image SHA is contained in the repo. + // If that was the case, cherry-picking the original commit + // should be preferred to apply a patch. + result.addErrorWithGitConflict("cannot apply hunk", fh.getOldPath(), hh); //$NON-NLS-1$ + newLines.add(Math.min(applyAt++, newLines.size()), + asBytes("<<<<<<< HEAD")); //$NON-NLS-1$ + applyAt += hh.getOldImage().lineCount; + newLines.add(Math.min(applyAt++, newLines.size()), + asBytes("=======")); //$NON-NLS-1$ + + int sz = hunkLines.size(); + for (int j = 1; j < sz; j++) { + ByteBuffer hunkLine = hunkLines.get(j); + if (!hunkLine.hasRemaining()) { + // Completely empty line; accept as empty context + // line + applyAt++; + lastWasRemoval = false; + continue; + } + switch (hunkLine.array()[hunkLine.position()]) { + case ' ': + case '+': + newLines.add(Math.min(applyAt++, newLines.size()), + slice(hunkLine, 1)); + break; + case '-': + case '\\': + default: + break; + } + } + newLines.add(Math.min(applyAt++, newLines.size()), + asBytes(">>>>>>> PATCH")); //$NON-NLS-1$ + continue; } // Hunk applies at applyAt. Apply it, and update afterLastHunk and // lineNumberShift @@ -1010,7 +1096,11 @@ public class PatchApplier { } else if (!rt.isMissingNewlineAtEnd()) { newLines.add(null); } + return toContentStreamLoader(newLines); + } + private static ContentStreamLoader toContentStreamLoader( + List<ByteBuffer> newLines) throws IOException { // We could check if old == new, but the short-circuiting complicates // logic for inCore patching, so just write the new thing regardless. TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(null); @@ -1034,6 +1124,10 @@ public class PatchApplier { } } + private ByteBuffer asBytes(String str) { + return ByteBuffer.wrap(str.getBytes(charset)); + } + @SuppressWarnings("ByteBufferBackingArray") private boolean canApplyAt(List<ByteBuffer> hunkLines, List<ByteBuffer> newLines, int line) { @@ -1123,4 +1217,4 @@ public class PatchApplier { in.close(); } } -} +}
\ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java index 82671d9abb..7c763bc9b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java @@ -43,7 +43,7 @@ import org.eclipse.jgit.util.RawParseUtils; * Tree and blob objects reachable from interesting commits are automatically * scheduled for inclusion in the results of {@link #nextObject()}, returning * each object exactly once. Objects are sorted and returned according to the - * the commits that reference them and the order they appear within a tree. + * commits that reference them and the order they appear within a tree. * Ordering can be affected by changing the * {@link org.eclipse.jgit.revwalk.RevSort} used to order the commits that are * returned first. @@ -164,29 +164,6 @@ public class ObjectWalk extends RevWalk { } /** - * Create an object reachability checker that will use bitmaps if possible. - * - * This reachability checker accepts any object as target. For checks - * exclusively between commits, see - * {@link RevWalk#createReachabilityChecker()}. - * - * @return an object reachability checker, using bitmaps if possible. - * - * @throws IOException - * when the index fails to load. - * - * @since 5.8 - * @deprecated use - * {@code ObjectReader#createObjectReachabilityChecker(ObjectWalk)} - * instead. - */ - @Deprecated - public final ObjectReachabilityChecker createObjectReachabilityChecker() - throws IOException { - return reader.createObjectReachabilityChecker(this); - } - - /** * Mark an object or commit to start graph traversal from. * <p> * Callers are encouraged to use diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java index 1a869a0703..5afb669a15 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java @@ -26,40 +26,6 @@ import org.eclipse.jgit.errors.MissingObjectException; * @since 5.4 */ public interface ReachabilityChecker { - - /** - * Check if all targets are reachable from the {@code starters} commits. - * <p> - * Caller should parse the objectIds (preferably with - * {@code walk.parseCommit()} and handle missing/incorrect type objects - * before calling this method. - * - * @param targets - * commits to reach. - * @param starters - * known starting points. - * @return An unreachable target if at least one of the targets is - * unreachable. An empty optional if all targets are reachable from - * the starters. - * - * @throws MissingObjectException - * if any of the incoming objects doesn't exist in the - * repository. - * @throws IncorrectObjectTypeException - * if any of the incoming objects is not a commit or a tag. - * @throws IOException - * if any of the underlying indexes or readers can not be - * opened. - * - * @deprecated see {{@link #areAllReachable(Collection, Stream)} - */ - @Deprecated - default Optional<RevCommit> areAllReachable(Collection<RevCommit> targets, - Collection<RevCommit> starters) throws MissingObjectException, - IncorrectObjectTypeException, IOException { - return areAllReachable(targets, starters.stream()); - } - /** * Check if all targets are reachable from the {@code starters} commits. * <p> diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java index 743a8ccce0..871545fca2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java @@ -1,6 +1,6 @@ /* - * Copyright (C) 2008-2009, Google Inc. - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2009 Google Inc. + * Copyright (C) 2008, 2024 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -401,13 +401,13 @@ public class RevCommit extends RevObject { * @since 5.1 */ public final byte[] getRawGpgSignature() { - final byte[] raw = buffer; - final byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; - final int start = RawParseUtils.headerStart(header, raw, 0); + byte[] raw = buffer; + byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; + int start = RawParseUtils.headerStart(header, raw, 0); if (start < 0) { return null; } - final int end = RawParseUtils.headerEnd(raw, start); + int end = RawParseUtils.nextLfSkippingSplitLines(raw, start); return RawParseUtils.headerValue(raw, start, end); } @@ -524,6 +524,30 @@ public class RevCommit extends RevObject { } /** + * Parse the commit message and return its first line, i.e., everything up + * to but not including the first newline, if any. + * + * @return the first line of the decoded commit message as a string; never + * {@code null}. + * @since 7.2 + */ + public final String getFirstMessageLine() { + int msgB = RawParseUtils.commitMessage(buffer, 0); + if (msgB < 0) { + return ""; //$NON-NLS-1$ + } + int msgE = msgB; + byte[] raw = buffer; + while (msgE < raw.length && raw[msgE] != '\n') { + msgE++; + } + if (msgE > msgB && msgE > 0 && raw[msgE - 1] == '\r') { + msgE--; + } + return RawParseUtils.decode(guessEncoding(buffer), buffer, msgB, msgE); + } + + /** * Determine the encoding of the commit message buffer. * <p> * Locates the "encoding" header (if present) and returns its value. Due to diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java index 75dbd57740..0737a78085 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java @@ -38,8 +38,17 @@ import org.eclipse.jgit.util.StringUtils; */ public class RevTag extends RevObject { - private static final byte[] hSignature = Constants - .encodeASCII("-----BEGIN PGP SIGNATURE-----"); //$NON-NLS-1$ + private static final byte[] SIGNATURE_START = Constants + .encodeASCII("-----BEGIN"); //$NON-NLS-1$ + + private static final byte[] GPG_SIGNATURE_START = Constants + .encodeASCII(Constants.GPG_SIGNATURE_PREFIX); + + private static final byte[] CMS_SIGNATURE_START = Constants + .encodeASCII(Constants.CMS_SIGNATURE_PREFIX); + + private static final byte[] SSH_SIGNATURE_START = Constants + .encodeASCII(Constants.SSH_SIGNATURE_PREFIX); /** * Parse an annotated tag from its canonical format. @@ -208,20 +217,27 @@ public class RevTag extends RevObject { return msgB; } // Find the last signature start and return the rest - int start = nextStart(hSignature, raw, msgB); + int start = nextStart(SIGNATURE_START, raw, msgB); if (start < 0) { return start; } int next = RawParseUtils.nextLF(raw, start); while (next < raw.length) { - int newStart = nextStart(hSignature, raw, next); + int newStart = nextStart(SIGNATURE_START, raw, next); if (newStart < 0) { break; } start = newStart; next = RawParseUtils.nextLF(raw, start); } - return start; + // SIGNATURE_START is just a prefix. Check that it is one of the known + // full signature start tags. + if (RawParseUtils.match(raw, start, GPG_SIGNATURE_START) > 0 + || RawParseUtils.match(raw, start, CMS_SIGNATURE_START) > 0 + || RawParseUtils.match(raw, start, SSH_SIGNATURE_START) > 0) { + return start; + } + return -1; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index 76c14e9c26..41f98bad84 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -19,9 +19,14 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Optional; +import java.util.Map; +import java.util. +Optional; +import java.util.Set; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; @@ -31,9 +36,9 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.RevWalkException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AsyncObjectLoaderQueue; -import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -279,23 +284,6 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { } /** - * Get a reachability checker for commits over this revwalk. - * - * @return the most efficient reachability checker for this repository. - * @throws IOException - * if it cannot open any of the underlying indices. - * - * @since 5.4 - * @deprecated use {@code ObjectReader#createReachabilityChecker(RevWalk)} - * instead. - */ - @Deprecated - public final ReachabilityChecker createReachabilityChecker() - throws IOException { - return reader.createReachabilityChecker(this); - } - - /** * {@inheritDoc} * <p> * Release any resources used by this walker's reader. @@ -540,6 +528,27 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { } /** + * Determine if a <code>commit</code> is merged into any of the given + * <code>revs</code>. + * + * @param commit + * commit the caller thinks is reachable from <code>revs</code>. + * @param revs + * commits to start iteration from, and which is most likely a + * descendant (child) of <code>commit</code>. + * @return true if commit is merged into any of the revs; false otherwise. + * @throws java.io.IOException + * a pack file or loose object could not be read. + * @since 6.10.1 + */ + public boolean isMergedIntoAnyCommit(RevCommit commit, Collection<RevCommit> revs) + throws IOException { + return getCommitsMergedInto(commit, revs, + GetMergedIntoStrategy.RETURN_ON_FIRST_FOUND, + NullProgressMonitor.INSTANCE).size() > 0; + } + + /** * Determine if a <code>commit</code> is merged into all of the given * <code>refs</code>. * @@ -562,7 +571,26 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { private List<Ref> getMergedInto(RevCommit needle, Collection<Ref> haystacks, Enum returnStrategy, ProgressMonitor monitor) throws IOException { + Map<RevCommit, List<Ref>> refsByCommit = new HashMap<>(); + for (Ref r : haystacks) { + RevObject o = peel(parseAny(r.getObjectId())); + if (!(o instanceof RevCommit)) { + continue; + } + refsByCommit.computeIfAbsent((RevCommit) o, c -> new ArrayList<>()).add(r); + } + monitor.update(1); List<Ref> result = new ArrayList<>(); + for (RevCommit c : getCommitsMergedInto(needle, refsByCommit.keySet(), + returnStrategy, monitor)) { + result.addAll(refsByCommit.get(c)); + } + return result; + } + + private Set<RevCommit> getCommitsMergedInto(RevCommit needle, Collection<RevCommit> haystacks, + Enum returnStrategy, ProgressMonitor monitor) throws IOException { + Set<RevCommit> result = new HashSet<>(); List<RevCommit> uninteresting = new ArrayList<>(); List<RevCommit> marked = new ArrayList<>(); RevFilter oldRF = filter; @@ -578,16 +606,11 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { needle.parseHeaders(this); } int cutoff = needle.getGeneration(); - for (Ref r : haystacks) { + for (RevCommit c : haystacks) { if (monitor.isCancelled()) { return result; } monitor.update(1); - RevObject o = peel(parseAny(r.getObjectId())); - if (!(o instanceof RevCommit)) { - continue; - } - RevCommit c = (RevCommit) o; reset(UNINTERESTING | TEMP_MARK); markStart(c); boolean commitFound = false; @@ -599,7 +622,7 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { } if (References.isSameObject(next, needle) || (next.flags & TEMP_MARK) != 0) { - result.add(r); + result.add(c); if (returnStrategy == GetMergedIntoStrategy.RETURN_ON_FIRST_FOUND) { return result; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java index 4100e877df..c9186b5491 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.revwalk.filter; import java.io.IOException; +import java.time.Instant; import java.util.Date; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -30,9 +31,24 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param ts * the point in time to cut on. * @return a new filter to select commits on or before <code>ts</code>. + * + * @deprecated Use {@link #before(Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter before(Date ts) { - return before(ts.getTime()); + return before(ts.toInstant()); + } + + /** + * Create a new filter to select commits before a given date/time. + * + * @param ts + * the point in time to cut on. + * @return a new filter to select commits on or before <code>ts</code>. + * @since 7.2 + */ + public static RevFilter before(Instant ts) { + return new Before(ts); } /** @@ -43,7 +59,7 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @return a new filter to select commits on or before <code>ts</code>. */ public static final RevFilter before(long ts) { - return new Before(ts); + return new Before(Instant.ofEpochMilli(ts)); } /** @@ -52,9 +68,24 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param ts * the point in time to cut on. * @return a new filter to select commits on or after <code>ts</code>. + * + * @deprecated Use {@link #after(Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter after(Date ts) { - return after(ts.getTime()); + return after(ts.toInstant()); + } + + /** + * Create a new filter to select commits after a given date/time. + * + * @param ts + * the point in time to cut on. + * @return a new filter to select commits on or after <code>ts</code>. + * @since 7.2 + */ + public static RevFilter after(Instant ts) { + return new After(ts); } /** @@ -65,7 +96,7 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @return a new filter to select commits on or after <code>ts</code>. */ public static final RevFilter after(long ts) { - return new After(ts); + return after(Instant.ofEpochMilli(ts)); } /** @@ -75,9 +106,28 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param since the point in time to cut on. * @param until the point in time to cut off. * @return a new filter to select commits between the given date/times. + * + * @deprecated Use {@link #between(Instant, Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter between(Date since, Date until) { - return between(since.getTime(), until.getTime()); + return between(since.toInstant(), until.toInstant()); + } + + /** + * Create a new filter to select commits after or equal a given date/time + * <code>since</code> and before or equal a given date/time + * <code>until</code>. + * + * @param since + * the point in time to cut on. + * @param until + * the point in time to cut off. + * @return a new filter to select commits between the given date/times. + * @since 7.2 + */ + public static RevFilter between(Instant since, Instant until) { + return new Between(since, until); } /** @@ -87,9 +137,12 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param since the point in time to cut on, in milliseconds. * @param until the point in time to cut off, in millisconds. * @return a new filter to select commits between the given date/times. + * + * @deprecated Use {@link #between(Instant, Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter between(long since, long until) { - return new Between(since, until); + return new Between(Instant.ofEpochMilli(since), Instant.ofEpochMilli(until)); } final int when; @@ -98,6 +151,10 @@ public abstract class CommitTimeRevFilter extends RevFilter { when = (int) (ts / 1000); } + CommitTimeRevFilter(Instant t) { + when = (int) t.getEpochSecond(); + } + @Override public RevFilter clone() { return this; @@ -109,8 +166,8 @@ public abstract class CommitTimeRevFilter extends RevFilter { } private static class Before extends CommitTimeRevFilter { - Before(long ts) { - super(ts); + Before(Instant t) { + super(t); } @Override @@ -123,14 +180,12 @@ public abstract class CommitTimeRevFilter extends RevFilter { @SuppressWarnings("nls") @Override public String toString() { - return super.toString() + "(" + new Date(when * 1000L) + ")"; + return super.toString() + "(" + Instant.ofEpochSecond(when) + ")"; } } private static class After extends CommitTimeRevFilter { - After(long ts) { - super(ts); - } + After(Instant t) { super(t); } @Override public boolean include(RevWalk walker, RevCommit cmit) @@ -148,16 +203,16 @@ public abstract class CommitTimeRevFilter extends RevFilter { @SuppressWarnings("nls") @Override public String toString() { - return super.toString() + "(" + new Date(when * 1000L) + ")"; + return super.toString() + "(" + Instant.ofEpochSecond(when) + ")"; } } private static class Between extends CommitTimeRevFilter { private final int until; - Between(long since, long until) { + Between(Instant since, Instant until) { super(since); - this.until = (int) (until / 1000); + this.until = (int) until.getEpochSecond(); } @Override @@ -170,8 +225,8 @@ public abstract class CommitTimeRevFilter extends RevFilter { @SuppressWarnings("nls") @Override public String toString() { - return super.toString() + "(" + new Date(when * 1000L) + " - " - + new Date(until * 1000L) + ")"; + return super.toString() + "(" + Instant.ofEpochSecond(when) + " - " + + Instant.ofEpochSecond(until) + ")"; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java index 7cb8618302..668b92cf53 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java @@ -22,27 +22,6 @@ import org.eclipse.jgit.internal.storage.file.WindowCache; */ @MXBean public interface WindowCacheStats { - /** - * Get number of open files - * - * @return the number of open files. - * @deprecated use {@link #getOpenFileCount()} instead - */ - @Deprecated - public static int getOpenFiles() { - return (int) WindowCache.getInstance().getStats().getOpenFileCount(); - } - - /** - * Get number of open bytes - * - * @return the number of open bytes. - * @deprecated use {@link #getOpenByteCount()} instead - */ - @Deprecated - public static long getOpenBytes() { - return WindowCache.getInstance().getStats().getOpenByteCount(); - } /** * Get cache statistics diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 8373d6809a..863b79466a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -50,7 +50,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.zip.Deflater; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Repository; @@ -995,7 +995,7 @@ public class PackConfig { * * @return the index version, the special version 0 designates the oldest * (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public int getIndexVersion() { return indexVersion; @@ -1009,7 +1009,7 @@ public class PackConfig { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java index becc8082ba..105cba7d28 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java @@ -787,14 +787,14 @@ public class SubmoduleWalk implements AutoCloseable { IgnoreSubmoduleMode mode = repoConfig.getEnum( IgnoreSubmoduleMode.values(), ConfigConstants.CONFIG_SUBMODULE_SECTION, getModuleName(), - ConfigConstants.CONFIG_KEY_IGNORE, null); + ConfigConstants.CONFIG_KEY_IGNORE); if (mode != null) { return mode; } lazyLoadModulesConfig(); - return modulesConfig.getEnum(IgnoreSubmoduleMode.values(), - ConfigConstants.CONFIG_SUBMODULE_SECTION, getModuleName(), - ConfigConstants.CONFIG_KEY_IGNORE, IgnoreSubmoduleMode.NONE); + return modulesConfig.getEnum(ConfigConstants.CONFIG_SUBMODULE_SECTION, + getModuleName(), ConfigConstants.CONFIG_KEY_IGNORE, + IgnoreSubmoduleMode.NONE); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java index b873925316..aaf9f8a08a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java @@ -757,8 +757,10 @@ public class AmazonS3 { final XMLReader xr; try { - xr = SAXParserFactory.newInstance().newSAXParser() - .getXMLReader(); + SAXParserFactory saxParserFactory = SAXParserFactory + .newInstance(); + saxParserFactory.setNamespaceAware(true); + xr = saxParserFactory.newSAXParser().getXMLReader(); } catch (SAXException | ParserConfigurationException e) { throw new IOException( JGitText.get().noXMLParserAvailable, e); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index 469a3d6015..be0d37b96e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -12,10 +12,10 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DELIM; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DEEPEN; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DEEPEN_NOT; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DEEPEN_SINCE; +import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DELIM; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DONE; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_END; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_ERR; @@ -32,7 +32,6 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -715,7 +714,7 @@ public abstract class BasePackFetchConnection extends BasePackConnection // wind up later matching up against things we want and we // can avoid asking for something we already happen to have. // - final Date maxWhen = new Date(maxTime * 1000L); + Instant maxWhen = Instant.ofEpochSecond(maxTime); walk.sort(RevSort.COMMIT_TIME_DESC); walk.markStart(reachableCommits); walk.setRevFilter(CommitTimeRevFilter.after(maxWhen)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java index 73eddb8e21..f10b7bf452 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java @@ -302,8 +302,7 @@ public class HttpConfig { int postBufferSize = config.getInt(HTTP, POST_BUFFER_KEY, 1 * 1024 * 1024); boolean sslVerifyFlag = config.getBoolean(HTTP, SSL_VERIFY_KEY, true); - HttpRedirectMode followRedirectsMode = config.getEnum( - HttpRedirectMode.values(), HTTP, null, + HttpRedirectMode followRedirectsMode = config.getEnum(HTTP, null, FOLLOW_REDIRECTS_KEY, HttpRedirectMode.INITIAL); int redirectLimit = config.getInt(HTTP, MAX_REDIRECTS_KEY, MAX_REDIRECTS); @@ -335,8 +334,8 @@ public class HttpConfig { postBufferSize); sslVerifyFlag = config.getBoolean(HTTP, match, SSL_VERIFY_KEY, sslVerifyFlag); - followRedirectsMode = config.getEnum(HttpRedirectMode.values(), - HTTP, match, FOLLOW_REDIRECTS_KEY, followRedirectsMode); + followRedirectsMode = config.getEnum(HTTP, match, + FOLLOW_REDIRECTS_KEY, followRedirectsMode); int newMaxRedirects = config.getInt(HTTP, match, MAX_REDIRECTS_KEY, redirectLimit); if (newMaxRedirects >= 0) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java index 7224405df7..e1f2b19ce5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java @@ -530,7 +530,7 @@ public abstract class PackParser { receiving.beginTask(JGitText.get().receivingObjects, (int) expectedObjectCount); try { - for (int done = 0; done < expectedObjectCount; done++) { + for (long done = 0; done < expectedObjectCount; done++) { indexOneObject(); receiving.update(1); if (receiving.isCancelled()) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java index ed33eaed07..614ad88246 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java @@ -43,24 +43,13 @@ public class PacketLineIn { /** * Magic return from {@link #readString()} when a flush packet is found. - * - * @deprecated Callers should use {@link #isEnd(String)} to check if a - * string is the end marker, or - * {@link PacketLineIn#readStrings()} to iterate over all - * strings in the input stream until the marker is reached. */ - @Deprecated - public static final String END = new String(); /* must not string pool */ + private static final String END = new String(); /* must not string pool */ /** * Magic return from {@link #readString()} when a delim packet is found. - * - * @since 5.0 - * @deprecated Callers should use {@link #isDelimiter(String)} to check if a - * string is the delimiter. */ - @Deprecated - public static final String DELIM = new String(); /* must not string pool */ + private static final String DELIM = new String(); /* must not string pool */ enum AckNackResult { /** NAK */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java index a9e93b6be6..6bdaf0e234 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java @@ -24,6 +24,7 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -329,7 +330,7 @@ public class PushCertificateStore implements AutoCloseable { if (newId == null) { return RefUpdate.Result.NO_CHANGE; } - try (ObjectInserter inserter = db.newObjectInserter()) { + try { RefUpdate.Result result = updateRef(newId); switch (result) { case FAST_FORWARD: @@ -404,8 +405,8 @@ public class PushCertificateStore implements AutoCloseable { } private static void sortPending(List<PendingCert> pending) { - Collections.sort(pending, (PendingCert a, PendingCert b) -> Long.signum( - a.ident.getWhen().getTime() - b.ident.getWhen().getTime())); + Collections.sort(pending, + Comparator.comparing((PendingCert a) -> a.ident.getWhenAsInstant())); } private DirCache newDirCache() throws IOException { @@ -503,7 +504,7 @@ public class PushCertificateStore implements AutoCloseable { } else { sb.append(MessageFormat.format( JGitText.get().storePushCertMultipleRefs, - Integer.valueOf(cert.getCommands().size()))); + cert.getCommands().size())); } return sb.append('\n').toString(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index ddde6038e9..6f211e0794 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java @@ -88,52 +88,6 @@ import org.eclipse.jgit.util.io.TimeoutOutputStream; * Implements the server side of a push connection, receiving objects. */ public class ReceivePack { - /** - * Data in the first line of a request, the line itself plus capabilities. - * - * @deprecated Use {@link FirstCommand} instead. - * @since 5.6 - */ - @Deprecated - public static class FirstLine { - private final FirstCommand command; - - /** - * Parse the first line of a receive-pack request. - * - * @param line - * line from the client. - */ - public FirstLine(String line) { - command = FirstCommand.fromLine(line); - } - - /** - * Get non-capabilities part of the line - * - * @return non-capabilities part of the line. - */ - public String getLine() { - return command.getLine(); - } - - /** - * Get capabilities parsed from the line - * - * @return capabilities parsed from the line. - */ - public Set<String> getCapabilities() { - Set<String> reconstructedCapabilites = new HashSet<>(); - for (Map.Entry<String, String> e : command.getCapabilities() - .entrySet()) { - String cap = e.getValue() == null ? e.getKey() - : e.getKey() + "=" + e.getValue(); //$NON-NLS-1$ - reconstructedCapabilites.add(cap); - } - - return reconstructedCapabilites; - } - } /** Database we write the stored objects into. */ private final Repository db; @@ -2149,22 +2103,6 @@ public class ReceivePack { } /** - * Set whether this class will report command failures as warning messages - * before sending the command results. - * - * @param echo - * if true this class will report command failures as warning - * messages before sending the command results. This is usually - * not necessary, but may help buggy Git clients that discard the - * errors when all branches fail. - * @deprecated no widely used Git versions need this any more - */ - @Deprecated - public void setEchoCommandFailures(boolean echo) { - // No-op. - } - - /** * Get the client session-id * * @return The client session-id. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java index f72c421920..3d4bea2e48 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java @@ -178,7 +178,6 @@ public abstract class RefAdvertiser { * * This method must be invoked prior to any of the following: * <ul> - * <li>{@link #send(Map)}</li> * <li>{@link #send(Collection)}</li> * </ul> * @@ -195,7 +194,6 @@ public abstract class RefAdvertiser { * <p> * This method must be invoked prior to any of the following: * <ul> - * <li>{@link #send(Map)}</li> * <li>{@link #send(Collection)}</li> * <li>{@link #advertiseHave(AnyObjectId)}</li> * </ul> @@ -230,7 +228,6 @@ public abstract class RefAdvertiser { * <p> * This method must be invoked prior to any of the following: * <ul> - * <li>{@link #send(Map)}</li> * <li>{@link #send(Collection)}</li> * <li>{@link #advertiseHave(AnyObjectId)}</li> * </ul> @@ -260,24 +257,6 @@ public abstract class RefAdvertiser { * @throws java.io.IOException * the underlying output stream failed to write out an * advertisement record. - * @deprecated use {@link #send(Collection)} instead. - */ - @Deprecated - public Set<ObjectId> send(Map<String, Ref> refs) throws IOException { - return send(refs.values()); - } - - /** - * Format an advertisement for the supplied refs. - * - * @param refs - * zero or more refs to format for the client. The collection is - * sorted before display if necessary, and therefore may appear - * in any order. - * @return set of ObjectIds that were advertised to the client. - * @throws java.io.IOException - * the underlying output stream failed to write out an - * advertisement record. * @since 5.0 */ public Set<ObjectId> send(Collection<Ref> refs) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java index a0194ea8b1..8120df0698 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java @@ -11,8 +11,6 @@ package org.eclipse.jgit.transport; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.Iterator; import java.util.ServiceLoader; @@ -99,9 +97,8 @@ public abstract class SshSessionFactory { * @since 5.2 */ public static String getLocalUserName() { - return AccessController - .doPrivileged((PrivilegedAction<String>) () -> SystemReader - .getInstance().getProperty(Constants.OS_USER_NAME_KEY)); + return SystemReader.getInstance() + .getProperty(Constants.OS_USER_NAME_KEY); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index b335675da5..ac76e83d34 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -1121,28 +1121,6 @@ public abstract class Transport implements AutoCloseable { } /** - * @return the blob limit value set with {@link #setFilterBlobLimit} or - * {@link #setFilterSpec(FilterSpec)}, or -1 if no blob limit value - * was set - * @since 5.0 - * @deprecated Use {@link #getFilterSpec()} instead - */ - @Deprecated - public final long getFilterBlobLimit() { - return filterSpec.getBlobLimit(); - } - - /** - * @param bytes exclude blobs of size greater than this - * @since 5.0 - * @deprecated Use {@link #setFilterSpec(FilterSpec)} instead - */ - @Deprecated - public final void setFilterBlobLimit(long bytes) { - setFilterSpec(FilterSpec.withBlobLimit(bytes)); - } - - /** * Get filter spec * * @return the last filter spec set with {@link #setFilterSpec(FilterSpec)}, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java index 0fc9710ecb..f77b04110d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java @@ -254,6 +254,12 @@ public class TransportGitSsh extends SshTransport implements PackTransport { pb.environment().put(Constants.GIT_DIR_KEY, directory.getPath()); } + File commonDirectory = local != null ? local.getCommonDirectory() + : null; + if (commonDirectory != null) { + pb.environment().put(Constants.GIT_COMMON_DIR_KEY, + commonDirectory.getPath()); + } return pb; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java index 3a06ce5b63..1b9431ce6e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java @@ -225,6 +225,7 @@ class TransportLocal extends Transport implements PackTransport { env.remove("GIT_CONFIG"); //$NON-NLS-1$ env.remove("GIT_CONFIG_PARAMETERS"); //$NON-NLS-1$ env.remove("GIT_DIR"); //$NON-NLS-1$ + env.remove("GIT_COMMON_DIR"); //$NON-NLS-1$ env.remove("GIT_WORK_TREE"); //$NON-NLS-1$ env.remove("GIT_GRAFT_FILE"); //$NON-NLS-1$ env.remove("GIT_INDEX_FILE"); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java index 4de6ff825f..7b5842b712 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java @@ -82,7 +82,7 @@ public class URIish implements Serializable { * Part of a pattern which matches a relative path. Relative paths don't * start with slash or drive letters. Defines no capturing group. */ - private static final String RELATIVE_PATH_P = "(?:(?:[^\\\\/]+[\\\\/]+)*[^\\\\/]+[\\\\/]*)"; //$NON-NLS-1$ + private static final String RELATIVE_PATH_P = "(?:(?:[^\\\\/]+[\\\\/]+)*+[^\\\\/]*)"; //$NON-NLS-1$ /** * Part of a pattern which matches a relative or absolute path. Defines no @@ -120,7 +120,7 @@ public class URIish implements Serializable { * path (maybe even containing windows drive-letters) or a relative path. */ private static final Pattern LOCAL_FILE = Pattern.compile("^" // //$NON-NLS-1$ - + "([\\\\/]?" + PATH_P + ")" // //$NON-NLS-1$ //$NON-NLS-2$ + + "([\\\\/]?+" + PATH_P + ")" // //$NON-NLS-1$ //$NON-NLS-2$ + "$"); //$NON-NLS-1$ /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 9318871520..41ab8acf05 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -30,11 +30,11 @@ import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK_D import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_NO_DONE; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_NO_PROGRESS; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_OFS_DELTA; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SESSION_ID; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SHALLOW; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDEBAND_ALL; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND_64K; -import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SESSION_ID; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_THIN_PACK; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_WAIT_FOR_DONE; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_ACK; @@ -80,7 +80,6 @@ import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.internal.transport.parser.FirstWant; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -118,13 +117,13 @@ public class UploadPack implements Closeable { /** Policy the server uses to validate client requests */ public enum RequestPolicy { /** Client may only ask for objects the server advertised a reference for. */ - ADVERTISED, + ADVERTISED(0x08), /** * Client may ask for any commit reachable from a reference advertised by * the server. */ - REACHABLE_COMMIT, + REACHABLE_COMMIT(0x02), /** * Client may ask for objects that are the tip of any reference, even if not @@ -134,18 +133,36 @@ public class UploadPack implements Closeable { * * @since 3.1 */ - TIP, + TIP(0x01), /** * Client may ask for any commit reachable from any reference, even if that - * reference wasn't advertised. + * reference wasn't advertised, implies REACHABLE_COMMIT and TIP. * * @since 3.1 */ - REACHABLE_COMMIT_TIP, + REACHABLE_COMMIT_TIP(0x03), + + /** Client may ask for any SHA-1 in the repository, implies REACHABLE_COMMIT_TIP. */ + ANY(0x07); + + private final int bitmask; + + RequestPolicy(int bitmask) { + this.bitmask = bitmask; + } - /** Client may ask for any SHA-1 in the repository. */ - ANY; + /** + * Check if the current policy implies another, based on its bitmask. + * + * @param implied + * the implied policy based on its bitmask. + * @return true if the policy is implied. + * @since 6.10.1 + */ + public boolean implies(RequestPolicy implied) { + return (bitmask & implied.bitmask) != 0; + } } /** @@ -172,52 +189,6 @@ public class UploadPack implements Closeable { throws PackProtocolException, IOException; } - /** - * Data in the first line of a want-list, the line itself plus options. - * - * @deprecated Use {@link FirstWant} instead - */ - @Deprecated - public static class FirstLine { - - private final FirstWant firstWant; - - /** - * @param line - * line from the client. - */ - public FirstLine(String line) { - try { - firstWant = FirstWant.fromLine(line); - } catch (PackProtocolException e) { - throw new UncheckedIOException(e); - } - } - - /** - * Get non-capabilities part of the line - * - * @return non-capabilities part of the line. - */ - public String getLine() { - return firstWant.getLine(); - } - - /** - * Get capabilities parsed from the line - * - * @return capabilities parsed from the line. - */ - public Set<String> getOptions() { - if (firstWant.getAgent() != null) { - Set<String> caps = new HashSet<>(firstWant.getCapabilities()); - caps.add(OPTION_AGENT + '=' + firstWant.getAgent()); - return caps; - } - return firstWant.getCapabilities(); - } - } - /* * {@link java.util.function.Consumer} doesn't allow throwing checked * exceptions. Define our own to propagate IOExceptions. @@ -1423,6 +1394,7 @@ public class UploadPack implements Closeable { if (transferConfig.isAdvertiseObjectInfo()) { caps.add(COMMAND_OBJECT_INFO); } + caps.add(OPTION_AGENT + "=" + UserAgent.get()); return caps; } @@ -1629,13 +1601,9 @@ public class UploadPack implements Closeable { if (!biDirectionalPipe) adv.advertiseCapability(OPTION_NO_DONE); RequestPolicy policy = getRequestPolicy(); - if (policy == RequestPolicy.TIP - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.TIP)) adv.advertiseCapability(OPTION_ALLOW_TIP_SHA1_IN_WANT); - if (policy == RequestPolicy.REACHABLE_COMMIT - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.REACHABLE_COMMIT)) adv.advertiseCapability(OPTION_ALLOW_REACHABLE_SHA1_IN_WANT); adv.advertiseCapability(OPTION_AGENT, UserAgent.get()); if (transferConfig.isAllowFilter()) { @@ -1693,18 +1661,6 @@ public class UploadPack implements Closeable { } /** - * Deprecated synonym for {@code getFilterSpec().getBlobLimit()}. - * - * @return filter blob limit requested by the client, or -1 if no limit - * @since 5.3 - * @deprecated Use {@link #getFilterSpec()} instead - */ - @Deprecated - public final long getFilterBlobLimit() { - return getFilterSpec().getBlobLimit(); - } - - /** * Returns the filter spec for the current request. Valid only after * calling recvWants(). This may be a no-op filter spec, but it won't be * null. @@ -1996,10 +1952,9 @@ public class UploadPack implements Closeable { @Override public void checkWants(UploadPack up, List<ObjectId> wants) throws PackProtocolException, IOException { - if (!up.isBiDirectionalPipe()) + if (!up.isBiDirectionalPipe() || !wants.isEmpty()) { new ReachableCommitRequestValidator().checkWants(up, wants); - else if (!wants.isEmpty()) - throw new WantNotValidException(wants.iterator().next()); + } } } @@ -2271,7 +2226,7 @@ public class UploadPack implements Closeable { walk.resetRetain(SAVE); walk.markStart((RevCommit) want); if (oldestTime != 0) - walk.setRevFilter(CommitTimeRevFilter.after(oldestTime * 1000L)); + walk.setRevFilter(CommitTimeRevFilter.after(Instant.ofEpochSecond(oldestTime))); for (;;) { final RevCommit c = walk.next(); if (c == null) @@ -2429,7 +2384,8 @@ public class UploadPack implements Closeable { : req.getDepth() - 1; pw.setShallowPack(req.getDepth(), unshallowCommits); - // Ownership is transferred below + // dw borrows the reader from walk which is closed by #close + @SuppressWarnings("resource") DepthWalk.RevWalk dw = new DepthWalk.RevWalk( walk.getObjectReader(), walkDepth); dw.setDeepenSince(req.getDeepenSince()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java index 7b052ad4a7..b23ee97dcb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java @@ -10,10 +10,6 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT; - -import java.util.Set; - import org.eclipse.jgit.util.StringUtils; /** @@ -91,43 +87,6 @@ public class UserAgent { userAgent = StringUtils.isEmptyOrNull(agent) ? null : clean(agent); } - /** - * - * @param options - * options - * @param transportAgent - * name of transport agent - * @return The transport agent. - * @deprecated Capabilities with <key>=<value> shape are now - * parsed alongside other capabilities in the ReceivePack flow. - */ - @Deprecated - static String getAgent(Set<String> options, String transportAgent) { - if (options == null || options.isEmpty()) { - return transportAgent; - } - for (String o : options) { - if (o.startsWith(OPTION_AGENT) - && o.length() > OPTION_AGENT.length() - && o.charAt(OPTION_AGENT.length()) == '=') { - return o.substring(OPTION_AGENT.length() + 1); - } - } - return transportAgent; - } - - /** - * - * @param options - * options - * @return True if the transport agent is set. False otherwise. - * @deprecated Capabilities with <key>=<value> shape are now - * parsed alongside other capabilities in the ReceivePack flow. - */ - @Deprecated - static boolean hasAgent(Set<String> options) { - return getAgent(options, null) != null; - } private UserAgent() { } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java index 3da76f38fa..b7bb0cbce3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java @@ -22,8 +22,10 @@ import java.util.Collection; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.eclipse.jgit.errors.CompoundException; @@ -122,7 +124,7 @@ class WalkFetchConnection extends BaseFetchConnection { private final Deque<WalkRemoteObjectDatabase> noAlternatesYet; /** Packs we have discovered, but have not yet fetched locally. */ - private final Deque<RemotePack> unfetchedPacks; + private final Map<String, RemotePack> unfetchedPacks; /** * Packs whose indexes we have looked at in {@link #unfetchedPacks}. @@ -164,7 +166,7 @@ class WalkFetchConnection extends BaseFetchConnection { remotes = new ArrayList<>(); remotes.add(w); - unfetchedPacks = new ArrayDeque<>(); + unfetchedPacks = new LinkedHashMap<>(); packsConsidered = new HashSet<>(); noPacksYet = new ArrayDeque<>(); @@ -227,7 +229,7 @@ class WalkFetchConnection extends BaseFetchConnection { public void close() { inserter.close(); reader.close(); - for (RemotePack p : unfetchedPacks) { + for (RemotePack p : unfetchedPacks.values()) { if (p.tmpIdx != null) p.tmpIdx.delete(); } @@ -422,8 +424,9 @@ class WalkFetchConnection extends BaseFetchConnection { if (packNameList == null || packNameList.isEmpty()) continue; for (String packName : packNameList) { - if (packsConsidered.add(packName)) - unfetchedPacks.add(new RemotePack(wrr, packName)); + if (packsConsidered.add(packName)) { + unfetchedPacks.put(packName, new RemotePack(wrr, packName)); + } } if (downloadPackedObject(pm, id)) return; @@ -466,15 +469,27 @@ class WalkFetchConnection extends BaseFetchConnection { } } + private boolean downloadPackedObject(ProgressMonitor monitor, + AnyObjectId id) throws TransportException { + Set<String> brokenPacks = new HashSet<>(); + try { + return downloadPackedObject(monitor, id, brokenPacks); + } finally { + brokenPacks.forEach(unfetchedPacks::remove); + } + } + @SuppressWarnings("Finally") private boolean downloadPackedObject(final ProgressMonitor monitor, - final AnyObjectId id) throws TransportException { + final AnyObjectId id, Set<String> brokenPacks) throws TransportException { // Search for the object in a remote pack whose index we have, // but whose pack we do not yet have. // - final Iterator<RemotePack> packItr = unfetchedPacks.iterator(); - while (packItr.hasNext() && !monitor.isCancelled()) { - final RemotePack pack = packItr.next(); + for (Entry<String, RemotePack> entry : unfetchedPacks.entrySet()) { + if (monitor.isCancelled()) { + break; + } + final RemotePack pack = entry.getValue(); try { pack.openIndex(monitor); } catch (IOException err) { @@ -484,7 +499,7 @@ class WalkFetchConnection extends BaseFetchConnection { // another source, so don't consider it a failure. // recordError(id, err); - packItr.remove(); + brokenPacks.add(entry.getKey()); continue; } @@ -535,7 +550,7 @@ class WalkFetchConnection extends BaseFetchConnection { } throw new TransportException(e.getMessage(), e); } - packItr.remove(); + brokenPacks.add(entry.getKey()); } if (!alreadyHave(id)) { @@ -550,11 +565,9 @@ class WalkFetchConnection extends BaseFetchConnection { // Complete any other objects that we can. // - final Iterator<ObjectId> pending = swapFetchQueue(); - while (pending.hasNext()) { - final ObjectId p = pending.next(); + final Deque<ObjectId> pending = swapFetchQueue(); + for (ObjectId p : pending) { if (pack.index.hasObject(p)) { - pending.remove(); process(p); } else { workQueue.add(p); @@ -566,8 +579,8 @@ class WalkFetchConnection extends BaseFetchConnection { return false; } - private Iterator<ObjectId> swapFetchQueue() { - final Iterator<ObjectId> r = workQueue.iterator(); + private Deque<ObjectId> swapFetchQueue() { + final Deque<ObjectId> r = workQueue; workQueue = new ArrayDeque<>(); return r; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java index 125ee6cbe9..95b8221a8b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java @@ -36,29 +36,39 @@ import org.eclipse.jgit.annotations.NonNull; */ public interface HttpConnection { /** + * HttpURLConnection#HTTP_OK + * * @see HttpURLConnection#HTTP_OK */ int HTTP_OK = java.net.HttpURLConnection.HTTP_OK; /** + * HttpURLConnection#HTTP_NOT_AUTHORITATIVE + * * @see HttpURLConnection#HTTP_NOT_AUTHORITATIVE * @since 5.8 */ int HTTP_NOT_AUTHORITATIVE = java.net.HttpURLConnection.HTTP_NOT_AUTHORITATIVE; /** + * HttpURLConnection#HTTP_MOVED_PERM + * * @see HttpURLConnection#HTTP_MOVED_PERM * @since 4.7 */ int HTTP_MOVED_PERM = java.net.HttpURLConnection.HTTP_MOVED_PERM; /** + * HttpURLConnection#HTTP_MOVED_TEMP + * * @see HttpURLConnection#HTTP_MOVED_TEMP * @since 4.9 */ int HTTP_MOVED_TEMP = java.net.HttpURLConnection.HTTP_MOVED_TEMP; /** + * HttpURLConnection#HTTP_SEE_OTHER + * * @see HttpURLConnection#HTTP_SEE_OTHER * @since 4.9 */ @@ -85,16 +95,22 @@ public interface HttpConnection { int HTTP_11_MOVED_PERM = 308; /** + * HttpURLConnection#HTTP_NOT_FOUND + * * @see HttpURLConnection#HTTP_NOT_FOUND */ int HTTP_NOT_FOUND = java.net.HttpURLConnection.HTTP_NOT_FOUND; /** + * HttpURLConnection#HTTP_UNAUTHORIZED + * * @see HttpURLConnection#HTTP_UNAUTHORIZED */ int HTTP_UNAUTHORIZED = java.net.HttpURLConnection.HTTP_UNAUTHORIZED; /** + * HttpURLConnection#HTTP_FORBIDDEN + * * @see HttpURLConnection#HTTP_FORBIDDEN */ int HTTP_FORBIDDEN = java.net.HttpURLConnection.HTTP_FORBIDDEN; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java index 36fa72028e..0cac374844 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java @@ -371,12 +371,6 @@ public class FileTreeIterator extends WorkingTreeIterator { return attributes.getLength(); } - @Override - @Deprecated - public long getLastModified() { - return attributes.getLastModifiedInstant().toEpochMilli(); - } - /** * @since 5.1.9 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java index aaac2a72e0..31c216b4a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java @@ -38,12 +38,12 @@ import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.treewalk.filter.TreeFilter; @@ -1587,10 +1587,16 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { */ private String getFilterCommandDefinition(String filterDriverName, String filterCommandType) { + if (config == null) { + return null; + } String key = filterDriverName + "." + filterCommandType; //$NON-NLS-1$ String filterCommand = filterCommandsByNameDotType.get(key); if (filterCommand != null) return filterCommand; + if (config == null) { + return null; + } filterCommand = config.getString(ConfigConstants.CONFIG_FILTER_SECTION, filterDriverName, filterCommandType); boolean useBuiltin = config.getBoolean( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 73a3ddaae7..f16d800f63 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -498,6 +498,8 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { filterProcessBuilder.directory(repository.getWorkTree()); filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, repository.getDirectory().getAbsolutePath()); + filterProcessBuilder.environment().put(Constants.GIT_COMMON_DIR_KEY, + repository.getCommonDirectory().getAbsolutePath()); ExecutionResult result; try { result = fs.execute(filterProcessBuilder, in); @@ -620,18 +622,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { /** * Get the last modified time of this entry. * - * @return last modified time of this file, in milliseconds since the epoch - * (Jan 1, 1970 UTC). - * @deprecated use {@link #getEntryLastModifiedInstant()} instead - */ - @Deprecated - public long getEntryLastModified() { - return current().getLastModified(); - } - - /** - * Get the last modified time of this entry. - * * @return last modified time of this file * @since 5.1.9 */ @@ -1229,21 +1219,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { * needs to compute the value they should cache the reference within an * instance member instead. * - * @return time since the epoch (in ms) of the last change. - * @deprecated use {@link #getLastModifiedInstant()} instead - */ - @Deprecated - public abstract long getLastModified(); - - /** - * Get the last modified time of this entry. - * <p> - * <b>Note: Efficient implementation required.</b> - * <p> - * The implementation of this method must be efficient. If a subclass - * needs to compute the value they should cache the reference within an - * instance member instead. - * * @return time of the last change. * @since 5.1.9 */ @@ -1332,7 +1307,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { IgnoreNode infoExclude = new IgnoreNodeWithParent( coreExclude); - File exclude = fs.resolve(repository.getDirectory(), + File exclude = fs.resolve(repository.getCommonDirectory(), Constants.INFO_EXCLUDE); if (fs.exists(exclude)) { loadRulesFromFile(infoExclude, exclude); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java index bcf79a285d..33db6ea661 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java @@ -13,12 +13,12 @@ package org.eclipse.jgit.treewalk.filter; -import org.eclipse.jgit.util.RawParseUtils; - import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; +import org.eclipse.jgit.util.RawParseUtils; + /** * Specialized set for byte arrays, interpreted as strings for use in * {@link PathFilterGroup.Group}. Most methods assume the hash is already know @@ -141,13 +141,19 @@ class ByteArraySet { } /** + * Returns number of arrays in the set + * * @return number of arrays in the set */ int size() { return size; } - /** @return true if {@link #size()} is 0. */ + /** + * Returns true if {@link #size()} is 0 + * + * @return true if {@link #size()} is 0 + */ boolean isEmpty() { return size == 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java index 12af374b2e..c8421d6012 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java @@ -86,8 +86,8 @@ public class ChangeIdUtil { } } - private static final Pattern issuePattern = Pattern - .compile("^(Bug|Issue)[a-zA-Z0-9-]*:.*$"); //$NON-NLS-1$ + private static final Pattern signedOffByPattern = Pattern + .compile("^Signed-off-by:.*$"); //$NON-NLS-1$ private static final Pattern footerPattern = Pattern .compile("(^[a-zA-Z0-9-]+:(?!//).*$)"); //$NON-NLS-1$ @@ -159,7 +159,7 @@ public class ChangeIdUtil { int footerFirstLine = indexOfFirstFooterLine(lines); int insertAfter = footerFirstLine; for (int i = footerFirstLine; i < lines.length; ++i) { - if (issuePattern.matcher(lines[i]).matches()) { + if (!signedOffByPattern.matcher(lines[i]).matches()) { insertAfter = i + 1; continue; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index a8e1dae10e..59bbacfa76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -30,7 +30,6 @@ import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; -import java.security.AccessControlException; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; @@ -262,31 +261,6 @@ public abstract class FS { private static final AtomicInteger threadNumber = new AtomicInteger(1); /** - * Don't use the default thread factory of the ForkJoinPool for the - * CompletableFuture; it runs without any privileges, which causes - * trouble if a SecurityManager is present. - * <p> - * Instead use normal daemon threads. They'll belong to the - * SecurityManager's thread group, or use the one of the calling thread, - * as appropriate. - * </p> - * - * @see java.util.concurrent.Executors#newCachedThreadPool() - */ - private static final ExecutorService FUTURE_RUNNER = new ThreadPoolExecutor( - 5, 5, 30L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - runnable -> { - Thread t = new Thread(runnable, - "JGit-FileStoreAttributeReader-" //$NON-NLS-1$ - + threadNumber.getAndIncrement()); - // Make sure these threads don't prevent application/JVM - // shutdown. - t.setDaemon(true); - return t; - }); - - /** * Use a separate executor with at most one thread to synchronize * writing to the config. We write asynchronously since the config * itself might be on a different file system, which might otherwise @@ -463,7 +437,7 @@ public abstract class FS { locks.remove(s); } return attributes; - }, FUTURE_RUNNER); + }); f = f.exceptionally(e -> { LOG.error(e.getLocalizedMessage(), e); return Optional.empty(); @@ -898,21 +872,6 @@ public abstract class FS { } /** - * Whether FileStore attributes should be determined asynchronously - * - * @param asynch - * whether FileStore attributes should be determined - * asynchronously. If false access to cached attributes may block - * for some seconds for the first call per FileStore - * @since 5.1.9 - * @deprecated Use {@link FileStoreAttributes#setBackground} instead - */ - @Deprecated - public static void setAsyncFileStoreAttributes(boolean asynch) { - FileStoreAttributes.setBackground(asynch); - } - - /** * Auto-detect the appropriate file system abstraction, taking into account * the presence of a Cygwin installation on the system. Using jgit in * combination with Cygwin requires a more elaborate (and possibly slower) @@ -1085,24 +1044,6 @@ public abstract class FS { * symbolic links, the modification time of the link is returned, rather * than that of the link target. * - * @param f - * a {@link java.io.File} object. - * @return last modified time of f - * @throws java.io.IOException - * if an IO error occurred - * @since 3.0 - * @deprecated use {@link #lastModifiedInstant(Path)} instead - */ - @Deprecated - public long lastModified(File f) throws IOException { - return FileUtils.lastModified(f); - } - - /** - * Get the last modified time of a file system object. If the OS/JRE support - * symbolic links, the modification time of the link is returned, rather - * than that of the link target. - * * @param p * a {@link Path} object. * @return last modified time of p @@ -1131,25 +1072,6 @@ public abstract class FS { * <p> * For symlinks it sets the modified time of the link target. * - * @param f - * a {@link java.io.File} object. - * @param time - * last modified time - * @throws java.io.IOException - * if an IO error occurred - * @since 3.0 - * @deprecated use {@link #setLastModified(Path, Instant)} instead - */ - @Deprecated - public void setLastModified(File f, long time) throws IOException { - FileUtils.setLastModified(f, time); - } - - /** - * Set the last modified time of a file system object. - * <p> - * For symlinks it sets the modified time of the link target. - * * @param p * a {@link Path} object. * @param time @@ -1443,13 +1365,6 @@ public abstract class FS { } } catch (IOException e) { LOG.error("Caught exception in FS.readPipe()", e); //$NON-NLS-1$ - } catch (AccessControlException e) { - LOG.warn(MessageFormat.format( - JGitText.get().readPipeIsNotAllowedRequiredPermission, - command, dir, e.getPermission())); - } catch (SecurityException e) { - LOG.warn(MessageFormat.format(JGitText.get().readPipeIsNotAllowed, - command, dir)); } if (debug) { LOG.debug("readpipe returns null"); //$NON-NLS-1$ @@ -1800,25 +1715,6 @@ public abstract class FS { } /** - * Create a new file. See {@link java.io.File#createNewFile()}. Subclasses - * of this class may take care to provide a safe implementation for this - * even if {@link #supportsAtomicCreateNewFile()} is <code>false</code> - * - * @param path - * the file to be created - * @return <code>true</code> if the file was created, <code>false</code> if - * the file already existed - * @throws java.io.IOException - * if an IO error occurred - * @deprecated use {@link #createNewFileAtomic(File)} instead - * @since 4.5 - */ - @Deprecated - public boolean createNewFile(File path) throws IOException { - return path.createNewFile(); - } - - /** * A token representing a file created by * {@link #createNewFileAtomic(File)}. The token must be retained until the * file has been deleted in order to guarantee that the unique file was @@ -2042,6 +1938,8 @@ public abstract class FS { environment.put(Constants.GIT_DIR_KEY, repository.getDirectory().getAbsolutePath()); if (!repository.isBare()) { + environment.put(Constants.GIT_COMMON_DIR_KEY, + repository.getCommonDirectory().getAbsolutePath()); environment.put(Constants.GIT_WORK_TREE_KEY, repository.getWorkTree().getAbsolutePath()); } @@ -2137,7 +2035,7 @@ public abstract class FS { case "post-receive": //$NON-NLS-1$ case "post-update": //$NON-NLS-1$ case "push-to-checkout": //$NON-NLS-1$ - return repository.getDirectory(); + return repository.getCommonDirectory(); default: return repository.getWorkTree(); } @@ -2150,7 +2048,7 @@ public abstract class FS { if (hooksDir != null) { return new File(hooksDir); } - File dir = repository.getDirectory(); + File dir = repository.getCommonDirectory(); return dir == null ? null : new File(dir, Constants.HOOKS); } @@ -2424,19 +2322,6 @@ public abstract class FS { } /** - * Get the time when the file was last modified in milliseconds since - * the epoch - * - * @return the time (milliseconds since 1970-01-01) when this object was - * last modified - * @deprecated use getLastModifiedInstant instead - */ - @Deprecated - public long getLastModifiedTime() { - return lastModifiedInstant.toEpochMilli(); - } - - /** * Get the time when this object was last modified * * @return the time when this object was last modified @@ -2578,6 +2463,33 @@ public abstract class FS { } /** + * Get common dir path. + * + * @param dir + * the .git folder + * @return common dir path + * @throws IOException + * if commondir file can't be read + * + * @since 7.0 + */ + public File getCommonDir(File dir) throws IOException { + // first the GIT_COMMON_DIR is same as GIT_DIR + File commonDir = dir; + // now check if commondir file exists (e.g. worktree repository) + File commonDirFile = new File(dir, Constants.COMMONDIR_FILE); + if (commonDirFile.isFile()) { + String commonDirPath = new String(IO.readFully(commonDirFile)) + .trim(); + commonDir = new File(commonDirPath); + if (!commonDir.isAbsolute()) { + commonDir = new File(dir, commonDirPath).getCanonicalFile(); + } + } + return commonDir; + } + + /** * This runnable will consume an input stream's content into an output * stream as soon as it gets available. * <p> diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java index ee907f2ee6..db2b5b4f71 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Robin Rosenberg and others + * Copyright (C) 2010, 2024, Robin Rosenberg and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -203,7 +203,16 @@ public class FS_POSIX extends FS { /** {@inheritDoc} */ @Override public boolean canExecute(File f) { - return FileUtils.canExecute(f); + if (!isFile(f)) { + return false; + } + try { + Path path = FileUtils.toPath(f); + Set<PosixFilePermission> pset = Files.getPosixFilePermissions(path); + return pset.contains(PosixFilePermission.OWNER_EXECUTE); + } catch (IOException ex) { + return false; + } } /** {@inheritDoc} */ @@ -332,73 +341,6 @@ public class FS_POSIX extends FS { return supportsAtomicFileCreation == AtomicFileCreation.SUPPORTED; } - @Override - @SuppressWarnings("boxing") - /** - * {@inheritDoc} - * <p> - * An implementation of the File#createNewFile() semantics which works also - * on NFS. If the config option - * {@code core.supportsAtomicCreateNewFile = true} (which is the default) - * then simply File#createNewFile() is called. - * - * But if {@code core.supportsAtomicCreateNewFile = false} then after - * successful creation of the lock file a hard link to that lock file is - * created and the attribute nlink of the lock file is checked to be 2. If - * multiple clients manage to create the same lock file nlink would be - * greater than 2 showing the error. - * - * @see "https://www.time-travellers.org/shane/papers/NFS_considered_harmful.html" - * - * @deprecated use {@link FS_POSIX#createNewFileAtomic(File)} instead - * @since 4.5 - */ - @Deprecated - public boolean createNewFile(File lock) throws IOException { - if (!lock.createNewFile()) { - return false; - } - if (supportsAtomicCreateNewFile()) { - return true; - } - Path lockPath = lock.toPath(); - Path link = null; - FileStore store = null; - try { - store = Files.getFileStore(lockPath); - } catch (SecurityException e) { - return true; - } - try { - Boolean canLink = CAN_HARD_LINK.computeIfAbsent(store, - s -> Boolean.TRUE); - if (Boolean.FALSE.equals(canLink)) { - return true; - } - link = Files.createLink( - Paths.get(lock.getAbsolutePath() + ".lnk"), //$NON-NLS-1$ - lockPath); - Integer nlink = (Integer) Files.getAttribute(lockPath, - "unix:nlink"); //$NON-NLS-1$ - if (nlink > 2) { - LOG.warn(MessageFormat.format( - JGitText.get().failedAtomicFileCreation, lockPath, - nlink)); - return false; - } else if (nlink < 2) { - CAN_HARD_LINK.put(store, Boolean.FALSE); - } - return true; - } catch (UnsupportedOperationException | IllegalArgumentException e) { - CAN_HARD_LINK.put(store, Boolean.FALSE); - return true; - } finally { - if (link != null) { - Files.delete(link); - } - } - } - /** * {@inheritDoc} * <p> diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java index 635351ac84..237879110a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java @@ -14,8 +14,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.OutputStream; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -43,10 +41,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { * @return true if cygwin is found */ public static boolean isCygwin() { - final String path = AccessController - .doPrivileged((PrivilegedAction<String>) () -> System - .getProperty("java.library.path") //$NON-NLS-1$ - ); + final String path = System.getProperty("java.library.path"); //$NON-NLS-1$ if (path == null) return false; File found = FS.searchPath(path, "cygpath.exe"); //$NON-NLS-1$ @@ -99,9 +94,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { @Override protected File userHomeImpl() { - final String home = AccessController.doPrivileged( - (PrivilegedAction<String>) () -> System.getenv("HOME") //$NON-NLS-1$ - ); + final String home = System.getenv("HOME"); //$NON-NLS-1$ if (home == null || home.length() == 0) return super.userHomeImpl(); return resolve(new File("."), home); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index cab0e6a0a9..39c67f1b86 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -771,24 +771,6 @@ public class FileUtils { } /** - * Get the lastModified attribute for a given file - * - * @param file - * the file - * @return lastModified attribute for given file, not following symbolic - * links - * @throws IOException - * if an IO error occurred - * @deprecated use {@link #lastModifiedInstant(Path)} instead which returns - * FileTime - */ - @Deprecated - static long lastModified(File file) throws IOException { - return Files.getLastModifiedTime(toPath(file), LinkOption.NOFOLLOW_LINKS) - .toMillis(); - } - - /** * Get last modified timestamp of a file * * @param path @@ -830,21 +812,6 @@ public class FileUtils { /** * Set the last modified time of a file system object. * - * @param file - * the file - * @param time - * last modified timestamp - * @throws IOException - * if an IO error occurred - */ - @Deprecated - static void setLastModified(File file, long time) throws IOException { - Files.setLastModifiedTime(toPath(file), FileTime.fromMillis(time)); - } - - /** - * Set the last modified time of a file system object. - * * @param path * file path * @param time diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java index e6bf497ac4..332e65985e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java @@ -10,10 +10,10 @@ package org.eclipse.jgit.util; -import java.text.DateFormat; -import java.text.SimpleDateFormat; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; import java.util.Locale; -import java.util.TimeZone; import org.eclipse.jgit.lib.PersonIdent; @@ -26,9 +26,9 @@ import org.eclipse.jgit.lib.PersonIdent; */ public class GitDateFormatter { - private DateFormat dateTimeInstance; + private DateTimeFormatter dateTimeFormat; - private DateFormat dateTimeInstance2; + private DateTimeFormatter dateTimeFormat2; private final Format format; @@ -96,30 +96,34 @@ public class GitDateFormatter { default: break; case DEFAULT: // Not default: - dateTimeInstance = new SimpleDateFormat( + dateTimeFormat = DateTimeFormatter.ofPattern( "EEE MMM dd HH:mm:ss yyyy Z", Locale.US); //$NON-NLS-1$ break; case ISO: - dateTimeInstance = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter.ofPattern( + "yyyy-MM-dd HH:mm:ss Z", //$NON-NLS-1$ Locale.US); break; case LOCAL: - dateTimeInstance = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy", //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter.ofPattern( + "EEE MMM dd HH:mm:ss yyyy", //$NON-NLS-1$ Locale.US); break; case RFC: - dateTimeInstance = new SimpleDateFormat( + dateTimeFormat = DateTimeFormatter.ofPattern( "EEE, dd MMM yyyy HH:mm:ss Z", Locale.US); //$NON-NLS-1$ break; case SHORT: - dateTimeInstance = new SimpleDateFormat("yyyy-MM-dd", Locale.US); //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd", //$NON-NLS-1$ + Locale.US); break; case LOCALE: case LOCALELOCAL: - SystemReader systemReader = SystemReader.getInstance(); - dateTimeInstance = systemReader.getDateTimeInstance( - DateFormat.DEFAULT, DateFormat.DEFAULT); - dateTimeInstance2 = systemReader.getSimpleDateFormat("Z"); //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter + .ofLocalizedDateTime(FormatStyle.MEDIUM) + .withLocale(Locale.US); + dateTimeFormat2 = DateTimeFormatter.ofPattern("Z", //$NON-NLS-1$ + Locale.US); break; } } @@ -135,39 +139,45 @@ public class GitDateFormatter { @SuppressWarnings("boxing") public String formatDate(PersonIdent ident) { switch (format) { - case RAW: - int offset = ident.getTimeZoneOffset(); + case RAW: { + int offset = ident.getZoneOffset().getTotalSeconds(); String sign = offset < 0 ? "-" : "+"; //$NON-NLS-1$ //$NON-NLS-2$ int offset2; - if (offset < 0) + if (offset < 0) { offset2 = -offset; - else + } else { offset2 = offset; - int hours = offset2 / 60; - int minutes = offset2 % 60; + } + int minutes = (offset2 / 60) % 60; + int hours = offset2 / 60 / 60; return String.format("%d %s%02d%02d", //$NON-NLS-1$ - ident.getWhen().getTime() / 1000, sign, hours, minutes); + ident.getWhenAsInstant().getEpochSecond(), sign, hours, + minutes); + } case RELATIVE: - return RelativeDateFormatter.format(ident.getWhen()); + return RelativeDateFormatter.format(ident.getWhenAsInstant()); case LOCALELOCAL: case LOCAL: - dateTimeInstance.setTimeZone(SystemReader.getInstance() - .getTimeZone()); - return dateTimeInstance.format(ident.getWhen()); - case LOCALE: - TimeZone tz = ident.getTimeZone(); - if (tz == null) - tz = SystemReader.getInstance().getTimeZone(); - dateTimeInstance.setTimeZone(tz); - dateTimeInstance2.setTimeZone(tz); - return dateTimeInstance.format(ident.getWhen()) + " " //$NON-NLS-1$ - + dateTimeInstance2.format(ident.getWhen()); - default: - tz = ident.getTimeZone(); - if (tz == null) - tz = SystemReader.getInstance().getTimeZone(); - dateTimeInstance.setTimeZone(ident.getTimeZone()); - return dateTimeInstance.format(ident.getWhen()); + return dateTimeFormat + .withZone(SystemReader.getInstance().getTimeZoneId()) + .format(ident.getWhenAsInstant()); + case LOCALE: { + ZoneId tz = ident.getZoneId(); + if (tz == null) { + tz = SystemReader.getInstance().getTimeZoneId(); + } + return dateTimeFormat.withZone(tz).format(ident.getWhenAsInstant()) + + " " //$NON-NLS-1$ + + dateTimeFormat2.withZone(tz) + .format(ident.getWhenAsInstant()); + } + default: { + ZoneId tz = ident.getZoneId(); + if (tz == null) { + tz = SystemReader.getInstance().getTimeZoneId(); + } + return dateTimeFormat.withZone(tz).format(ident.getWhenAsInstant()); + } } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java index 6a4b39652a..f080056546 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java @@ -28,7 +28,10 @@ import org.eclipse.jgit.internal.JGitText; * used. One example is the parsing of the config parameter gc.pruneexpire. The * parser can handle only subset of what native gits approxidate parser * understands. + * + * @deprecated Use {@link GitTimeParser} instead. */ +@Deprecated(since = "7.1") public class GitDateParser { /** * The Date representing never. Though this is a concrete value, most diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java new file mode 100644 index 0000000000..acaa1ce563 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java @@ -0,0 +1,248 @@ +/* + * Copyright (C) 2024 Christian Halstrick and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import java.text.MessageFormat; +import java.text.ParseException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.EnumMap; +import java.util.Map; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.JGitText; + +/** + * Parses strings with time and date specifications into + * {@link java.time.Instant}. + * + * When git needs to parse strings specified by the user this parser can be + * used. One example is the parsing of the config parameter gc.pruneexpire. The + * parser can handle only subset of what native gits approxidate parser + * understands. + * + * @since 7.1 + */ +public class GitTimeParser { + + private static final Map<ParseableSimpleDateFormat, DateTimeFormatter> formatCache = new EnumMap<>( + ParseableSimpleDateFormat.class); + + // An enum of all those formats which this parser can parse with the help of + // a DateTimeFormatter. There are other formats (e.g. the relative formats + // like "yesterday" or "1 week ago") which this parser can parse but which + // are not listed here because they are parsed without the help of a + // DateTimeFormatter. + enum ParseableSimpleDateFormat { + ISO("yyyy-MM-dd HH:mm:ss Z"), // //$NON-NLS-1$ + RFC("EEE, dd MMM yyyy HH:mm:ss Z"), // //$NON-NLS-1$ + SHORT("yyyy-MM-dd"), // //$NON-NLS-1$ + SHORT_WITH_DOTS_REVERSE("dd.MM.yyyy"), // //$NON-NLS-1$ + SHORT_WITH_DOTS("yyyy.MM.dd"), // //$NON-NLS-1$ + SHORT_WITH_SLASH("MM/dd/yyyy"), // //$NON-NLS-1$ + DEFAULT("EEE MMM dd HH:mm:ss yyyy Z"), // //$NON-NLS-1$ + LOCAL("EEE MMM dd HH:mm:ss yyyy"); //$NON-NLS-1$ + + private final String formatStr; + + ParseableSimpleDateFormat(String formatStr) { + this.formatStr = formatStr; + } + } + + private GitTimeParser() { + // This class is not supposed to be instantiated + } + + /** + * Parses a string into a {@link java.time.LocalDateTime} using the default + * locale. Since this parser also supports relative formats (e.g. + * "yesterday") the caller can specify the reference date. These types of + * strings can be parsed: + * <ul> + * <li>"never"</li> + * <li>"now"</li> + * <li>"yesterday"</li> + * <li>"(x) years|months|weeks|days|hours|minutes|seconds ago"<br> + * Multiple specs can be combined like in "2 weeks 3 days ago". Instead of ' + * ' one can use '.' to separate the words</li> + * <li>"yyyy-MM-dd HH:mm:ss Z" (ISO)</li> + * <li>"EEE, dd MMM yyyy HH:mm:ss Z" (RFC)</li> + * <li>"yyyy-MM-dd"</li> + * <li>"yyyy.MM.dd"</li> + * <li>"MM/dd/yyyy",</li> + * <li>"dd.MM.yyyy"</li> + * <li>"EEE MMM dd HH:mm:ss yyyy Z" (DEFAULT)</li> + * <li>"EEE MMM dd HH:mm:ss yyyy" (LOCAL)</li> + * </ul> + * + * @param dateStr + * the string to be parsed + * @return the parsed {@link java.time.LocalDateTime} + * @throws java.text.ParseException + * if the given dateStr was not recognized + */ + public static LocalDateTime parse(String dateStr) throws ParseException { + return parse(dateStr, SystemReader.getInstance().civilNow()); + } + + /** + * Parses a string into a {@link java.time.Instant} using the default + * locale. Since this parser also supports relative formats (e.g. + * "yesterday") the caller can specify the reference date. These types of + * strings can be parsed: + * <ul> + * <li>"never"</li> + * <li>"now"</li> + * <li>"yesterday"</li> + * <li>"(x) years|months|weeks|days|hours|minutes|seconds ago"<br> + * Multiple specs can be combined like in "2 weeks 3 days ago". Instead of ' + * ' one can use '.' to separate the words</li> + * <li>"yyyy-MM-dd HH:mm:ss Z" (ISO)</li> + * <li>"EEE, dd MMM yyyy HH:mm:ss Z" (RFC)</li> + * <li>"yyyy-MM-dd"</li> + * <li>"yyyy.MM.dd"</li> + * <li>"MM/dd/yyyy",</li> + * <li>"dd.MM.yyyy"</li> + * <li>"EEE MMM dd HH:mm:ss yyyy Z" (DEFAULT)</li> + * <li>"EEE MMM dd HH:mm:ss yyyy" (LOCAL)</li> + * </ul> + * + * @param dateStr + * the string to be parsed + * @return the parsed {@link java.time.Instant} + * @throws java.text.ParseException + * if the given dateStr was not recognized + * @since 7.2 + */ + public static Instant parseInstant(String dateStr) throws ParseException { + return parse(dateStr).atZone(SystemReader.getInstance().getTimeZoneId()) + .toInstant(); + } + + // Only tests seem to use this method + static LocalDateTime parse(String dateStr, LocalDateTime now) + throws ParseException { + dateStr = dateStr.trim(); + + if (dateStr.equalsIgnoreCase("never")) { //$NON-NLS-1$ + return LocalDateTime.MAX; + } + LocalDateTime ret = parseRelative(dateStr, now); + if (ret != null) { + return ret; + } + for (ParseableSimpleDateFormat f : ParseableSimpleDateFormat.values()) { + try { + return parseSimple(dateStr, f); + } catch (DateTimeParseException e) { + // simply proceed with the next parser + } + } + ParseableSimpleDateFormat[] values = ParseableSimpleDateFormat.values(); + StringBuilder allFormats = new StringBuilder("\"") //$NON-NLS-1$ + .append(values[0].formatStr); + for (int i = 1; i < values.length; i++) { + allFormats.append("\", \"").append(values[i].formatStr); //$NON-NLS-1$ + } + allFormats.append("\""); //$NON-NLS-1$ + throw new ParseException( + MessageFormat.format(JGitText.get().cannotParseDate, dateStr, + allFormats.toString()), + 0); + } + + // tries to parse a string with the formats supported by DateTimeFormatter + private static LocalDateTime parseSimple(String dateStr, + ParseableSimpleDateFormat f) throws DateTimeParseException { + DateTimeFormatter dateFormat = formatCache.computeIfAbsent(f, + format -> DateTimeFormatter + .ofPattern(f.formatStr) + .withLocale(SystemReader.getInstance().getLocale())); + TemporalAccessor parsed = dateFormat.parse(dateStr); + return parsed.isSupported(ChronoField.HOUR_OF_DAY) + ? LocalDateTime.from(parsed) + : LocalDate.from(parsed).atStartOfDay(); + } + + // tries to parse a string with a relative time specification + @SuppressWarnings("nls") + @Nullable + private static LocalDateTime parseRelative(String dateStr, + LocalDateTime now) { + // check for the static words "yesterday" or "now" + if (dateStr.equals("now")) { + return now; + } + + if (dateStr.equals("yesterday")) { + return now.minusDays(1); + } + + // parse constructs like "3 days ago", "5.week.2.day.ago" + String[] parts = dateStr.split("\\.| ", -1); + int partsLength = parts.length; + // check we have an odd number of parts (at least 3) and that the last + // part is "ago" + if (partsLength < 3 || (partsLength & 1) == 0 + || !parts[parts.length - 1].equals("ago")) { + return null; + } + int number; + for (int i = 0; i < parts.length - 2; i += 2) { + try { + number = Integer.parseInt(parts[i]); + } catch (NumberFormatException e) { + return null; + } + if (parts[i + 1] == null) { + return null; + } + switch (parts[i + 1]) { + case "year": + case "years": + now = now.minusYears(number); + break; + case "month": + case "months": + now = now.minusMonths(number); + break; + case "week": + case "weeks": + now = now.minusWeeks(number); + break; + case "day": + case "days": + now = now.minusDays(number); + break; + case "hour": + case "hours": + now = now.minusHours(number); + break; + case "minute": + case "minutes": + now = now.minusMinutes(number); + break; + case "second": + case "seconds": + now = now.minusSeconds(number); + break; + default: + return null; + } + } + return now; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java index 46d0bc85ff..3ed72516c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java @@ -13,6 +13,8 @@ package org.eclipse.jgit.util; import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.Instant.EPOCH; +import static java.time.ZoneOffset.UTC; import static org.eclipse.jgit.lib.ObjectChecker.author; import static org.eclipse.jgit.lib.ObjectChecker.committer; import static org.eclipse.jgit.lib.ObjectChecker.encoding; @@ -30,6 +32,10 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.CodingErrorAction; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -44,14 +50,6 @@ import org.eclipse.jgit.lib.PersonIdent; * Handy utility functions to parse raw object contents. */ public final class RawParseUtils { - /** - * UTF-8 charset constant. - * - * @since 2.2 - * @deprecated use {@link java.nio.charset.StandardCharsets#UTF_8} instead - */ - @Deprecated - public static final Charset UTF8_CHARSET = UTF_8; private static final byte[] digits10; @@ -467,6 +465,29 @@ public final class RawParseUtils { } /** + * Parse a Git style timezone string in [+-]hhmm format + * + * @param b + * buffer to scan. + * @param ptr + * position within buffer to start parsing digits at. + * @param ptrResult + * optional location to return the new ptr value through. If null + * the ptr value will be discarded. + * @return the ZoneOffset represention of the timezone offset string. + * Invalid offsets default to UTC. + */ + private static ZoneId parseZoneOffset(final byte[] b, int ptr, + MutableInteger ptrResult) { + int hhmm = parseBase10(b, ptr, ptrResult); + try { + return ZoneOffset.ofHoursMinutes(hhmm / 100, hhmm % 100); + } catch (DateTimeException e) { + return UTC; + } + } + + /** * Locate the first position after a given character. * * @param b @@ -1035,17 +1056,19 @@ public final class RawParseUtils { // character if there is no trailing LF. final int tzBegin = lastIndexOfTrim(raw, ' ', nextLF(raw, emailE - 1) - 2) + 1; - if (tzBegin <= emailE) // No time/zone, still valid - return new PersonIdent(name, email, 0, 0); + if (tzBegin <= emailE) { // No time/zone, still valid + return new PersonIdent(name, email, EPOCH, UTC); + } final int whenBegin = Math.max(emailE, lastIndexOfTrim(raw, ' ', tzBegin - 1) + 1); - if (whenBegin >= tzBegin - 1) // No time/zone, still valid - return new PersonIdent(name, email, 0, 0); + if (whenBegin >= tzBegin - 1) { // No time/zone, still valid + return new PersonIdent(name, email, EPOCH, UTC); + } - final long when = parseLongBase10(raw, whenBegin, null); - final int tz = parseTimeZoneOffset(raw, tzBegin); - return new PersonIdent(name, email, when * 1000L, tz); + long when = parseLongBase10(raw, whenBegin, null); + return new PersonIdent(name, email, Instant.ofEpochSecond(when), + parseZoneOffset(raw, tzBegin, null)); } /** @@ -1083,16 +1106,16 @@ public final class RawParseUtils { name = decode(raw, nameB, stop); final MutableInteger ptrout = new MutableInteger(); - long when; - int tz; + Instant when; + ZoneId tz; if (emailE < stop) { - when = parseLongBase10(raw, emailE + 1, ptrout); - tz = parseTimeZoneOffset(raw, ptrout.value); + when = Instant.ofEpochSecond(parseLongBase10(raw, emailE + 1, ptrout)); + tz = parseZoneOffset(raw, ptrout.value, null); } else { - when = 0; - tz = 0; + when = EPOCH; + tz = UTC; } - return new PersonIdent(name, email, when * 1000L, tz); + return new PersonIdent(name, email, when, tz); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java index 5611b1e78e..b6b19e0e7b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.util; import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; import java.util.Date; import org.eclipse.jgit.internal.JGitText; @@ -42,12 +44,29 @@ public class RelativeDateFormatter { * @return age of given {@link java.util.Date} compared to now formatted in * the same relative format as returned by * {@code git log --relative-date} + * @deprecated Use {@link #format(Instant)} instead. */ + @Deprecated(since = "7.2") @SuppressWarnings("boxing") public static String format(Date when) { + return format(when.toInstant()); + } - long ageMillis = SystemReader.getInstance().getCurrentTime() - - when.getTime(); + /** + * Get age of given {@link java.time.Instant} compared to now formatted in the + * same relative format as returned by {@code git log --relative-date} + * + * @param when + * an instant to format + * @return age of given instant compared to now formatted in + * the same relative format as returned by + * {@code git log --relative-date} + * @since 7.2 + */ + @SuppressWarnings("boxing") + public static String format(Instant when) { + long ageMillis = Duration + .between(when, SystemReader.getInstance().now()).toMillis(); // shouldn't happen in a perfect world if (ageMillis < 0) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java index cf06172c17..e3e3e04fd9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java @@ -13,8 +13,8 @@ import java.text.MessageFormat; import java.util.Locale; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; -import org.eclipse.jgit.lib.GpgSignatureVerifier.TrustLevel; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.SignatureVerifier.TrustLevel; import org.eclipse.jgit.lib.PersonIdent; /** @@ -39,29 +39,34 @@ public final class SignatureUtils { * to use for dates * @return a textual representation of the {@link SignatureVerification}, * using LF as line separator + * + * @since 7.0 */ public static String toString(SignatureVerification verification, PersonIdent creator, GitDateFormatter formatter) { StringBuilder result = new StringBuilder(); - // Use the creator's timezone for the signature date - PersonIdent dateId = new PersonIdent(creator, - verification.getCreationDate()); - result.append(MessageFormat.format(JGitText.get().verifySignatureMade, - formatter.formatDate(dateId))); - result.append('\n'); + if (verification.creationDate() != null) { + // Use the creator's timezone for the signature date + PersonIdent dateId = new PersonIdent(creator, + verification.creationDate().toInstant()); + result.append( + MessageFormat.format(JGitText.get().verifySignatureMade, + formatter.formatDate(dateId))); + result.append('\n'); + } result.append(MessageFormat.format( JGitText.get().verifySignatureKey, - verification.getKeyFingerprint().toUpperCase(Locale.ROOT))); + verification.keyFingerprint().toUpperCase(Locale.ROOT))); result.append('\n'); - if (!StringUtils.isEmptyOrNull(verification.getSigner())) { + if (!StringUtils.isEmptyOrNull(verification.signer())) { result.append( MessageFormat.format(JGitText.get().verifySignatureIssuer, - verification.getSigner())); + verification.signer())); result.append('\n'); } String msg; - if (verification.getVerified()) { - if (verification.isExpired()) { + if (verification.verified()) { + if (verification.expired()) { msg = JGitText.get().verifySignatureExpired; } else { msg = JGitText.get().verifySignatureGood; @@ -69,14 +74,14 @@ public final class SignatureUtils { } else { msg = JGitText.get().verifySignatureBad; } - result.append(MessageFormat.format(msg, verification.getKeyUser())); - if (!TrustLevel.UNKNOWN.equals(verification.getTrustLevel())) { + result.append(MessageFormat.format(msg, verification.keyUser())); + if (!TrustLevel.UNKNOWN.equals(verification.trustLevel())) { result.append(' ' + MessageFormat .format(JGitText.get().verifySignatureTrust, verification - .getTrustLevel().name().toLowerCase(Locale.ROOT))); + .trustLevel().name().toLowerCase(Locale.ROOT))); } result.append('\n'); - msg = verification.getMessage(); + msg = verification.message(); if (!StringUtils.isEmptyOrNull(msg)) { result.append(msg); result.append('\n'); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java index d957deb34c..efa6e7ddc3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java @@ -43,14 +43,18 @@ public class Stats { } /** - * @return number of the added values + * Returns the number of added values + * + * @return the number of added values */ public int count() { return n; } /** - * @return minimum of the added values + * Returns the smallest value added + * + * @return the smallest value added */ public double min() { if (n < 1) { @@ -60,7 +64,9 @@ public class Stats { } /** - * @return maximum of the added values + * Returns the biggest value added + * + * @return the biggest value added */ public double max() { if (n < 1) { @@ -70,9 +76,10 @@ public class Stats { } /** - * @return average of the added values + * Returns the average of the added values + * + * @return the average of the added values */ - public double avg() { if (n < 1) { return Double.NaN; @@ -81,7 +88,9 @@ public class Stats { } /** - * @return variance of the added values + * Returns the variance of the added values + * + * @return the variance of the added values */ public double var() { if (n < 2) { @@ -91,7 +100,9 @@ public class Stats { } /** - * @return standard deviation of the added values + * Returns the standard deviation of the added values + * + * @return the standard deviation of the added values */ public double stddev() { return Math.sqrt(this.var()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java index 2fbd12dcc5..e381a3bcc9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java @@ -278,6 +278,44 @@ public final class StringUtils { } /** + * Remove the specified character from beginning and end of a string + * <p> + * If the character repeats, all copies + * + * @param str input string + * @param c character to remove + * @return the input string with c + * @since 7.2 + */ + public static String trim(String str, char c) { + if (str == null || str.length() == 0) { + return str; + } + + int endPos = str.length()-1; + while (endPos >= 0 && str.charAt(endPos) == c) { + endPos--; + } + + // Whole string is c + if (endPos == -1) { + return EMPTY; + } + + int startPos = 0; + while (startPos < endPos && str.charAt(startPos) == c) { + startPos++; + } + + if (startPos == 0 && endPos == str.length()-1) { + // No need to copy + return str; + } + + return str.substring(startPos, endPos+1); + } + + /** * Appends {@link Constants#DOT_GIT_EXT} unless the given name already ends * with that suffix. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index ed62c71371..22b82b3610 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -23,10 +23,12 @@ import java.nio.charset.UnsupportedCharsetException; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Locale; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicReference; @@ -169,6 +171,11 @@ public abstract class SystemReader { } @Override + public Instant now() { + return Instant.now(); + } + + @Override public int getTimezone(long when) { return getTimeZone().getOffset(when) / (60 * 1000); } @@ -230,9 +237,19 @@ public abstract class SystemReader { } @Override + public Instant now() { + return delegate.now(); + } + + @Override public int getTimezone(long when) { return delegate.getTimezone(when); } + + @Override + public ZoneOffset getTimeZoneAt(Instant when) { + return delegate.getTimeZoneAt(when); + } } private static volatile SystemReader INSTANCE = DEFAULT; @@ -503,10 +520,37 @@ public abstract class SystemReader { * Get the current system time * * @return the current system time + * + * @deprecated Use {@link #now()} */ + @Deprecated(since = "7.1") public abstract long getCurrentTime(); /** + * Get the current system time + * + * @return the current system time + * + * @since 7.1 + */ + public Instant now() { + // Subclasses overriding getCurrentTime should keep working + // TODO(ifrade): Once we remove getCurrentTime, use Instant.now() + return Instant.ofEpochMilli(getCurrentTime()); + } + + /** + * Get "now" as civil time, in the System timezone + * + * @return the current system time + * + * @since 7.1 + */ + public LocalDateTime civilNow() { + return LocalDateTime.ofInstant(now(), getTimeZoneId()); + } + + /** * Get clock instance preferred by this system. * * @return clock instance preferred by this system. @@ -522,20 +566,48 @@ public abstract class SystemReader { * @param when * a system timestamp * @return the local time zone + * + * @deprecated Use {@link #getTimeZoneAt(Instant)} instead. */ + @Deprecated(since = "7.1") public abstract int getTimezone(long when); /** + * Get the local time zone offset at "when" time + * + * @param when + * a system timestamp + * @return the local time zone + * @since 7.1 + */ + public ZoneOffset getTimeZoneAt(Instant when) { + return getTimeZoneId().getRules().getOffset(when); + } + + /** * Get system time zone, possibly mocked for testing * * @return system time zone, possibly mocked for testing * @since 1.2 + * + * @deprecated Use {@link #getTimeZoneId()} */ + @Deprecated(since = "7.1") public TimeZone getTimeZone() { return TimeZone.getDefault(); } /** + * Get system time zone, possibly mocked for testing + * + * @return system time zone, possibly mocked for testing + * @since 7.1 + */ + public ZoneId getTimeZoneId() { + return ZoneId.systemDefault(); + } + + /** * Get the locale to use * * @return the locale to use @@ -670,9 +742,7 @@ public abstract class SystemReader { } private String getOsName() { - return AccessController.doPrivileged( - (PrivilegedAction<String>) () -> getProperty("os.name") //$NON-NLS-1$ - ); + return getProperty("os.name"); //$NON-NLS-1$ } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java index 2385865674..4b9706a3ce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java @@ -147,44 +147,6 @@ public class AutoLFInputStream extends InputStream { && flags.contains(StreamFlag.FOR_CHECKOUT); } - /** - * Creates a new InputStream, wrapping the specified stream. - * - * @param in - * raw input stream - * @param detectBinary - * whether binaries should be detected - * @since 2.0 - * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} - * instead - */ - @Deprecated - public AutoLFInputStream(InputStream in, boolean detectBinary) { - this(in, detectBinary, false); - } - - /** - * Creates a new InputStream, wrapping the specified stream. - * - * @param in - * raw input stream - * @param detectBinary - * whether binaries should be detected - * @param abortIfBinary - * throw an IOException if the file is binary - * @since 3.3 - * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} - * instead - */ - @Deprecated - public AutoLFInputStream(InputStream in, boolean detectBinary, - boolean abortIfBinary) { - this.in = in; - this.detectBinary = detectBinary; - this.abortIfBinary = abortIfBinary; - this.forCheckout = false; - } - @Override public int read() throws IOException { final int read = read(single, 0, 1); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java index 4764676c88..13982b133c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java @@ -11,8 +11,6 @@ package org.eclipse.jgit.util.io; import java.io.IOException; import java.io.Writer; -import java.security.AccessController; -import java.security.PrivilegedAction; import org.eclipse.jgit.util.SystemReader; @@ -35,10 +33,7 @@ public class ThrowingPrintWriter extends Writer { */ public ThrowingPrintWriter(Writer out) { this.out = out; - LF = AccessController - .doPrivileged((PrivilegedAction<String>) () -> SystemReader - .getInstance().getProperty("line.separator") //$NON-NLS-1$ - ); + LF = SystemReader.getInstance().getProperty("line.separator"); //$NON-NLS-1$ } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java index c3a1c4e3bb..7e950f6529 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java @@ -14,7 +14,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayDeque; import java.util.Deque; -import java.util.Iterator; /** * An InputStream which reads from one or more InputStreams. @@ -164,14 +163,14 @@ public class UnionInputStream extends InputStream { public void close() throws IOException { IOException err = null; - for (Iterator<InputStream> i = streams.iterator(); i.hasNext();) { + for (InputStream stream : streams) { try { - i.next().close(); + stream.close(); } catch (IOException closeError) { err = closeError; } - i.remove(); } + streams.clear(); if (err != null) throw err; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java index a5ee1070d0..a20eaaf908 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java @@ -138,7 +138,10 @@ public abstract class ProposedTimestamp implements AutoCloseable { * Get time since epoch, with up to microsecond resolution. * * @return time since epoch, with up to microsecond resolution. + * + * @deprecated Use instant() instead */ + @Deprecated(since = "7.2") public Timestamp timestamp() { return Timestamp.from(instant()); } @@ -147,7 +150,10 @@ public abstract class ProposedTimestamp implements AutoCloseable { * Get time since epoch, with up to millisecond resolution. * * @return time since epoch, with up to millisecond resolution. + * + * @deprecated Use instant() instead */ + @Deprecated(since = "7.2") public Date date() { return new Date(millis()); } |