diff options
Diffstat (limited to 'org.eclipse.jgit/src/org/eclipse')
68 files changed, 1883 insertions, 2728 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java index 680f2babcc..e228e8276a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ApplyCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2012, IBM Corporation and others. and others + * Copyright (C) 2011, 2020 IBM Corporation and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -9,17 +9,15 @@ */ package org.eclipse.jgit.api; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStreamWriter; import java.io.Writer; +import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import org.eclipse.jgit.api.errors.GitAPIException; @@ -34,7 +32,6 @@ import org.eclipse.jgit.patch.FileHeader; import org.eclipse.jgit.patch.HunkHeader; import org.eclipse.jgit.patch.Patch; import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.IO; /** * Apply a patch to files and/or to the index. @@ -114,24 +111,21 @@ public class ApplyCommand extends GitCommand<ApplyResult> { f = getFile(fh.getOldPath(), false); File dest = getFile(fh.getNewPath(), false); try { + FileUtils.mkdirs(dest.getParentFile(), true); FileUtils.rename(f, dest, StandardCopyOption.ATOMIC_MOVE); } catch (IOException e) { throw new PatchApplyException(MessageFormat.format( JGitText.get().renameFileFailed, f, dest), e); } + apply(dest, fh); break; case COPY: f = getFile(fh.getOldPath(), false); - byte[] bs = IO.readFully(f); - FileOutputStream fos = new FileOutputStream(getFile( - fh.getNewPath(), - true)); - try { - fos.write(bs); - } finally { - fos.close(); - } + File target = getFile(fh.getNewPath(), false); + FileUtils.mkdirs(target.getParentFile(), true); + Files.copy(f.toPath(), target.toPath()); + apply(target, fh); } r.addUpdatedFile(f); } @@ -171,71 +165,156 @@ public class ApplyCommand extends GitCommand<ApplyResult> { for (int i = 0; i < rt.size(); i++) oldLines.add(rt.getString(i)); List<String> newLines = new ArrayList<>(oldLines); + int afterLastHunk = 0; + int lineNumberShift = 0; + int lastHunkNewLine = -1; for (HunkHeader hh : fh.getHunks()) { + // We assume hunks to be ordered + if (hh.getNewStartLine() <= lastHunkNewLine) { + throw new PatchApplyException(MessageFormat + .format(JGitText.get().patchApplyException, hh)); + } + lastHunkNewLine = hh.getNewStartLine(); + byte[] b = new byte[hh.getEndOffset() - hh.getStartOffset()]; System.arraycopy(hh.getBuffer(), hh.getStartOffset(), b, 0, b.length); RawText hrt = new RawText(b); List<String> hunkLines = new ArrayList<>(hrt.size()); - for (int i = 0; i < hrt.size(); i++) + for (int i = 0; i < hrt.size(); i++) { hunkLines.add(hrt.getString(i)); - int pos = 0; - for (int j = 1; j < hunkLines.size(); j++) { + } + + if (hh.getNewStartLine() == 0) { + // Must be the single hunk for clearing all content + if (fh.getHunks().size() == 1 + && canApplyAt(hunkLines, newLines, 0)) { + newLines.clear(); + break; + } + throw new PatchApplyException(MessageFormat + .format(JGitText.get().patchApplyException, hh)); + } + // Hunk lines as reported by the hunk may be off, so don't rely on + // them. + int applyAt = hh.getNewStartLine() - 1 + lineNumberShift; + // But they definitely should not go backwards. + if (applyAt < afterLastHunk && lineNumberShift < 0) { + applyAt = hh.getNewStartLine() - 1; + lineNumberShift = 0; + } + if (applyAt < afterLastHunk) { + throw new PatchApplyException(MessageFormat + .format(JGitText.get().patchApplyException, hh)); + } + boolean applies = false; + int oldLinesInHunk = hh.getLinesContext() + + hh.getOldImage().getLinesDeleted(); + if (oldLinesInHunk <= 1) { + // Don't shift hunks without context lines. Just try the + // position corrected by the current lineNumberShift, and if + // that fails, the position recorded in the hunk header. + applies = canApplyAt(hunkLines, newLines, applyAt); + if (!applies && lineNumberShift != 0) { + applyAt = hh.getNewStartLine() - 1; + applies = applyAt >= afterLastHunk + && canApplyAt(hunkLines, newLines, applyAt); + } + } else { + int maxShift = applyAt - afterLastHunk; + for (int shift = 0; shift <= maxShift; shift++) { + if (canApplyAt(hunkLines, newLines, applyAt - shift)) { + applies = true; + applyAt -= shift; + break; + } + } + if (!applies) { + // Try shifting the hunk downwards + applyAt = hh.getNewStartLine() - 1 + lineNumberShift; + maxShift = newLines.size() - applyAt - oldLinesInHunk; + for (int shift = 1; shift <= maxShift; shift++) { + if (canApplyAt(hunkLines, newLines, applyAt + shift)) { + applies = true; + applyAt += shift; + break; + } + } + } + } + if (!applies) { + throw new PatchApplyException(MessageFormat + .format(JGitText.get().patchApplyException, hh)); + } + // Hunk applies at applyAt. Apply it, and update afterLastHunk and + // lineNumberShift + lineNumberShift = applyAt - hh.getNewStartLine() + 1; + int sz = hunkLines.size(); + for (int j = 1; j < sz; j++) { String hunkLine = hunkLines.get(j); switch (hunkLine.charAt(0)) { case ' ': - if (!newLines.get(hh.getNewStartLine() - 1 + pos).equals( - hunkLine.substring(1))) { - throw new PatchApplyException(MessageFormat.format( - JGitText.get().patchApplyException, hh)); - } - pos++; + applyAt++; break; case '-': - if (hh.getNewStartLine() == 0) { - newLines.clear(); - } else { - if (!newLines.get(hh.getNewStartLine() - 1 + pos) - .equals(hunkLine.substring(1))) { - throw new PatchApplyException(MessageFormat.format( - JGitText.get().patchApplyException, hh)); - } - newLines.remove(hh.getNewStartLine() - 1 + pos); - } + newLines.remove(applyAt); break; case '+': - newLines.add(hh.getNewStartLine() - 1 + pos, - hunkLine.substring(1)); - pos++; + newLines.add(applyAt++, hunkLine.substring(1)); + break; + default: break; } } + afterLastHunk = applyAt; } - if (!isNoNewlineAtEndOfFile(fh)) + if (!isNoNewlineAtEndOfFile(fh)) { newLines.add(""); //$NON-NLS-1$ - if (!rt.isMissingNewlineAtEnd()) + } + if (!rt.isMissingNewlineAtEnd()) { oldLines.add(""); //$NON-NLS-1$ - if (!isChanged(oldLines, newLines)) - return; // don't touch the file - StringBuilder sb = new StringBuilder(); - for (String l : newLines) { - // don't bother handling line endings - if it was windows, the \r is - // still there! - sb.append(l).append('\n'); } - if (sb.length() > 0) { - sb.deleteCharAt(sb.length() - 1); + if (!isChanged(oldLines, newLines)) { + return; // Don't touch the file } - try (Writer fw = new OutputStreamWriter(new FileOutputStream(f), - UTF_8)) { - fw.write(sb.toString()); + try (Writer fw = Files.newBufferedWriter(f.toPath())) { + for (Iterator<String> l = newLines.iterator(); l.hasNext();) { + fw.write(l.next()); + if (l.hasNext()) { + // Don't bother handling line endings - if it was Windows, + // the \r is still there! + fw.write('\n'); + } + } } - getRepository().getFS().setExecute(f, fh.getNewMode() == FileMode.EXECUTABLE_FILE); } + private boolean canApplyAt(List<String> hunkLines, List<String> newLines, + int line) { + int sz = hunkLines.size(); + int limit = newLines.size(); + int pos = line; + for (int j = 1; j < sz; j++) { + String hunkLine = hunkLines.get(j); + switch (hunkLine.charAt(0)) { + case ' ': + case '-': + if (pos >= limit + || !newLines.get(pos).equals(hunkLine.substring(1))) { + return false; + } + pos++; + break; + default: + break; + } + } + return true; + } + private static boolean isChanged(List<String> ol, List<String> nl) { if (ol.size() != nl.size()) return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java index c5bc8587ef..5d0154c6dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java @@ -142,12 +142,14 @@ public class CherryPickCommand extends GitCommand<CherryPickResult> { dco.setProgressMonitor(monitor); dco.checkout(); if (!noCommit) { - newHead = new Git(getRepository()).commit() - .setMessage(srcCommit.getFullMessage()) - .setReflogComment(reflogPrefix + " " //$NON-NLS-1$ - + srcCommit.getShortMessage()) - .setAuthor(srcCommit.getAuthorIdent()) - .setNoVerify(true).call(); + try (Git git = new Git(getRepository())) { + newHead = git.commit() + .setMessage(srcCommit.getFullMessage()) + .setReflogComment(reflogPrefix + " " //$NON-NLS-1$ + + srcCommit.getShortMessage()) + .setAuthor(srcCommit.getAuthorIdent()) + .setNoVerify(true).call(); + } } cherryPickedRefs.add(src); } else { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java index 78afe18f39..30d7f9adc4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java @@ -89,6 +89,8 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { private FETCH_TYPE fetchType; + private TagOpt tagOption; + private enum FETCH_TYPE { MULTIPLE_BRANCHES, ALL_BRANCHES, MIRROR } @@ -278,6 +280,9 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { config.setFetchRefSpecs(calculateRefSpecs(fetchType, config.getName())); config.setMirror(fetchType == FETCH_TYPE.MIRROR); + if (tagOption != null) { + config.setTagOpt(tagOption); + } config.update(clonedRepo.getConfig()); clonedRepo.getConfig().save(); @@ -286,7 +291,12 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { FetchCommand command = new FetchCommand(clonedRepo); command.setRemote(remote); command.setProgressMonitor(monitor); - command.setTagOpt(fetchAll ? TagOpt.FETCH_TAGS : TagOpt.AUTO_FOLLOW); + if (tagOption != null) { + command.setTagOpt(tagOption); + } else { + command.setTagOpt( + fetchAll ? TagOpt.FETCH_TAGS : TagOpt.AUTO_FOLLOW); + } configure(command); return command.call(); @@ -664,6 +674,30 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { } /** + * Set the tag option used for the remote configuration explicitly. + * + * @param tagOption + * tag option to be used for the remote config + * @return {@code this} + * @since 5.8 + */ + public CloneCommand setTagOption(TagOpt tagOption) { + this.tagOption = tagOption; + return this; + } + + /** + * Set the --no-tags option. Tags are not cloned now and the remote + * configuration is initialized with the --no-tags option as well. + * + * @return {@code this} + * @since 5.8 + */ + public CloneCommand setNoTags() { + return setTagOption(TagOpt.NO_TAGS); + } + + /** * Set whether to skip checking out a branch * * @param noCheckout diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index 4e18b5994d..b4f7175036 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -27,6 +27,7 @@ import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.api.errors.NoFilepatternException; import org.eclipse.jgit.api.errors.NoHeadException; import org.eclipse.jgit.api.errors.NoMessageException; +import org.eclipse.jgit.api.errors.ServiceUnavailableException; import org.eclipse.jgit.api.errors.UnmergedPathsException; import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.api.errors.WrongRepositoryStateException; @@ -55,7 +56,6 @@ import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryState; -import org.eclipse.jgit.lib.internal.BouncyCastleGpgSigner; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevTag; @@ -140,12 +140,16 @@ public class CommitCommand extends GitCommand<RevCommit> { * collected by the setter methods of this class. Each instance of this * class should only be used for one invocation of the command (means: one * call to {@link #call()}) + * + * @throws ServiceUnavailableException + * if signing service is not available e.g. since it isn't + * installed */ @Override - public RevCommit call() throws GitAPIException, NoHeadException, - NoMessageException, UnmergedPathsException, - ConcurrentRefUpdateException, WrongRepositoryStateException, - AbortedByHookException { + public RevCommit call() throws GitAPIException, AbortedByHookException, + ConcurrentRefUpdateException, NoHeadException, NoMessageException, + ServiceUnavailableException, UnmergedPathsException, + WrongRepositoryStateException { checkCallable(); Collections.sort(only); @@ -239,6 +243,10 @@ public class CommitCommand extends GitCommand<RevCommit> { commit.setTreeId(indexTreeId); if (signCommit.booleanValue()) { + if (gpgSigner == null) { + throw new ServiceUnavailableException( + JGitText.get().signingServiceUnavailable); + } gpgSigner.sign(commit, signingKey, committer, credentialsProvider); } @@ -510,7 +518,8 @@ public class CommitCommand extends GitCommand<RevCommit> { * * @throws NoMessageException * if the commit message has not been specified - * @throws UnsupportedSigningFormatException if the configured gpg.format is not supported + * @throws UnsupportedSigningFormatException + * if the configured gpg.format is not supported */ private void processOptions(RepositoryState state, RevWalk rw) throws NoMessageException, UnsupportedSigningFormatException { @@ -581,9 +590,6 @@ public class CommitCommand extends GitCommand<RevCommit> { JGitText.get().onlyOpenPgpSupportedForSigning); } gpgSigner = GpgSigner.getDefault(); - if (gpgSigner == null) { - gpgSigner = new BouncyCastleGpgSigner(); - } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java index 01306f4129..64314772b7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java @@ -18,6 +18,8 @@ import java.io.IOException; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryBuilder; import org.eclipse.jgit.lib.RepositoryCache; +import org.eclipse.jgit.lib.internal.WorkQueue; +import org.eclipse.jgit.nls.NLS; import org.eclipse.jgit.util.FS; /** @@ -171,6 +173,15 @@ public class Git implements AutoCloseable { } /** + * Shutdown JGit and release resources it holds like NLS and thread pools + * @since 5.8 + */ + public static void shutdown() { + WorkQueue.getExecutor().shutdownNow(); + NLS.clear(); + } + + /** * Construct a new {@link org.eclipse.jgit.api.Git} object which can * interact with the specified git repository. * <p> diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/ServiceUnavailableException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/ServiceUnavailableException.java new file mode 100644 index 0000000000..207ded0262 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/ServiceUnavailableException.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2020, Matthias Sohn <matthias.sohn@sap.com> and + * other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api.errors; + +/** + * Exception thrown when an optional service is not available + * + * @since 5.8 + */ +public class ServiceUnavailableException extends GitAPIException { + private static final long serialVersionUID = 1L; + + /** + * Constructor for ServiceUnavailableException + * + * @param message + * error message + * @param cause + * a {@link java.lang.Throwable} + */ + public ServiceUnavailableException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructor for ServiceUnavailableException + * + * @param message + * error message + */ + public ServiceUnavailableException(String message) { + super(message); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawTextComparator.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawTextComparator.java index 508d07c200..0c41b8598b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawTextComparator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawTextComparator.java @@ -191,21 +191,15 @@ public abstract class RawTextComparator extends SequenceComparator<RawText> { be = trimTrailingWhitespace(b.content, bs, be); while (as < ae && bs < be) { - byte ac = a.content[as]; - byte bc = b.content[bs]; + byte ac = a.content[as++]; + byte bc = b.content[bs++]; - if (ac != bc) - return false; - - if (isWhitespace(ac)) + if (isWhitespace(ac) && isWhitespace(bc)) { as = trimLeadingWhitespace(a.content, as, ae); - else - as++; - - if (isWhitespace(bc)) bs = trimLeadingWhitespace(b.content, bs, be); - else - bs++; + } else if (ac != bc) { + return false; + } } return as == ae && bs == be; } @@ -215,12 +209,12 @@ public abstract class RawTextComparator extends SequenceComparator<RawText> { int hash = 5381; end = trimTrailingWhitespace(raw, ptr, end); while (ptr < end) { - byte c = raw[ptr]; - hash = ((hash << 5) + hash) + (c & 0xff); - if (isWhitespace(c)) + byte c = raw[ptr++]; + if (isWhitespace(c)) { ptr = trimLeadingWhitespace(raw, ptr, end); - else - ptr++; + c = ' '; + } + hash = ((hash << 5) + hash) + (c & 0xff); } return hash; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index e8e1984306..8c51a7ac2f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -1217,7 +1217,7 @@ public class DirCacheCheckout { if (e != null && !FileMode.TREE.equals(e.getFileMode())) builder.add(e); if (force) { - if (f.isModified(e, true, walk.getObjectReader())) { + if (f == null || f.isModified(e, true, walk.getObjectReader())) { kept.add(path); checkoutEntry(repo, e, walk.getObjectReader(), false, new CheckoutMetadata(walk.getEolStreamType(CHECKOUT_OP), diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/fnmatch/CharacterHead.java b/org.eclipse.jgit/src/org/eclipse/jgit/fnmatch/CharacterHead.java index ebffa19b1d..faf4ee66c9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/fnmatch/CharacterHead.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/fnmatch/CharacterHead.java @@ -19,7 +19,7 @@ final class CharacterHead extends AbstractHead { * @param expectedCharacter * expected {@code char} */ - protected CharacterHead(char expectedCharacter) { + CharacterHead(char expectedCharacter) { super(false); this.expectedCharacter = expectedCharacter; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index b2a6b2a31e..ef0d477af1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -82,6 +82,7 @@ public class JGitText extends TranslationBundle { /***/ public String cannotCheckoutFromUnbornBranch; /***/ public String cannotCheckoutOursSwitchBranch; /***/ public String cannotCombineSquashWithNoff; + /***/ public String cannotCombineTopoSortWithTopoKeepBranchTogetherSort; /***/ public String cannotCombineTreeFilterWithRevFilter; /***/ public String cannotCommitOnARepoWithState; /***/ public String cannotCommitWriteTo; @@ -297,6 +298,7 @@ public class JGitText extends TranslationBundle { /***/ public String exceptionHookExecutionInterrupted; /***/ public String exceptionOccurredDuringAddingOfOptionToALogCommand; /***/ public String exceptionOccurredDuringReadingOfGIT_DIR; + /***/ public String exceptionWhileFindingUserHome; /***/ public String exceptionWhileReadingPack; /***/ public String expectedACKNAKFoundEOF; /***/ public String expectedACKNAKGot; @@ -332,15 +334,6 @@ public class JGitText extends TranslationBundle { /***/ public String funnyRefname; /***/ public String gcFailed; /***/ public String gcTooManyUnpruned; - /***/ public String gpgFailedToParseSecretKey; - /***/ public String gpgNoCredentialsProvider; - /***/ public String gpgNoKeyring; - /***/ public String gpgNoKeyInLegacySecring; - /***/ public String gpgNoPublicKeyFound; - /***/ public String gpgNoSecretKeyForPublicKey; - /***/ public String gpgNotASigningKey; - /***/ public String gpgKeyInfo; - /***/ public String gpgSigningCancelled; /***/ public String headRequiredToStash; /***/ public String hoursAgo; /***/ public String httpConfigCannotNormalizeURL; @@ -384,6 +377,7 @@ public class JGitText extends TranslationBundle { /***/ public String invalidGitModules; /***/ public String invalidGitType; /***/ public String invalidHexString; + /***/ public String invalidHomeDirectory; /***/ public String invalidHooksPath; /***/ public String invalidId; /***/ public String invalidId0; @@ -648,6 +642,7 @@ public class JGitText extends TranslationBundle { /***/ public String shortReadOfOptionalDIRCExtensionExpectedAnotherBytes; /***/ public String shortSkipOfBlock; /***/ public String signingNotSupportedOnTag; + /***/ public String signingServiceUnavailable; /***/ public String similarityScoreMustBeWithinBounds; /***/ public String skipMustBeNonNegative; /***/ public String skipNotAccessiblePath; @@ -658,7 +653,6 @@ public class JGitText extends TranslationBundle { /***/ public String sourceRefNotSpecifiedForRefspec; /***/ public String squashCommitNotUpdatingHEAD; /***/ public String sshCommandFailed; - /***/ public String sshUserNameError; /***/ public String sslFailureExceptionMessage; /***/ public String sslFailureInfo; /***/ public String sslFailureCause; @@ -719,7 +713,6 @@ public class JGitText extends TranslationBundle { /***/ public String transportProtoSSH; /***/ public String transportProtoTest; /***/ public String transportProvidedRefWithNoObjectId; - /***/ public String transportSSHRetryInterrupt; /***/ public String treeEntryAlreadyExists; /***/ public String treeFilterMarkerTooManyFilters; /***/ public String treeWalkMustHaveExactlyTwoTrees; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/AddToBitmapWithCacheFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/AddToBitmapWithCacheFilter.java new file mode 100644 index 0000000000..d7ccadfbe7 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/AddToBitmapWithCacheFilter.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2020, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.revwalk; + +import org.eclipse.jgit.lib.BitmapIndex.Bitmap; +import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevFlag; + +/** + * A RevFilter that adds the visited commits to {@code bitmap} as a side effect. + * <p> + * When the walk hits a commit that is the same as {@code cachedCommit} or is + * part of {@code bitmap}'s BitmapIndex, that entire bitmap is ORed into + * {@code bitmap} and the commit and its parents are marked as SEEN so that the + * walk does not have to visit its ancestors. This ensures the walk is very + * short if there is good bitmap coverage. + */ +public class AddToBitmapWithCacheFilter extends RevFilter { + private final AnyObjectId cachedCommit; + + private final Bitmap cachedBitmap; + + private final BitmapBuilder bitmap; + + /** + * Create a filter with a cached BitmapCommit that adds visited commits to + * the given bitmap. + * + * @param cachedCommit + * the cached commit + * @param cachedBitmap + * the bitmap corresponds to {@code cachedCommit}} + * @param bitmap + * bitmap to write visited commits to + */ + public AddToBitmapWithCacheFilter(AnyObjectId cachedCommit, + Bitmap cachedBitmap, + BitmapBuilder bitmap) { + this.cachedCommit = cachedCommit; + this.cachedBitmap = cachedBitmap; + this.bitmap = bitmap; + } + + /** {@inheritDoc} */ + @Override + public final boolean include(RevWalk rw, RevCommit c) { + Bitmap visitedBitmap; + + if (bitmap.contains(c)) { + // already included + } else if ((visitedBitmap = bitmap.getBitmapIndex() + .getBitmap(c)) != null) { + bitmap.or(visitedBitmap); + } else if (cachedCommit.equals(c)) { + bitmap.or(cachedBitmap); + } else { + bitmap.addObject(c, Constants.OBJ_COMMIT); + return true; + } + + for (RevCommit p : c.getParents()) { + p.add(RevFlag.SEEN); + } + return false; + } + + /** {@inheritDoc} */ + @Override + public final RevFilter clone() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override + public final boolean requiresCommitBody() { + return false; + } +} + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java index 6aa1a0e8ea..0d3a2b4f12 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java @@ -252,6 +252,11 @@ public class BitmapIndexImpl implements BitmapIndex { return bitmapIndex; } + @Override + public EWAHCompressedBitmap retrieveCompressed() { + return build().retrieveCompressed(); + } + private EWAHCompressedBitmap ewahBitmap(Bitmap other) { if (other instanceof CompressedBitmap) { CompressedBitmap b = (CompressedBitmap) other; @@ -372,7 +377,8 @@ public class BitmapIndexImpl implements BitmapIndex { }; } - EWAHCompressedBitmap getEwahCompressedBitmap() { + @Override + public EWAHCompressedBitmap retrieveCompressed() { return bitmap; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java index 9538cc5e0a..5666b57609 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java @@ -11,17 +11,16 @@ package org.eclipse.jgit.internal.storage.file; import java.text.MessageFormat; +import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; +import java.util.LinkedList; import java.util.List; -import java.util.NoSuchElementException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap; +import org.eclipse.jgit.internal.storage.pack.BitmapCommit; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BitmapIndex.Bitmap; -import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; @@ -41,8 +40,12 @@ public class PackBitmapIndexBuilder extends BasePackBitmapIndex { private final EWAHCompressedBitmap blobs; private final EWAHCompressedBitmap tags; private final BlockList<PositionEntry> byOffset; - final BlockList<StoredBitmap> - byAddOrder = new BlockList<>(); + + private final LinkedList<StoredBitmap> + bitmapsToWriteXorBuffer = new LinkedList<>(); + + private List<StoredEntry> bitmapsToWrite = new ArrayList<>(); + final ObjectIdOwnerMap<PositionEntry> positionEntries = new ObjectIdOwnerMap<>(); @@ -134,16 +137,64 @@ public class PackBitmapIndexBuilder extends BasePackBitmapIndex { * the flags to be stored with the bitmap */ public void addBitmap(AnyObjectId objectId, Bitmap bitmap, int flags) { - if (bitmap instanceof BitmapBuilder) - bitmap = ((BitmapBuilder) bitmap).build(); + addBitmap(objectId, bitmap.retrieveCompressed(), flags); + } - EWAHCompressedBitmap compressed; - if (bitmap instanceof CompressedBitmap) - compressed = ((CompressedBitmap) bitmap).getEwahCompressedBitmap(); - else - throw new IllegalArgumentException(bitmap.getClass().toString()); + /** + * Processes a commit and prepares its bitmap to write to the bitmap index + * file. + * + * @param c + * the commit corresponds to the bitmap. + * @param bitmap + * the bitmap to be written. + * @param flags + * the flags of the commit. + */ + public void processBitmapForWrite(BitmapCommit c, Bitmap bitmap, + int flags) { + EWAHCompressedBitmap compressed = bitmap.retrieveCompressed(); + compressed.trim(); + StoredBitmap newest = new StoredBitmap(c, compressed, null, flags); + + bitmapsToWriteXorBuffer.add(newest); + if (bitmapsToWriteXorBuffer.size() > MAX_XOR_OFFSET_SEARCH) { + bitmapsToWrite.add( + generateStoredEntry(bitmapsToWriteXorBuffer.pollFirst())); + } - addBitmap(objectId, compressed, flags); + if (c.isAddToIndex()) { + // The Bitmap map in the base class is used to make revwalks + // efficient, so only add bitmaps that keep it efficient without + // bloating memory. + addBitmap(c, bitmap, flags); + } + } + + private StoredEntry generateStoredEntry(StoredBitmap bitmapToWrite) { + int bestXorOffset = 0; + EWAHCompressedBitmap bestBitmap = bitmapToWrite.getBitmap(); + + int offset = 1; + for (StoredBitmap curr : bitmapsToWriteXorBuffer) { + EWAHCompressedBitmap bitmap = curr.getBitmap() + .xor(bitmapToWrite.getBitmap()); + if (bitmap.sizeInBytes() < bestBitmap.sizeInBytes()) { + bestBitmap = bitmap; + bestXorOffset = offset; + } + offset++; + } + + PositionEntry entry = positionEntries.get(bitmapToWrite); + if (entry == null) { + throw new IllegalStateException(); + } + bestBitmap.trim(); + StoredEntry result = new StoredEntry(entry.namePosition, bestBitmap, + bestXorOffset, bitmapToWrite.getFlags()); + + return result; } /** @@ -161,7 +212,6 @@ public class PackBitmapIndexBuilder extends BasePackBitmapIndex { bitmap.trim(); StoredBitmap result = new StoredBitmap(objectId, bitmap, null, flags); getBitmaps().add(result); - byAddOrder.add(result); } /** {@inheritDoc} */ @@ -247,15 +297,18 @@ public class PackBitmapIndexBuilder extends BasePackBitmapIndex { /** {@inheritDoc} */ @Override public int getBitmapCount() { - return getBitmaps().size(); + return bitmapsToWriteXorBuffer.size() + bitmapsToWrite.size(); } /** * Remove all the bitmaps entries added. + * + * @param size + * the expected number of bitmap entries to be written. */ - public void clearBitmaps() { - byAddOrder.clear(); + public void resetBitmaps(int size) { getBitmaps().clear(); + bitmapsToWrite = new ArrayList<>(size); } /** {@inheritDoc} */ @@ -265,64 +318,18 @@ public class PackBitmapIndexBuilder extends BasePackBitmapIndex { } /** - * Get an iterator over the xor compressed entries. + * Get list of xor compressed entries that need to be written. * - * @return an iterator over the xor compressed entries. + * @return a list of the xor compressed entries. */ - public Iterable<StoredEntry> getCompressedBitmaps() { - // Add order is from oldest to newest. The reverse add order is the - // output order. - return () -> new Iterator<StoredEntry>() { - - private int index = byAddOrder.size() - 1; - - @Override - public boolean hasNext() { - return index >= 0; - } - - @Override - public StoredEntry next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - StoredBitmap item = byAddOrder.get(index); - int bestXorOffset = 0; - EWAHCompressedBitmap bestBitmap = item.getBitmap(); - - // Attempt to compress the bitmap with an XOR of the - // previously written entries. - for (int i = 1; i <= MAX_XOR_OFFSET_SEARCH; i++) { - int curr = i + index; - if (curr >= byAddOrder.size()) { - break; - } - - StoredBitmap other = byAddOrder.get(curr); - EWAHCompressedBitmap bitmap = other.getBitmap() - .xor(item.getBitmap()); - - if (bitmap.sizeInBytes() < bestBitmap.sizeInBytes()) { - bestBitmap = bitmap; - bestXorOffset = i; - } - } - index--; - - PositionEntry entry = positionEntries.get(item); - if (entry == null) { - throw new IllegalStateException(); - } - bestBitmap.trim(); - return new StoredEntry(entry.namePosition, bestBitmap, - bestXorOffset, item.getFlags()); - } + public List<StoredEntry> getCompressedBitmaps() { + while (!bitmapsToWriteXorBuffer.isEmpty()) { + bitmapsToWrite.add( + generateStoredEntry(bitmapsToWriteXorBuffer.pollFirst())); + } - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; + Collections.reverse(bitmapsToWrite); + return bitmapsToWrite; } /** Data object for the on disk representation of a bitmap entry. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java index 273eeef7e5..4b25284517 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java @@ -18,7 +18,6 @@ import org.eclipse.jgit.internal.storage.file.BasePackBitmapIndex.StoredBitmap; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BitmapIndex; import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdOwnerMap; import com.googlecode.javaewah.EWAHCompressedBitmap; import com.googlecode.javaewah.IntIterator; @@ -34,7 +33,6 @@ public class PackBitmapIndexRemapper extends PackBitmapIndex private final BasePackBitmapIndex oldPackIndex; final PackBitmapIndex newPackIndex; - private final ObjectIdOwnerMap<StoredBitmap> convertedBitmaps; private final BitSet inflated; private final int[] prevToNewMapping; @@ -65,7 +63,6 @@ public class PackBitmapIndexRemapper extends PackBitmapIndex private PackBitmapIndexRemapper(PackBitmapIndex newPackIndex) { this.oldPackIndex = null; this.newPackIndex = newPackIndex; - this.convertedBitmaps = null; this.inflated = null; this.prevToNewMapping = null; } @@ -74,7 +71,6 @@ public class PackBitmapIndexRemapper extends PackBitmapIndex BasePackBitmapIndex oldPackIndex, PackBitmapIndex newPackIndex) { this.oldPackIndex = oldPackIndex; this.newPackIndex = newPackIndex; - convertedBitmaps = new ObjectIdOwnerMap<>(); inflated = new BitSet(newPackIndex.getObjectCount()); prevToNewMapping = new int[oldPackIndex.getObjectCount()]; @@ -152,10 +148,6 @@ public class PackBitmapIndexRemapper extends PackBitmapIndex if (bitmap != null || oldPackIndex == null) return bitmap; - StoredBitmap stored = convertedBitmaps.get(objectId); - if (stored != null) - return stored.getBitmap(); - StoredBitmap oldBitmap = oldPackIndex.getBitmaps().get(objectId); if (oldBitmap == null) return null; @@ -168,8 +160,6 @@ public class PackBitmapIndexRemapper extends PackBitmapIndex inflated.set(prevToNewMapping[i.next()]); bitmap = inflated.toEWAHCompressedBitmap(); bitmap.trim(); - convertedBitmaps.add( - new StoredBitmap(objectId, bitmap, null, oldBitmap.getFlags())); return bitmap; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 852302f00c..80c8e10dec 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -470,7 +470,9 @@ public class WindowCache { mbean = new StatsRecorderImpl(); statsRecorder = mbean; - Monitoring.registerMBean(mbean, "block_cache"); //$NON-NLS-1$ + if (cfg.getExposeStatsViaJmx()) { + Monitoring.registerMBean(mbean, "block_cache"); //$NON-NLS-1$ + } if (maxFiles < 1) throw new IllegalArgumentException(JGitText.get().openFilesMustBeAtLeast1); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java new file mode 100644 index 0000000000..33c478e4ab --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BitmapCommit.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2020, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.pack; + +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; + +/** + * A commit object for which a bitmap index should be built. + */ +public final class BitmapCommit extends ObjectId { + + private final boolean reuseWalker; + + private final int flags; + + private final boolean addToIndex; + + BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags) { + super(objectId); + this.reuseWalker = reuseWalker; + this.flags = flags; + this.addToIndex = false; + } + + BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags, + boolean addToIndex) { + super(objectId); + this.reuseWalker = reuseWalker; + this.flags = flags; + this.addToIndex = addToIndex; + } + + boolean isReuseWalker() { + return reuseWalker; + } + + int getFlags() { + return flags; + } + + /** + * Whether corresponding bitmap should be added to PackBitmapIndexBuilder. + * + * @return true if the corresponding bitmap should be added to + * PackBitmapIndexBuilder. + */ + public boolean isAddToIndex() { + return addToIndex; + } + + /** + * Get a builder of BitmapCommit whose object id is {@code objId}. + * + * @param objId + * the object id of the BitmapCommit + * @return a BitmapCommit builder with object id set. + */ + public static Builder newBuilder(AnyObjectId objId) { + return new Builder().setId(objId); + } + + /** + * Get a builder of BitmapCommit whose fields are copied from + * {@code commit}. + * + * @param commit + * the bitmap commit the builder is copying from + * @return a BitmapCommit build with fields copied from an existing bitmap + * commit. + */ + public static Builder copyFrom(BitmapCommit commit) { + return new Builder().setId(commit) + .setReuseWalker(commit.isReuseWalker()) + .setFlags(commit.getFlags()) + .setAddToIndex(commit.isAddToIndex()); + } + + /** + * Builder of BitmapCommit. + */ + public static class Builder { + private AnyObjectId objectId; + + private boolean reuseWalker; + + private int flags; + + private boolean addToIndex; + + // Prevent default constructor. + private Builder() { + } + + /** + * Set objectId of the builder. + * + * @param objectId + * the object id of the BitmapCommit + * @return the builder itself + */ + public Builder setId(AnyObjectId objectId) { + this.objectId = objectId; + return this; + } + + /** + * Set reuseWalker of the builder. + * + * @param reuseWalker + * whether the BitmapCommit should reuse bitmap walker when + * walking objects + * @return the builder itself + */ + public Builder setReuseWalker(boolean reuseWalker) { + this.reuseWalker = reuseWalker; + return this; + } + + /** + * Set flags of the builder. + * + * @param flags + * the flags of the BitmapCommit + * @return the builder itself + */ + public Builder setFlags(int flags) { + this.flags = flags; + return this; + } + + /** + * Set whether whether the bitmap of the BitmapCommit should be added to + * PackBitmapIndexBuilder when building bitmap index file. + * + * @param addToIndex + * whether the bitmap of the BitmapCommit should be added to + * PackBitmapIndexBuilder when building bitmap index file + * @return the builder itself + */ + public Builder setAddToIndex(boolean addToIndex) { + this.addToIndex = addToIndex; + return this; + } + + /** + * Builds BitmapCommit from the builder. + * + * @return the new BitmapCommit. + */ + public BitmapCommit build() { + return new BitmapCommit(objectId, reuseWalker, flags, + addToIndex); + } + } +}
\ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 75dd345f46..824c62ad9a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -2313,14 +2313,14 @@ public class PackWriter implements AutoCloseable { PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer( reader, writeBitmaps, pm, stats.interestingObjects, config); - Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits = bitmapPreparer + Collection<BitmapCommit> selectedCommits = bitmapPreparer .selectCommits(numCommits, excludeFromBitmapSelection); beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size()); BitmapWalker walker = bitmapPreparer.newBitmapWalker(); AnyObjectId last = null; - for (PackWriterBitmapPreparer.BitmapCommit cmit : selectedCommits) { + for (BitmapCommit cmit : selectedCommits) { if (!cmit.isReuseWalker()) { walker = bitmapPreparer.newBitmapWalker(); } @@ -2331,8 +2331,14 @@ public class PackWriter implements AutoCloseable { throw new IllegalStateException(MessageFormat.format( JGitText.get().bitmapMissingObject, cmit.name(), last.name())); - last = cmit; - writeBitmaps.addBitmap(cmit, bitmap.build(), cmit.getFlags()); + last = BitmapCommit.copyFrom(cmit).build(); + writeBitmaps.processBitmapForWrite(cmit, bitmap.build(), + cmit.getFlags()); + + // The bitmap walker should stop when the walk hits the previous + // commit, which saves time. + walker.setPrevCommit(last); + walker.setPrevBitmap(bitmap); pm.update(1); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java index 51b4993e26..f1ede2acff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java @@ -58,6 +58,8 @@ class PackWriterBitmapPreparer { private static final int DAY_IN_SECONDS = 24 * 60 * 60; + private static final int DISTANCE_THRESHOLD = 2000; + private static final Comparator<RevCommit> ORDER_BY_REVERSE_TIMESTAMP = ( RevCommit a, RevCommit b) -> Integer .signum(b.getCommitTime() - a.getCommitTime()); @@ -244,6 +246,7 @@ class PackWriterBitmapPreparer { // This commit is selected. // Calculate where to look for the next one. int flags = nextFlg; + int currDist = distanceFromTip; nextIn = nextSpan(distanceFromTip); nextFlg = nextIn == distantCommitSpan ? PackBitmapIndex.FLAG_REUSE @@ -279,8 +282,17 @@ class PackWriterBitmapPreparer { longestAncestorChain = new ArrayList<>(); chains.add(longestAncestorChain); } - longestAncestorChain.add(new BitmapCommit(c, - !longestAncestorChain.isEmpty(), flags)); + + // The commit bc should reuse bitmap walker from its close + // ancestor. And the bitmap of bc should only be added to + // PackBitmapIndexBuilder when it's an old enough + // commit,i.e. distance from tip should be greater than + // DISTANCE_THRESHOLD, to save memory. + BitmapCommit bc = BitmapCommit.newBuilder(c).setFlags(flags) + .setAddToIndex(currDist >= DISTANCE_THRESHOLD) + .setReuseWalker(!longestAncestorChain.isEmpty()) + .build(); + longestAncestorChain.add(bc); writeBitmaps.addBitmap(c, bitmap, 0); } @@ -288,12 +300,12 @@ class PackWriterBitmapPreparer { selections.addAll(chain); } } - writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps. // Add the remaining peeledWant for (AnyObjectId remainingWant : selectionHelper.newWants) { selections.add(new BitmapCommit(remainingWant, false, 0)); } + writeBitmaps.resetBitmaps(selections.size()); // Remove the temporary commit bitmaps. pm.endTask(); return selections; @@ -468,28 +480,6 @@ class PackWriterBitmapPreparer { } /** - * A commit object for which a bitmap index should be built. - */ - static final class BitmapCommit extends ObjectId { - private final boolean reuseWalker; - private final int flags; - - BitmapCommit(AnyObjectId objectId, boolean reuseWalker, int flags) { - super(objectId); - this.reuseWalker = reuseWalker; - this.flags = flags; - } - - boolean isReuseWalker() { - return reuseWalker; - } - - int getFlags() { - return flags; - } - } - - /** * Container for state used in the first phase of selecting commits, which * walks all of the reachable commits via the branch tips that are not * covered by a previous pack's bitmaps ({@code newWants}) and stores them diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java index 4de6c28709..4747be3544 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java @@ -280,7 +280,7 @@ public abstract class ReftableDatabase { /** * Returns all refs that resolve directly to the given {@link ObjectId}. - * Includes peeled {@linkObjectId}s. + * Includes peeled {@link ObjectId}s. * * @param id * {@link ObjectId} to resolve diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java index c5560b97bc..b154b95adc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java @@ -88,8 +88,9 @@ class RefTreeBatch extends BatchRefUpdate { tree = RefTree.read(rw.getObjectReader(), c.getTree()); } else { parentCommitId = ObjectId.zeroId(); - parentTreeId = new ObjectInserter.Formatter() - .idFor(OBJ_TREE, new byte[] {}); + try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { + parentTreeId = fmt.idFor(OBJ_TREE, new byte[] {}); + } tree = RefTree.newEmptyTree(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/internal/FullConnectivityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/FullConnectivityChecker.java index 60d8f452ba..b76e3a3caa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/internal/FullConnectivityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/FullConnectivityChecker.java @@ -8,7 +8,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.transport.internal; +package org.eclipse.jgit.internal.transport.connectivity; import java.io.IOException; import java.util.Set; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/IterativeConnectivityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/IterativeConnectivityChecker.java new file mode 100644 index 0000000000..b44c4ae5cb --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/connectivity/IterativeConnectivityChecker.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2019, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.transport.connectivity; + +import static java.util.stream.Collectors.toList; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.stream.Stream; + +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.ConnectivityChecker; +import org.eclipse.jgit.transport.ReceiveCommand; + +/** + * Implementation of connectivity checker which tries to do check with smaller + * set of references first and if it fails will fall back to check against all + * advertised references. + * + * This is useful for big repos with enormous number of references. + */ +public class IterativeConnectivityChecker implements ConnectivityChecker { + private static final int MAXIMUM_PARENTS_TO_CHECK = 128; + + private final ConnectivityChecker delegate; + + private Set<ObjectId> forcedHaves = Collections.emptySet(); + + /** + * @param delegate + * Delegate checker which will be called for actual checks. + */ + public IterativeConnectivityChecker(ConnectivityChecker delegate) { + this.delegate = delegate; + } + + @Override + public void checkConnectivity(ConnectivityCheckInfo connectivityCheckInfo, + Set<ObjectId> advertisedHaves, ProgressMonitor pm) + throws MissingObjectException, IOException { + try { + Set<ObjectId> newRefs = new HashSet<>(); + Set<ObjectId> expectedParents = new HashSet<>(); + + getAllObjectIds(connectivityCheckInfo.getCommands()) + .forEach(oid -> { + if (advertisedHaves.contains(oid)) { + expectedParents.add(oid); + } else { + newRefs.add(oid); + } + }); + if (!newRefs.isEmpty()) { + expectedParents.addAll(extractAdvertisedParentCommits(newRefs, + advertisedHaves, connectivityCheckInfo.getWalk())); + } + + expectedParents.addAll(forcedHaves); + + if (!expectedParents.isEmpty()) { + delegate.checkConnectivity(connectivityCheckInfo, + expectedParents, pm); + return; + } + } catch (MissingObjectException e) { + // This is fine, retry with all haves. + } + delegate.checkConnectivity(connectivityCheckInfo, advertisedHaves, pm); + } + + private static Stream<ObjectId> getAllObjectIds( + List<ReceiveCommand> commands) { + return commands.stream().flatMap(cmd -> { + if (cmd.getType() == ReceiveCommand.Type.UPDATE || cmd + .getType() == ReceiveCommand.Type.UPDATE_NONFASTFORWARD) { + return Stream.of(cmd.getOldId(), cmd.getNewId()); + } else if (cmd.getType() == ReceiveCommand.Type.CREATE) { + return Stream.of(cmd.getNewId()); + } + return Stream.of(); + }); + } + + /** + * Sets additional haves that client can depend on (e.g. gerrit changes). + * + * @param forcedHaves + * Haves server expects client to depend on. + */ + public void setForcedHaves(Set<ObjectId> forcedHaves) { + this.forcedHaves = Collections.unmodifiableSet(forcedHaves); + } + + private static Set<ObjectId> extractAdvertisedParentCommits( + Set<ObjectId> newRefs, Set<ObjectId> advertisedHaves, RevWalk rw) + throws MissingObjectException, IOException { + Set<ObjectId> advertisedParents = new HashSet<>(); + for (ObjectId newRef : newRefs) { + RevObject object = rw.parseAny(newRef); + if (object instanceof RevCommit) { + int numberOfParentsToCheck = 0; + Queue<RevCommit> parents = new ArrayDeque<>( + MAXIMUM_PARENTS_TO_CHECK); + parents.addAll( + parseParents(((RevCommit) object).getParents(), rw)); + // Looking through a chain of ancestors handles the case where a + // series of commits is sent in a single push for a new branch. + while (!parents.isEmpty()) { + RevCommit parentCommit = parents.poll(); + if (advertisedHaves.contains(parentCommit.getId())) { + advertisedParents.add(parentCommit.getId()); + } else if (numberOfParentsToCheck < MAXIMUM_PARENTS_TO_CHECK) { + RevCommit[] grandParents = parentCommit.getParents(); + numberOfParentsToCheck += grandParents.length; + parents.addAll(parseParents(grandParents, rw)); + } + } + } + } + return advertisedParents; + } + + private static List<RevCommit> parseParents(RevCommit[] parents, + RevWalk rw) { + return Arrays.stream(parents).map((commit) -> { + try { + return rw.parseCommit(commit); + } catch (Exception e) { + throw new RuntimeException(e); + } + }).collect(toList()); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/internal/DelegatingSSLSocketFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/http/DelegatingSSLSocketFactory.java index d25ecd459d..5aab61ad05 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/internal/DelegatingSSLSocketFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/http/DelegatingSSLSocketFactory.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.transport.internal; +package org.eclipse.jgit.internal.transport.http; import java.io.IOException; import java.net.InetAddress; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java index 2fbc9122f1..98c63cdcdd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java @@ -32,6 +32,7 @@ import java.util.TreeSet; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.InvalidPatternException; import org.eclipse.jgit.fnmatch.FileNameMatcher; +import org.eclipse.jgit.transport.SshConfigStore; import org.eclipse.jgit.transport.SshConstants; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.StringUtils; @@ -80,7 +81,7 @@ import org.eclipse.jgit.util.SystemReader; * @see <a href="http://man.openbsd.org/OpenBSD-current/man5/ssh_config.5">man * ssh-config</a> */ -public class OpenSshConfigFile { +public class OpenSshConfigFile implements SshConfigStore { /** * "Host" name of the HostEntry for the default options before the first @@ -152,8 +153,9 @@ public class OpenSshConfigFile { * the user supplied; <= 0 if none * @param userName * the user supplied, may be {@code null} or empty if none given - * @return r configuration for the requested name. + * @return the configuration for the requested name. */ + @Override @NonNull public HostEntry lookup(@NonNull String hostName, int port, String userName) { @@ -446,7 +448,7 @@ public class OpenSshConfigFile { * of several matching host entries, %-substitutions, and ~ replacement have * all been done. */ - public static class HostEntry { + public static class HostEntry implements SshConfigStore.HostConfig { /** * Keys that can be specified multiple times, building up a list. (I.e., @@ -489,7 +491,7 @@ public class OpenSshConfigFile { private Map<String, List<String>> listOptions; /** - * Retrieves the value of a single-valued key, or the first is the key + * Retrieves the value of a single-valued key, or the first if the key * has multiple values. Keys are case-insensitive, so * {@code getValue("HostName") == getValue("HOSTNAME")}. * @@ -497,6 +499,7 @@ public class OpenSshConfigFile { * to get the value of * @return the value, or {@code null} if none */ + @Override public String getValue(String key) { String result = options != null ? options.get(key) : null; if (result == null) { @@ -524,6 +527,7 @@ public class OpenSshConfigFile { * to get the values of * @return a possibly empty list of values */ + @Override public List<String> getValues(String key) { List<String> values = listOptions != null ? listOptions.get(key) : null; @@ -778,6 +782,7 @@ public class OpenSshConfigFile { * * @return all single-valued options */ + @Override @NonNull public Map<String, String> getOptions() { if (options == null) { @@ -792,6 +797,7 @@ public class OpenSshConfigFile { * * @return all multi-valued options */ + @Override @NonNull public Map<String, List<String>> getMultiValuedOptions() { if (listOptions == null && multiOptions == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java index f61286d6da..f6695bdf7d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BitmapIndex.java @@ -14,6 +14,8 @@ import java.util.Iterator; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; +import com.googlecode.javaewah.EWAHCompressedBitmap; + /** * A compressed bitmap representation of the entire object graph. * @@ -81,6 +83,14 @@ public interface BitmapIndex { */ @Override Iterator<BitmapObject> iterator(); + + /** + * Returns the corresponding raw compressed EWAH bitmap of the bitmap. + * + * @return the corresponding {@code EWAHCompressedBitmap} + * @since 5.8 + */ + EWAHCompressedBitmap retrieveCompressed(); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index e607edc2ea..eef822fa4b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -542,4 +542,124 @@ public final class ConfigConstants { * @since 5.1.13 */ public static final String CONFIG_JMX_SECTION = "jmx"; + + /** + * The "pack.bigfilethreshold" key + * @since 5.8 + */ + public static final String CONFIG_KEY_BIGFILE_THRESHOLD = "bigfilethreshold"; + + /** + * The "pack.bitmapContiguousCommitCount" key + * @since 5.8 + */ + public static final String CONFIG_KEY_BITMAP_CONTIGUOUS_COMMIT_COUNT = "bitmapcontiguouscommitcount"; + + /** + * The "pack.bitmapDistantCommitSpan" key + * @since 5.8 + */ + public static final String CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN = "bitmapdistantcommitspan"; + + /** + * The "pack.bitmapExcessiveBranchCount" key + * @since 5.8 + */ + public static final String CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT = "bitmapexcessivebranchcount"; + + /** + * The "pack.bitmapInactiveBranchAgeInDays" key + * @since 5.8 + */ + public static final String CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS = "bitmapinactivebranchageindays"; + + /** + * The "pack.bitmapRecentCommitSpan" key + * @since 5.8 + */ + public static final String CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT = "bitmaprecentcommitspan"; + + /** + * The "pack.buildBitmaps" key + * @since 5.8 + */ + public static final String CONFIG_KEY_BUILD_BITMAPS = "buildbitmaps"; + + /** + * The "pack.cutDeltaChains" key + * @since 5.8 + */ + public static final String CONFIG_KEY_CUT_DELTACHAINS = "cutdeltachains"; + + /** + * The "pack.deltaCacheLimit" key + * @since 5.8 + */ + public static final String CONFIG_KEY_DELTA_CACHE_LIMIT = "deltacachelimit"; + + /** + * The "pack.deltaCacheSize" key + * @since 5.8 + */ + public static final String CONFIG_KEY_DELTA_CACHE_SIZE = "deltacachesize"; + + /** + * The "pack.deltaCompression" key + * @since 5.8 + */ + public static final String CONFIG_KEY_DELTA_COMPRESSION = "deltacompression"; + + /** + * The "pack.depth" key + * @since 5.8 + */ + public static final String CONFIG_KEY_DEPTH = "depth"; + + /** + * The "pack.minSizePreventRacyPack" key + * @since 5.8 + */ + public static final String CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK = "minsizepreventracypack"; + + /** + * The "pack.reuseDeltas" key + * @since 5.8 + */ + public static final String CONFIG_KEY_REUSE_DELTAS = "reusedeltas"; + + /** + * The "pack.reuseObjects" key + * @since 5.8 + */ + public static final String CONFIG_KEY_REUSE_OBJECTS = "reuseobjects"; + + /** + * The "pack.singlePack" key + * @since 5.8 + */ + public static final String CONFIG_KEY_SINGLE_PACK = "singlepack"; + + /** + * The "pack.threads" key + * @since 5.8 + */ + public static final String CONFIG_KEY_THREADS = "threads"; + + /** + * The "pack.waitPreventRacyPack" key + * @since 5.8 + */ + public static final String CONFIG_KEY_WAIT_PREVENT_RACYPACK = "waitpreventracypack"; + + /** + * The "pack.window" key + * @since 5.8 + */ + public static final String CONFIG_KEY_WINDOW = "window"; + + /** + * The "pack.windowMemory" key + * @since 5.8 + */ + public static final String CONFIG_KEY_WINDOW_MEMORY = "windowmemory"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 459ca2f7ff..92367ebd0c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -386,6 +386,13 @@ public final class Constants { public static final String DOT_GIT_EXT = ".git"; /** + * The default extension for local bundle files + * + * @since 5.8 + */ + public static final String DOT_BUNDLE_EXT = ".bundle"; + + /** * Name of the attributes file * * @since 3.7 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java index d953a01945..5b32cf0b5f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java @@ -9,11 +9,16 @@ */ package org.eclipse.jgit.lib; +import java.util.Iterator; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; + import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.lib.internal.BouncyCastleGpgSigner; import org.eclipse.jgit.transport.CredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Creates GPG signatures for Git objects. @@ -21,8 +26,23 @@ import org.eclipse.jgit.transport.CredentialsProvider; * @since 5.3 */ public abstract class GpgSigner { + private static final Logger LOG = LoggerFactory.getLogger(GpgSigner.class); + + private static GpgSigner defaultSigner = loadGpgSigner(); - private static GpgSigner defaultSigner = new BouncyCastleGpgSigner(); + private static GpgSigner loadGpgSigner() { + try { + ServiceLoader<GpgSigner> loader = ServiceLoader + .load(GpgSigner.class); + Iterator<GpgSigner> iter = loader.iterator(); + if (iter.hasNext()) { + return iter.next(); + } + } catch (ServiceConfigurationError e) { + LOG.error(e.getMessage(), e); + } + return null; + } /** * Get the default signer, or <code>null</code>. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index ff5a84ca6e..6832c9cd80 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -441,7 +441,7 @@ public abstract class RefDatabase { /** * Returns all refs that resolve directly to the given {@link ObjectId}. - * Includes peeled {@linkObjectId}s. This is the inverse lookup of + * Includes peeled {@link ObjectId}s. This is the inverse lookup of * {@link #exactRef(String...)}. * * <p> diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKey.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKey.java deleted file mode 100644 index 8601d7c94f..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKey.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (C) 2018, Salesforce. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib.internal; - -import java.nio.file.Path; - -import org.bouncycastle.openpgp.PGPSecretKey; - -/** - * Container which holds a {@link #getSecretKey()} together with the - * {@link #getOrigin() path it was loaded from}. - */ -class BouncyCastleGpgKey { - - private PGPSecretKey secretKey; - - private Path origin; - - public BouncyCastleGpgKey(PGPSecretKey secretKey, Path origin) { - this.secretKey = secretKey; - this.origin = origin; - } - - public PGPSecretKey getSecretKey() { - return secretKey; - } - - public Path getOrigin() { - return origin; - } -}
\ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKeyLocator.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKeyLocator.java deleted file mode 100644 index 8a32299dd3..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKeyLocator.java +++ /dev/null @@ -1,614 +0,0 @@ -/* - * Copyright (C) 2018, Salesforce. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib.internal; - -import static java.nio.file.Files.exists; -import static java.nio.file.Files.newInputStream; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.text.MessageFormat; -import java.util.Iterator; -import java.util.Locale; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.bouncycastle.gpg.SExprParser; -import org.bouncycastle.gpg.keybox.BlobType; -import org.bouncycastle.gpg.keybox.KeyBlob; -import org.bouncycastle.gpg.keybox.KeyBox; -import org.bouncycastle.gpg.keybox.KeyInformation; -import org.bouncycastle.gpg.keybox.PublicKeyRingBlob; -import org.bouncycastle.gpg.keybox.UserID; -import org.bouncycastle.gpg.keybox.jcajce.JcaKeyBoxBuilder; -import org.bouncycastle.openpgp.PGPException; -import org.bouncycastle.openpgp.PGPKeyFlags; -import org.bouncycastle.openpgp.PGPPublicKey; -import org.bouncycastle.openpgp.PGPPublicKeyRing; -import org.bouncycastle.openpgp.PGPPublicKeyRingCollection; -import org.bouncycastle.openpgp.PGPSecretKey; -import org.bouncycastle.openpgp.PGPSecretKeyRing; -import org.bouncycastle.openpgp.PGPSecretKeyRingCollection; -import org.bouncycastle.openpgp.PGPSignature; -import org.bouncycastle.openpgp.PGPUtil; -import org.bouncycastle.openpgp.operator.PBEProtectionRemoverFactory; -import org.bouncycastle.openpgp.operator.PGPDigestCalculatorProvider; -import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator; -import org.bouncycastle.openpgp.operator.jcajce.JcaPGPDigestCalculatorProviderBuilder; -import org.bouncycastle.openpgp.operator.jcajce.JcePBEProtectionRemoverFactory; -import org.bouncycastle.util.encoders.Hex; -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.errors.UnsupportedCredentialItem; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.util.FS; -import org.eclipse.jgit.util.StringUtils; -import org.eclipse.jgit.util.SystemReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Locates GPG keys from either <code>~/.gnupg/private-keys-v1.d</code> or - * <code>~/.gnupg/secring.gpg</code> - */ -class BouncyCastleGpgKeyLocator { - - /** Thrown if a keybox file exists but doesn't contain an OpenPGP key. */ - private static class NoOpenPgpKeyException extends Exception { - - private static final long serialVersionUID = 1L; - - } - - private static final Logger log = LoggerFactory - .getLogger(BouncyCastleGpgKeyLocator.class); - - private static final Path GPG_DIRECTORY = findGpgDirectory(); - - private static final Path USER_KEYBOX_PATH = GPG_DIRECTORY - .resolve("pubring.kbx"); //$NON-NLS-1$ - - private static final Path USER_SECRET_KEY_DIR = GPG_DIRECTORY - .resolve("private-keys-v1.d"); //$NON-NLS-1$ - - private static final Path USER_PGP_PUBRING_FILE = GPG_DIRECTORY - .resolve("pubring.gpg"); //$NON-NLS-1$ - - private static final Path USER_PGP_LEGACY_SECRING_FILE = GPG_DIRECTORY - .resolve("secring.gpg"); //$NON-NLS-1$ - - private final String signingKey; - - private BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt; - - private static Path findGpgDirectory() { - SystemReader system = SystemReader.getInstance(); - if (system.isWindows()) { - // On Windows prefer %APPDATA%\gnupg if it exists, even if Cygwin is - // used. - String appData = system.getenv("APPDATA"); //$NON-NLS-1$ - if (appData != null && !appData.isEmpty()) { - try { - Path directory = Paths.get(appData).resolve("gnupg"); //$NON-NLS-1$ - if (Files.isDirectory(directory)) { - return directory; - } - } catch (SecurityException | InvalidPathException e) { - // Ignore and return the default location below. - } - } - } - // All systems, including Cygwin and even Windows if - // %APPDATA%\gnupg doesn't exist: ~/.gnupg - File home = FS.DETECTED.userHome(); - if (home == null) { - // Oops. What now? - home = new File(".").getAbsoluteFile(); //$NON-NLS-1$ - } - return home.toPath().resolve(".gnupg"); //$NON-NLS-1$ - } - - /** - * Create a new key locator for the specified signing key. - * <p> - * The signing key must either be a hex representation of a specific key or - * a user identity substring (eg., email address). All keys in the KeyBox - * will be looked up in the order as returned by the KeyBox. A key id will - * be searched before attempting to find a key by user id. - * </p> - * - * @param signingKey - * the signing key to search for - * @param passphrasePrompt - * the provider to use when asking for key passphrase - */ - public BouncyCastleGpgKeyLocator(String signingKey, - @NonNull BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt) { - this.signingKey = signingKey; - this.passphrasePrompt = passphrasePrompt; - } - - private PGPSecretKey attemptParseSecretKey(Path keyFile, - PGPDigestCalculatorProvider calculatorProvider, - PBEProtectionRemoverFactory passphraseProvider, - PGPPublicKey publicKey) { - try (InputStream in = newInputStream(keyFile)) { - return new SExprParser(calculatorProvider).parseSecretKey( - new BufferedInputStream(in), passphraseProvider, publicKey); - } catch (IOException | PGPException | ClassCastException e) { - if (log.isDebugEnabled()) - log.debug("Ignoring unreadable file '{}': {}", keyFile, //$NON-NLS-1$ - e.getMessage(), e); - return null; - } - } - - /** - * Checks whether a given OpenPGP {@code userId} matches a given - * {@code signingKeySpec}, which is supposed to have one of the formats - * defined by GPG. - * <p> - * Not all formats are supported; only formats starting with '=', '<', - * '@', and '*' are handled. Any other format results in a case-insensitive - * substring match. - * </p> - * - * @param userId - * of a key - * @param signingKeySpec - * GPG key identification - * @return whether the {@code userId} matches - * @see <a href= - * "https://www.gnupg.org/documentation/manuals/gnupg/Specify-a-User-ID.html">GPG - * Documentation: How to Specify a User ID</a> - */ - static boolean containsSigningKey(String userId, String signingKeySpec) { - if (StringUtils.isEmptyOrNull(userId) - || StringUtils.isEmptyOrNull(signingKeySpec)) { - return false; - } - String toMatch = signingKeySpec; - if (toMatch.startsWith("0x") && toMatch.trim().length() > 2) { //$NON-NLS-1$ - return false; // Explicit fingerprint - } - int command = toMatch.charAt(0); - switch (command) { - case '=': - case '<': - case '@': - case '*': - toMatch = toMatch.substring(1); - if (toMatch.isEmpty()) { - return false; - } - break; - default: - break; - } - switch (command) { - case '=': - return userId.equals(toMatch); - case '<': { - int begin = userId.indexOf('<'); - int end = userId.indexOf('>', begin + 1); - int stop = toMatch.indexOf('>'); - return begin >= 0 && end > begin + 1 && stop > 0 - && userId.substring(begin + 1, end) - .equals(toMatch.substring(0, stop)); - } - case '@': { - int begin = userId.indexOf('<'); - int end = userId.indexOf('>', begin + 1); - return begin >= 0 && end > begin + 1 - && userId.substring(begin + 1, end).contains(toMatch); - } - default: - if (toMatch.trim().isEmpty()) { - return false; - } - return userId.toLowerCase(Locale.ROOT) - .contains(toMatch.toLowerCase(Locale.ROOT)); - } - } - - private String toFingerprint(String keyId) { - if (keyId.startsWith("0x")) { //$NON-NLS-1$ - return keyId.substring(2); - } - return keyId; - } - - private PGPPublicKey findPublicKeyByKeyId(KeyBlob keyBlob) - throws IOException { - String keyId = toFingerprint(signingKey).toLowerCase(Locale.ROOT); - if (keyId.isEmpty()) { - return null; - } - for (KeyInformation keyInfo : keyBlob.getKeyInformation()) { - String fingerprint = Hex.toHexString(keyInfo.getFingerprint()) - .toLowerCase(Locale.ROOT); - if (fingerprint.endsWith(keyId)) { - return getPublicKey(keyBlob, keyInfo.getFingerprint()); - } - } - return null; - } - - private PGPPublicKey findPublicKeyByUserId(KeyBlob keyBlob) - throws IOException { - for (UserID userID : keyBlob.getUserIds()) { - if (containsSigningKey(userID.getUserIDAsString(), signingKey)) { - return getSigningPublicKey(keyBlob); - } - } - return null; - } - - /** - * Finds a public key associated with the signing key. - * - * @param keyboxFile - * the KeyBox file - * @return publicKey the public key (maybe <code>null</code>) - * @throws IOException - * in case of problems reading the file - * @throws NoSuchAlgorithmException - * @throws NoSuchProviderException - * @throws NoOpenPgpKeyException - * if the file does not contain any OpenPGP key - */ - private PGPPublicKey findPublicKeyInKeyBox(Path keyboxFile) - throws IOException, NoSuchAlgorithmException, - NoSuchProviderException, NoOpenPgpKeyException { - KeyBox keyBox = readKeyBoxFile(keyboxFile); - boolean hasOpenPgpKey = false; - for (KeyBlob keyBlob : keyBox.getKeyBlobs()) { - if (keyBlob.getType() == BlobType.OPEN_PGP_BLOB) { - hasOpenPgpKey = true; - PGPPublicKey key = findPublicKeyByKeyId(keyBlob); - if (key != null) { - return key; - } - key = findPublicKeyByUserId(keyBlob); - if (key != null) { - return key; - } - } - } - if (!hasOpenPgpKey) { - throw new NoOpenPgpKeyException(); - } - return null; - } - - /** - * If there is a private key directory containing keys, use pubring.kbx or - * pubring.gpg to find the public key; then try to find the secret key in - * the directory. - * <p> - * If there is no private key directory (or it doesn't contain any keys), - * try to find the key in secring.gpg directly. - * </p> - * - * @return the secret key - * @throws IOException - * in case of issues reading key files - * @throws NoSuchAlgorithmException - * @throws NoSuchProviderException - * @throws PGPException - * in case of issues finding a key, including no key found - * @throws CanceledException - * @throws URISyntaxException - * @throws UnsupportedCredentialItem - */ - @NonNull - public BouncyCastleGpgKey findSecretKey() throws IOException, - NoSuchAlgorithmException, NoSuchProviderException, PGPException, - CanceledException, UnsupportedCredentialItem, URISyntaxException { - BouncyCastleGpgKey key; - PGPPublicKey publicKey = null; - if (hasKeyFiles(USER_SECRET_KEY_DIR)) { - // Use pubring.kbx or pubring.gpg to find the public key, then try - // the key files in the directory. If the public key was found in - // pubring.gpg also try secring.gpg to find the secret key. - if (exists(USER_KEYBOX_PATH)) { - try { - publicKey = findPublicKeyInKeyBox(USER_KEYBOX_PATH); - if (publicKey != null) { - key = findSecretKeyForKeyBoxPublicKey(publicKey, - USER_KEYBOX_PATH); - if (key != null) { - return key; - } - throw new PGPException(MessageFormat.format( - JGitText.get().gpgNoSecretKeyForPublicKey, - Long.toHexString(publicKey.getKeyID()))); - } - throw new PGPException(MessageFormat.format( - JGitText.get().gpgNoPublicKeyFound, signingKey)); - } catch (NoOpenPgpKeyException e) { - // There are no OpenPGP keys in the keybox at all: try the - // pubring.gpg, if it exists. - if (log.isDebugEnabled()) { - log.debug("{} does not contain any OpenPGP keys", //$NON-NLS-1$ - USER_KEYBOX_PATH); - } - } - } - if (exists(USER_PGP_PUBRING_FILE)) { - publicKey = findPublicKeyInPubring(USER_PGP_PUBRING_FILE); - if (publicKey != null) { - // GPG < 2.1 may have both; the agent using the directory - // and gpg using secring.gpg. GPG >= 2.1 delegates all - // secret key handling to the agent and doesn't use - // secring.gpg at all, even if it exists. Which means for us - // we have to try both since we don't know which GPG version - // the user has. - key = findSecretKeyForKeyBoxPublicKey(publicKey, - USER_PGP_PUBRING_FILE); - if (key != null) { - return key; - } - } - } - if (publicKey == null) { - throw new PGPException(MessageFormat.format( - JGitText.get().gpgNoPublicKeyFound, signingKey)); - } - // We found a public key, but didn't find the secret key in the - // private key directory. Go try the secring.gpg. - } - boolean hasSecring = false; - if (exists(USER_PGP_LEGACY_SECRING_FILE)) { - hasSecring = true; - key = loadKeyFromSecring(USER_PGP_LEGACY_SECRING_FILE); - if (key != null) { - return key; - } - } - if (publicKey != null) { - throw new PGPException(MessageFormat.format( - JGitText.get().gpgNoSecretKeyForPublicKey, - Long.toHexString(publicKey.getKeyID()))); - } else if (hasSecring) { - // publicKey == null: user has _only_ pubring.gpg/secring.gpg. - throw new PGPException(MessageFormat.format( - JGitText.get().gpgNoKeyInLegacySecring, signingKey)); - } else { - throw new PGPException(JGitText.get().gpgNoKeyring); - } - } - - private boolean hasKeyFiles(Path dir) { - try (DirectoryStream<Path> contents = Files.newDirectoryStream(dir, - "*.key")) { //$NON-NLS-1$ - return contents.iterator().hasNext(); - } catch (IOException e) { - // Not a directory, or something else - return false; - } - } - - private BouncyCastleGpgKey loadKeyFromSecring(Path secring) - throws IOException, PGPException { - PGPSecretKey secretKey = findSecretKeyInLegacySecring(signingKey, - secring); - - if (secretKey != null) { - if (!secretKey.isSigningKey()) { - throw new PGPException(MessageFormat - .format(JGitText.get().gpgNotASigningKey, signingKey)); - } - return new BouncyCastleGpgKey(secretKey, secring); - } - return null; - } - - private BouncyCastleGpgKey findSecretKeyForKeyBoxPublicKey( - PGPPublicKey publicKey, Path userKeyboxPath) - throws PGPException, CanceledException, UnsupportedCredentialItem, - URISyntaxException { - /* - * this is somewhat brute-force but there doesn't seem to be another - * way; we have to walk all private key files we find and try to open - * them - */ - - PGPDigestCalculatorProvider calculatorProvider = new JcaPGPDigestCalculatorProviderBuilder() - .build(); - - PBEProtectionRemoverFactory passphraseProvider = new JcePBEProtectionRemoverFactory( - passphrasePrompt.getPassphrase(publicKey.getFingerprint(), - userKeyboxPath)); - - try (Stream<Path> keyFiles = Files.walk(USER_SECRET_KEY_DIR)) { - for (Path keyFile : keyFiles.filter(Files::isRegularFile) - .collect(Collectors.toList())) { - PGPSecretKey secretKey = attemptParseSecretKey(keyFile, - calculatorProvider, passphraseProvider, publicKey); - if (secretKey != null) { - if (!secretKey.isSigningKey()) { - throw new PGPException(MessageFormat.format( - JGitText.get().gpgNotASigningKey, signingKey)); - } - return new BouncyCastleGpgKey(secretKey, userKeyboxPath); - } - } - - passphrasePrompt.clear(); - return null; - } catch (RuntimeException e) { - passphrasePrompt.clear(); - throw e; - } catch (IOException e) { - passphrasePrompt.clear(); - throw new PGPException(MessageFormat.format( - JGitText.get().gpgFailedToParseSecretKey, - USER_SECRET_KEY_DIR.toAbsolutePath()), e); - } - } - - /** - * Return the first suitable key for signing in the key ring collection. For - * this case we only expect there to be one key available for signing. - * </p> - * - * @param signingkey - * @param secringFile - * - * @return the first suitable PGP secret key found for signing - * @throws IOException - * on I/O related errors - * @throws PGPException - * on BouncyCastle errors - */ - private PGPSecretKey findSecretKeyInLegacySecring(String signingkey, - Path secringFile) throws IOException, PGPException { - - try (InputStream in = newInputStream(secringFile)) { - PGPSecretKeyRingCollection pgpSec = new PGPSecretKeyRingCollection( - PGPUtil.getDecoderStream(new BufferedInputStream(in)), - new JcaKeyFingerprintCalculator()); - - String keyId = toFingerprint(signingkey).toLowerCase(Locale.ROOT); - Iterator<PGPSecretKeyRing> keyrings = pgpSec.getKeyRings(); - while (keyrings.hasNext()) { - PGPSecretKeyRing keyRing = keyrings.next(); - Iterator<PGPSecretKey> keys = keyRing.getSecretKeys(); - while (keys.hasNext()) { - PGPSecretKey key = keys.next(); - // try key id - String fingerprint = Hex - .toHexString(key.getPublicKey().getFingerprint()) - .toLowerCase(Locale.ROOT); - if (fingerprint.endsWith(keyId)) { - return key; - } - // try user id - Iterator<String> userIDs = key.getUserIDs(); - while (userIDs.hasNext()) { - String userId = userIDs.next(); - if (containsSigningKey(userId, signingKey)) { - return key; - } - } - } - } - } - return null; - } - - /** - * Return the first public key matching the key id ({@link #signingKey}. - * - * @param pubringFile - * - * @return the PGP public key, or {@code null} if none found - * @throws IOException - * on I/O related errors - * @throws PGPException - * on BouncyCastle errors - */ - private PGPPublicKey findPublicKeyInPubring(Path pubringFile) - throws IOException, PGPException { - try (InputStream in = newInputStream(pubringFile)) { - PGPPublicKeyRingCollection pgpPub = new PGPPublicKeyRingCollection( - new BufferedInputStream(in), - new JcaKeyFingerprintCalculator()); - - String keyId = toFingerprint(signingKey).toLowerCase(Locale.ROOT); - Iterator<PGPPublicKeyRing> keyrings = pgpPub.getKeyRings(); - while (keyrings.hasNext()) { - PGPPublicKeyRing keyRing = keyrings.next(); - Iterator<PGPPublicKey> keys = keyRing.getPublicKeys(); - while (keys.hasNext()) { - PGPPublicKey key = keys.next(); - // try key id - String fingerprint = Hex.toHexString(key.getFingerprint()) - .toLowerCase(Locale.ROOT); - if (fingerprint.endsWith(keyId)) { - return key; - } - // try user id - Iterator<String> userIDs = key.getUserIDs(); - while (userIDs.hasNext()) { - String userId = userIDs.next(); - if (containsSigningKey(userId, signingKey)) { - return key; - } - } - } - } - } - return null; - } - - private PGPPublicKey getPublicKey(KeyBlob blob, byte[] fingerprint) - throws IOException { - return ((PublicKeyRingBlob) blob).getPGPPublicKeyRing() - .getPublicKey(fingerprint); - } - - private PGPPublicKey getSigningPublicKey(KeyBlob blob) throws IOException { - PGPPublicKey masterKey = null; - Iterator<PGPPublicKey> keys = ((PublicKeyRingBlob) blob) - .getPGPPublicKeyRing().getPublicKeys(); - while (keys.hasNext()) { - PGPPublicKey key = keys.next(); - // only consider keys that have the [S] usage flag set - if (isSigningKey(key)) { - if (key.isMasterKey()) { - masterKey = key; - } else { - return key; - } - } - } - // return the master key if no other signing key was found or null if - // the master key did not have the signing flag set - return masterKey; - } - - private boolean isSigningKey(PGPPublicKey key) { - Iterator signatures = key.getSignatures(); - while (signatures.hasNext()) { - PGPSignature sig = (PGPSignature) signatures.next(); - if ((sig.getHashedSubPackets().getKeyFlags() - & PGPKeyFlags.CAN_SIGN) > 0) { - return true; - } - } - return false; - } - - private KeyBox readKeyBoxFile(Path keyboxFile) throws IOException, - NoSuchAlgorithmException, NoSuchProviderException, - NoOpenPgpKeyException { - if (keyboxFile.toFile().length() == 0) { - throw new NoOpenPgpKeyException(); - } - KeyBox keyBox; - try (InputStream in = new BufferedInputStream( - newInputStream(keyboxFile))) { - keyBox = new JcaKeyBoxBuilder().build(in); - } - return keyBox; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKeyPassphrasePrompt.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKeyPassphrasePrompt.java deleted file mode 100644 index 6e29af51d8..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgKeyPassphrasePrompt.java +++ /dev/null @@ -1,101 +0,0 @@ -/*- - * Copyright (C) 2019, Salesforce. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib.internal; - -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.text.MessageFormat; - -import org.bouncycastle.openpgp.PGPException; -import org.bouncycastle.util.encoders.Hex; -import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.errors.UnsupportedCredentialItem; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.transport.CredentialItem.CharArrayType; -import org.eclipse.jgit.transport.CredentialItem.InformationalMessage; -import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.URIish; - -/** - * Prompts for a passphrase and caches it until {@link #clear() cleared}. - * <p> - * Implements {@link AutoCloseable} so it can be used within a - * try-with-resources block. - * </p> - */ -class BouncyCastleGpgKeyPassphrasePrompt implements AutoCloseable { - - private CharArrayType passphrase; - - private CredentialsProvider credentialsProvider; - - public BouncyCastleGpgKeyPassphrasePrompt( - CredentialsProvider credentialsProvider) { - this.credentialsProvider = credentialsProvider; - } - - /** - * Clears any cached passphrase - */ - public void clear() { - if (passphrase != null) { - passphrase.clear(); - passphrase = null; - } - } - - @Override - public void close() { - clear(); - } - - private URIish createURI(Path keyLocation) throws URISyntaxException { - return new URIish(keyLocation.toUri().toString()); - } - - /** - * Prompts use for a passphrase unless one was cached from a previous - * prompt. - * - * @param keyFingerprint - * the fingerprint to show to the user during prompting - * @param keyLocation - * the location the key was loaded from - * @return the passphrase (maybe <code>null</code>) - * @throws PGPException - * @throws CanceledException - * in case passphrase was not entered by user - * @throws URISyntaxException - * @throws UnsupportedCredentialItem - */ - public char[] getPassphrase(byte[] keyFingerprint, Path keyLocation) - throws PGPException, CanceledException, UnsupportedCredentialItem, - URISyntaxException { - if (passphrase == null) { - passphrase = new CharArrayType(JGitText.get().credentialPassphrase, - true); - } - - if (credentialsProvider == null) { - throw new PGPException(JGitText.get().gpgNoCredentialsProvider); - } - - if (passphrase.getValue() == null - && !credentialsProvider.get(createURI(keyLocation), - new InformationalMessage( - MessageFormat.format(JGitText.get().gpgKeyInfo, - Hex.toHexString(keyFingerprint))), - passphrase)) { - throw new CanceledException(JGitText.get().gpgSigningCancelled); - } - return passphrase.getValue(); - } - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgSigner.java deleted file mode 100644 index 388169637e..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/internal/BouncyCastleGpgSigner.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (C) 2018, Salesforce. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib.internal; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URISyntaxException; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.Security; - -import org.bouncycastle.bcpg.ArmoredOutputStream; -import org.bouncycastle.bcpg.BCPGOutputStream; -import org.bouncycastle.bcpg.HashAlgorithmTags; -import org.bouncycastle.jce.provider.BouncyCastleProvider; -import org.bouncycastle.openpgp.PGPException; -import org.bouncycastle.openpgp.PGPPrivateKey; -import org.bouncycastle.openpgp.PGPSecretKey; -import org.bouncycastle.openpgp.PGPSignature; -import org.bouncycastle.openpgp.PGPSignatureGenerator; -import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentSignerBuilder; -import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyDecryptorBuilder; -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.errors.UnsupportedCredentialItem; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.GpgSignature; -import org.eclipse.jgit.lib.GpgSigner; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.transport.CredentialsProvider; - -/** - * GPG Signer using BouncyCastle library - */ -public class BouncyCastleGpgSigner extends GpgSigner { - - private static void registerBouncyCastleProviderIfNecessary() { - if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { - Security.addProvider(new BouncyCastleProvider()); - } - } - - /** - * Create a new instance. - * <p> - * The BounceCastleProvider will be registered if necessary. - * </p> - */ - public BouncyCastleGpgSigner() { - registerBouncyCastleProviderIfNecessary(); - } - - @Override - public boolean canLocateSigningKey(@Nullable String gpgSigningKey, - PersonIdent committer, CredentialsProvider credentialsProvider) - throws CanceledException { - try (BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt = new BouncyCastleGpgKeyPassphrasePrompt( - credentialsProvider)) { - BouncyCastleGpgKey gpgKey = locateSigningKey(gpgSigningKey, - committer, passphrasePrompt); - return gpgKey != null; - } catch (PGPException | IOException | NoSuchAlgorithmException - | NoSuchProviderException | URISyntaxException e) { - return false; - } - } - - private BouncyCastleGpgKey locateSigningKey(@Nullable String gpgSigningKey, - PersonIdent committer, - BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt) - throws CanceledException, UnsupportedCredentialItem, IOException, - NoSuchAlgorithmException, NoSuchProviderException, PGPException, - URISyntaxException { - if (gpgSigningKey == null || gpgSigningKey.isEmpty()) { - gpgSigningKey = '<' + committer.getEmailAddress() + '>'; - } - - BouncyCastleGpgKeyLocator keyHelper = new BouncyCastleGpgKeyLocator( - gpgSigningKey, passphrasePrompt); - - return keyHelper.findSecretKey(); - } - - @Override - public void sign(@NonNull CommitBuilder commit, - @Nullable String gpgSigningKey, @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider) throws CanceledException { - try (BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt = new BouncyCastleGpgKeyPassphrasePrompt( - credentialsProvider)) { - BouncyCastleGpgKey gpgKey = locateSigningKey(gpgSigningKey, - committer, passphrasePrompt); - PGPSecretKey secretKey = gpgKey.getSecretKey(); - if (secretKey == null) { - throw new JGitInternalException( - JGitText.get().unableToSignCommitNoSecretKey); - } - char[] passphrase = passphrasePrompt.getPassphrase( - secretKey.getPublicKey().getFingerprint(), - gpgKey.getOrigin()); - PGPPrivateKey privateKey = secretKey - .extractPrivateKey(new JcePBESecretKeyDecryptorBuilder() - .setProvider(BouncyCastleProvider.PROVIDER_NAME) - .build(passphrase)); - PGPSignatureGenerator signatureGenerator = new PGPSignatureGenerator( - new JcaPGPContentSignerBuilder( - secretKey.getPublicKey().getAlgorithm(), - HashAlgorithmTags.SHA256).setProvider( - BouncyCastleProvider.PROVIDER_NAME)); - signatureGenerator.init(PGPSignature.BINARY_DOCUMENT, privateKey); - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (BCPGOutputStream out = new BCPGOutputStream( - new ArmoredOutputStream(buffer))) { - signatureGenerator.update(commit.build()); - signatureGenerator.generate().encode(out); - } - commit.setGpgSignature(new GpgSignature(buffer.toByteArray())); - } catch (PGPException | IOException | NoSuchAlgorithmException - | NoSuchProviderException | URISyntaxException e) { - throw new JGitInternalException(e.getMessage(), e); - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index 575e7bd285..506d333120 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -789,27 +789,37 @@ public class ResolveMerger extends ThreeWayMerger { MergeResult<RawText> result = contentMerge(base, ours, theirs, attributes); - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); - DirCacheEntry e = add(tw.getRawPath(), theirs, - DirCacheEntry.STAGE_3, EPOCH, 0); + if (ignoreConflicts) { + // In case a conflict is detected the working tree file is + // again filled with new content (containing conflict + // markers). But also stage 0 of the index is filled with + // that content. + result.setContainsConflicts(false); + updateIndex(base, ours, theirs, result, attributes); + } else { + add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); + DirCacheEntry e = add(tw.getRawPath(), theirs, + DirCacheEntry.STAGE_3, EPOCH, 0); - // OURS was deleted checkout THEIRS - if (modeO == 0) { - // Check worktree before checking out THEIRS - if (isWorktreeDirty(work, ourDce)) - return false; - if (nonTree(modeT)) { - if (e != null) { - addToCheckout(tw.getPathString(), e, attributes); + // OURS was deleted checkout THEIRS + if (modeO == 0) { + // Check worktree before checking out THEIRS + if (isWorktreeDirty(work, ourDce)) { + return false; + } + if (nonTree(modeT)) { + if (e != null) { + addToCheckout(tw.getPathString(), e, attributes); + } } } - } - unmergedPaths.add(tw.getPathString()); + unmergedPaths.add(tw.getPathString()); - // generate a MergeResult for the deleted file - mergeResults.put(tw.getPathString(), result); + // generate a MergeResult for the deleted file + mergeResults.put(tw.getPathString(), result); + } } } return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/nls/GlobalBundleCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/nls/GlobalBundleCache.java index 6d4437f4c0..9b556393e2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/nls/GlobalBundleCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/nls/GlobalBundleCache.java @@ -70,4 +70,8 @@ class GlobalBundleCache { throw new Error(e); } } + + static void clear() { + cachedBundles.clear(); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java b/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java index daa039d347..d7dd3bee52 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java @@ -100,6 +100,15 @@ public class NLS { return b.get(type); } + /** + * Release resources held by NLS + * @since 5.8 + */ + public static void clear() { + local.remove(); + GlobalBundleCache.clear(); + } + private final Locale locale; private final ConcurrentHashMap<Class, TranslationBundle> map = new ConcurrentHashMap<>(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java index 023962e251..8cd5eb2238 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmapWalker.java @@ -16,6 +16,7 @@ import java.util.Arrays; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.revwalk.AddToBitmapFilter; +import org.eclipse.jgit.internal.revwalk.AddToBitmapWithCacheFilter; import org.eclipse.jgit.internal.revwalk.AddUnseenToBitmapFilter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BitmapIndex; @@ -41,6 +42,11 @@ public final class BitmapWalker { private long countOfBitmapIndexMisses; + // Cached bitmap and commit to save walk time. + private AnyObjectId prevCommit; + + private Bitmap prevBitmap; + /** * Create a BitmapWalker. * @@ -56,6 +62,28 @@ public final class BitmapWalker { } /** + * Set the cached commit for the walker. + * + * @param prevCommit + * the cached commit. + * @since 5.8 + */ + public void setPrevCommit(AnyObjectId prevCommit) { + this.prevCommit = prevCommit; + } + + /** + * Set the bitmap associated with the cached commit for the walker. + * + * @param prevBitmap + * the bitmap associated with the cached commit. + * @since 5.8 + */ + public void setPrevBitmap(Bitmap prevBitmap) { + this.prevBitmap = prevBitmap; + } + + /** * Return the number of objects that had to be walked because they were not covered by a * bitmap. * @@ -169,7 +197,10 @@ public final class BitmapWalker { } if (marked) { - if (seen == null) { + if (prevCommit != null) { + walker.setRevFilter(new AddToBitmapWithCacheFilter(prevCommit, + prevBitmap, bitmapResult)); + } else if (seen == null) { walker.setRevFilter(new AddToBitmapFilter(bitmapResult)); } else { walker.setRevFilter( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java new file mode 100644 index 0000000000..89aef7dc41 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2020, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.revwalk; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; + +/** + * Checks if all objects are reachable from certain starting points using + * bitmaps. + */ +class BitmappedObjectReachabilityChecker + implements ObjectReachabilityChecker { + + private final ObjectWalk walk; + + /** + * New instance of the reachability checker using a existing walk. + * + * @param walk + * ObjectWalk instance to reuse. Caller retains ownership. + */ + public BitmappedObjectReachabilityChecker(ObjectWalk walk) { + this.walk = walk; + } + + /** + * {@inheritDoc} + * + * This implementation tries to shortcut the check adding starters + * incrementally. Ordering the starters by relevance can improve performance + * in the average case. + */ + @Override + public Optional<RevObject> areAllReachable(Collection<RevObject> targets, + Stream<RevObject> starters) throws IOException { + + try { + List<RevObject> remainingTargets = new ArrayList<>(targets); + BitmapWalker bitmapWalker = new BitmapWalker(walk, + walk.getObjectReader().getBitmapIndex(), null); + + Iterator<RevObject> starterIt = starters.iterator(); + BitmapBuilder seen = null; + while (starterIt.hasNext()) { + List<RevObject> asList = Arrays.asList(starterIt.next()); + BitmapBuilder visited = bitmapWalker.findObjects(asList, seen, + true); + seen = seen == null ? visited : seen.or(visited); + + remainingTargets.removeIf(seen::contains); + if (remainingTargets.isEmpty()) { + return Optional.empty(); + } + } + + return Optional.of(remainingTargets.get(0)); + } catch (MissingObjectException | IncorrectObjectTypeException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java index 02514526d0..0d9c4593bf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java @@ -42,7 +42,7 @@ class BitmappedReachabilityChecker implements ReachabilityChecker { * @throws IOException * if the index or the object reader cannot be opened. */ - public BitmappedReachabilityChecker(RevWalk walk) + BitmappedReachabilityChecker(RevWalk walk) throws IOException { this.walk = walk; if (walk.getObjectReader().getBitmapIndex() == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectReachabilityChecker.java new file mode 100644 index 0000000000..48e908e884 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectReachabilityChecker.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2020, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.revwalk; + +import java.io.IOException; +import java.util.Collection; +import java.util.Optional; +import java.util.stream.Stream; + +/** + * Checks if all objects are reachable from certain starting points. + * + * This is an expensive check that browses commits, trees, blobs and tags. For + * reachability just between commits see {@link ReachabilityChecker} + * implementations. + * + * @since 5.8 + */ +public interface ObjectReachabilityChecker { + + /** + * Checks that all targets are reachable from the starters. + * + * @implSpec Missing or invalid objects are reported as illegal state. + * Caller should have found them while translating ObjectIds into + * RevObjects. They can only happen here if the caller is mixing + * revwalks. + * + * @param targets + * objects to check for reachability from the starters + * @param starters + * objects known to be reachable to the caller + * @return Optional a single unreachable target if there are any (there + * could be more). Empty optional means all targets are reachable. + * @throws IOException + * Cannot access underlying storage + */ + Optional<RevObject> areAllReachable(Collection<RevObject> targets, + Stream<RevObject> starters) throws IOException; + +}
\ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java index 04a4b4c631..4c7a6f556e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java @@ -160,6 +160,29 @@ public class ObjectWalk extends RevWalk { } /** + * Create an object reachability checker that will use bitmaps if possible. + * + * This reachability checker accepts any object as target. For checks + * exclusively between commits, see + * {@link RevWalk#createReachabilityChecker()}. + * + * @return an object reachability checker, using bitmaps if possible. + * + * @throws IOException + * when the index fails to load. + * + * @since 5.8 + */ + public ObjectReachabilityChecker createObjectReachabilityChecker() + throws IOException { + if (reader.getBitmapIndex() != null) { + return new BitmappedObjectReachabilityChecker(this); + } + + return new PedestrianObjectReachabilityChecker(this); + } + + /** * Mark an object or commit to start graph traversal from. * <p> * Callers are encouraged to use diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java new file mode 100644 index 0000000000..df5d68a66e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2020, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.revwalk; + +import java.io.IOException; +import java.io.InvalidObjectException; +import java.util.Collection; +import java.util.Iterator; +import java.util.Optional; +import java.util.stream.Stream; + +import org.eclipse.jgit.errors.MissingObjectException; + +/** + * Checks if all objects are reachable from certain starting points doing a + * walk. + */ +class PedestrianObjectReachabilityChecker implements ObjectReachabilityChecker { + private final ObjectWalk walk; + + /** + * New instance of the reachability checker using a existing walk. + * + * @param walk + * ObjectWalk instance to reuse. Caller retains ownership. + */ + PedestrianObjectReachabilityChecker(ObjectWalk walk) { + this.walk = walk; + } + + /** + * {@inheritDoc} + */ + @Override + public Optional<RevObject> areAllReachable(Collection<RevObject> targets, + Stream<RevObject> starters) throws IOException { + try { + walk.reset(); + walk.sort(RevSort.TOPO); + for (RevObject target : targets) { + walk.markStart(target); + } + + Iterator<RevObject> iterator = starters.iterator(); + while (iterator.hasNext()) { + RevObject o = iterator.next(); + walk.markUninteresting(o); + + RevObject peeled = walk.peel(o); + if (peeled instanceof RevCommit) { + // By default, for performance reasons, ObjectWalk does not + // mark + // a tree as uninteresting when we mark a commit. Mark it + // ourselves so that we can determine reachability exactly. + walk.markUninteresting(((RevCommit) peeled).getTree()); + } + } + + RevCommit commit = walk.next(); + if (commit != null) { + return Optional.of(commit); + } + + RevObject object = walk.nextObject(); + if (object != null) { + return Optional.of(object); + } + + return Optional.empty(); + } catch (MissingObjectException | InvalidObjectException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevObject.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevObject.java index 5ce4bc33b5..4d2684a0f0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevObject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevObject.java @@ -152,6 +152,7 @@ public abstract class RevObject extends ObjectIdOwnerMap.Entry { */ protected void appendCoreFlags(StringBuilder s) { s.append((flags & RevWalk.TOPO_DELAY) != 0 ? 'o' : '-'); + s.append((flags & RevWalk.TOPO_QUEUED) != 0 ? 'q' : '-'); s.append((flags & RevWalk.TEMP_MARK) != 0 ? 't' : '-'); s.append((flags & RevWalk.REWRITE) != 0 ? 'r' : '-'); s.append((flags & RevWalk.UNINTERESTING) != 0 ? 'u' : '-'); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevSort.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevSort.java index fc6ae28dc9..08396a8061 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevSort.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevSort.java @@ -40,6 +40,18 @@ public enum RevSort { TOPO, /** + * Topological sorting (all children before parents) without intermixing + * lines of history. + * <p> + * This behavior more closely resembles C Git's git-log --topo-order than + * {@link #TOPO}. See C Git's topo-order <a href= + * "https://git-scm.com/docs/git-log#Documentation/git-log.txt---topo-order">description</a>. + * + * @since 5.8 + */ + TOPO_KEEP_BRANCH_TOGETHER, + + /** * Flip the output into the reverse ordering. * <p> * This strategy can be combined with the others described by this type as diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index f425e87618..6b62fcdf6d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -131,8 +131,17 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { */ static final int TOPO_DELAY = 1 << 5; + /** + * Temporary mark for use within {@link TopoNonIntermixSortGenerator}. + * <p> + * This mark indicates the commit has been queued for emission in + * {@link TopoSortGenerator} and can be produced. This mark is removed when + * the commit has been produced. + */ + static final int TOPO_QUEUED = 1 << 6; + /** Number of flag bits we keep internal for our own use. See above flags. */ - static final int RESERVED_FLAGS = 6; + static final int RESERVED_FLAGS = 7; private static final int APP_FLAGS = -1 & ~((1 << RESERVED_FLAGS) - 1); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/StartGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/StartGenerator.java index 2c1b0a59ee..bfcea6ea8f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/StartGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/StartGenerator.java @@ -135,8 +135,18 @@ class StartGenerator extends Generator { } if (walker.hasRevSort(RevSort.TOPO) - && (g.outputType() & SORT_TOPO) == 0) + && walker.hasRevSort(RevSort.TOPO_KEEP_BRANCH_TOGETHER)) { + throw new IllegalStateException(JGitText + .get().cannotCombineTopoSortWithTopoKeepBranchTogetherSort); + } + + if (walker.hasRevSort(RevSort.TOPO) + && (g.outputType() & SORT_TOPO) == 0) { g = new TopoSortGenerator(g); + } else if (walker.hasRevSort(RevSort.TOPO_KEEP_BRANCH_TOGETHER) + && (g.outputType() & SORT_TOPO) == 0) { + g = new TopoNonIntermixSortGenerator(g); + } if (walker.hasRevSort(RevSort.REVERSE)) g = new LIFORevQueue(g); if (boundary) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/TopoNonIntermixSortGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/TopoNonIntermixSortGenerator.java new file mode 100644 index 0000000000..4f6d417ed1 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/TopoNonIntermixSortGenerator.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2020, Google LLC. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.revwalk; + +import java.io.IOException; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; + +/** Sorts commits in topological order without intermixing lines of history. */ +class TopoNonIntermixSortGenerator extends Generator { + private static final int TOPO_QUEUED = RevWalk.TOPO_QUEUED; + + private final FIFORevQueue pending; + + private final int outputType; + + /** + * Create a new sorter and completely spin the generator. + * <p> + * When the constructor completes the supplied generator will have no + * commits remaining, as all of the commits will be held inside of this + * generator's internal buffer. + * + * @param s + * generator to pull all commits out of, and into this buffer. + * @throws MissingObjectException + * @throws IncorrectObjectTypeException + * @throws IOException + */ + TopoNonIntermixSortGenerator(Generator s) throws MissingObjectException, + IncorrectObjectTypeException, IOException { + super(s.firstParent); + pending = new FIFORevQueue(firstParent); + outputType = s.outputType() | SORT_TOPO; + s.shareFreeList(pending); + for (;;) { + final RevCommit c = s.next(); + if (c == null) { + break; + } + if ((c.flags & TOPO_QUEUED) == 0) { + for (RevCommit p : c.parents) { + p.inDegree++; + + if (firstParent) { + break; + } + } + } + c.flags |= TOPO_QUEUED; + pending.add(c); + } + } + + @Override + int outputType() { + return outputType; + } + + @Override + void shareFreeList(BlockRevQueue q) { + q.shareFreeList(pending); + } + + @Override + RevCommit next() throws MissingObjectException, + IncorrectObjectTypeException, IOException { + for (;;) { + final RevCommit c = pending.next(); + if (c == null) { + return null; + } + + if (c.inDegree > 0) { + // At least one of our children is missing. We delay + // production until all of our children are output. + // + continue; + } + + if ((c.flags & TOPO_QUEUED) == 0) { + // c is a parent that already produced or a parent that + // was never in the priority queue and should never produce. + // + continue; + } + + for (RevCommit p : c.parents) { + if (--p.inDegree == 0 && (p.flags & TOPO_QUEUED) != 0) { + // The parent has no unproduced interesting children. unpop + // the parent so it goes right behind this child. This means + // that this parent commit may appear in "pending" more than + // once, but this is safe since upon the second and + // subsequent iterations with this commit, it will no longer + // have TOPO_QUEUED set, and thus will be skipped. + // + pending.unpop(p); + } + if (firstParent) { + break; + } + } + + c.flags &= ~TOPO_QUEUED; + return c; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileRepositoryBuilder.java index 553d875bba..d476a0d2fb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileRepositoryBuilder.java @@ -32,7 +32,7 @@ import org.eclipse.jgit.lib.Repository; * <pre> * new FileRepositoryBuilder() // * .setGitDir(gitDirArgument) // --git-dir if supplied, no-op if null - * .readEnviroment() // scan environment GIT_* variables + * .readEnvironment() // scan environment GIT_* variables * .findGitDir() // scan up the file system tree * .build() * </pre> diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java index 221353a91b..a12f652598 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheConfig.java @@ -47,6 +47,8 @@ public class WindowCacheConfig { private int streamFileThreshold; + private boolean exposeStats; + /** * Create a default configuration. */ @@ -58,6 +60,7 @@ public class WindowCacheConfig { packedGitMMAP = false; deltaBaseCacheLimit = 10 * MB; streamFileThreshold = PackConfig.DEFAULT_BIG_FILE_THRESHOLD; + exposeStats = true; } /** @@ -220,6 +223,39 @@ public class WindowCacheConfig { } /** + * Tell whether the statistics JMX bean should be automatically registered. + * <p> + * Registration of that bean via JMX is additionally subject to a boolean + * JGit-specific user config "jmx.WindowCacheStats". The bean will be + * registered only if this user config is {@code true} <em>and</em> + * {@code getExposeStatsViaJmx() == true}. + * </p> + * <p> + * By default, this returns {@code true} unless changed via + * {@link #setExposeStatsViaJmx(boolean)}. + * + * @return whether to expose WindowCacheStats statistics via JMX upon + * {@link #install()} + * @since 5.8 + */ + public boolean getExposeStatsViaJmx() { + return exposeStats; + } + + /** + * Defines whether the statistics JMX MBean should be automatically set up. + * (By default {@code true}.) If set to {@code false}, the JMX monitoring + * bean is not registered. + * + * @param expose + * whether to register the JMX Bean + * @since 5.8 + */ + public void setExposeStatsViaJmx(boolean expose) { + exposeStats = expose; + } + + /** * Update properties by setting fields from the configuration. * <p> * If a property is not defined in the configuration, then it is left diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 259f011757..f76dd2721f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -11,6 +11,31 @@ package org.eclipse.jgit.storage.pack; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BIGFILE_THRESHOLD; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_CONTIGUOUS_COMMIT_COUNT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BUILD_BITMAPS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_COMPRESSION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CUT_DELTACHAINS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_CACHE_LIMIT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_CACHE_SIZE; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_COMPRESSION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DEPTH; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_INDEXVERSION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_DELTAS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REUSE_OBJECTS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_SINGLE_PACK; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_THREADS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WAIT_PREVENT_RACYPACK; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WINDOW_MEMORY; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; + import java.util.concurrent.Executor; import java.util.zip.Deflater; @@ -1101,52 +1126,63 @@ public class PackConfig { * configuration to read properties from. */ public void fromConfig(Config rc) { - setMaxDeltaDepth(rc.getInt("pack", "depth", getMaxDeltaDepth())); //$NON-NLS-1$ //$NON-NLS-2$ - setDeltaSearchWindowSize(rc.getInt( - "pack", "window", getDeltaSearchWindowSize())); //$NON-NLS-1$ //$NON-NLS-2$ - setDeltaSearchMemoryLimit(rc.getLong( - "pack", "windowmemory", getDeltaSearchMemoryLimit())); //$NON-NLS-1$ //$NON-NLS-2$ - setDeltaCacheSize(rc.getLong( - "pack", "deltacachesize", getDeltaCacheSize())); //$NON-NLS-1$ //$NON-NLS-2$ - setDeltaCacheLimit(rc.getInt( - "pack", "deltacachelimit", getDeltaCacheLimit())); //$NON-NLS-1$ //$NON-NLS-2$ - setCompressionLevel(rc.getInt("pack", "compression", //$NON-NLS-1$ //$NON-NLS-2$ - rc.getInt("core", "compression", getCompressionLevel()))); //$NON-NLS-1$ //$NON-NLS-2$ - setIndexVersion(rc.getInt("pack", "indexversion", getIndexVersion())); //$NON-NLS-1$ //$NON-NLS-2$ - setBigFileThreshold(rc.getInt( - "core", "bigfilethreshold", getBigFileThreshold())); //$NON-NLS-1$ //$NON-NLS-2$ - setThreads(rc.getInt("pack", "threads", getThreads())); //$NON-NLS-1$ //$NON-NLS-2$ + setMaxDeltaDepth(rc.getInt(CONFIG_PACK_SECTION, CONFIG_KEY_DEPTH, + getMaxDeltaDepth())); + setDeltaSearchWindowSize(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_WINDOW, getDeltaSearchWindowSize())); + setDeltaSearchMemoryLimit(rc.getLong(CONFIG_PACK_SECTION, + CONFIG_KEY_WINDOW_MEMORY, getDeltaSearchMemoryLimit())); + setDeltaCacheSize(rc.getLong(CONFIG_PACK_SECTION, + CONFIG_KEY_DELTA_CACHE_SIZE, getDeltaCacheSize())); + setDeltaCacheLimit(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_DELTA_CACHE_LIMIT, getDeltaCacheLimit())); + setCompressionLevel(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_COMPRESSION, rc.getInt(CONFIG_CORE_SECTION, + CONFIG_KEY_COMPRESSION, getCompressionLevel()))); + setIndexVersion(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_INDEXVERSION, + getIndexVersion())); + setBigFileThreshold(rc.getInt(CONFIG_CORE_SECTION, + CONFIG_KEY_BIGFILE_THRESHOLD, getBigFileThreshold())); + setThreads(rc.getInt(CONFIG_PACK_SECTION, CONFIG_KEY_THREADS, + getThreads())); // These variables aren't standardized - // - setReuseDeltas(rc.getBoolean("pack", "reusedeltas", isReuseDeltas())); //$NON-NLS-1$ //$NON-NLS-2$ - setReuseObjects( - rc.getBoolean("pack", "reuseobjects", isReuseObjects())); //$NON-NLS-1$ //$NON-NLS-2$ - setDeltaCompress( - rc.getBoolean("pack", "deltacompression", isDeltaCompress())); //$NON-NLS-1$ //$NON-NLS-2$ - setCutDeltaChains( - rc.getBoolean("pack", "cutdeltachains", getCutDeltaChains())); //$NON-NLS-1$ //$NON-NLS-2$ - setSinglePack( - rc.getBoolean("pack", "singlepack", getSinglePack())); //$NON-NLS-1$ //$NON-NLS-2$ - setBuildBitmaps( - rc.getBoolean("pack", "buildbitmaps", isBuildBitmaps())); //$NON-NLS-1$ //$NON-NLS-2$ - setBitmapContiguousCommitCount( - rc.getInt("pack", "bitmapcontiguouscommitcount", //$NON-NLS-1$ //$NON-NLS-2$ - getBitmapContiguousCommitCount())); - setBitmapRecentCommitCount(rc.getInt("pack", "bitmaprecentcommitcount", //$NON-NLS-1$ //$NON-NLS-2$ + setReuseDeltas(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_REUSE_DELTAS, isReuseDeltas())); + setReuseObjects(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_REUSE_OBJECTS, isReuseObjects())); + setDeltaCompress(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_DELTA_COMPRESSION, isDeltaCompress())); + setCutDeltaChains(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_CUT_DELTACHAINS, getCutDeltaChains())); + setSinglePack(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_SINGLE_PACK, + getSinglePack())); + setBuildBitmaps(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_BUILD_BITMAPS, isBuildBitmaps())); + setBitmapContiguousCommitCount(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_BITMAP_CONTIGUOUS_COMMIT_COUNT, + getBitmapContiguousCommitCount())); + setBitmapRecentCommitCount(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT, getBitmapRecentCommitCount())); - setBitmapRecentCommitSpan(rc.getInt("pack", "bitmaprecentcommitspan", //$NON-NLS-1$ //$NON-NLS-2$ + setBitmapRecentCommitSpan(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_BITMAP_RECENT_COMMIT_COUNT, getBitmapRecentCommitSpan())); - setBitmapDistantCommitSpan(rc.getInt("pack", "bitmapdistantcommitspan", //$NON-NLS-1$ //$NON-NLS-2$ + setBitmapDistantCommitSpan(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN, getBitmapDistantCommitSpan())); - setBitmapExcessiveBranchCount(rc.getInt("pack", //$NON-NLS-1$ - "bitmapexcessivebranchcount", getBitmapExcessiveBranchCount())); //$NON-NLS-1$ - setBitmapInactiveBranchAgeInDays( - rc.getInt("pack", "bitmapinactivebranchageindays", //$NON-NLS-1$ //$NON-NLS-2$ - getBitmapInactiveBranchAgeInDays())); - setWaitPreventRacyPack(rc.getBoolean("pack", "waitpreventracypack", //$NON-NLS-1$ //$NON-NLS-2$ - isWaitPreventRacyPack())); - setMinSizePreventRacyPack(rc.getLong("pack", "minsizepreventracypack", //$NON-NLS-1$//$NON-NLS-2$ + setBitmapExcessiveBranchCount(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_BITMAP_EXCESSIVE_BRANCH_COUNT, + getBitmapExcessiveBranchCount())); + setBitmapInactiveBranchAgeInDays(rc.getInt(CONFIG_PACK_SECTION, + CONFIG_KEY_BITMAP_INACTIVE_BRANCH_AGE_INDAYS, + getBitmapInactiveBranchAgeInDays())); + setWaitPreventRacyPack(rc.getBoolean(CONFIG_PACK_SECTION, + CONFIG_KEY_WAIT_PREVENT_RACYPACK, isWaitPreventRacyPack())); + setMinSizePreventRacyPack(rc.getLong(CONFIG_PACK_SECTION, + CONFIG_KEY_MIN_SIZE_PREVENT_RACYPACK, getMinSizePreventRacyPack())); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/CredentialsProviderUserInfo.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/CredentialsProviderUserInfo.java deleted file mode 100644 index 10646b9e7a..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/CredentialsProviderUserInfo.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (C) 2010, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.transport; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.jcraft.jsch.Session; -import com.jcraft.jsch.UIKeyboardInteractive; -import com.jcraft.jsch.UserInfo; - -/** - * A JSch {@link com.jcraft.jsch.UserInfo} adapter for a - * {@link org.eclipse.jgit.transport.CredentialsProvider}. - */ -public class CredentialsProviderUserInfo implements UserInfo, - UIKeyboardInteractive { - private final URIish uri; - - private final CredentialsProvider provider; - - private String password; - - private String passphrase; - - /** - * Wrap a CredentialsProvider to make it suitable for use with JSch. - * - * @param session - * the JSch session this UserInfo will support authentication on. - * @param credentialsProvider - * the provider that will perform the authentication. - */ - public CredentialsProviderUserInfo(Session session, - CredentialsProvider credentialsProvider) { - this.uri = createURI(session); - this.provider = credentialsProvider; - } - - private static URIish createURI(Session session) { - URIish uri = new URIish(); - uri = uri.setScheme("ssh"); //$NON-NLS-1$ - uri = uri.setUser(session.getUserName()); - uri = uri.setHost(session.getHost()); - uri = uri.setPort(session.getPort()); - return uri; - } - - /** {@inheritDoc} */ - @Override - public String getPassword() { - return password; - } - - /** {@inheritDoc} */ - @Override - public String getPassphrase() { - return passphrase; - } - - /** {@inheritDoc} */ - @Override - public boolean promptPassphrase(String msg) { - CredentialItem.StringType v = newPrompt(msg); - if (provider.get(uri, v)) { - passphrase = v.getValue(); - return true; - } - passphrase = null; - return false; - } - - /** {@inheritDoc} */ - @Override - public boolean promptPassword(String msg) { - CredentialItem.Password p = new CredentialItem.Password(msg); - if (provider.get(uri, p)) { - password = new String(p.getValue()); - return true; - } - password = null; - return false; - } - - private CredentialItem.StringType newPrompt(String msg) { - return new CredentialItem.StringType(msg, true); - } - - /** {@inheritDoc} */ - @Override - public boolean promptYesNo(String msg) { - CredentialItem.YesNoType v = new CredentialItem.YesNoType(msg); - return provider.get(uri, v) && v.getValue(); - } - - /** {@inheritDoc} */ - @Override - public void showMessage(String msg) { - provider.get(uri, new CredentialItem.InformationalMessage(msg)); - } - - /** {@inheritDoc} */ - @Override - public String[] promptKeyboardInteractive(String destination, String name, - String instruction, String[] prompt, boolean[] echo) { - CredentialItem.StringType[] v = new CredentialItem.StringType[prompt.length]; - for (int i = 0; i < prompt.length; i++) - v[i] = new CredentialItem.StringType(prompt[i], !echo[i]); - - List<CredentialItem> items = new ArrayList<>(); - if (instruction != null && instruction.length() > 0) - items.add(new CredentialItem.InformationalMessage(instruction)); - items.addAll(Arrays.asList(v)); - - if (!provider.get(uri, items)) - return null; // cancel - - String[] result = new String[v.length]; - for (int i = 0; i < v.length; i++) - result[i] = v[i].getValue(); - return result; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/DefaultSshSessionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/DefaultSshSessionFactory.java deleted file mode 100644 index afa0a11c24..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/DefaultSshSessionFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2009, Constantine Plotnikov <constantine.plotnikov@gmail.com> - * Copyright (C) 2009, Google Inc. - * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.transport; - -import com.jcraft.jsch.Session; - -/** - * Loads known hosts and private keys from <code>$HOME/.ssh</code>. - * <p> - * This is the default implementation used by JGit and provides most of the - * compatibility necessary to match OpenSSH, a popular implementation of SSH - * used by C Git. - * <p> - * If user interactivity is required by SSH (e.g. to obtain a password), the - * connection will immediately fail. - * - * @since 5.7 - */ -public class DefaultSshSessionFactory extends JschConfigSessionFactory { - /** {@inheritDoc} */ - @Override - protected void configure(OpenSshConfig.Host hc, Session session) { - // No additional configuration required. - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschConfigSessionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschConfigSessionFactory.java deleted file mode 100644 index 718c8f6115..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschConfigSessionFactory.java +++ /dev/null @@ -1,530 +0,0 @@ -/* - * Copyright (C) 2018, Sasa Zivkov <sasa.zivkov@sap.com> - * Copyright (C) 2016, Mark Ingram <markdingram@gmail.com> - * Copyright (C) 2009, Constantine Plotnikov <constantine.plotnikov@gmail.com> - * Copyright (C) 2008-2009, Google Inc. - * Copyright (C) 2009, Google, Inc. - * Copyright (C) 2009, JetBrains s.r.o. - * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.transport; - -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.ConnectException; -import java.net.UnknownHostException; -import java.text.MessageFormat; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -import org.eclipse.jgit.errors.TransportException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.util.FS; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.jcraft.jsch.ConfigRepository; -import com.jcraft.jsch.ConfigRepository.Config; -import com.jcraft.jsch.HostKey; -import com.jcraft.jsch.HostKeyRepository; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.Session; - -/** - * The base session factory that loads known hosts and private keys from - * <code>$HOME/.ssh</code>. - * <p> - * This is the default implementation used by JGit and provides most of the - * compatibility necessary to match OpenSSH, a popular implementation of SSH - * used by C Git. - * <p> - * The factory does not provide UI behavior. Override the method - * {@link #configure(org.eclipse.jgit.transport.OpenSshConfig.Host, Session)} to - * supply appropriate {@link com.jcraft.jsch.UserInfo} to the session. - */ -public abstract class JschConfigSessionFactory extends SshSessionFactory { - - private static final Logger LOG = LoggerFactory - .getLogger(JschConfigSessionFactory.class); - - /** - * We use different Jsch instances for hosts that have an IdentityFile - * configured in ~/.ssh/config. Jsch by default would cache decrypted keys - * only per session, which results in repeated password prompts. Using - * different Jsch instances, we can cache the keys on these instances so - * that they will be re-used for successive sessions, and thus the user is - * prompted for a key password only once while Eclipse runs. - */ - private final Map<String, JSch> byIdentityFile = new HashMap<>(); - - private JSch defaultJSch; - - private OpenSshConfig config; - - /** {@inheritDoc} */ - @Override - public synchronized RemoteSession getSession(URIish uri, - CredentialsProvider credentialsProvider, FS fs, int tms) - throws TransportException { - - String user = uri.getUser(); - final String pass = uri.getPass(); - String host = uri.getHost(); - int port = uri.getPort(); - - try { - if (config == null) - config = OpenSshConfig.get(fs); - - final OpenSshConfig.Host hc = config.lookup(host); - if (port <= 0) - port = hc.getPort(); - if (user == null) - user = hc.getUser(); - - Session session = createSession(credentialsProvider, fs, user, - pass, host, port, hc); - - int retries = 0; - while (!session.isConnected()) { - try { - retries++; - session.connect(tms); - } catch (JSchException e) { - session.disconnect(); - session = null; - // Make sure our known_hosts is not outdated - knownHosts(getJSch(hc, fs), fs); - - if (isAuthenticationCanceled(e)) { - throw e; - } else if (isAuthenticationFailed(e) - && credentialsProvider != null) { - // if authentication failed maybe credentials changed at - // the remote end therefore reset credentials and retry - if (retries < 3) { - credentialsProvider.reset(uri); - session = createSession(credentialsProvider, fs, - user, pass, host, port, hc); - } else - throw e; - } else if (retries >= hc.getConnectionAttempts()) { - throw e; - } else { - try { - Thread.sleep(1000); - session = createSession(credentialsProvider, fs, - user, pass, host, port, hc); - } catch (InterruptedException e1) { - throw new TransportException( - JGitText.get().transportSSHRetryInterrupt, - e1); - } - } - } - } - - return new JschSession(session, uri); - - } catch (JSchException je) { - final Throwable c = je.getCause(); - if (c instanceof UnknownHostException) { - throw new TransportException(uri, JGitText.get().unknownHost, - je); - } - if (c instanceof ConnectException) { - throw new TransportException(uri, c.getMessage(), je); - } - throw new TransportException(uri, je.getMessage(), je); - } - - } - - private static boolean isAuthenticationFailed(JSchException e) { - return e.getCause() == null && e.getMessage().equals("Auth fail"); //$NON-NLS-1$ - } - - private static boolean isAuthenticationCanceled(JSchException e) { - return e.getCause() == null && e.getMessage().equals("Auth cancel"); //$NON-NLS-1$ - } - - // Package visibility for tests - Session createSession(CredentialsProvider credentialsProvider, - FS fs, String user, final String pass, String host, int port, - final OpenSshConfig.Host hc) throws JSchException { - final Session session = createSession(hc, user, host, port, fs); - // Jsch will have overridden the explicit user by the one from the SSH - // config file... - setUserName(session, user); - // Jsch will also have overridden the port. - if (port > 0 && port != session.getPort()) { - session.setPort(port); - } - // We retry already in getSession() method. JSch must not retry - // on its own. - session.setConfig("MaxAuthTries", "1"); //$NON-NLS-1$ //$NON-NLS-2$ - if (pass != null) - session.setPassword(pass); - final String strictHostKeyCheckingPolicy = hc - .getStrictHostKeyChecking(); - if (strictHostKeyCheckingPolicy != null) - session.setConfig("StrictHostKeyChecking", //$NON-NLS-1$ - strictHostKeyCheckingPolicy); - final String pauth = hc.getPreferredAuthentications(); - if (pauth != null) - session.setConfig("PreferredAuthentications", pauth); //$NON-NLS-1$ - if (credentialsProvider != null - && (!hc.isBatchMode() || !credentialsProvider.isInteractive())) { - session.setUserInfo(new CredentialsProviderUserInfo(session, - credentialsProvider)); - } - safeConfig(session, hc.getConfig()); - if (hc.getConfig().getValue("HostKeyAlgorithms") == null) { //$NON-NLS-1$ - setPreferredKeyTypesOrder(session); - } - configure(hc, session); - return session; - } - - private void safeConfig(Session session, Config cfg) { - // Ensure that Jsch checks all configured algorithms, not just its - // built-in ones. Otherwise it may propose an algorithm for which it - // doesn't have an implementation, and then run into an NPE if that - // algorithm ends up being chosen. - copyConfigValueToSession(session, cfg, "Ciphers", "CheckCiphers"); //$NON-NLS-1$ //$NON-NLS-2$ - copyConfigValueToSession(session, cfg, "KexAlgorithms", "CheckKexes"); //$NON-NLS-1$ //$NON-NLS-2$ - copyConfigValueToSession(session, cfg, "HostKeyAlgorithms", //$NON-NLS-1$ - "CheckSignatures"); //$NON-NLS-1$ - } - - private static void setPreferredKeyTypesOrder(Session session) { - HostKeyRepository hkr = session.getHostKeyRepository(); - HostKey[] hostKeys = hkr.getHostKey(hostName(session), null); - - if (hostKeys == null) { - return; - } - - List<String> known = Stream.of(hostKeys) - .map(HostKey::getType) - .collect(toList()); - - if (!known.isEmpty()) { - String serverHostKey = "server_host_key"; //$NON-NLS-1$ - String current = session.getConfig(serverHostKey); - if (current == null) { - session.setConfig(serverHostKey, String.join(",", known)); //$NON-NLS-1$ - return; - } - - String knownFirst = Stream.concat( - known.stream(), - Stream.of(current.split(",")) //$NON-NLS-1$ - .filter(s -> !known.contains(s))) - .collect(joining(",")); //$NON-NLS-1$ - session.setConfig(serverHostKey, knownFirst); - } - } - - private static String hostName(Session s) { - if (s.getPort() == SshConstants.SSH_DEFAULT_PORT) { - return s.getHost(); - } - return String.format("[%s]:%d", s.getHost(), //$NON-NLS-1$ - Integer.valueOf(s.getPort())); - } - - private void copyConfigValueToSession(Session session, Config cfg, - String from, String to) { - String value = cfg.getValue(from); - if (value != null) { - session.setConfig(to, value); - } - } - - private void setUserName(Session session, String userName) { - // Jsch 0.1.54 picks up the user name from the ssh config, even if an - // explicit user name was given! We must correct that if ~/.ssh/config - // has a different user name. - if (userName == null || userName.isEmpty() - || userName.equals(session.getUserName())) { - return; - } - try { - Class<?>[] parameterTypes = { String.class }; - Method method = Session.class.getDeclaredMethod("setUserName", //$NON-NLS-1$ - parameterTypes); - method.setAccessible(true); - method.invoke(session, userName); - } catch (NullPointerException | IllegalAccessException - | IllegalArgumentException | InvocationTargetException - | NoSuchMethodException | SecurityException e) { - LOG.error(MessageFormat.format(JGitText.get().sshUserNameError, - userName, session.getUserName()), e); - } - } - - /** - * Create a new remote session for the requested address. - * - * @param hc - * host configuration - * @param user - * login to authenticate as. - * @param host - * server name to connect to. - * @param port - * port number of the SSH daemon (typically 22). - * @param fs - * the file system abstraction which will be necessary to - * perform certain file system operations. - * @return new session instance, but otherwise unconfigured. - * @throws com.jcraft.jsch.JSchException - * the session could not be created. - */ - protected Session createSession(final OpenSshConfig.Host hc, - final String user, final String host, final int port, FS fs) - throws JSchException { - return getJSch(hc, fs).getSession(user, host, port); - } - - /** - * Provide additional configuration for the JSch instance. This method could - * be overridden to supply a preferred - * {@link com.jcraft.jsch.IdentityRepository}. - * - * @param jsch - * jsch instance - * @since 4.5 - */ - protected void configureJSch(JSch jsch) { - // No additional configuration required. - } - - /** - * Provide additional configuration for the session based on the host - * information. This method could be used to supply - * {@link com.jcraft.jsch.UserInfo}. - * - * @param hc - * host configuration - * @param session - * session to configure - */ - protected abstract void configure(OpenSshConfig.Host hc, Session session); - - /** - * Obtain the JSch used to create new sessions. - * - * @param hc - * host configuration - * @param fs - * the file system abstraction which will be necessary to - * perform certain file system operations. - * @return the JSch instance to use. - * @throws com.jcraft.jsch.JSchException - * the user configuration could not be created. - */ - protected JSch getJSch(OpenSshConfig.Host hc, FS fs) throws JSchException { - if (defaultJSch == null) { - defaultJSch = createDefaultJSch(fs); - if (defaultJSch.getConfigRepository() == null) { - defaultJSch.setConfigRepository( - new JschBugFixingConfigRepository(config)); - } - for (Object name : defaultJSch.getIdentityNames()) - byIdentityFile.put((String) name, defaultJSch); - } - - final File identityFile = hc.getIdentityFile(); - if (identityFile == null) - return defaultJSch; - - final String identityKey = identityFile.getAbsolutePath(); - JSch jsch = byIdentityFile.get(identityKey); - if (jsch == null) { - jsch = new JSch(); - configureJSch(jsch); - if (jsch.getConfigRepository() == null) { - jsch.setConfigRepository(defaultJSch.getConfigRepository()); - } - jsch.setHostKeyRepository(defaultJSch.getHostKeyRepository()); - jsch.addIdentity(identityKey); - byIdentityFile.put(identityKey, jsch); - } - return jsch; - } - - /** - * Create default instance of jsch - * - * @param fs - * the file system abstraction which will be necessary to perform - * certain file system operations. - * @return the new default JSch implementation. - * @throws com.jcraft.jsch.JSchException - * known host keys cannot be loaded. - */ - protected JSch createDefaultJSch(FS fs) throws JSchException { - final JSch jsch = new JSch(); - JSch.setConfig("ssh-rsa", JSch.getConfig("signature.rsa")); //$NON-NLS-1$ //$NON-NLS-2$ - JSch.setConfig("ssh-dss", JSch.getConfig("signature.dss")); //$NON-NLS-1$ //$NON-NLS-2$ - configureJSch(jsch); - knownHosts(jsch, fs); - identities(jsch, fs); - return jsch; - } - - private static void knownHosts(JSch sch, FS fs) throws JSchException { - final File home = fs.userHome(); - if (home == null) - return; - final File known_hosts = new File(new File(home, ".ssh"), "known_hosts"); //$NON-NLS-1$ //$NON-NLS-2$ - try (FileInputStream in = new FileInputStream(known_hosts)) { - sch.setKnownHosts(in); - } catch (FileNotFoundException none) { - // Oh well. They don't have a known hosts in home. - } catch (IOException err) { - // Oh well. They don't have a known hosts in home. - } - } - - private static void identities(JSch sch, FS fs) { - final File home = fs.userHome(); - if (home == null) - return; - final File sshdir = new File(home, ".ssh"); //$NON-NLS-1$ - if (sshdir.isDirectory()) { - loadIdentity(sch, new File(sshdir, "identity")); //$NON-NLS-1$ - loadIdentity(sch, new File(sshdir, "id_rsa")); //$NON-NLS-1$ - loadIdentity(sch, new File(sshdir, "id_dsa")); //$NON-NLS-1$ - } - } - - private static void loadIdentity(JSch sch, File priv) { - if (priv.isFile()) { - try { - sch.addIdentity(priv.getAbsolutePath()); - } catch (JSchException e) { - // Instead, pretend the key doesn't exist. - } - } - } - - private static class JschBugFixingConfigRepository - implements ConfigRepository { - - private final ConfigRepository base; - - public JschBugFixingConfigRepository(ConfigRepository base) { - this.base = base; - } - - @Override - public Config getConfig(String host) { - return new JschBugFixingConfig(base.getConfig(host)); - } - - /** - * A {@link com.jcraft.jsch.ConfigRepository.Config} that transforms - * some values from the config file into the format Jsch 0.1.54 expects. - * This is a work-around for bugs in Jsch. - * <p> - * Additionally, this config hides the IdentityFile config entries from - * Jsch; we manage those ourselves. Otherwise Jsch would cache passwords - * (or rather, decrypted keys) only for a single session, resulting in - * multiple password prompts for user operations that use several Jsch - * sessions. - */ - private static class JschBugFixingConfig implements Config { - - private static final String[] NO_IDENTITIES = {}; - - private final Config real; - - public JschBugFixingConfig(Config delegate) { - real = delegate; - } - - @Override - public String getHostname() { - return real.getHostname(); - } - - @Override - public String getUser() { - return real.getUser(); - } - - @Override - public int getPort() { - return real.getPort(); - } - - @Override - public String getValue(String key) { - String k = key.toUpperCase(Locale.ROOT); - if ("IDENTITYFILE".equals(k)) { //$NON-NLS-1$ - return null; - } - String result = real.getValue(key); - if (result != null) { - if ("SERVERALIVEINTERVAL".equals(k) //$NON-NLS-1$ - || "CONNECTTIMEOUT".equals(k)) { //$NON-NLS-1$ - // These values are in seconds. Jsch 0.1.54 passes them - // on as is to java.net.Socket.setSoTimeout(), which - // expects milliseconds. So convert here to - // milliseconds. - try { - int timeout = Integer.parseInt(result); - result = Long.toString( - TimeUnit.SECONDS.toMillis(timeout)); - } catch (NumberFormatException e) { - // Ignore - } - } - } - return result; - } - - @Override - public String[] getValues(String key) { - String k = key.toUpperCase(Locale.ROOT); - if ("IDENTITYFILE".equals(k)) { //$NON-NLS-1$ - return NO_IDENTITIES; - } - return real.getValues(key); - } - } - } - - /** - * Set the {@link OpenSshConfig} to use. Intended for use in tests. - * - * @param config - * to use - */ - synchronized void setConfig(OpenSshConfig config) { - this.config = config; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschSession.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschSession.java deleted file mode 100644 index d7270343cb..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschSession.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Copyright (C) 2009, Constantine Plotnikov <constantine.plotnikov@gmail.com> - * Copyright (C) 2008-2009, Google Inc. - * Copyright (C) 2009, Google, Inc. - * Copyright (C) 2009, JetBrains s.r.o. - * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.transport; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; - -import org.eclipse.jgit.errors.TransportException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.util.io.IsolatedOutputStream; - -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelExec; -import com.jcraft.jsch.ChannelSftp; -import com.jcraft.jsch.JSchException; -import com.jcraft.jsch.Session; -import com.jcraft.jsch.SftpException; - -/** - * Run remote commands using Jsch. - * <p> - * This class is the default session implementation using Jsch. Note that - * {@link org.eclipse.jgit.transport.JschConfigSessionFactory} is used to create - * the actual session passed to the constructor. - */ -public class JschSession implements RemoteSession { - final Session sock; - final URIish uri; - - /** - * Create a new session object by passing the real Jsch session and the URI - * information. - * - * @param session - * the real Jsch session created elsewhere. - * @param uri - * the URI information for the remote connection - */ - public JschSession(Session session, URIish uri) { - sock = session; - this.uri = uri; - } - - /** {@inheritDoc} */ - @Override - public Process exec(String command, int timeout) throws IOException { - return new JschProcess(command, timeout); - } - - /** {@inheritDoc} */ - @Override - public void disconnect() { - if (sock.isConnected()) - sock.disconnect(); - } - - /** - * A kludge to allow {@link org.eclipse.jgit.transport.TransportSftp} to get - * an Sftp channel from Jsch. Ideally, this method would be generic, which - * would require implementing generic Sftp channel operations in the - * RemoteSession class. - * - * @return a channel suitable for Sftp operations. - * @throws com.jcraft.jsch.JSchException - * on problems getting the channel. - * @deprecated since 5.2; use {@link #getFtpChannel()} instead - */ - @Deprecated - public Channel getSftpChannel() throws JSchException { - return sock.openChannel("sftp"); //$NON-NLS-1$ - } - - /** - * {@inheritDoc} - * - * @since 5.2 - */ - @Override - public FtpChannel getFtpChannel() { - return new JschFtpChannel(); - } - - /** - * Implementation of Process for running a single command using Jsch. - * <p> - * Uses the Jsch session to do actual command execution and manage the - * execution. - */ - private class JschProcess extends Process { - private ChannelExec channel; - - final int timeout; - - private InputStream inputStream; - - private OutputStream outputStream; - - private InputStream errStream; - - /** - * Opens a channel on the session ("sock") for executing the given - * command, opens streams, and starts command execution. - * - * @param commandName - * the command to execute - * @param tms - * the timeout value, in seconds, for the command. - * @throws TransportException - * on problems opening a channel or connecting to the remote - * host - * @throws IOException - * on problems opening streams - */ - JschProcess(String commandName, int tms) - throws TransportException, IOException { - timeout = tms; - try { - channel = (ChannelExec) sock.openChannel("exec"); //$NON-NLS-1$ - channel.setCommand(commandName); - setupStreams(); - channel.connect(timeout > 0 ? timeout * 1000 : 0); - if (!channel.isConnected()) { - closeOutputStream(); - throw new TransportException(uri, - JGitText.get().connectionFailed); - } - } catch (JSchException e) { - closeOutputStream(); - throw new TransportException(uri, e.getMessage(), e); - } - } - - private void closeOutputStream() { - if (outputStream != null) { - try { - outputStream.close(); - } catch (IOException ioe) { - // ignore - } - } - } - - private void setupStreams() throws IOException { - inputStream = channel.getInputStream(); - - // JSch won't let us interrupt writes when we use our InterruptTimer - // to break out of a long-running write operation. To work around - // that we spawn a background thread to shuttle data through a pipe, - // as we can issue an interrupted write out of that. Its slower, so - // we only use this route if there is a timeout. - OutputStream out = channel.getOutputStream(); - if (timeout <= 0) { - outputStream = out; - } else { - IsolatedOutputStream i = new IsolatedOutputStream(out); - outputStream = new BufferedOutputStream(i, 16 * 1024); - } - - errStream = channel.getErrStream(); - } - - @Override - public InputStream getInputStream() { - return inputStream; - } - - @Override - public OutputStream getOutputStream() { - return outputStream; - } - - @Override - public InputStream getErrorStream() { - return errStream; - } - - @Override - public int exitValue() { - if (isRunning()) - throw new IllegalStateException(); - return channel.getExitStatus(); - } - - private boolean isRunning() { - return channel.getExitStatus() < 0 && channel.isConnected(); - } - - @Override - public void destroy() { - if (channel.isConnected()) - channel.disconnect(); - closeOutputStream(); - } - - @Override - public int waitFor() throws InterruptedException { - while (isRunning()) - Thread.sleep(100); - return exitValue(); - } - } - - private class JschFtpChannel implements FtpChannel { - - private ChannelSftp ftp; - - @Override - public void connect(int timeout, TimeUnit unit) throws IOException { - try { - ftp = (ChannelSftp) sock.openChannel("sftp"); //$NON-NLS-1$ - ftp.connect((int) unit.toMillis(timeout)); - } catch (JSchException e) { - ftp = null; - throw new IOException(e.getLocalizedMessage(), e); - } - } - - @Override - public void disconnect() { - ftp.disconnect(); - ftp = null; - } - - private <T> T map(Callable<T> op) throws IOException { - try { - return op.call(); - } catch (Exception e) { - if (e instanceof SftpException) { - throw new FtpChannel.FtpException(e.getLocalizedMessage(), - ((SftpException) e).id, e); - } - throw new IOException(e.getLocalizedMessage(), e); - } - } - - @Override - public boolean isConnected() { - return ftp != null && sock.isConnected(); - } - - @Override - public void cd(String path) throws IOException { - map(() -> { - ftp.cd(path); - return null; - }); - } - - @Override - public String pwd() throws IOException { - return map(() -> ftp.pwd()); - } - - @Override - public Collection<DirEntry> ls(String path) throws IOException { - return map(() -> { - List<DirEntry> result = new ArrayList<>(); - for (Object e : ftp.ls(path)) { - ChannelSftp.LsEntry entry = (ChannelSftp.LsEntry) e; - result.add(new DirEntry() { - - @Override - public String getFilename() { - return entry.getFilename(); - } - - @Override - public long getModifiedTime() { - return entry.getAttrs().getMTime(); - } - - @Override - public boolean isDirectory() { - return entry.getAttrs().isDir(); - } - }); - } - return result; - }); - } - - @Override - public void rmdir(String path) throws IOException { - map(() -> { - ftp.rm(path); - return null; - }); - } - - @Override - public void mkdir(String path) throws IOException { - map(() -> { - ftp.mkdir(path); - return null; - }); - } - - @Override - public InputStream get(String path) throws IOException { - return map(() -> ftp.get(path)); - } - - @Override - public OutputStream put(String path) throws IOException { - return map(() -> ftp.put(path)); - } - - @Override - public void rm(String path) throws IOException { - map(() -> { - ftp.rm(path); - return null; - }); - } - - @Override - public void rename(String from, String to) throws IOException { - map(() -> { - // Plain FTP rename will fail if "to" exists. Jsch knows about - // the FTP extension "posix-rename@openssh.com", which will - // remove "to" first if it exists. - if (hasPosixRename()) { - ftp.rename(from, to); - } else if (!to.equals(from)) { - // Try to remove "to" first. With git, we typically get this - // when a lock file is moved over the file locked. Note that - // the check for to being equal to from may still fail in - // the general case, but for use with JGit's TransportSftp - // it should be good enough. - delete(to); - ftp.rename(from, to); - } - return null; - }); - } - - /** - * Determine whether the server has the posix-rename extension. - * - * @return {@code true} if it is supported, {@code false} otherwise - * @see <a href= - * "https://cvsweb.openbsd.org/src/usr.bin/ssh/PROTOCOL?annotate=HEAD">OpenSSH - * deviations and extensions to the published SSH protocol</a> - * @see <a href= - * "http://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html">stdio.h: - * rename()</a> - */ - private boolean hasPosixRename() { - return "1".equals(ftp.getExtension("posix-rename@openssh.com")); //$NON-NLS-1$//$NON-NLS-2$ - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java deleted file mode 100644 index a628897a59..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Copyright (C) 2008, 2018, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.transport; - -import static org.eclipse.jgit.internal.transport.ssh.OpenSshConfigFile.positive; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import org.eclipse.jgit.internal.transport.ssh.OpenSshConfigFile; -import org.eclipse.jgit.internal.transport.ssh.OpenSshConfigFile.HostEntry; -import org.eclipse.jgit.util.FS; - -import com.jcraft.jsch.ConfigRepository; - -/** - * Fairly complete configuration parser for the OpenSSH ~/.ssh/config file. - * <p> - * JSch does have its own config file parser - * {@link com.jcraft.jsch.OpenSSHConfig} since version 0.1.50, but it has a - * number of problems: - * <ul> - * <li>it splits lines of the format "keyword = value" wrongly: you'd end up - * with the value "= value". - * <li>its "Host" keyword is not case insensitive. - * <li>it doesn't handle quoted values. - * <li>JSch's OpenSSHConfig doesn't monitor for config file changes. - * </ul> - * <p> - * This parser makes the critical options available to - * {@link org.eclipse.jgit.transport.SshSessionFactory} via - * {@link org.eclipse.jgit.transport.OpenSshConfig.Host} objects returned by - * {@link #lookup(String)}, and implements a fully conforming - * {@link com.jcraft.jsch.ConfigRepository} providing - * {@link com.jcraft.jsch.ConfigRepository.Config}s via - * {@link #getConfig(String)}. - * </p> - * - * @see OpenSshConfigFile - */ -public class OpenSshConfig implements ConfigRepository { - - /** - * Obtain the user's configuration data. - * <p> - * The configuration file is always returned to the caller, even if no file - * exists in the user's home directory at the time the call was made. Lookup - * requests are cached and are automatically updated if the user modifies - * the configuration file since the last time it was cached. - * - * @param fs - * the file system abstraction which will be necessary to - * perform certain file system operations. - * @return a caching reader of the user's configuration file. - */ - public static OpenSshConfig get(FS fs) { - File home = fs.userHome(); - if (home == null) - home = new File(".").getAbsoluteFile(); //$NON-NLS-1$ - - final File config = new File(new File(home, SshConstants.SSH_DIR), - SshConstants.CONFIG); - return new OpenSshConfig(home, config); - } - - /** The base file. */ - private OpenSshConfigFile configFile; - - OpenSshConfig(File h, File cfg) { - configFile = new OpenSshConfigFile(h, cfg, - SshSessionFactory.getLocalUserName()); - } - - /** - * Locate the configuration for a specific host request. - * - * @param hostName - * the name the user has supplied to the SSH tool. This may be a - * real host name, or it may just be a "Host" block in the - * configuration file. - * @return r configuration for the requested name. Never null. - */ - public Host lookup(String hostName) { - HostEntry entry = configFile.lookup(hostName, -1, null); - return new Host(entry, hostName, configFile.getLocalUserName()); - } - - /** - * Configuration of one "Host" block in the configuration file. - * <p> - * If returned from {@link OpenSshConfig#lookup(String)} some or all of the - * properties may not be populated. The properties which are not populated - * should be defaulted by the caller. - * <p> - * When returned from {@link OpenSshConfig#lookup(String)} any wildcard - * entries which appear later in the configuration file will have been - * already merged into this block. - */ - public static class Host { - String hostName; - - int port; - - File identityFile; - - String user; - - String preferredAuthentications; - - Boolean batchMode; - - String strictHostKeyChecking; - - int connectionAttempts; - - private HostEntry entry; - - private Config config; - - // See com.jcraft.jsch.OpenSSHConfig. Translates some command-line keys - // to ssh-config keys. - private static final Map<String, String> KEY_MAP = new TreeMap<>( - String.CASE_INSENSITIVE_ORDER); - - static { - KEY_MAP.put("kex", SshConstants.KEX_ALGORITHMS); //$NON-NLS-1$ - KEY_MAP.put("server_host_key", SshConstants.HOST_KEY_ALGORITHMS); //$NON-NLS-1$ - KEY_MAP.put("cipher.c2s", SshConstants.CIPHERS); //$NON-NLS-1$ - KEY_MAP.put("cipher.s2c", SshConstants.CIPHERS); //$NON-NLS-1$ - KEY_MAP.put("mac.c2s", SshConstants.MACS); //$NON-NLS-1$ - KEY_MAP.put("mac.s2c", SshConstants.MACS); //$NON-NLS-1$ - KEY_MAP.put("compression.s2c", SshConstants.COMPRESSION); //$NON-NLS-1$ - KEY_MAP.put("compression.c2s", SshConstants.COMPRESSION); //$NON-NLS-1$ - KEY_MAP.put("compression_level", "CompressionLevel"); //$NON-NLS-1$ //$NON-NLS-2$ - KEY_MAP.put("MaxAuthTries", //$NON-NLS-1$ - SshConstants.NUMBER_OF_PASSWORD_PROMPTS); - } - - private static String mapKey(String key) { - String k = KEY_MAP.get(key); - return k != null ? k : key; - } - - /** - * Creates a new uninitialized {@link Host}. - */ - public Host() { - // For API backwards compatibility with pre-4.9 JGit - } - - Host(HostEntry entry, String hostName, String localUserName) { - this.entry = entry; - complete(hostName, localUserName); - } - - /** - * @return the value StrictHostKeyChecking property, the valid values - * are "yes" (unknown hosts are not accepted), "no" (unknown - * hosts are always accepted), and "ask" (user should be asked - * before accepting the host) - */ - public String getStrictHostKeyChecking() { - return strictHostKeyChecking; - } - - /** - * @return the real IP address or host name to connect to; never null. - */ - public String getHostName() { - return hostName; - } - - /** - * @return the real port number to connect to; never 0. - */ - public int getPort() { - return port; - } - - /** - * @return path of the private key file to use for authentication; null - * if the caller should use default authentication strategies. - */ - public File getIdentityFile() { - return identityFile; - } - - /** - * @return the real user name to connect as; never null. - */ - public String getUser() { - return user; - } - - /** - * @return the preferred authentication methods, separated by commas if - * more than one authentication method is preferred. - */ - public String getPreferredAuthentications() { - return preferredAuthentications; - } - - /** - * @return true if batch (non-interactive) mode is preferred for this - * host connection. - */ - public boolean isBatchMode() { - return batchMode != null && batchMode.booleanValue(); - } - - /** - * @return the number of tries (one per second) to connect before - * exiting. The argument must be an integer. This may be useful - * in scripts if the connection sometimes fails. The default is - * 1. - * @since 3.4 - */ - public int getConnectionAttempts() { - return connectionAttempts; - } - - - private void complete(String initialHostName, String localUserName) { - // Try to set values from the options. - hostName = entry.getValue(SshConstants.HOST_NAME); - user = entry.getValue(SshConstants.USER); - port = positive(entry.getValue(SshConstants.PORT)); - connectionAttempts = positive( - entry.getValue(SshConstants.CONNECTION_ATTEMPTS)); - strictHostKeyChecking = entry - .getValue(SshConstants.STRICT_HOST_KEY_CHECKING); - batchMode = Boolean.valueOf(OpenSshConfigFile - .flag(entry.getValue(SshConstants.BATCH_MODE))); - preferredAuthentications = entry - .getValue(SshConstants.PREFERRED_AUTHENTICATIONS); - // Fill in defaults if still not set - if (hostName == null || hostName.isEmpty()) { - hostName = initialHostName; - } - if (user == null || user.isEmpty()) { - user = localUserName; - } - if (port <= 0) { - port = SshConstants.SSH_DEFAULT_PORT; - } - if (connectionAttempts <= 0) { - connectionAttempts = 1; - } - List<String> identityFiles = entry - .getValues(SshConstants.IDENTITY_FILE); - if (identityFiles != null && !identityFiles.isEmpty()) { - identityFile = new File(identityFiles.get(0)); - } - } - - Config getConfig() { - if (config == null) { - config = new Config() { - - @Override - public String getHostname() { - return Host.this.getHostName(); - } - - @Override - public String getUser() { - return Host.this.getUser(); - } - - @Override - public int getPort() { - return Host.this.getPort(); - } - - @Override - public String getValue(String key) { - // See com.jcraft.jsch.OpenSSHConfig.MyConfig.getValue() - // for this special case. - if (key.equals("compression.s2c") //$NON-NLS-1$ - || key.equals("compression.c2s")) { //$NON-NLS-1$ - if (!OpenSshConfigFile.flag( - Host.this.entry.getValue(mapKey(key)))) { - return "none,zlib@openssh.com,zlib"; //$NON-NLS-1$ - } - return "zlib@openssh.com,zlib,none"; //$NON-NLS-1$ - } - return Host.this.entry.getValue(mapKey(key)); - } - - @Override - public String[] getValues(String key) { - List<String> values = Host.this.entry - .getValues(mapKey(key)); - if (values == null) { - return new String[0]; - } - return values.toArray(new String[0]); - } - }; - } - return config; - } - - @Override - @SuppressWarnings("nls") - public String toString() { - return "Host [hostName=" + hostName + ", port=" + port - + ", identityFile=" + identityFile + ", user=" + user - + ", preferredAuthentications=" + preferredAuthentications - + ", batchMode=" + batchMode + ", strictHostKeyChecking=" - + strictHostKeyChecking + ", connectionAttempts=" - + connectionAttempts + ", entry=" + entry + "]"; - } - } - - /** - * {@inheritDoc} - * <p> - * Retrieves the full {@link com.jcraft.jsch.ConfigRepository.Config Config} - * for the given host name. Should be called only by Jsch and tests. - * - * @since 4.9 - */ - @Override - public Config getConfig(String hostName) { - Host host = lookup(hostName); - return host.getConfig(); - } - - /** {@inheritDoc} */ - @Override - public String toString() { - return "OpenSshConfig [configFile=" + configFile + ']'; //$NON-NLS-1$ - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java index 52a5576e43..350311ecc8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2010, Google Inc. - * Copyright (C) 2008-2009, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2010 Google Inc. + * Copyright (C) 2008, 2009 Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -50,7 +50,7 @@ public class PacketLineIn { * strings in the input stream until the marker is reached. */ @Deprecated - public static final String END = new StringBuilder(0).toString(); /* must not string pool */ + public static final String END = new String(); /* must not string pool */ /** * Magic return from {@link #readString()} when a delim packet is found. @@ -60,7 +60,7 @@ public class PacketLineIn { * string is the delimiter. */ @Deprecated - public static final String DELIM = new StringBuilder(0).toString(); /* must not string pool */ + public static final String DELIM = new String(); /* must not string pool */ enum AckNackResult { /** NAK */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index ec2b76938e..79f60c3202 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java @@ -48,6 +48,7 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackLock; import org.eclipse.jgit.internal.submodule.SubmoduleValidator; import org.eclipse.jgit.internal.submodule.SubmoduleValidator.SubmoduleValidationException; +import org.eclipse.jgit.internal.transport.connectivity.FullConnectivityChecker; import org.eclipse.jgit.internal.transport.parser.FirstCommand; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BatchRefUpdate; @@ -72,7 +73,6 @@ import org.eclipse.jgit.transport.ConnectivityChecker.ConnectivityCheckInfo; import org.eclipse.jgit.transport.PacketLineIn.InputOverLimitIOException; import org.eclipse.jgit.transport.ReceiveCommand.Result; import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser; -import org.eclipse.jgit.transport.internal.FullConnectivityChecker; import org.eclipse.jgit.util.io.InterruptTimer; import org.eclipse.jgit.util.io.LimitedInputStream; import org.eclipse.jgit.util.io.TimeoutInputStream; @@ -1814,55 +1814,59 @@ public class ReceivePack { .append(" ("); //$NON-NLS-1$ } - switch (cmd.getResult()) { - case NOT_ATTEMPTED: - r.append("server bug; ref not processed"); //$NON-NLS-1$ - break; - - case REJECTED_NOCREATE: - r.append("creation prohibited"); //$NON-NLS-1$ - break; - - case REJECTED_NODELETE: - r.append("deletion prohibited"); //$NON-NLS-1$ - break; - - case REJECTED_NONFASTFORWARD: - r.append("non-fast forward"); //$NON-NLS-1$ - break; - - case REJECTED_CURRENT_BRANCH: - r.append("branch is currently checked out"); //$NON-NLS-1$ - break; - - case REJECTED_MISSING_OBJECT: + if (cmd.getResult() == Result.REJECTED_MISSING_OBJECT) { if (cmd.getMessage() == null) r.append("missing object(s)"); //$NON-NLS-1$ else if (cmd.getMessage() .length() == Constants.OBJECT_ID_STRING_LENGTH) { + // TODO: Using get/setMessage to store an OID is a + // misuse. The caller should set a full error message. r.append("object "); //$NON-NLS-1$ r.append(cmd.getMessage()); r.append(" missing"); //$NON-NLS-1$ - } else + } else { r.append(cmd.getMessage()); - break; + } + } else if (cmd.getMessage() != null) { + r.append(cmd.getMessage()); + } else { + switch (cmd.getResult()) { + case NOT_ATTEMPTED: + r.append("server bug; ref not processed"); //$NON-NLS-1$ + break; - case REJECTED_OTHER_REASON: - if (cmd.getMessage() == null) + case REJECTED_NOCREATE: + r.append("creation prohibited"); //$NON-NLS-1$ + break; + + case REJECTED_NODELETE: + r.append("deletion prohibited"); //$NON-NLS-1$ + break; + + case REJECTED_NONFASTFORWARD: + r.append("non-fast forward"); //$NON-NLS-1$ + break; + + case REJECTED_CURRENT_BRANCH: + r.append("branch is currently checked out"); //$NON-NLS-1$ + break; + + case REJECTED_OTHER_REASON: r.append("unspecified reason"); //$NON-NLS-1$ - else - r.append(cmd.getMessage()); - break; + break; - case LOCK_FAILURE: - r.append("failed to lock"); //$NON-NLS-1$ - break; + case LOCK_FAILURE: + r.append("failed to lock"); //$NON-NLS-1$ + break; - case OK: - // We shouldn't have reached this case (see 'ok' case - // above). - continue; + case REJECTED_MISSING_OBJECT: + case OK: + // We shouldn't have reached this case (see 'ok' case + // above and if-statement above). + throw new AssertionError(); + } } + if (!reportStatus) { r.append(")"); //$NON-NLS-1$ } @@ -2057,6 +2061,16 @@ public class ReceivePack { } /** + * Get the current unpack error handler. + * + * @return the current unpack error handler. + * @since 5.8 + */ + public UnpackErrorHandler getUnpackErrorHandler() { + return unpackErrorHandler; + } + + /** * @param unpackErrorHandler * the unpackErrorHandler to set * @since 5.7 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshConfigStore.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshConfigStore.java new file mode 100644 index 0000000000..04a4922bb9 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshConfigStore.java @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2020, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.annotations.NonNull; + +/** + * An abstraction for a SSH config storage, like the OpenSSH ~/.ssh/config file. + * + * @since 5.8 + */ +public interface SshConfigStore { + + /** + * Locate the configuration for a specific host request. + * + * @param hostName + * to look up + * @param port + * the user supplied; <= 0 if none + * @param userName + * the user supplied, may be {@code null} or empty if none given + * @return the configuration for the requested name. + */ + @NonNull + HostConfig lookup(@NonNull String hostName, int port, String userName); + + /** + * A host entry from the ssh config. Any merging of global values and of + * several matching host entries, %-substitutions, and ~ replacement have + * all been done. + */ + interface HostConfig { + + /** + * Retrieves the value of a single-valued key, or the first if the key + * has multiple values. Keys are case-insensitive, so + * {@code getValue("HostName") == getValue("HOSTNAME")}. + * + * @param key + * to get the value of + * @return the value, or {@code null} if none + */ + String getValue(String key); + + /** + * Retrieves the values of a multi- or list-valued key. Keys are + * case-insensitive, so + * {@code getValue("HostName") == getValue("HOSTNAME")}. + * + * @param key + * to get the values of + * @return a possibly empty list of values + */ + List<String> getValues(String key); + + /** + * Retrieves an unmodifiable map of all single-valued options, with + * case-insensitive lookup by keys. + * + * @return all single-valued options + */ + @NonNull + Map<String, String> getOptions(); + + /** + * Retrieves an unmodifiable map of all multi- or list-valued options, + * with case-insensitive lookup by keys. + * + * @return all multi-valued options + */ + @NonNull + Map<String, List<String>> getMultiValuedOptions(); + + } + + /** + * An empty {@link HostConfig}. + */ + static final HostConfig EMPTY_CONFIG = new HostConfig() { + + @Override + public String getValue(String key) { + return null; + } + + @Override + public List<String> getValues(String key) { + return Collections.emptyList(); + } + + @Override + public Map<String, String> getOptions() { + return Collections.emptyMap(); + } + + @Override + public Map<String, List<String>> getMultiValuedOptions() { + return Collections.emptyMap(); + } + + }; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java index e6d2042422..ef845f4dce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java @@ -43,11 +43,12 @@ public abstract class SshSessionFactory { } return null; } + /** * Get the currently configured JVM-wide factory. * <p> - * A factory is always available. By default the factory will read from the - * user's <code>$HOME/.ssh</code> and assume OpenSSH compatibility. + * By default the factory will read from the user's <code>$HOME/.ssh</code> + * and assume OpenSSH compatibility. * * @return factory the current factory for this JVM. */ @@ -60,7 +61,7 @@ public abstract class SshSessionFactory { * * @param newFactory * factory for future sessions to be created through. If null the - * default factory will be restored.s + * default factory will be restored. */ public static void setInstance(SshSessionFactory newFactory) { if (newFactory != null) { @@ -110,6 +111,15 @@ public abstract class SshSessionFactory { throws TransportException; /** + * The name of the type of session factory. + * + * @return the name of the type of session factory. + * + * @since 5.8 + */ + public abstract String getType(); + + /** * Close (or recycle) a session to a host. * * @param session diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java index 947c4c3222..b9cb2484d8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java @@ -50,6 +50,8 @@ import org.eclipse.jgit.util.io.StreamCopyThread; * enumeration, save file modification and hook execution. */ public class TransportGitSsh extends SshTransport implements PackTransport { + private static final String EXT = "ext"; //$NON-NLS-1$ + static final TransportProtocol PROTO_SSH = new TransportProtocol() { private final String[] schemeNames = { "ssh", "ssh+git", "git+ssh" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ @@ -127,6 +129,11 @@ public class TransportGitSsh extends SshTransport implements PackTransport { throws TransportException { return new ExtSession(); } + + @Override + public String getType() { + return EXT; + } }); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 356f88d918..16169f028b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -39,11 +39,13 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.InterruptedIOException; import java.io.OutputStream; import java.net.HttpCookie; import java.net.MalformedURLException; import java.net.Proxy; import java.net.ProxySelector; +import java.net.SocketException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -549,6 +551,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, case HttpConnection.HTTP_MOVED_PERM: case HttpConnection.HTTP_MOVED_TEMP: case HttpConnection.HTTP_SEE_OTHER: + case HttpConnection.HTTP_11_MOVED_PERM: case HttpConnection.HTTP_11_MOVED_TEMP: // SEE_OTHER should actually never be sent by a git server, // and in general should occur only on POST requests. But it @@ -572,6 +575,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } } catch (NotSupportedException | TransportException e) { throw e; + } catch (InterruptedIOException e) { + // Timeout!? Don't try other authentication methods. + throw new TransportException(uri, MessageFormat.format( + JGitText.get().connectionTimeOut, u.getHost()), e); + } catch (SocketException e) { + // Nothing on other end, timeout, connection reset, ... + throw new TransportException(uri, + JGitText.get().connectionFailed, e); } catch (SSLHandshakeException e) { handleSslFailure(e); continue; // Re-try @@ -1412,6 +1423,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, case HttpConnection.HTTP_MOVED_PERM: case HttpConnection.HTTP_MOVED_TEMP: + case HttpConnection.HTTP_11_MOVED_PERM: case HttpConnection.HTTP_11_MOVED_TEMP: // SEE_OTHER after a POST doesn't make sense for a git // server, so we don't handle it here and thus we'll @@ -1499,6 +1511,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } catch (SSLHandshakeException e) { handleSslFailure(e); continue; // Re-try + } catch (SocketException | InterruptedIOException e) { + // Timeout!? Must propagate; don't try other authentication + // methods. + throw e; } catch (IOException e) { if (authenticator == null || authMethod .getType() != HttpAuthMethod.Type.NONE) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java index 06520ec4ca..c9bb89a436 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java @@ -738,6 +738,12 @@ public class URIish implements Serializable { else if (result.endsWith(Constants.DOT_GIT_EXT)) result = result.substring(0, result.length() - Constants.DOT_GIT_EXT.length()); + if (("file".equals(scheme) || LOCAL_FILE.matcher(s) //$NON-NLS-1$ + .matches()) + && result.endsWith(Constants.DOT_BUNDLE_EXT)) { + result = result.substring(0, + result.length() - Constants.DOT_BUNDLE_EXT.length()); + } return result; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 35196c6e34..9889015261 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -66,8 +66,6 @@ import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.internal.transport.parser.FirstWant; -import org.eclipse.jgit.lib.BitmapIndex; -import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -77,15 +75,14 @@ import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.AsyncRevObjectQueue; -import org.eclipse.jgit.revwalk.BitmapWalker; import org.eclipse.jgit.revwalk.DepthWalk; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.ReachabilityChecker; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevFlag; import org.eclipse.jgit.revwalk.RevFlagSet; import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevSort; import org.eclipse.jgit.revwalk.RevTag; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter; @@ -298,7 +295,7 @@ public class UploadPack { private boolean sentReady; - /** Objects we sent in our advertisement list, clients can ask for these. */ + /** Objects we sent in our advertisement list. */ private Set<ObjectId> advertised; /** Marked on objects the client has asked us to give them. */ @@ -381,8 +378,10 @@ public class UploadPack { /** * Get refs which were advertised to the client. * - * @return all refs which were advertised to the client, or null if - * {@link #setAdvertisedRefs(Map)} has not been called yet. + * @return all refs which were advertised to the client. Only valid during + * the negotiation phase. Will return {@code null} if + * {@link #setAdvertisedRefs(Map)} has not been called yet or if + * {@code #sendPack()} has been called. */ public final Map<String, Ref> getAdvertisedRefs() { return refs; @@ -1896,48 +1895,6 @@ public class UploadPack { } } - private static void checkNotAdvertisedWantsUsingBitmap(ObjectReader reader, - BitmapIndex bitmapIndex, List<ObjectId> notAdvertisedWants, - Set<ObjectId> reachableFrom) throws IOException { - BitmapWalker bitmapWalker = new BitmapWalker(new ObjectWalk(reader), bitmapIndex, null); - BitmapBuilder reachables = bitmapWalker.findObjects(reachableFrom, null, false); - for (ObjectId oid : notAdvertisedWants) { - if (!reachables.contains(oid)) { - throw new WantNotValidException(oid); - } - } - } - - private static void checkReachabilityByWalkingObjects(ObjectWalk walk, - List<RevObject> wants, Set<ObjectId> reachableFrom) throws IOException { - - walk.sort(RevSort.TOPO); - for (RevObject want : wants) { - walk.markStart(want); - } - for (ObjectId have : reachableFrom) { - RevObject o = walk.parseAny(have); - walk.markUninteresting(o); - - RevObject peeled = walk.peel(o); - if (peeled instanceof RevCommit) { - // By default, for performance reasons, ObjectWalk does not mark a - // tree as uninteresting when we mark a commit. Mark it ourselves so - // that we can determine reachability exactly. - walk.markUninteresting(((RevCommit) peeled).getTree()); - } - } - - RevCommit commit = walk.next(); - if (commit != null) { - throw new WantNotValidException(commit); - } - RevObject object = walk.nextObject(); - if (object != null) { - throw new WantNotValidException(object); - } - } - private static void checkNotAdvertisedWants(UploadPack up, List<ObjectId> notAdvertisedWants, Collection<Ref> visibleRefs) throws IOException { @@ -1946,7 +1903,6 @@ public class UploadPack { try (RevWalk walk = new RevWalk(reader)) { walk.setRetainBody(false); - Set<ObjectId> reachableFrom = refIdSet(visibleRefs); // Missing "wants" throw exception here List<RevObject> wantsAsObjs = objectIdsToRevObjects(walk, notAdvertisedWants); @@ -1959,33 +1915,33 @@ public class UploadPack { boolean repoHasBitmaps = reader.getBitmapIndex() != null; if (!allWantsAreCommits) { - if (!repoHasBitmaps) { - if (up.transferConfig.isAllowFilter()) { - // Use allowFilter as an indication that the server - // operator is willing to pay the cost of these - // reachability checks. - try (ObjectWalk objWalk = walk.toObjectWalkWithSameObjects()) { - checkReachabilityByWalkingObjects(objWalk, - wantsAsObjs, reachableFrom); - } - return; - } - - // If unadvertized non-commits are requested, use - // bitmaps. If there are no bitmaps, instead of - // incurring the expense of a manual walk, reject - // the request. + if (!repoHasBitmaps && !up.transferConfig.isAllowFilter()) { + // Checking unadvertised non-commits without bitmaps + // requires an expensive manual walk. Use allowFilter as an + // indication that the server operator is willing to pay + // this cost. Reject the request otherwise. RevObject nonCommit = wantsAsObjs .stream() .filter(obj -> !(obj instanceof RevCommit)) .limit(1) .collect(Collectors.toList()).get(0); throw new WantNotValidException(nonCommit); + } + try (ObjectWalk objWalk = walk.toObjectWalkWithSameObjects()) { + Stream<RevObject> startersAsObjs = importantRefsFirst(visibleRefs) + .map(UploadPack::refToObjectId) + .map(objId -> objectIdToRevObject(objWalk, objId)) + .filter(Objects::nonNull); // Ignore missing tips + + ObjectReachabilityChecker reachabilityChecker = objWalk + .createObjectReachabilityChecker(); + Optional<RevObject> unreachable = reachabilityChecker + .areAllReachable(wantsAsObjs, startersAsObjs); + if (unreachable.isPresent()) { + throw new WantNotValidException(unreachable.get()); + } } - checkNotAdvertisedWantsUsingBitmap(reader, - reader.getBitmapIndex(), notAdvertisedWants, - reachableFrom); return; } @@ -2053,6 +2009,29 @@ public class UploadPack { } } + /** + * Translate an object id to a RevObject. + * + * @param walk + * walk on the relevant object storage + * @param objectId + * Object Id + * @return RevObject instance or null if the object is missing + */ + @Nullable + private static RevObject objectIdToRevObject(RevWalk walk, + ObjectId objectId) { + if (objectId == null) { + return null; + } + + try { + return walk.parseAny(objectId); + } catch (IOException e) { + return null; + } + } + // Resolve the ObjectIds into RevObjects. Any missing object raises an // exception private static List<RevObject> objectIdsToRevObjects(RevWalk walk, @@ -2205,6 +2184,11 @@ public class UploadPack { } msgOut.flush(); + // Advertised objects and refs are not used from here on and can be + // cleared. + advertised = null; + refs = null; + PackConfig cfg = packConfig; if (cfg == null) cfg = new PackConfig(db); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java index a5f2bc605d..98c231a46d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java @@ -41,6 +41,12 @@ public interface HttpConnection { int HTTP_OK = java.net.HttpURLConnection.HTTP_OK; /** + * @see HttpURLConnection#HTTP_NOT_AUTHORITATIVE + * @since 5.8 + */ + int HTTP_NOT_AUTHORITATIVE = java.net.HttpURLConnection.HTTP_NOT_AUTHORITATIVE; + + /** * @see HttpURLConnection#HTTP_MOVED_PERM * @since 4.7 */ @@ -59,14 +65,26 @@ public interface HttpConnection { int HTTP_SEE_OTHER = java.net.HttpURLConnection.HTTP_SEE_OTHER; /** - * HTTP 1.1 additional MOVED_TEMP status code; value = 307. + * HTTP 1.1 additional "temporary redirect" status code; value = 307. * * @see #HTTP_MOVED_TEMP + * @see <a href="https://tools.ietf.org/html/rfc7231#section-6.4.7">RFC + * 7231, section 6.4.7: 307 Temporary Redirect</a> * @since 4.9 */ int HTTP_11_MOVED_TEMP = 307; /** + * HTTP 1.1 additional "permanent redirect" status code; value = 308. + * + * @see #HTTP_MOVED_TEMP + * @see <a href="https://tools.ietf.org/html/rfc7538#section-3">RFC 7538, + * section 3: 308 Permanent Redirect</a> + * @since 5.8 + */ + int HTTP_11_MOVED_PERM = 308; + + /** * @see HttpURLConnection#HTTP_NOT_FOUND */ int HTTP_NOT_FOUND = java.net.HttpURLConnection.HTTP_NOT_FOUND; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnection.java index 925c4e2f84..3b0bae21ef 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnection.java @@ -32,7 +32,7 @@ import javax.net.ssl.SSLSocket; import javax.net.ssl.TrustManager; import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.transport.internal.DelegatingSSLSocketFactory; +import org.eclipse.jgit.internal.transport.http.DelegatingSSLSocketFactory; import org.eclipse.jgit.util.HttpSupport; /** * A {@link org.eclipse.jgit.transport.http.HttpConnection} which simply diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 988953b00c..91574efec4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -1036,12 +1036,36 @@ public abstract class FS { public File userHome() { Holder<File> p = userHome; if (p == null) { - p = new Holder<>(userHomeImpl()); + p = new Holder<>(safeUserHomeImpl()); userHome = p; } return p.value; } + private File safeUserHomeImpl() { + File home; + try { + home = userHomeImpl(); + if (home != null) { + home.toPath(); + return home; + } + } catch (RuntimeException e) { + LOG.error(JGitText.get().exceptionWhileFindingUserHome, e); + } + home = defaultUserHomeImpl(); + if (home != null) { + try { + home.toPath(); + return home; + } catch (InvalidPathException e) { + LOG.error(MessageFormat + .format(JGitText.get().invalidHomeDirectory, home), e); + } + } + return null; + } + /** * Set the user's home directory location. * @@ -1081,6 +1105,10 @@ public abstract class FS { * @return the user's home directory; null if the user does not have one. */ protected File userHomeImpl() { + return defaultUserHomeImpl(); + } + + private File defaultUserHomeImpl() { final String home = AccessController.doPrivileged( (PrivilegedAction<String>) () -> System.getProperty("user.home") //$NON-NLS-1$ ); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index 4831fbb64e..c43956e53d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.CopyOption; +import java.nio.file.DirectoryNotEmptyException; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.LinkOption; @@ -180,21 +181,31 @@ public class FileUtils { } if (delete) { - Throwable t = null; + IOException t = null; Path p = f.toPath(); - try { - Files.delete(p); - return; - } catch (FileNotFoundException e) { - if ((options & (SKIP_MISSING | IGNORE_ERRORS)) == 0) { - throw new IOException(MessageFormat.format( - JGitText.get().deleteFileFailed, - f.getAbsolutePath()), e); + boolean tryAgain; + do { + tryAgain = false; + try { + Files.delete(p); + return; + } catch (NoSuchFileException | FileNotFoundException e) { + handleDeleteException(f, e, options, + SKIP_MISSING | IGNORE_ERRORS); + return; + } catch (DirectoryNotEmptyException e) { + handleDeleteException(f, e, options, IGNORE_ERRORS); + return; + } catch (IOException e) { + if (!f.canWrite()) { + tryAgain = f.setWritable(true); + } + if (!tryAgain) { + t = e; + } } - return; - } catch (IOException e) { - t = e; - } + } while (tryAgain); + if ((options & RETRY) != 0) { for (int i = 1; i < 10; i++) { try { @@ -210,11 +221,15 @@ public class FileUtils { } } } - if ((options & IGNORE_ERRORS) == 0) { - throw new IOException(MessageFormat.format( - JGitText.get().deleteFileFailed, f.getAbsolutePath()), - t); - } + handleDeleteException(f, t, options, IGNORE_ERRORS); + } + } + + private static void handleDeleteException(File f, IOException e, + int allOptions, int checkOptions) throws IOException { + if (e != null && (allOptions & checkOptions) == 0) { + throw new IOException(MessageFormat.format( + JGitText.get().deleteFileFailed, f.getAbsolutePath()), e); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java index deab4e67a0..c33c869b64 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/EolStreamTypeUtil.java @@ -140,19 +140,19 @@ public final class EolStreamTypeUtil { } // new git system + if ("auto".equals(attrs.getValue("text"))) { //$NON-NLS-1$ //$NON-NLS-2$ + return EolStreamType.AUTO_LF; + } + String eol = attrs.getValue("eol"); //$NON-NLS-1$ - if (eol != null) + if (eol != null) { // check-in is always normalized to LF return EolStreamType.TEXT_LF; - + } if (attrs.isSet("text")) { //$NON-NLS-1$ return EolStreamType.TEXT_LF; } - if ("auto".equals(attrs.getValue("text"))) { //$NON-NLS-1$ //$NON-NLS-2$ - return EolStreamType.AUTO_LF; - } - switch (options.getAutoCRLF()) { case TRUE: case INPUT: @@ -168,6 +168,8 @@ public final class EolStreamTypeUtil { switch (options.getAutoCRLF()) { case TRUE: return EolStreamType.TEXT_CRLF; + case INPUT: + return EolStreamType.DIRECT; default: // no decision } @@ -205,7 +207,10 @@ public final class EolStreamTypeUtil { // new git system String eol = attrs.getValue("eol"); //$NON-NLS-1$ if (eol != null) { - if ("crlf".equals(eol)) {//$NON-NLS-1$ + if ("crlf".equals(eol)) { //$NON-NLS-1$ + if ("auto".equals(attrs.getValue("text"))) { //$NON-NLS-1$ //$NON-NLS-2$ + return EolStreamType.AUTO_CRLF; + } return EolStreamType.TEXT_CRLF; } else if ("lf".equals(eol)) { //$NON-NLS-1$ return EolStreamType.DIRECT; |