diff options
author | Matthias Sohn <matthias.sohn@sap.com> | 2021-02-25 10:29:07 +0100 |
---|---|---|
committer | Matthias Sohn <matthias.sohn@sap.com> | 2021-02-28 00:58:04 +0100 |
commit | f6597971991e3350df568b0cde05c014dcd69c47 (patch) | |
tree | cb61592af3f53da45174beed517b3284d7bd55c6 /org.eclipse.jgit | |
parent | 286ad23cb56ffeac77d4bfd03be575358fd5217c (diff) | |
parent | 789c0479a9294417db0375cce9f1949fe9052d8c (diff) | |
download | jgit-f6597971991e3350df568b0cde05c014dcd69c47.tar.gz jgit-f6597971991e3350df568b0cde05c014dcd69c47.zip |
Merge branch 'master' into next
* master: (143 commits)
Prepare 5.11.0-SNAPSHOT builds
JGit v5.11.0.202102240950-m3
[releng] japicmp: update last release version
IgnoreNode: include path to file for invalid .gitignore patterns
FastIgnoreRule: include bad pattern in log message
init: add config option to set default for the initial branch name
init: allow specifying the initial branch name for the new repository
Fail clone if initial branch doesn't exist in remote repository
GPG: fix reading unprotected old-format secret keys
Update Orbit to S20210216215844
Add missing bazel dependency for o.e.j.gpg.bc.test
GPG: handle extended private key format
dfs: handle short copies
[GPG] Provide a factory for the BouncyCastleGpgSigner
Fix boxing warnings
GPG: compute the keygrip to find a secret key
GPG signature verification via BouncyCastle
Post commit hook failure should not cause commit failure
Allow to define additional Hook classes outside JGit
GitHook: use default charset for output and error streams
...
Change-Id: I689f4070e79f4a0ac1c02b35698ccaab68ad2f34
Diffstat (limited to 'org.eclipse.jgit')
155 files changed, 4935 insertions, 7904 deletions
diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 538c6f7bfb..d389ac5888 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,38 +1,56 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <component id="org.eclipse.jgit" version="2"> - <resource path="src/org/eclipse/jgit/lib/TypedConfigGetter.java" type="org.eclipse.jgit.lib.TypedConfigGetter"> - <filter id="404000815"> + <resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants"> + <filter id="338755678"> <message_arguments> - <message_argument value="org.eclipse.jgit.lib.TypedConfigGetter"/> - <message_argument value="getPath(Config, String, String, String, FS, File, Path)"/> + <message_argument value="org.eclipse.jgit.lib.ConfigConstants"/> + <message_argument value="CONFIG_REFSTORAGE_REFTREE"/> </message_arguments> </filter> </resource> - <resource path="src/org/eclipse/jgit/storage/pack/PackStatistics.java" type="org.eclipse.jgit.storage.pack.PackStatistics$Accumulator"> - <filter id="336658481"> + <resource path="src/org/eclipse/jgit/revwalk/ObjectWalk.java" type="org.eclipse.jgit.revwalk.ObjectWalk"> + <filter id="421654647"> <message_arguments> - <message_argument value="org.eclipse.jgit.storage.pack.PackStatistics.Accumulator"/> - <message_argument value="notAdvertisedWants"/> + <message_argument value="org.eclipse.jgit.revwalk.ObjectWalk"/> + <message_argument value="createObjectReachabilityChecker()"/> </message_arguments> </filter> - <filter id="336658481"> + </resource> + <resource path="src/org/eclipse/jgit/revwalk/RevWalk.java" type="org.eclipse.jgit.revwalk.RevWalk"> + <filter id="421654647"> <message_arguments> - <message_argument value="org.eclipse.jgit.storage.pack.PackStatistics.Accumulator"/> - <message_argument value="reachabilityCheckDuration"/> + <message_argument value="org.eclipse.jgit.revwalk.RevWalk"/> + <message_argument value="createReachabilityChecker()"/> </message_arguments> </filter> </resource> - <resource path="src/org/eclipse/jgit/transport/HttpConfig.java" type="org.eclipse.jgit.transport.HttpConfig"> - <filter id="336658481"> + <resource path="src/org/eclipse/jgit/util/FS.java" type="org.eclipse.jgit.util.FS"> + <filter id="338792546"> <message_arguments> - <message_argument value="org.eclipse.jgit.transport.HttpConfig"/> - <message_argument value="EXTRA_HEADER"/> + <message_argument value="org.eclipse.jgit.util.FS"/> + <message_argument value="internalRunHookIfPresent(Repository, String, String[], PrintStream, PrintStream, String)"/> </message_arguments> </filter> - <filter id="336658481"> + <filter id="338792546"> + <message_arguments> + <message_argument value="org.eclipse.jgit.util.FS"/> + <message_argument value="runHookIfPresent(Repository, String, String[], PrintStream, PrintStream, String)"/> + </message_arguments> + </filter> + </resource> + <resource path="src/org/eclipse/jgit/util/FS_POSIX.java" type="org.eclipse.jgit.util.FS_POSIX"> + <filter id="338792546"> + <message_arguments> + <message_argument value="org.eclipse.jgit.util.FS_POSIX"/> + <message_argument value="runHookIfPresent(Repository, String, String[], PrintStream, PrintStream, String)"/> + </message_arguments> + </filter> + </resource> + <resource path="src/org/eclipse/jgit/util/FS_Win32_Cygwin.java" type="org.eclipse.jgit.util.FS_Win32_Cygwin"> + <filter id="338792546"> <message_arguments> - <message_argument value="org.eclipse.jgit.transport.HttpConfig"/> - <message_argument value="USER_AGENT"/> + <message_argument value="org.eclipse.jgit.util.FS_Win32_Cygwin"/> + <message_argument value="runHookIfPresent(Repository, String, String[], PrintStream, PrintStream, String)"/> </message_arguments> </filter> </resource> diff --git a/org.eclipse.jgit/BUILD b/org.eclipse.jgit/BUILD index 2083372248..04873b0c72 100644 --- a/org.eclipse.jgit/BUILD +++ b/org.eclipse.jgit/BUILD @@ -38,7 +38,7 @@ genrule( "cd $$TMP", "unzip -q $$ROOT/$<", "echo \"Implementation-Version: $$GEN_VERSION\n$$(cat META-INF/MANIFEST.MF)\" > META-INF/MANIFEST.MF", - "find . -exec touch '{}' ';'", + "find . -exec touch -t 198001010000 '{}' ';'", "zip -Xqr $$ROOT/$@ .", "rm -rf $$TMP", ]), diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index 403b8eed0f..d755f83626 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -72,11 +72,8 @@ Export-Package: org.eclipse.jgit.annotations;version="6.0.0", org.eclipse.jgit.http.test", org.eclipse.jgit.internal.fsck;version="6.0.0"; x-friends:="org.eclipse.jgit.test", - org.eclipse.jgit.internal.ketch;version="6.0.0"; - x-friends:="org.eclipse.jgit.junit, - org.eclipse.jgit.test, - org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.revwalk;version="6.0.0";x-internal:=true, + org.eclipse.jgit.internal.revwalk;version="6.0.0"; + x-friends:="org.eclipse.jgit.test", org.eclipse.jgit.internal.storage.dfs;version="6.0.0"; x-friends:="org.eclipse.jgit.test, org.eclipse.jgit.http.server, @@ -104,10 +101,6 @@ Export-Package: org.eclipse.jgit.annotations;version="6.0.0", org.eclipse.jgit.junit, org.eclipse.jgit.test, org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.reftree;version="6.0.0"; - x-friends:="org.eclipse.jgit.junit, - org.eclipse.jgit.test, - org.eclipse.jgit.pgm", org.eclipse.jgit.internal.submodule;version="6.0.0";x-internal:=true, org.eclipse.jgit.internal.transport.connectivity;version="6.0.0"; x-friends:="org.eclipse.jgit.test", diff --git a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF index e1904d5d3f..134cbacbee 100644 --- a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF @@ -5,4 +5,3 @@ Bundle-SymbolicName: org.eclipse.jgit.source Bundle-Vendor: Eclipse.org - JGit Bundle-Version: 6.0.0.qualifier Eclipse-SourceBundle: org.eclipse.jgit;version="6.0.0.qualifier";roots="." - diff --git a/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml b/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml index 2efbb9c1a5..73a6685564 100644 --- a/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml +++ b/org.eclipse.jgit/findBugs/FindBugsExcludeFilter.xml @@ -9,6 +9,12 @@ <Bug pattern="DM_GC" /> </Match> + <Match> + <Class name="org.eclipse.jgit.internal.storage.pack.PackOutputStream" /> + <Method name="writeHeader" /> + <Bug pattern="NS_DANGEROUS_NON_SHORT_CIRCUIT" /> + </Match> + <!-- Silence ignoring return value of mkdirs --> <Match> <Class name="org.eclipse.jgit.dircache.DirCacheCheckout" /> @@ -19,8 +25,26 @@ <!-- Silence the construction of our magic String instance. --> <Match> - <Class name="org.eclipse.jgit.lib.Config" /> - <Bug pattern="DM_STRING_VOID_CTOR"/> + <Class name="org.eclipse.jgit.lib.Config" /> + <Bug pattern="DM_STRING_VOID_CTOR"/> + </Match> + + <Match> + <Class name="org.eclipse.jgit.lib.Config" /> + <Method name="isMissing" /> + <Bug pattern="ES_COMPARING_PARAMETER_STRING_WITH_EQ"/> + </Match> + + <Match> + <Class name="org.eclipse.jgit.transport.PacketLineIn" /> + <Method name="isDelimiter" /> + <Bug pattern="ES_COMPARING_PARAMETER_STRING_WITH_EQ"/> + </Match> + + <Match> + <Class name="org.eclipse.jgit.transport.PacketLineIn" /> + <Method name="isEnd" /> + <Bug pattern="ES_COMPARING_PARAMETER_STRING_WITH_EQ"/> </Match> <!-- Silence comparison of string by == or !=. This class is built @@ -53,6 +77,12 @@ <Bug pattern="NP_BOOLEAN_RETURN_NULL" /> </Match> + <Match> + <Class name="org.eclipse.jgit.ignore.IgnoreNode" /> + <Method name="checkIgnored" /> + <Bug pattern="NP_BOOLEAN_RETURN_NULL" /> + </Match> + <!-- Transport initialization works like this --> <Match> <Class name="org.eclipse.jgit.transport.Transport" /> diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 12902b9004..c00203dd07 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -30,6 +30,8 @@ badEntryDelimiter=Bad entry delimiter badEntryName=Bad entry name: {0} badEscape=Bad escape: {0} badGroupHeader=Bad group header +badIgnorePattern=Cannot parse .gitignore pattern ''{0}'' +badIgnorePatternFull=File {0} line {1}: cannot parse pattern ''{2}'': {3} badObjectType=Bad object type: {0} badRef=Bad ref: {0}: {1} badSectionEntry=Bad section entry: {0} @@ -233,6 +235,7 @@ downloadCancelled=Download cancelled downloadCancelledDuringIndexing=Download cancelled during indexing duplicateAdvertisementsOf=duplicate advertisements of {0} duplicateRef=Duplicate ref: {0} +duplicateRefAttribute=Duplicate ref attribute: {0} duplicateRemoteRefUpdateIsIllegal=Duplicate remote ref update is illegal. Affected remote name: {0} duplicateStagesNotAllowed=Duplicate stages not allowed eitherGitDirOrWorkTreeRequired=One of setGitDir or setWorkTree must be called. @@ -310,6 +313,10 @@ headRequiredToStash=HEAD required to stash local changes hoursAgo={0} hours ago httpConfigCannotNormalizeURL=Cannot normalize URL path {0}: too many .. segments httpConfigInvalidURL=Cannot parse URL from subsection http.{0} in git config; ignored. +httpFactoryInUse=Changing the HTTP connection factory after an HTTP connection has already been opened is not allowed. +httpPreAuthTooLate=HTTP Basic preemptive authentication cannot be set once an HTTP connection has already been opened. +httpUserInfoDecodeError=Cannot decode user info from URL {}; ignored. +httpWrongConnectionType=Wrong connection type: expected {0}, got {1}. hugeIndexesAreNotSupportedByJgitYet=Huge indexes are not supported by jgit, yet hunkBelongsToAnotherFile=Hunk belongs to another file hunkDisconnectedFromFile=Hunk disconnected from file @@ -563,6 +570,7 @@ refNotResolved=Ref {0} cannot be resolved reftableDirExists=reftable dir exists and is nonempty reftableRecordsMustIncrease=records must be increasing: last {0}, this {1} refUpdateReturnCodeWas=RefUpdate return code was: {0} +remoteBranchNotFound=Remote branch ''{0}'' not found in upstream origin remoteConfigHasNoURIAssociated=Remote config "{0}" has no URIs associated remoteDoesNotHaveSpec=Remote does not have {0} available for fetch. remoteDoesNotSupportSmartHTTPPush=remote does not support smart HTTP push @@ -617,7 +625,9 @@ shortCompressedStreamAt=Short compressed stream at {0} shortReadOfBlock=Short read of block. shortReadOfOptionalDIRCExtensionExpectedAnotherBytes=Short read of optional DIRC extension {0}; expected another {1} bytes within the section. shortSkipOfBlock=Short skip of block. -signingNotSupportedOnTag=Signing isn't supported on tag operations yet. +signatureVerificationError=Signature verification failed +signatureVerificationUnavailable=No signature verifier registered +signedTagMessageNoLf=A non-empty message of a signed tag must end in LF. signingServiceUnavailable=Signing service is not available similarityScoreMustBeWithinBounds=Similarity score must be between 0 and 100. skipMustBeNonNegative=skip must be >= 0 @@ -758,6 +768,13 @@ uriNotFoundWithMessage={0} not found: {1} URINotSupported=URI not supported: {0} userConfigInvalid=Git config in the user's home directory {0} is invalid {1} validatingGitModules=Validating .gitmodules files +verifySignatureBad=BAD signature from "{0}" +verifySignatureExpired=Expired signature from "{0}" +verifySignatureGood=Good signature from "{0}" +verifySignatureIssuer=issuer "{0}" +verifySignatureKey=using key {0} +verifySignatureMade=Signature made {0} +verifySignatureTrust=[{0}] walkFailure=Walk failure. wantNoSpaceWithCapabilities=No space between oid and first capability in first want line wantNotValid=want {0} not valid diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties deleted file mode 100644 index 1fbb7cb3b5..0000000000 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties +++ /dev/null @@ -1,13 +0,0 @@ -accepted=accepted. -cannotFetchFromLocalReplica=cannot fetch from LocalReplica -failed=failed! -invalidFollowerUri=invalid follower URI -leaderFailedToStore=leader failed to store -localReplicaRequired=LocalReplica instance is required -mismatchedTxnNamespace=mismatched txnNamespace; expected {0} found {1} -outsideTxnNamespace=ref {0} is outside of txnNamespace {1} -proposingUpdates=Proposing updates -queuedProposalFailedToApply=queued proposal failed to apply -starting=starting! -unsupportedVoterCount=unsupported voter count {0}, expected one of {1} -waitingForQueue=Waiting for queue diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java index 2c01c19c59..fdf8b80cd4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ArchiveCommand.java @@ -19,7 +19,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; @@ -202,7 +201,7 @@ public class ArchiveCommand extends GitCommand<OutputStream> { * Available archival formats (corresponding to values for * the --format= option) */ - private static final ConcurrentMap<String, FormatEntry> formats = + private static final Map<String, FormatEntry> formats = new ConcurrentHashMap<>(); /** @@ -215,7 +214,7 @@ public class ArchiveCommand extends GitCommand<OutputStream> { * @param newValue value to be associated with the key (null to remove). * @return true if the value was replaced */ - private static <K, V> boolean replace(ConcurrentMap<K, V> map, + private static <K, V> boolean replace(Map<K, V> map, K key, V oldValue, V newValue) { if (oldValue == null && newValue == null) // Nothing to do. return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java index aba86fc361..cf7bc1f263 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java @@ -297,6 +297,7 @@ public class CloneCommand extends TransportCommand<CloneCommand, Git> { command.setTagOpt( fetchAll ? TagOpt.FETCH_TAGS : TagOpt.AUTO_FOLLOW); } + command.setInitialBranch(branch); configure(command); return command.call(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index b4f7175036..31f6a31c75 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -47,6 +47,7 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.GpgConfig; import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.GpgObjectSigner; import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; @@ -120,6 +121,8 @@ public class CommitCommand extends GitCommand<RevCommit> { private GpgSigner gpgSigner; + private GpgConfig gpgConfig; + private CredentialsProvider credentialsProvider; /** @@ -247,8 +250,18 @@ public class CommitCommand extends GitCommand<RevCommit> { throw new ServiceUnavailableException( JGitText.get().signingServiceUnavailable); } - gpgSigner.sign(commit, signingKey, committer, - credentialsProvider); + if (gpgSigner instanceof GpgObjectSigner) { + ((GpgObjectSigner) gpgSigner).signObject(commit, + signingKey, committer, credentialsProvider, + gpgConfig); + } else { + if (gpgConfig.getKeyFormat() != GpgFormat.OPENPGP) { + throw new UnsupportedSigningFormatException(JGitText + .get().onlyOpenPgpSupportedForSigning); + } + gpgSigner.sign(commit, signingKey, committer, + credentialsProvider); + } } ObjectId commitId = odi.insert(commit); @@ -576,7 +589,9 @@ public class CommitCommand extends GitCommand<RevCommit> { // an explicit message throw new NoMessageException(JGitText.get().commitMessageNotSpecified); - GpgConfig gpgConfig = new GpgConfig(repo.getConfig()); + if (gpgConfig == null) { + gpgConfig = new GpgConfig(repo.getConfig()); + } if (signCommit == null) { signCommit = gpgConfig.isSignCommits() ? Boolean.TRUE : Boolean.FALSE; @@ -585,10 +600,6 @@ public class CommitCommand extends GitCommand<RevCommit> { signingKey = gpgConfig.getSigningKey(); } if (gpgSigner == null) { - if (gpgConfig.getKeyFormat() != GpgFormat.OPENPGP) { - throw new UnsupportedSigningFormatException( - JGitText.get().onlyOpenPgpSupportedForSigning); - } gpgSigner = GpgSigner.getDefault(); } } @@ -973,6 +984,36 @@ public class CommitCommand extends GitCommand<RevCommit> { } /** + * Sets the {@link GpgSigner} to use if the commit is to be signed. + * + * @param signer + * to use; if {@code null}, the default signer will be used + * @return {@code this} + * @since 5.11 + */ + public CommitCommand setGpgSigner(GpgSigner signer) { + checkCallable(); + this.gpgSigner = signer; + return this; + } + + /** + * Sets an external {@link GpgConfig} to use. Whether it will be used is at + * the discretion of the {@link #setGpgSigner(GpgSigner)}. + * + * @param config + * to set; if {@code null}, the config will be loaded from the + * git config of the repository + * @return {@code this} + * @since 5.11 + */ + public CommitCommand setGpgConfig(GpgConfig config) { + checkCallable(); + this.gpgConfig = config; + return this; + } + + /** * Sets a {@link CredentialsProvider} * * @param credentialsProvider diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java index 033dd60c3b..90c1515b06 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java @@ -74,6 +74,8 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> { private boolean isForceUpdate; + private String initialBranch; + /** * Callback for status of fetch operation. * @@ -209,7 +211,7 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> { transport.setFetchThin(thin); configure(transport); FetchResult result = transport.fetch(monitor, - applyOptions(refSpecs)); + applyOptions(refSpecs), initialBranch); if (!repo.isBare()) { fetchSubmodules(result); } @@ -488,6 +490,24 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> { } /** + * Set the initial branch + * + * @param branch + * the initial branch to check out when cloning the repository. + * Can be specified as ref name (<code>refs/heads/master</code>), + * branch name (<code>master</code>) or tag name + * (<code>v1.2.3</code>). The default is to use the branch + * pointed to by the cloned repository's HEAD and can be + * requested by passing {@code null} or <code>HEAD</code>. + * @return {@code this} + * @since 5.11 + */ + public FetchCommand setInitialBranch(String branch) { + this.initialBranch = branch; + return this; + } + + /** * Register a progress callback. * * @param callback diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java index 64314772b7..3b3e10e7b2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com> - * Copyright (C) 2010, Chris Aniszczyk <caniszczyk@gmail.com> and others + * Copyright (C) 2010, 2021 Chris Aniszczyk <caniszczyk@gmail.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -773,6 +773,16 @@ public class Git implements AutoCloseable { } /** + * Return a command to verify signatures of tags or commits. + * + * @return a {@link VerifySignatureCommand} + * @since 5.11 + */ + public VerifySignatureCommand verifySignature() { + return new VerifySignatureCommand(repo); + } + + /** * Get repository * * @return the git repository this class is interacting with; see diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java index 41fcf29ed0..240290f4f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java @@ -15,12 +15,16 @@ import java.text.MessageFormat; import java.util.concurrent.Callable; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryBuilder; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.SystemReader; /** @@ -38,6 +42,8 @@ public class InitCommand implements Callable<Git> { private FS fs; + private String initialBranch; + /** * {@inheritDoc} * <p> @@ -87,11 +93,16 @@ public class InitCommand implements Callable<Git> { builder.setWorkTree(new File(dStr)); } } + builder.setInitialBranch(StringUtils.isEmptyOrNull(initialBranch) + ? SystemReader.getInstance().getUserConfig().getString( + ConfigConstants.CONFIG_INIT_SECTION, null, + ConfigConstants.CONFIG_KEY_DEFAULT_BRANCH) + : initialBranch); Repository repository = builder.build(); if (!repository.getObjectDatabase().exists()) repository.create(bare); return new Git(repository, true); - } catch (IOException e) { + } catch (IOException | ConfigInvalidException e) { throw new JGitInternalException(e.getMessage(), e); } } @@ -184,4 +195,23 @@ public class InitCommand implements Callable<Git> { this.fs = fs; return this; } + + /** + * Set the initial branch of the new repository. If not specified + * ({@code null} or empty), fall back to the default name (currently + * master). + * + * @param branch + * initial branch name of the new repository + * @return {@code this} + * @throws InvalidRefNameException + * if the branch name is not valid + * + * @since 5.11 + */ + public InitCommand setInitialBranch(String branch) + throws InvalidRefNameException { + this.initialBranch = branch; + return this; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java index a4ca309095..0c691062f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/LsRemoteCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, Christoph Brill <egore911@egore911.de> and others + * Copyright (C) 2011, 2020 Christoph Brill <egore911@egore911.de> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -164,7 +164,7 @@ public class LsRemoteCommand extends refSpecs.add(new RefSpec("refs/heads/*:refs/remotes/origin/*")); //$NON-NLS-1$ Collection<Ref> refs; Map<String, Ref> refmap = new HashMap<>(); - try (FetchConnection fc = transport.openFetch()) { + try (FetchConnection fc = transport.openFetch(refSpecs)) { refs = fc.getRefs(); if (refSpecs.isEmpty()) for (Ref r : refs) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 6678af163a..836175dcea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -67,6 +67,7 @@ import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.merge.MergeStrategy; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevSort; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.submodule.SubmoduleWalk.IgnoreSubmoduleMode; @@ -1137,15 +1138,19 @@ public class RebaseCommand extends GitCommand<RebaseResult> { private List<RevCommit> calculatePickList(RevCommit headCommit) throws GitAPIException, NoHeadException, IOException { - Iterable<RevCommit> commitsToUse; - try (Git git = new Git(repo)) { - LogCommand cmd = git.log().addRange(upstreamCommit, headCommit); - commitsToUse = cmd.call(); - } List<RevCommit> cherryPickList = new ArrayList<>(); - for (RevCommit commit : commitsToUse) { - if (preserveMerges || commit.getParentCount() == 1) - cherryPickList.add(commit); + try (RevWalk r = new RevWalk(repo)) { + r.sort(RevSort.TOPO_KEEP_BRANCH_TOGETHER, true); + r.sort(RevSort.COMMIT_TIME_DESC, true); + r.markUninteresting(r.lookupCommit(upstreamCommit)); + r.markStart(r.lookupCommit(headCommit)); + Iterator<RevCommit> commitsToUse = r.iterator(); + while (commitsToUse.hasNext()) { + RevCommit commit = commitsToUse.next(); + if (preserveMerges || commit.getParentCount() == 1) { + cherryPickList.add(commit); + } + } } Collections.reverse(cherryPickList); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java index 9a328a6eaa..58c18b38d1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2013 Chris Aniszczyk <caniszczyk@gmail.com> and others + * Copyright (C) 2010, 2020 Chris Aniszczyk <caniszczyk@gmail.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -18,8 +18,14 @@ import org.eclipse.jgit.api.errors.InvalidTagNameException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.api.errors.NoHeadException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.eclipse.jgit.api.errors.ServiceUnavailableException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.GpgObjectSigner; +import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; @@ -31,6 +37,7 @@ import org.eclipse.jgit.lib.RepositoryState; import org.eclipse.jgit.lib.TagBuilder; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.CredentialsProvider; /** * Create/update an annotated tag object or a simple unannotated tag @@ -56,6 +63,7 @@ import org.eclipse.jgit.revwalk.RevWalk; * >Git documentation about Tag</a> */ public class TagCommand extends GitCommand<Ref> { + private RevObject id; private String name; @@ -64,11 +72,19 @@ public class TagCommand extends GitCommand<Ref> { private PersonIdent tagger; - private boolean signed; + private Boolean signed; private boolean forceUpdate; - private boolean annotated = true; + private Boolean annotated; + + private String signingKey; + + private GpgConfig gpgConfig; + + private GpgObjectSigner gpgSigner; + + private CredentialsProvider credentialsProvider; /** * <p>Constructor for TagCommand.</p> @@ -77,6 +93,7 @@ public class TagCommand extends GitCommand<Ref> { */ protected TagCommand(Repository repo) { super(repo); + this.credentialsProvider = CredentialsProvider.getDefault(); } /** @@ -108,10 +125,7 @@ public class TagCommand extends GitCommand<Ref> { id = revWalk.parseCommit(objectId); } - if (!annotated) { - if (message != null || tagger != null) - throw new JGitInternalException( - JGitText.get().messageAndTaggerNotAllowedInUnannotatedTags); + if (!isAnnotated()) { return updateTagRef(id, revWalk, name, "SimpleTag[" + name + " : " + id //$NON-NLS-1$ //$NON-NLS-2$ + "]"); //$NON-NLS-1$ @@ -124,6 +138,11 @@ public class TagCommand extends GitCommand<Ref> { newTag.setTagger(tagger); newTag.setObjectId(id); + if (gpgSigner != null) { + gpgSigner.signObject(newTag, signingKey, tagger, + credentialsProvider, gpgConfig); + } + // write the tag object try (ObjectInserter inserter = repo.newObjectInserter()) { ObjectId tagId = inserter.insert(newTag); @@ -158,9 +177,17 @@ public class TagCommand extends GitCommand<Ref> { throw new ConcurrentRefUpdateException( JGitText.get().couldNotLockHEAD, tagRef.getRef(), updateResult); + case NO_CHANGE: + if (forceUpdate) { + return repo.exactRef(refName); + } + throw new RefAlreadyExistsException(MessageFormat + .format(JGitText.get().tagAlreadyExists, newTagToString), + updateResult); case REJECTED: throw new RefAlreadyExistsException(MessageFormat.format( - JGitText.get().tagAlreadyExists, newTagToString)); + JGitText.get().tagAlreadyExists, newTagToString), + updateResult); default: throw new JGitInternalException(MessageFormat.format( JGitText.get().updatingRefFailed, refName, newTagToString, @@ -177,20 +204,60 @@ public class TagCommand extends GitCommand<Ref> { * * @throws InvalidTagNameException * if the tag name is null or invalid - * @throws UnsupportedOperationException - * if the tag is signed (not supported yet) + * @throws ServiceUnavailableException + * if the tag should be signed but no signer can be found + * @throws UnsupportedSigningFormatException + * if the tag should be signed but {@code gpg.format} is not + * {@link GpgFormat#OPENPGP} */ private void processOptions(RepositoryState state) - throws InvalidTagNameException { - if (tagger == null && annotated) - tagger = new PersonIdent(repo); - if (name == null || !Repository.isValidRefName(Constants.R_TAGS + name)) + throws InvalidTagNameException, ServiceUnavailableException, + UnsupportedSigningFormatException { + if (name == null + || !Repository.isValidRefName(Constants.R_TAGS + name)) { throw new InvalidTagNameException( MessageFormat.format(JGitText.get().tagNameInvalid, name == null ? "<null>" : name)); //$NON-NLS-1$ - if (signed) - throw new UnsupportedOperationException( - JGitText.get().signingNotSupportedOnTag); + } + if (!isAnnotated()) { + if ((message != null && !message.isEmpty()) || tagger != null) { + throw new JGitInternalException(JGitText + .get().messageAndTaggerNotAllowedInUnannotatedTags); + } + } else { + if (tagger == null) { + tagger = new PersonIdent(repo); + } + // Figure out whether to sign. + if (!(Boolean.FALSE.equals(signed) && signingKey == null)) { + if (gpgConfig == null) { + gpgConfig = new GpgConfig(repo.getConfig()); + } + boolean doSign = isSigned() || gpgConfig.isSignAllTags(); + if (!Boolean.TRUE.equals(annotated) && !doSign) { + doSign = gpgConfig.isSignAnnotated(); + } + if (doSign) { + if (signingKey == null) { + signingKey = gpgConfig.getSigningKey(); + } + if (gpgSigner == null) { + GpgSigner signer = GpgSigner.getDefault(); + if (!(signer instanceof GpgObjectSigner)) { + throw new ServiceUnavailableException( + JGitText.get().signingServiceUnavailable); + } + gpgSigner = (GpgObjectSigner) signer; + } + // The message of a signed tag must end in a newline because + // the signature will be appended. + if (message != null && !message.isEmpty() + && !message.endsWith("\n")) { //$NON-NLS-1$ + message += '\n'; + } + } + } + } } /** @@ -238,24 +305,61 @@ public class TagCommand extends GitCommand<Ref> { } /** - * Whether this tag is signed + * Whether {@link #setSigned(boolean) setSigned(true)} has been called or + * whether a {@link #setSigningKey(String) signing key ID} has been set; + * i.e., whether -s or -u was specified explicitly. * * @return whether the tag is signed */ public boolean isSigned() { - return signed; + return Boolean.TRUE.equals(signed) || signingKey != null; } /** * If set to true the Tag command creates a signed tag object. This - * corresponds to the parameter -s on the command line. + * corresponds to the parameter -s (--sign or --no-sign) on the command + * line. + * <p> + * If {@code true}, the tag will be a signed annotated tag. + * </p> * * @param signed - * a boolean. + * whether to sign * @return {@code this} */ public TagCommand setSigned(boolean signed) { - this.signed = signed; + checkCallable(); + this.signed = Boolean.valueOf(signed); + return this; + } + + /** + * Sets the {@link GpgSigner} to use if the commit is to be signed. + * + * @param signer + * to use; if {@code null}, the default signer will be used + * @return {@code this} + * @since 5.11 + */ + public TagCommand setGpgSigner(GpgObjectSigner signer) { + checkCallable(); + this.gpgSigner = signer; + return this; + } + + /** + * Sets an external {@link GpgConfig} to use. Whether it will be used is at + * the discretion of the {@link #setGpgSigner(GpgObjectSigner)}. + * + * @param config + * to set; if {@code null}, the config will be loaded from the + * git config of the repository + * @return {@code this} + * @since 5.11 + */ + public TagCommand setGpgConfig(GpgConfig config) { + checkCallable(); + this.gpgConfig = config; return this; } @@ -268,6 +372,7 @@ public class TagCommand extends GitCommand<Ref> { * @return {@code this} */ public TagCommand setTagger(PersonIdent tagger) { + checkCallable(); this.tagger = tagger; return this; } @@ -291,14 +396,15 @@ public class TagCommand extends GitCommand<Ref> { } /** - * Sets the object id of the tag. If the object id is null, the commit - * pointed to from HEAD will be used. + * Sets the object id of the tag. If the object id is {@code null}, the + * commit pointed to from HEAD will be used. * * @param id * a {@link org.eclipse.jgit.revwalk.RevObject} object. * @return {@code this} */ public TagCommand setObjectId(RevObject id) { + checkCallable(); this.id = id; return this; } @@ -321,6 +427,7 @@ public class TagCommand extends GitCommand<Ref> { * @return {@code this} */ public TagCommand setForceUpdate(boolean forceUpdate) { + checkCallable(); this.forceUpdate = forceUpdate; return this; } @@ -334,18 +441,77 @@ public class TagCommand extends GitCommand<Ref> { * @since 3.0 */ public TagCommand setAnnotated(boolean annotated) { - this.annotated = annotated; + checkCallable(); + this.annotated = Boolean.valueOf(annotated); return this; } /** - * Whether this will create an annotated command + * Whether this will create an annotated tag. * * @return true if this command will create an annotated tag (default is * true) * @since 3.0 */ public boolean isAnnotated() { - return annotated; + boolean setExplicitly = Boolean.TRUE.equals(annotated) || isSigned(); + if (setExplicitly) { + return true; + } + // Annotated at default (not set explicitly) + return annotated == null; } + + /** + * Sets the signing key. + * <p> + * Per spec of {@code user.signingKey}: this will be sent to the GPG program + * as is, i.e. can be anything supported by the GPG program. + * </p> + * <p> + * Note, if none was set or {@code null} is specified a default will be + * obtained from the configuration. + * </p> + * <p> + * If set to a non-{@code null} value, the tag will be a signed annotated + * tag. + * </p> + * + * @param signingKey + * signing key; {@code null} allowed + * @return {@code this} + * @since 5.11 + */ + public TagCommand setSigningKey(String signingKey) { + checkCallable(); + this.signingKey = signingKey; + return this; + } + + /** + * Retrieves the signing key ID. + * + * @return the key ID set, or {@code null} if none is set + * @since 5.11 + */ + public String getSigningKey() { + return signingKey; + } + + /** + * Sets a {@link CredentialsProvider} + * + * @param credentialsProvider + * the provider to use when querying for credentials (eg., during + * signing) + * @return {@code this} + * @since 5.11 + */ + public TagCommand setCredentialsProvider( + CredentialsProvider credentialsProvider) { + checkCallable(); + this.credentialsProvider = credentialsProvider; + return this; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java new file mode 100644 index 0000000000..21cddf75b7 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api; + +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.revwalk.RevObject; + +/** + * A {@code VerificationResult} describes the outcome of a signature + * verification. + * + * @see VerifySignatureCommand + * + * @since 5.11 + */ +public interface VerificationResult { + + /** + * If an error occurred during signature verification, this retrieves the + * exception. + * + * @return the exception, or {@code null} if none occurred + */ + Throwable getException(); + + /** + * Retrieves the signature verification result. + * + * @return the result, or {@code null} if none was computed + */ + GpgSignatureVerifier.SignatureVerification getVerification(); + + /** + * Retrieves the git object of which the signature was verified. + * + * @return the git object + */ + RevObject getObject(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java new file mode 100644 index 0000000000..6a2a44ea2d --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java @@ -0,0 +1,307 @@ +/* + * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.ServiceUnavailableException; +import org.eclipse.jgit.api.errors.WrongObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevWalk; + +/** + * A command to verify GPG signatures on tags or commits. + * + * @since 5.11 + */ +public class VerifySignatureCommand extends GitCommand<Map<String, VerificationResult>> { + + /** + * Describes what kind of objects shall be handled by a + * {@link VerifySignatureCommand}. + */ + public enum VerifyMode { + /** + * Handle any object type, ignore anything that is not a commit or tag. + */ + ANY, + /** + * Handle only commits; throw a {@link WrongObjectTypeException} for + * anything else. + */ + COMMITS, + /** + * Handle only tags; throw a {@link WrongObjectTypeException} for + * anything else. + */ + TAGS + } + + private final Set<String> namesToCheck = new HashSet<>(); + + private VerifyMode mode = VerifyMode.ANY; + + private GpgSignatureVerifier verifier; + + private GpgConfig config; + + private boolean ownVerifier; + + /** + * Creates a new {@link VerifySignatureCommand} for the given {@link Repository}. + * + * @param repo + * to operate on + */ + public VerifySignatureCommand(Repository repo) { + super(repo); + } + + /** + * Add a name of an object (SHA-1, ref name; anything that can be + * {@link Repository#resolve(String) resolved}) to the command to have its + * signature verified. + * + * @param name + * to add + * @return {@code this} + */ + public VerifySignatureCommand addName(String name) { + checkCallable(); + namesToCheck.add(name); + return this; + } + + /** + * Add names of objects (SHA-1, ref name; anything that can be + * {@link Repository#resolve(String) resolved}) to the command to have their + * signatures verified. + * + * @param names + * to add; duplicates will be ignored + * @return {@code this} + */ + public VerifySignatureCommand addNames(String... names) { + checkCallable(); + namesToCheck.addAll(Arrays.asList(names)); + return this; + } + + /** + * Add names of objects (SHA-1, ref name; anything that can be + * {@link Repository#resolve(String) resolved}) to the command to have their + * signatures verified. + * + * @param names + * to add; duplicates will be ignored + * @return {@code this} + */ + public VerifySignatureCommand addNames(Collection<String> names) { + checkCallable(); + namesToCheck.addAll(names); + return this; + } + + /** + * Sets the mode of operation for this command. + * + * @param mode + * the {@link VerifyMode} to set + * @return {@code this} + */ + public VerifySignatureCommand setMode(@NonNull VerifyMode mode) { + checkCallable(); + this.mode = mode; + return this; + } + + /** + * Sets the {@link GpgSignatureVerifier} to use. + * + * @param verifier + * the {@link GpgSignatureVerifier} to use, or {@code null} to + * use the default verifier + * @return {@code this} + */ + public VerifySignatureCommand setVerifier(GpgSignatureVerifier verifier) { + checkCallable(); + this.verifier = verifier; + return this; + } + + /** + * Sets an external {@link GpgConfig} to use. Whether it will be used it at + * the discretion of the {@link #setVerifier(GpgSignatureVerifier)}. + * + * @param config + * to set; if {@code null}, the config will be loaded from the + * git config of the repository + * @return {@code this} + * @since 5.11 + */ + public VerifySignatureCommand setGpgConfig(GpgConfig config) { + checkCallable(); + this.config = config; + return this; + } + + /** + * Retrieves the currently set {@link GpgSignatureVerifier}. Can be used + * after a successful {@link #call()} to get the verifier that was used. + * + * @return the {@link GpgSignatureVerifier} + */ + public GpgSignatureVerifier getVerifier() { + return verifier; + } + + /** + * {@link Repository#resolve(String) Resolves} all names added to the + * command to git objects and verifies their signature. Non-existing objects + * are ignored. + * <p> + * Depending on the {@link #setMode(VerifyMode)}, only tags or commits or + * any kind of objects are allowed. + * </p> + * <p> + * Unsigned objects are silently skipped. + * </p> + * + * @return a map of the given names to the corresponding + * {@link VerificationResult}, excluding ignored or skipped objects. + * @throws ServiceUnavailableException + * if no {@link GpgSignatureVerifier} was set and no + * {@link GpgSignatureVerifierFactory} is available + * @throws WrongObjectTypeException + * if a name resolves to an object of a type not allowed by the + * {@link #setMode(VerifyMode)} mode + */ + @Override + @NonNull + public Map<String, VerificationResult> call() + throws ServiceUnavailableException, WrongObjectTypeException { + checkCallable(); + setCallable(false); + Map<String, VerificationResult> result = new HashMap<>(); + if (verifier == null) { + GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory + .getDefault(); + if (factory == null) { + throw new ServiceUnavailableException( + JGitText.get().signatureVerificationUnavailable); + } + verifier = factory.getVerifier(); + ownVerifier = true; + } + if (config == null) { + config = new GpgConfig(repo.getConfig()); + } + try (RevWalk walk = new RevWalk(repo)) { + for (String toCheck : namesToCheck) { + ObjectId id = repo.resolve(toCheck); + if (id != null && !ObjectId.zeroId().equals(id)) { + RevObject object; + try { + object = walk.parseAny(id); + } catch (MissingObjectException e) { + continue; + } + VerificationResult verification = verifyOne(object); + if (verification != null) { + result.put(toCheck, verification); + } + } + } + } catch (IOException e) { + throw new JGitInternalException( + JGitText.get().signatureVerificationError, e); + } finally { + if (ownVerifier) { + verifier.clear(); + } + } + return result; + } + + private VerificationResult verifyOne(RevObject object) + throws WrongObjectTypeException, IOException { + int type = object.getType(); + if (VerifyMode.TAGS.equals(mode) && type != Constants.OBJ_TAG) { + throw new WrongObjectTypeException(object, Constants.OBJ_TAG); + } else if (VerifyMode.COMMITS.equals(mode) + && type != Constants.OBJ_COMMIT) { + throw new WrongObjectTypeException(object, Constants.OBJ_COMMIT); + } + if (type == Constants.OBJ_COMMIT || type == Constants.OBJ_TAG) { + try { + GpgSignatureVerifier.SignatureVerification verification = verifier + .verifySignature(object, config); + if (verification == null) { + // Not signed + return null; + } + // Create new result + return new Result(object, verification, null); + } catch (JGitInternalException e) { + return new Result(object, null, e); + } + } + return null; + } + + private static class Result implements VerificationResult { + + private final Throwable throwable; + + private final SignatureVerification verification; + + private final RevObject object; + + public Result(RevObject object, SignatureVerification verification, + Throwable throwable) { + this.object = object; + this.verification = verification; + this.throwable = throwable; + } + + @Override + public Throwable getException() { + return throwable; + } + + @Override + public SignatureVerification getVerification() { + return verification; + } + + @Override + public RevObject getObject() { + return object; + } + + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java index 7e39361eff..81b7bd84cb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/RefAlreadyExistsException.java @@ -1,42 +1,17 @@ /* - * Copyright (C) 2010,Mathias Kinzler <mathias.kinzler@sap.com> and - * other copyright owners as documented in the project's IP log. + * Copyright (C) 2010, 2020 Mathias Kinzler <mathias.kinzler@sap.com> and others * * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v1.0 which accompanies this - * distribution, is reproduced below, and is available at - * http://www.eclipse.org/org/documents/edl-v10.php + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * - Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * - Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * - Neither the name of the Eclipse Foundation, Inc. nor the names of its - * contributors may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. + * SPDX-License-Identifier: BSD-3-Clause */ package org.eclipse.jgit.api.errors; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.lib.RefUpdate; + /** * Thrown when trying to create a {@link org.eclipse.jgit.lib.Ref} with the same * name as an existing one @@ -44,13 +19,43 @@ package org.eclipse.jgit.api.errors; public class RefAlreadyExistsException extends GitAPIException { private static final long serialVersionUID = 1L; + private final RefUpdate.Result updateResult; + /** - * Constructor for RefAlreadyExistsException + * Creates a new instance with the given message. * * @param message * error message */ public RefAlreadyExistsException(String message) { + this(message, null); + } + + /** + * Constructor for RefAlreadyExistsException + * + * @param message + * error message + * @param updateResult + * that caused the exception; may be {@code null} + * @since 5.11 + */ + public RefAlreadyExistsException(String message, + @Nullable RefUpdate.Result updateResult) { super(message); + this.updateResult = updateResult; + } + + /** + * Retrieves the {@link org.eclipse.jgit.lib.RefUpdate.Result + * RefUpdate.Result} that caused the exception. + * + * @return the {@link org.eclipse.jgit.lib.RefUpdate.Result + * RefUpdate.Result} or {@code null} if unknown + * @since 5.11 + */ + @Nullable + public RefUpdate.Result getUpdateResult() { + return updateResult; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/WrongObjectTypeException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/WrongObjectTypeException.java new file mode 100644 index 0000000000..f639c2f838 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/WrongObjectTypeException.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api.errors; + +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; + +/** + * A given object is not of an expected object type. + * + * @since 5.11 + */ +public class WrongObjectTypeException extends GitAPIException { + + private static final long serialVersionUID = 1L; + + private String name; + + private int type; + + /** + * Construct a {@link WrongObjectTypeException} for the specified object id, + * giving the expected type. + * + * @param id + * {@link ObjectId} of the object with the unexpected type + * @param type + * expected object type code; see + * {@link Constants}{@code .OBJ_*}. + */ + public WrongObjectTypeException(ObjectId id, int type) { + super(MessageFormat.format(JGitText.get().objectIsNotA, id.name(), + Constants.typeString(type))); + this.name = id.name(); + this.type = type; + } + + /** + * Retrieves the name (SHA-1) of the object. + * + * @return the name + */ + public String getObjectId() { + return name; + } + + /** + * Retrieves the expected type code. See {@link Constants}{@code .OBJ_*}. + * + * @return the type code + */ + public int getExpectedType() { + return type; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java index 2698e23035..1c9e9d7f71 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.attributes; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -23,7 +24,7 @@ import org.eclipse.jgit.lib.Repository; * @since 4.6 */ public class FilterCommandRegistry { - private static ConcurrentHashMap<String, FilterCommandFactory> filterCommandRegistry = new ConcurrentHashMap<>(); + private static Map<String, FilterCommandFactory> filterCommandRegistry = new ConcurrentHashMap<>(); /** * Register a {@link org.eclipse.jgit.attributes.FilterCommandFactory} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 8c51a7ac2f..671475ed47 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -946,12 +946,14 @@ public class DirCacheCheckout { // called before). Ignore the cached deletion and use what we // find in Merge. Potentially updates the file. if (equalIdAndMode(hId, hMode, mId, mMode)) { - if (initialCheckout) + if (initialCheckout || force) { update(name, mId, mMode); - else + } else { keep(name, dce, f); - } else + } + } else { conflict(name, dce, h, m); + } } } else { // Something in Index @@ -1214,8 +1216,12 @@ public class DirCacheCheckout { private void keep(String path, DirCacheEntry e, WorkingTreeIterator f) throws IOException { - if (e != null && !FileMode.TREE.equals(e.getFileMode())) + if (e == null) { + return; + } + if (!FileMode.TREE.equals(e.getFileMode())) { builder.add(e); + } if (force) { if (f == null || f.isModified(e, true, walk.getObjectReader())) { kept.add(path); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java index c3b1df9928..a37b8bee24 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/NoPackSignatureException.java @@ -13,8 +13,7 @@ package org.eclipse.jgit.errors; import java.io.IOException; /** - * Thrown when a PackFile is found not to contain the pack signature defined by - * git. + * Thrown when a Pack is found not to contain the pack signature defined by git. * * @since 4.5 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java index c484984f82..1fd80867b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java @@ -17,7 +17,7 @@ import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; /** - * Thrown when a PackFile previously failed and is known to be unusable + * Thrown when a Pack previously failed and is known to be unusable */ public class PackInvalidException extends IOException { private static final long serialVersionUID = 1L; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java index ad5664ceb2..44b8e0193c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackMismatchException.java @@ -13,7 +13,7 @@ package org.eclipse.jgit.errors; import java.io.IOException; /** - * Thrown when a PackFile no longer matches the PackIndex. + * Thrown when a Pack no longer matches the PackIndex. */ public class PackMismatchException extends IOException { private static final long serialVersionUID = 1L; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java index 7538229950..07aa7564dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java @@ -16,7 +16,7 @@ import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; /** - * Thrown when a PackFile uses a pack version not supported by JGit. + * Thrown when a Pack uses a pack version not supported by JGit. * * @since 4.5 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java b/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java index 32c3a1d28f..476c37c1c3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/events/ListenerList.java @@ -11,15 +11,15 @@ package org.eclipse.jgit.events; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; /** * Manages a thread-safe list of {@link org.eclipse.jgit.events.RepositoryListener}s. */ public class ListenerList { - private final ConcurrentMap<Class<? extends RepositoryListener>, CopyOnWriteArrayList<ListenerHandle>> lists = new ConcurrentHashMap<>(); + private final Map<Class<? extends RepositoryListener>, CopyOnWriteArrayList<ListenerHandle>> lists = new ConcurrentHashMap<>(); /** * Register a {@link org.eclipse.jgit.events.WorkingTreeModifiedListener}. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java index 4059b16b39..ce3ad2239a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java @@ -9,12 +9,10 @@ */ package org.eclipse.jgit.hooks; -import static java.nio.charset.StandardCharsets.UTF_8; - import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; +import java.io.OutputStream; +import java.nio.charset.Charset; import java.util.concurrent.Callable; import org.eclipse.jgit.api.errors.AbortedByHookException; @@ -35,21 +33,21 @@ import org.eclipse.jgit.util.io.TeeOutputStream; * the return type which is expected from {@link #call()} * @see <a href="http://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks">Git * Hooks on the git-scm official site</a> - * @since 4.0 + * @since 5.11 */ -abstract class GitHook<T> implements Callable<T> { +public abstract class GitHook<T> implements Callable<T> { private final Repository repo; /** * The output stream to be used by the hook. */ - protected final PrintStream outputStream; + private final OutputStream outputStream; /** * The error stream to be used by the hook. */ - protected final PrintStream errorStream; + private final OutputStream errorStream; /** * Constructor for GitHook. @@ -63,7 +61,7 @@ abstract class GitHook<T> implements Callable<T> { * The output stream the hook must use. {@code null} is allowed, * in which case the hook will use {@code System.out}. */ - protected GitHook(Repository repo, PrintStream outputStream) { + protected GitHook(Repository repo, OutputStream outputStream) { this(repo, outputStream, null); } @@ -79,8 +77,8 @@ abstract class GitHook<T> implements Callable<T> { * The error stream the hook must use. {@code null} is allowed, * in which case the hook will use {@code System.err}. */ - protected GitHook(Repository repo, PrintStream outputStream, - PrintStream errorStream) { + protected GitHook(Repository repo, OutputStream outputStream, + OutputStream errorStream) { this.repo = repo; this.outputStream = outputStream; this.errorStream = errorStream; @@ -137,7 +135,7 @@ abstract class GitHook<T> implements Callable<T> { * @return The output stream the hook must use. Never {@code null}, * {@code System.out} is returned by default. */ - protected PrintStream getOutputStream() { + protected OutputStream getOutputStream() { return outputStream == null ? System.out : outputStream; } @@ -147,7 +145,7 @@ abstract class GitHook<T> implements Callable<T> { * @return The error stream the hook must use. Never {@code null}, * {@code System.err} is returned by default. */ - protected PrintStream getErrorStream() { + protected OutputStream getErrorStream() { return errorStream == null ? System.err : errorStream; } @@ -156,34 +154,48 @@ abstract class GitHook<T> implements Callable<T> { * * @throws org.eclipse.jgit.api.errors.AbortedByHookException * If the underlying hook script exited with non-zero. + * @throws IOException + * if an IO error occurred */ - protected void doRun() throws AbortedByHookException { + protected void doRun() throws AbortedByHookException, IOException { final ByteArrayOutputStream errorByteArray = new ByteArrayOutputStream(); final TeeOutputStream stderrStream = new TeeOutputStream(errorByteArray, getErrorStream()); - PrintStream hookErrRedirect = null; - try { - hookErrRedirect = new PrintStream(stderrStream, false, - UTF_8.name()); - } catch (UnsupportedEncodingException e) { - // UTF-8 is guaranteed to be available - } Repository repository = getRepository(); FS fs = repository.getFS(); if (fs == null) { fs = FS.DETECTED; } ProcessResult result = fs.runHookIfPresent(repository, getHookName(), - getParameters(), getOutputStream(), hookErrRedirect, + getParameters(), getOutputStream(), stderrStream, getStdinArgs()); if (result.isExecutedWithError()) { - throw new AbortedByHookException( - new String(errorByteArray.toByteArray(), UTF_8), - getHookName(), result.getExitCode()); + handleError(new String(errorByteArray.toByteArray(), + Charset.defaultCharset().name()), result); } } /** + * Process that the hook exited with an error. This default implementation + * throws an {@link AbortedByHookException }. Hooks which need a different + * behavior can overwrite this method. + * + * @param message + * error message + * @param result + * The process result of the hook + * @throws AbortedByHookException + * When the hook should be aborted + * @since 5.11 + */ + protected void handleError(String message, + final ProcessResult result) + throws AbortedByHookException { + throw new AbortedByHookException(message, getHookName(), + result.getExitCode()); + } + + /** * Check whether a 'native' (i.e. script) hook is installed in the * repository. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java index 0b61ebea3f..b9dafcca31 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/PostCommitHook.java @@ -14,6 +14,7 @@ import java.io.PrintStream; import org.eclipse.jgit.api.errors.AbortedByHookException; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.ProcessResult; /** * The <code>post-commit</code> hook implementation. This hook is run after the @@ -73,4 +74,16 @@ public class PostCommitHook extends GitHook<Void> { return NAME; } + + /** + * Overwrites the default implementation to never throw an + * {@link AbortedByHookException}, as the commit has already been done and + * the exit code of the post-commit hook has no effect. + */ + @Override + protected void handleError(String message, ProcessResult result) + throws AbortedByHookException { + // Do nothing as the exit code of the post-commit hook has no effect. + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java index d7e4f79d26..9dd565ff0a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014, Andrey Loskutov <loskutov@gmx.de> and others + * Copyright (C) 2014, 2021 Andrey Loskutov <loskutov@gmx.de> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -14,8 +14,11 @@ import static org.eclipse.jgit.ignore.internal.Strings.isDirectoryPattern; import static org.eclipse.jgit.ignore.internal.Strings.stripTrailing; import static org.eclipse.jgit.ignore.internal.Strings.stripTrailingWhitespace; +import java.text.MessageFormat; + import org.eclipse.jgit.errors.InvalidPatternException; import org.eclipse.jgit.ignore.internal.PathMatcher; +import org.eclipse.jgit.internal.JGitText; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,11 +39,11 @@ public class FastIgnoreRule { */ public static final char PATH_SEPARATOR = '/'; - private final IMatcher matcher; + private IMatcher matcher; - private final boolean inverse; + private boolean inverse; - private final boolean dirOnly; + private boolean dirOnly; /** * Constructor for FastIgnoreRule @@ -52,8 +55,23 @@ public class FastIgnoreRule { * (comment), this rule doesn't match anything. */ public FastIgnoreRule(String pattern) { - if (pattern == null) + this(); + try { + parse(pattern); + } catch (InvalidPatternException e) { + LOG.error(MessageFormat.format(JGitText.get().badIgnorePattern, + e.getPattern()), e); + } + } + + FastIgnoreRule() { + matcher = IMatcher.NO_MATCH; + } + + void parse(String pattern) throws InvalidPatternException { + if (pattern == null) { throw new IllegalArgumentException("Pattern must not be null!"); //$NON-NLS-1$ + } if (pattern.length() == 0) { dirOnly = false; inverse = false; @@ -90,15 +108,8 @@ public class FastIgnoreRule { return; } } - IMatcher m; - try { - m = PathMatcher.createPathMatcher(pattern, - Character.valueOf(PATH_SEPARATOR), dirOnly); - } catch (InvalidPatternException e) { - m = NO_MATCH; - LOG.error(e.getMessage(), e); - } - this.matcher = m; + this.matcher = PathMatcher.createPathMatcher(pattern, + Character.valueOf(PATH_SEPARATOR), dirOnly); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java index 0bc6124912..4e7f126a60 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Red Hat Inc. and others + * Copyright (C) 2010, 2021 Red Hat Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -15,16 +15,26 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.errors.InvalidPatternException; +import org.eclipse.jgit.internal.JGitText; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Represents a bundle of ignore rules inherited from a base directory. * * This class is not thread safe, it maintains state about the last match. */ public class IgnoreNode { + + private static final Logger LOG = LoggerFactory.getLogger(IgnoreNode.class); + /** Result from {@link IgnoreNode#isIgnored(String, boolean)}. */ public enum MatchResult { /** The file is not ignored, due to a rule saying its not ignored. */ @@ -52,7 +62,7 @@ public class IgnoreNode { * Create an empty ignore node with no rules. */ public IgnoreNode() { - rules = new ArrayList<>(); + this(new ArrayList<>()); } /** @@ -75,15 +85,47 @@ public class IgnoreNode { * Error thrown when reading an ignore file. */ public void parse(InputStream in) throws IOException { + parse(null, in); + } + + /** + * Parse files according to gitignore standards. + * + * @param sourceName + * identifying the source of the stream + * @param in + * input stream holding the standard ignore format. The caller is + * responsible for closing the stream. + * @throws java.io.IOException + * Error thrown when reading an ignore file. + * @since 5.11 + */ + public void parse(String sourceName, InputStream in) throws IOException { BufferedReader br = asReader(in); String txt; + int lineNumber = 1; while ((txt = br.readLine()) != null) { if (txt.length() > 0 && !txt.startsWith("#") && !txt.equals("/")) { //$NON-NLS-1$ //$NON-NLS-2$ - FastIgnoreRule rule = new FastIgnoreRule(txt); + FastIgnoreRule rule = new FastIgnoreRule(); + try { + rule.parse(txt); + } catch (InvalidPatternException e) { + if (sourceName != null) { + LOG.error(MessageFormat.format( + JGitText.get().badIgnorePatternFull, sourceName, + Integer.toString(lineNumber), e.getPattern(), + e.getLocalizedMessage()), e); + } else { + LOG.error(MessageFormat.format( + JGitText.get().badIgnorePattern, + e.getPattern()), e); + } + } if (!rule.isEmpty()) { rules.add(rule); } } + lineNumber++; } } @@ -135,7 +177,8 @@ public class IgnoreNode { * undetermined * @since 4.11 */ - public Boolean checkIgnored(String entryPath, boolean isDirectory) { + public @Nullable Boolean checkIgnored(String entryPath, + boolean isDirectory) { // Parse rules in the reverse order that they were read because later // rules have higher priority for (int i = rules.size() - 1; i > -1; i--) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 892657d5d3..9d215ca455 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, 2013 Sasa Zivkov <sasa.zivkov@sap.com> - * Copyright (C) 2012, Research In Motion Limited and others + * Copyright (C) 2012, 2021 Research In Motion Limited and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -58,6 +58,8 @@ public class JGitText extends TranslationBundle { /***/ public String badEntryName; /***/ public String badEscape; /***/ public String badGroupHeader; + /***/ public String badIgnorePattern; + /***/ public String badIgnorePatternFull; /***/ public String badObjectType; /***/ public String badRef; /***/ public String badSectionEntry; @@ -261,6 +263,7 @@ public class JGitText extends TranslationBundle { /***/ public String downloadCancelledDuringIndexing; /***/ public String duplicateAdvertisementsOf; /***/ public String duplicateRef; + /***/ public String duplicateRefAttribute; /***/ public String duplicateRemoteRefUpdateIsIllegal; /***/ public String duplicateStagesNotAllowed; /***/ public String eitherGitDirOrWorkTreeRequired; @@ -338,6 +341,10 @@ public class JGitText extends TranslationBundle { /***/ public String hoursAgo; /***/ public String httpConfigCannotNormalizeURL; /***/ public String httpConfigInvalidURL; + /***/ public String httpFactoryInUse; + /***/ public String httpPreAuthTooLate; + /***/ public String httpUserInfoDecodeError; + /***/ public String httpWrongConnectionType; /***/ public String hugeIndexesAreNotSupportedByJgitYet; /***/ public String hunkBelongsToAnotherFile; /***/ public String hunkDisconnectedFromFile; @@ -591,6 +598,7 @@ public class JGitText extends TranslationBundle { /***/ public String reftableDirExists; /***/ public String reftableRecordsMustIncrease; /***/ public String refUpdateReturnCodeWas; + /***/ public String remoteBranchNotFound; /***/ public String remoteConfigHasNoURIAssociated; /***/ public String remoteDoesNotHaveSpec; /***/ public String remoteDoesNotSupportSmartHTTPPush; @@ -645,7 +653,9 @@ public class JGitText extends TranslationBundle { /***/ public String shortReadOfBlock; /***/ public String shortReadOfOptionalDIRCExtensionExpectedAnotherBytes; /***/ public String shortSkipOfBlock; - /***/ public String signingNotSupportedOnTag; + /***/ public String signatureVerificationError; + /***/ public String signatureVerificationUnavailable; + /***/ public String signedTagMessageNoLf; /***/ public String signingServiceUnavailable; /***/ public String similarityScoreMustBeWithinBounds; /***/ public String skipMustBeNonNegative; @@ -786,6 +796,13 @@ public class JGitText extends TranslationBundle { /***/ public String URINotSupported; /***/ public String userConfigInvalid; /***/ public String validatingGitModules; + /***/ public String verifySignatureBad; + /***/ public String verifySignatureExpired; + /***/ public String verifySignatureGood; + /***/ public String verifySignatureIssuer; + /***/ public String verifySignatureKey; + /***/ public String verifySignatureMade; + /***/ public String verifySignatureTrust; /***/ public String walkFailure; /***/ public String wantNoSpaceWithCapabilities; /***/ public String wantNotValid; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java deleted file mode 100644 index 5ddbcbd0ed..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.KetchConstants.TERM; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeoutException; - -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.TreeFormatter; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.time.ProposedTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The initial {@link Round} for a leaderless repository, used to establish a - * leader. - */ -class ElectionRound extends Round { - private static final Logger log = LoggerFactory.getLogger(ElectionRound.class); - - private long term; - - ElectionRound(KetchLeader leader, LogIndex head) { - super(leader, head); - } - - @Override - void start() throws IOException { - ObjectId id; - try (Repository git = leader.openRepository(); - ProposedTimestamp ts = getSystem().getClock().propose(); - ObjectInserter inserter = git.newObjectInserter()) { - id = bumpTerm(git, ts, inserter); - inserter.flush(); - blockUntil(ts); - } - runAsync(id); - } - - @Override - void success() { - // Do nothing upon election, KetchLeader will copy the term. - } - - long getTerm() { - return term; - } - - private ObjectId bumpTerm(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException { - CommitBuilder b = new CommitBuilder(); - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(acceptedOldIndex); - if (getSystem().requireMonotonicLeaderElections()) { - if (ts.read(SECONDS) < c.getCommitTime()) { - throw new TimeIsUncertainException(); - } - } - b.setTreeId(c.getTree()); - b.setParentId(acceptedOldIndex); - term = parseTerm(c.getFooterLines(TERM)) + 1; - } - } else { - term = 1; - b.setTreeId(inserter.insert(new TreeFormatter())); - } - - StringBuilder msg = new StringBuilder(); - msg.append(KetchConstants.TERM.getName()) - .append(": ") //$NON-NLS-1$ - .append(term); - - String tag = leader.getSystem().newLeaderTag(); - if (tag != null && !tag.isEmpty()) { - msg.append(' ').append(tag); - } - - b.setAuthor(leader.getSystem().newCommitter(ts)); - b.setCommitter(b.getAuthor()); - b.setMessage(msg.toString()); - - if (log.isDebugEnabled()) { - log.debug("Trying to elect myself " + b.getMessage()); //$NON-NLS-1$ - } - return inserter.insert(b); - } - - private static long parseTerm(List<String> footer) { - if (footer.isEmpty()) { - return 0; - } - - String s = footer.get(0); - int p = s.indexOf(' '); - if (p > 0) { - s = s.substring(0, p); - } - return Long.parseLong(s, 10); - } - - private void blockUntil(ProposedTimestamp ts) throws IOException { - try { - ts.blockUntil(getSystem().getMaxWaitForMonotonicClock()); - } catch (InterruptedException | TimeoutException e) { - throw new TimeIsUncertainException(e); - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java deleted file mode 100644 index f4a7f592f0..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.revwalk.FooterKey; - -/** - * Frequently used constants in a Ketch system. - */ -public class KetchConstants { - /** - * Default reference namespace holding {@link #ACCEPTED} and - * {@link #COMMITTED} references and the {@link #STAGE} sub-namespace. - */ - public static final String DEFAULT_TXN_NAMESPACE = "refs/txn/"; //$NON-NLS-1$ - - /** Reference name holding the RefTree accepted by a follower. */ - public static final String ACCEPTED = "accepted"; //$NON-NLS-1$ - - /** Reference name holding the RefTree known to be committed. */ - public static final String COMMITTED = "committed"; //$NON-NLS-1$ - - /** Reference subdirectory holding proposed heads. */ - public static final String STAGE = "stage/"; //$NON-NLS-1$ - - /** Footer containing the current term. */ - public static final FooterKey TERM = new FooterKey("Term"); //$NON-NLS-1$ - - /** Section for Ketch configuration ({@code ketch}). */ - public static final String CONFIG_SECTION_KETCH = "ketch"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-type}) */ - public static final String CONFIG_KEY_TYPE = "ketch-type"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-commit}) */ - public static final String CONFIG_KEY_COMMIT = "ketch-commit"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-speed}) */ - public static final String CONFIG_KEY_SPEED = "ketch-speed"; //$NON-NLS-1$ - - private KetchConstants() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java deleted file mode 100644 index 743d1939c8..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java +++ /dev/null @@ -1,604 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.CANDIDATE; -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.LEADER; -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.SHUTDOWN; -import static org.eclipse.jgit.internal.ketch.KetchReplica.Participation.FOLLOWER_ONLY; -import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A leader managing consensus across remote followers. - * <p> - * A leader instance starts up in - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#CANDIDATE} and tries - * to begin a new term by sending an - * {@link org.eclipse.jgit.internal.ketch.ElectionRound} to all replicas. Its - * term starts if a majority of replicas have accepted this leader instance for - * the term. - * <p> - * Once elected by a majority the instance enters - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER} and runs - * proposals offered to {@link #queueProposal(Proposal)}. This continues until - * the leader is timed out for inactivity, or is deposed by a competing leader - * gaining its own majority. - * <p> - * Once timed out or deposed this {@code KetchLeader} instance should be - * discarded, and a new instance takes over. - * <p> - * Each leader instance coordinates a group of - * {@link org.eclipse.jgit.internal.ketch.KetchReplica}s. Replica instances are - * owned by the leader instance and must be discarded when the leader is - * discarded. - * <p> - * In Ketch all push requests are issued through the leader. The steps are as - * follows (see {@link org.eclipse.jgit.internal.ketch.KetchPreReceive} for an - * example): - * <ul> - * <li>Create a {@link org.eclipse.jgit.internal.ketch.Proposal} with the - * {@link org.eclipse.jgit.transport.ReceiveCommand}s that represent the push. - * <li>Invoke {@link #queueProposal(Proposal)} on the leader instance. - * <li>Wait for consensus with - * {@link org.eclipse.jgit.internal.ketch.Proposal#await()}. - * <li>To examine the status of the push, check - * {@link org.eclipse.jgit.internal.ketch.Proposal#getCommands()}, looking at - * {@link org.eclipse.jgit.internal.storage.reftree.Command#getResult()}. - * </ul> - * <p> - * The leader gains consensus by first pushing the needed objects and a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree} representing the - * desired target repository state to the {@code refs/txn/accepted} branch on - * each of the replicas. Once a majority has succeeded, the leader commits the - * state by either pushing the {@code refs/txn/accepted} value to - * {@code refs/txn/committed} (for Ketch-aware replicas) or by pushing updates - * to {@code refs/heads/master}, etc. for stock Git replicas. - * <p> - * Internally, the actual transport to replicas is performed on background - * threads via the {@link org.eclipse.jgit.internal.ketch.KetchSystem}'s - * executor service. For performance, the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}, - * {@link org.eclipse.jgit.internal.ketch.KetchReplica} and - * {@link org.eclipse.jgit.internal.ketch.Proposal} objects share some state, - * and may invoke each other's methods on different threads. This access is - * protected by the leader's {@link #lock} object. Care must be taken to prevent - * concurrent access by correctly obtaining the leader's lock. - */ -public abstract class KetchLeader { - private static final Logger log = LoggerFactory.getLogger(KetchLeader.class); - - /** Current state of the leader instance. */ - public enum State { - /** Newly created instance trying to elect itself leader. */ - CANDIDATE, - - /** Leader instance elected by a majority. */ - LEADER, - - /** Instance has been deposed by another with a more recent term. */ - DEPOSED, - - /** Leader has been gracefully shutdown, e.g. due to inactivity. */ - SHUTDOWN; - } - - private final KetchSystem system; - - /** Leader's knowledge of replicas for this repository. */ - private KetchReplica[] voters; - private KetchReplica[] followers; - private LocalReplica self; - - /** - * Lock protecting all data within this leader instance. - * <p> - * This lock extends into the {@link KetchReplica} instances used by the - * leader. They share the same lock instance to simplify concurrency. - */ - final Lock lock; - - private State state = CANDIDATE; - - /** Term of this leader, once elected. */ - private long term; - - /** - * Pending proposals accepted into the queue in FIFO order. - * <p> - * These proposals were preflighted and do not contain any conflicts with - * each other and their expectations matched the leader's local view of the - * agreed upon {@code refs/txn/accepted} tree. - */ - private final List<Proposal> queued; - - /** - * State of the repository's RefTree after applying all entries in - * {@link #queued}. New proposals must be consistent with this tree to be - * appended to the end of {@link #queued}. - * <p> - * Must be deep-copied with {@link RefTree#copy()} if - * {@link #roundHoldsReferenceToRefTree} is {@code true}. - */ - private RefTree refTree; - - /** - * If {@code true} {@link #refTree} must be duplicated before queuing the - * next proposal. The {@link #refTree} was passed into the constructor of a - * {@link ProposalRound}, and that external reference to the {@link RefTree} - * object is held by the proposal until it materializes the tree object in - * the object store. This field is set {@code true} when the proposal begins - * execution and set {@code false} once tree objects are persisted in the - * local repository's object store or {@link #refTree} is replaced with a - * copy to isolate it from any running rounds. - * <p> - * If proposals arrive less frequently than the {@code RefTree} is written - * out to the repository the {@link #roundHoldsReferenceToRefTree} behavior - * avoids duplicating {@link #refTree}, reducing both time and memory used. - * However if proposals arrive more frequently {@link #refTree} must be - * duplicated to prevent newly queued proposals from corrupting the - * {@link #runningRound}. - */ - volatile boolean roundHoldsReferenceToRefTree; - - /** End of the leader's log. */ - private LogIndex headIndex; - - /** Leader knows this (and all prior) states are committed. */ - private LogIndex committedIndex; - - /** - * Is the leader idle with no work pending? If {@code true} there is no work - * for the leader (normal state). This field is {@code false} when the - * leader thread is scheduled for execution, or while {@link #runningRound} - * defines a round in progress. - */ - private boolean idle; - - /** Current round the leader is preparing and waiting for a vote on. */ - private Round runningRound; - - /** - * Construct a leader for a Ketch instance. - * - * @param system - * Ketch system configuration the leader must adhere to. - */ - protected KetchLeader(KetchSystem system) { - this.system = system; - this.lock = new ReentrantLock(true /* fair */); - this.queued = new ArrayList<>(4); - this.idle = true; - } - - /** @return system configuration. */ - KetchSystem getSystem() { - return system; - } - - /** - * Configure the replicas used by this Ketch instance. - * <p> - * Replicas should be configured once at creation before any proposals are - * executed. Once elections happen, <b>reconfiguration is a complicated - * concept that is not currently supported</b>. - * - * @param replicas - * members participating with the same repository. - */ - public void setReplicas(Collection<KetchReplica> replicas) { - List<KetchReplica> v = new ArrayList<>(5); - List<KetchReplica> f = new ArrayList<>(5); - for (KetchReplica r : replicas) { - switch (r.getParticipation()) { - case FULL: - v.add(r); - break; - - case FOLLOWER_ONLY: - f.add(r); - break; - } - } - - Collection<Integer> validVoters = validVoterCounts(); - if (!validVoters.contains(Integer.valueOf(v.size()))) { - throw new IllegalArgumentException(MessageFormat.format( - KetchText.get().unsupportedVoterCount, - Integer.valueOf(v.size()), - validVoters)); - } - - LocalReplica me = findLocal(v); - if (me == null) { - throw new IllegalArgumentException( - KetchText.get().localReplicaRequired); - } - - lock.lock(); - try { - voters = v.toArray(new KetchReplica[0]); - followers = f.toArray(new KetchReplica[0]); - self = me; - } finally { - lock.unlock(); - } - } - - private static Collection<Integer> validVoterCounts() { - @SuppressWarnings("boxing") - Integer[] valid = { - // An odd number of voting replicas is required. - 1, 3, 5, 7, 9 }; - return Arrays.asList(valid); - } - - private static LocalReplica findLocal(Collection<KetchReplica> voters) { - for (KetchReplica r : voters) { - if (r instanceof LocalReplica) { - return (LocalReplica) r; - } - } - return null; - } - - /** - * Get an instance of the repository for use by a leader thread. - * <p> - * The caller will close the repository. - * - * @return opened repository for use by the leader thread. - * @throws java.io.IOException - * cannot reopen the repository for the leader. - */ - protected abstract Repository openRepository() throws IOException; - - /** - * Queue a reference update proposal for consensus. - * <p> - * This method does not wait for consensus to be reached. The proposal is - * checked to look for risks of conflicts, and then submitted into the queue - * for distribution as soon as possible. - * <p> - * Callers must use {@link org.eclipse.jgit.internal.ketch.Proposal#await()} - * to see if the proposal is done. - * - * @param proposal - * the proposed reference updates to queue for consideration. - * Once execution is complete the individual reference result - * fields will be populated with the outcome. - * @throws java.lang.InterruptedException - * current thread was interrupted. The proposal may have been - * aborted if it was not yet queued for execution. - * @throws java.io.IOException - * unrecoverable error preventing proposals from being attempted - * by this leader. - */ - public void queueProposal(Proposal proposal) - throws InterruptedException, IOException { - try { - lock.lockInterruptibly(); - } catch (InterruptedException e) { - proposal.abort(); - throw e; - } - try { - if (refTree == null) { - initialize(); - for (Proposal p : queued) { - refTree.apply(p.getCommands()); - } - } else if (roundHoldsReferenceToRefTree) { - refTree = refTree.copy(); - roundHoldsReferenceToRefTree = false; - } - - if (!refTree.apply(proposal.getCommands())) { - // A conflict exists so abort the proposal. - proposal.abort(); - return; - } - - queued.add(proposal); - proposal.notifyState(QUEUED); - - if (idle) { - scheduleLeader(); - } - } finally { - lock.unlock(); - } - } - - private void initialize() throws IOException { - try (Repository git = openRepository(); RevWalk rw = new RevWalk(git)) { - self.initialize(git); - - ObjectId accepted = self.getTxnAccepted(); - if (!ObjectId.zeroId().equals(accepted)) { - RevCommit c = rw.parseCommit(accepted); - headIndex = LogIndex.unknown(accepted); - refTree = RefTree.read(rw.getObjectReader(), c.getTree()); - } else { - headIndex = LogIndex.unknown(ObjectId.zeroId()); - refTree = RefTree.newEmptyTree(); - } - } - } - - private void scheduleLeader() { - idle = false; - system.getExecutor().execute(this::runLeader); - } - - private void runLeader() { - Round round; - lock.lock(); - try { - switch (state) { - case CANDIDATE: - round = new ElectionRound(this, headIndex); - break; - - case LEADER: - round = newProposalRound(); - break; - - case DEPOSED: - case SHUTDOWN: - default: - log.warn("Leader cannot run {}", state); //$NON-NLS-1$ - // TODO(sop): Redirect proposals. - return; - } - } finally { - lock.unlock(); - } - - try { - round.start(); - } catch (IOException e) { - // TODO(sop) Depose leader if it cannot use its repository. - log.error(KetchText.get().leaderFailedToStore, e); - lock.lock(); - try { - nextRound(); - } finally { - lock.unlock(); - } - } - } - - private ProposalRound newProposalRound() { - List<Proposal> todo = new ArrayList<>(queued); - queued.clear(); - roundHoldsReferenceToRefTree = true; - return new ProposalRound(this, headIndex, todo, refTree); - } - - /** @return term of this leader's reign. */ - long getTerm() { - return term; - } - - /** @return end of the leader's log. */ - LogIndex getHead() { - return headIndex; - } - - /** - * @return state leader knows it has committed across a quorum of replicas. - */ - LogIndex getCommitted() { - return committedIndex; - } - - boolean isIdle() { - return idle; - } - - void runAsync(Round round) { - lock.lock(); - try { - // End of the log is this round. Once transport begins it is - // reasonable to assume at least one replica will eventually get - // this, and there is reasonable probability it commits. - headIndex = round.acceptedNewIndex; - runningRound = round; - - for (KetchReplica replica : voters) { - replica.pushTxnAcceptedAsync(round); - } - for (KetchReplica replica : followers) { - replica.pushTxnAcceptedAsync(round); - } - } finally { - lock.unlock(); - } - } - - /** - * Asynchronous signal from a replica after completion. - * <p> - * Must be called while {@link #lock} is held by the replica. - * - * @param replica - * replica posting a completion event. - */ - void onReplicaUpdate(KetchReplica replica) { - if (log.isDebugEnabled()) { - log.debug("Replica {} finished:\n{}", //$NON-NLS-1$ - replica.describeForLog(), snapshot()); - } - - if (replica.getParticipation() == FOLLOWER_ONLY) { - // Followers cannot vote, so votes haven't changed. - return; - } else if (runningRound == null) { - // No round running, no need to tally votes. - return; - } - - assert headIndex.equals(runningRound.acceptedNewIndex); - int matching = 0; - for (KetchReplica r : voters) { - if (r.hasAccepted(headIndex)) { - matching++; - } - } - - int quorum = voters.length / 2 + 1; - boolean success = matching >= quorum; - if (!success) { - return; - } - - switch (state) { - case CANDIDATE: - term = ((ElectionRound) runningRound).getTerm(); - state = LEADER; - if (log.isDebugEnabled()) { - log.debug("Won election, running term " + term); //$NON-NLS-1$ - } - - //$FALL-THROUGH$ - case LEADER: - committedIndex = headIndex; - if (log.isDebugEnabled()) { - log.debug("Committed {} in term {}", //$NON-NLS-1$ - committedIndex.describeForLog(), - Long.valueOf(term)); - } - nextRound(); - commitAsync(replica); - notifySuccess(runningRound); - if (log.isDebugEnabled()) { - log.debug("Leader state:\n{}", snapshot()); //$NON-NLS-1$ - } - break; - - default: - log.debug("Leader ignoring replica while in {}", state); //$NON-NLS-1$ - break; - } - } - - private void notifySuccess(Round round) { - // Drop the leader lock while notifying Proposal listeners. - lock.unlock(); - try { - round.success(); - } finally { - lock.lock(); - } - } - - private void commitAsync(KetchReplica caller) { - for (KetchReplica r : voters) { - if (r == caller) { - continue; - } - if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { - r.pushCommitAsync(committedIndex); - } - } - for (KetchReplica r : followers) { - if (r == caller) { - continue; - } - if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { - r.pushCommitAsync(committedIndex); - } - } - } - - /** Schedule the next round; invoked while {@link #lock} is held. */ - void nextRound() { - runningRound = null; - - if (queued.isEmpty()) { - idle = true; - } else { - // Caller holds lock. Reschedule leader on a new thread so - // the call stack can unwind and lock is not held unexpectedly - // during prepare for the next round. - scheduleLeader(); - } - } - - /** - * Snapshot this leader - * - * @return snapshot of this leader - */ - public LeaderSnapshot snapshot() { - lock.lock(); - try { - LeaderSnapshot s = new LeaderSnapshot(); - s.state = state; - s.term = term; - s.headIndex = headIndex; - s.committedIndex = committedIndex; - s.idle = isIdle(); - for (KetchReplica r : voters) { - s.replicas.add(r.snapshot()); - } - for (KetchReplica r : followers) { - s.replicas.add(r.snapshot()); - } - return s; - } finally { - lock.unlock(); - } - } - - /** - * Gracefully shutdown this leader and cancel outstanding operations. - */ - public void shutdown() { - lock.lock(); - try { - if (state != SHUTDOWN) { - state = SHUTDOWN; - for (KetchReplica r : voters) { - r.shutdown(); - } - for (KetchReplica r : followers) { - r.shutdown(); - } - } - } finally { - lock.unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public String toString() { - return snapshot().toString(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java deleted file mode 100644 index e01fb3ae53..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.net.URISyntaxException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.eclipse.jgit.internal.storage.dfs.DfsRepository; -import org.eclipse.jgit.lib.Repository; - -/** - * A cache of live leader instances, keyed by repository. - * <p> - * Ketch only assigns a leader to a repository when needed. If - * {@link #get(Repository)} is called for a repository that does not have a - * leader, the leader is created and added to the cache. - */ -public class KetchLeaderCache { - private final KetchSystem system; - private final ConcurrentMap<String, KetchLeader> leaders; - private final Lock startLock; - - /** - * Initialize a new leader cache. - * - * @param system - * system configuration for the leaders - */ - public KetchLeaderCache(KetchSystem system) { - this.system = system; - leaders = new ConcurrentHashMap<>(); - startLock = new ReentrantLock(true /* fair */); - } - - /** - * Lookup the leader instance for a given repository. - * - * @param repo - * repository to get the leader for. - * @return the leader instance for the repository. - * @throws java.net.URISyntaxException - * remote configuration contains an invalid URL. - */ - public KetchLeader get(Repository repo) - throws URISyntaxException { - String key = computeKey(repo); - KetchLeader leader = leaders.get(key); - if (leader != null) { - return leader; - } - return startLeader(key, repo); - } - - private KetchLeader startLeader(String key, Repository repo) - throws URISyntaxException { - startLock.lock(); - try { - KetchLeader leader = leaders.get(key); - if (leader != null) { - return leader; - } - leader = system.createLeader(repo); - leaders.put(key, leader); - return leader; - } finally { - startLock.unlock(); - } - } - - private static String computeKey(Repository repo) { - if (repo instanceof DfsRepository) { - DfsRepository dfs = (DfsRepository) repo; - return dfs.getDescription().getRepositoryName(); - } - - if (repo.getDirectory() != null) { - return repo.getDirectory().toURI().toString(); - } - - throw new IllegalArgumentException(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java deleted file mode 100644 index 1c9535f7be..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.Collection; - -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.transport.PreReceiveHook; -import org.eclipse.jgit.transport.ProgressSpinner; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.ReceivePack; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * PreReceiveHook for handling push traffic in a Ketch system. - * <p> - * Install an instance on {@link org.eclipse.jgit.transport.ReceivePack} to - * capture the commands and other connection state and relay them through the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}, allowing the leader to - * gain consensus about the new reference state. - */ -public class KetchPreReceive implements PreReceiveHook { - private static final Logger log = LoggerFactory.getLogger(KetchPreReceive.class); - - private final KetchLeader leader; - - /** - * Construct a hook executing updates through a - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. - * - * @param leader - * leader for this repository. - */ - public KetchPreReceive(KetchLeader leader) { - this.leader = leader; - } - - /** {@inheritDoc} */ - @Override - public void onPreReceive(ReceivePack rp, Collection<ReceiveCommand> cmds) { - cmds = ReceiveCommand.filter(cmds, NOT_ATTEMPTED); - if (cmds.isEmpty()) { - return; - } - - try { - Proposal proposal = new Proposal(rp.getRevWalk(), cmds) - .setPushCertificate(rp.getPushCertificate()) - .setAuthor(rp.getRefLogIdent()) - .setMessage("push"); //$NON-NLS-1$ - leader.queueProposal(proposal); - if (proposal.isDone()) { - // This failed fast, e.g. conflict or bad precondition. - return; - } - - ProgressSpinner spinner = new ProgressSpinner( - rp.getMessageOutputStream()); - if (proposal.getState() == QUEUED) { - waitForQueue(proposal, spinner); - } - if (!proposal.isDone()) { - waitForPropose(proposal, spinner); - } - } catch (IOException | InterruptedException e) { - String msg = JGitText.get().transactionAborted; - for (ReceiveCommand cmd : cmds) { - if (cmd.getResult() == NOT_ATTEMPTED) { - cmd.setResult(REJECTED_OTHER_REASON, msg); - } - } - log.error(msg, e); - } - } - - private void waitForQueue(Proposal proposal, ProgressSpinner spinner) - throws InterruptedException { - spinner.beginTask(KetchText.get().waitingForQueue, 1, SECONDS); - while (!proposal.awaitStateChange(QUEUED, 250, MILLISECONDS)) { - spinner.update(); - } - switch (proposal.getState()) { - case RUNNING: - default: - spinner.endTask(KetchText.get().starting); - break; - - case EXECUTED: - spinner.endTask(KetchText.get().accepted); - break; - - case ABORTED: - spinner.endTask(KetchText.get().failed); - break; - } - } - - private void waitForPropose(Proposal proposal, ProgressSpinner spinner) - throws InterruptedException { - spinner.beginTask(KetchText.get().proposingUpdates, 2, SECONDS); - while (!proposal.await(250, MILLISECONDS)) { - spinner.update(); - } - spinner.endTask(proposal.getState() == EXECUTED - ? KetchText.get().accepted - : KetchText.get().failed); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java deleted file mode 100644 index a9a694a565..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java +++ /dev/null @@ -1,758 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.BATCHED; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.FAST; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.CURRENT; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.CREATE; - -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.Future; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.SystemReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A Ketch replica, either {@link org.eclipse.jgit.internal.ketch.LocalReplica} - * or {@link org.eclipse.jgit.internal.ketch.RemoteGitReplica}. - * <p> - * Replicas can be either a stock Git replica, or a Ketch-aware replica. - * <p> - * A stock Git replica has no special knowledge of Ketch and simply stores - * objects and references. Ketch communicates with the stock Git replica using - * the Git push wire protocol. The - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} commits an agreed upon - * state by pushing all references to the Git replica, for example - * {@code "refs/heads/master"} is pushed during commit. Stock Git replicas use - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS} to - * record the final state. - * <p> - * Ketch-aware replicas understand the {@code RefTree} sent during the proposal - * and during commit are able to update their own reference space to match the - * state represented by the {@code RefTree}. Ketch-aware replicas typically use - * a {@link org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase} and - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#TXN_COMMITTED} - * to record the final state. - * <p> - * KetchReplica instances are tightly coupled with a single - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. Some state may be - * accessed by the leader thread and uses the leader's own - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} to protect shared - * data. - */ -public abstract class KetchReplica { - static final Logger log = LoggerFactory.getLogger(KetchReplica.class); - private static final byte[] PEEL = { ' ', '^' }; - - /** Participation of a replica in establishing consensus. */ - public enum Participation { - /** Replica can vote. */ - FULL, - - /** Replica does not vote, but tracks leader. */ - FOLLOWER_ONLY; - } - - /** How this replica wants to receive Ketch commit operations. */ - public enum CommitMethod { - /** All references are pushed to the peer as standard Git. */ - ALL_REFS, - - /** Only {@code refs/txn/committed} is written/updated. */ - TXN_COMMITTED; - } - - /** Delay before committing to a replica. */ - public enum CommitSpeed { - /** - * Send the commit immediately, even if it could be batched with the - * next proposal. - */ - FAST, - - /** - * If the next proposal is available, batch the commit with it, - * otherwise just send the commit. This generates less network use, but - * may provide slower consistency on the replica. - */ - BATCHED; - } - - /** Current state of a replica. */ - public enum State { - /** Leader has not yet contacted the replica. */ - UNKNOWN, - - /** Replica is behind the consensus. */ - LAGGING, - - /** Replica matches the consensus. */ - CURRENT, - - /** Replica has a different (or unknown) history. */ - DIVERGENT, - - /** Replica's history contains the leader's history. */ - AHEAD, - - /** Replica can not be contacted. */ - OFFLINE; - } - - private final KetchLeader leader; - private final String replicaName; - private final Participation participation; - private final CommitMethod commitMethod; - private final CommitSpeed commitSpeed; - private final long minRetryMillis; - private final long maxRetryMillis; - private final Map<ObjectId, List<ReceiveCommand>> staged; - private final Map<String, ReceiveCommand> running; - private final Map<String, ReceiveCommand> waiting; - private final List<ReplicaPushRequest> queued; - - /** - * Value known for {@code "refs/txn/accepted"}. - * <p> - * Raft literature refers to this as {@code matchIndex}. - */ - private ObjectId txnAccepted; - - /** - * Value known for {@code "refs/txn/committed"}. - * <p> - * Raft literature refers to this as {@code commitIndex}. In traditional - * Raft this is a state variable inside the follower implementation, but - * Ketch keeps it in the leader. - */ - private ObjectId txnCommitted; - - /** What is happening with this replica. */ - private State state = UNKNOWN; - private String error; - - /** Scheduled retry due to communication failure. */ - private Future<?> retryFuture; - private long lastRetryMillis; - private long retryAtMillis; - - /** - * Configure a replica representation. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this replica for debugging. - * @param cfg - * how Ketch should treat the replica. - */ - protected KetchReplica(KetchLeader leader, String name, ReplicaConfig cfg) { - this.leader = leader; - this.replicaName = name; - this.participation = cfg.getParticipation(); - this.commitMethod = cfg.getCommitMethod(); - this.commitSpeed = cfg.getCommitSpeed(); - this.minRetryMillis = cfg.getMinRetry(MILLISECONDS); - this.maxRetryMillis = cfg.getMaxRetry(MILLISECONDS); - this.staged = new HashMap<>(); - this.running = new HashMap<>(); - this.waiting = new HashMap<>(); - this.queued = new ArrayList<>(4); - } - - /** - * Get system configuration. - * - * @return system configuration. - */ - public KetchSystem getSystem() { - return getLeader().getSystem(); - } - - /** - * Get leader instance this replica follows. - * - * @return leader instance this replica follows. - */ - public KetchLeader getLeader() { - return leader; - } - - /** - * Get unique-ish name for debugging. - * - * @return unique-ish name for debugging. - */ - public String getName() { - return replicaName; - } - - /** - * Get description of this replica for error/debug logging purposes. - * - * @return description of this replica for error/debug logging purposes. - */ - protected String describeForLog() { - return getName(); - } - - /** - * Get how the replica participates in this Ketch system. - * - * @return how the replica participates in this Ketch system. - */ - public Participation getParticipation() { - return participation; - } - - /** - * Get how Ketch will commit to the repository. - * - * @return how Ketch will commit to the repository. - */ - public CommitMethod getCommitMethod() { - return commitMethod; - } - - /** - * Get when Ketch will commit to the repository. - * - * @return when Ketch will commit to the repository. - */ - public CommitSpeed getCommitSpeed() { - return commitSpeed; - } - - /** - * Called by leader to perform graceful shutdown. - * <p> - * Default implementation cancels any scheduled retry. Subclasses may add - * additional logic before or after calling {@code super.shutdown()}. - * <p> - * Called with {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held - * by caller. - */ - protected void shutdown() { - Future<?> f = retryFuture; - if (f != null) { - retryFuture = null; - f.cancel(true); - } - } - - ReplicaSnapshot snapshot() { - ReplicaSnapshot s = new ReplicaSnapshot(this); - s.accepted = txnAccepted; - s.committed = txnCommitted; - s.state = state; - s.error = error; - s.retryAtMillis = waitingForRetry() ? retryAtMillis : 0; - return s; - } - - /** - * Update the leader's view of the replica after a poll. - * <p> - * Called with {@link KetchLeader#lock} held by caller. - * - * @param refs - * map of refs from the replica. - */ - void initialize(Map<String, Ref> refs) { - if (txnAccepted == null) { - txnAccepted = getId(refs.get(getSystem().getTxnAccepted())); - } - if (txnCommitted == null) { - txnCommitted = getId(refs.get(getSystem().getTxnCommitted())); - } - } - - ObjectId getTxnAccepted() { - return txnAccepted; - } - - boolean hasAccepted(LogIndex id) { - return equals(txnAccepted, id); - } - - private static boolean equals(@Nullable ObjectId a, LogIndex b) { - return a != null && b != null && AnyObjectId.isEqual(a, b); - } - - /** - * Schedule a proposal round with the replica. - * <p> - * Called with {@link KetchLeader#lock} held by caller. - * - * @param round - * current round being run by the leader. - */ - void pushTxnAcceptedAsync(Round round) { - List<ReceiveCommand> cmds = new ArrayList<>(); - if (commitSpeed == BATCHED) { - LogIndex committedIndex = leader.getCommitted(); - if (equals(txnAccepted, committedIndex) - && !equals(txnCommitted, committedIndex)) { - prepareTxnCommitted(cmds, committedIndex); - } - } - - // TODO(sop) Lagging replicas should build accept on the fly. - if (round.stageCommands != null) { - for (ReceiveCommand cmd : round.stageCommands) { - // TODO(sop): Do not send certain object graphs to replica. - cmds.add(copy(cmd)); - } - } - cmds.add(new ReceiveCommand( - round.acceptedOldIndex, round.acceptedNewIndex, - getSystem().getTxnAccepted())); - pushAsync(new ReplicaPushRequest(this, cmds)); - } - - private static ReceiveCommand copy(ReceiveCommand c) { - return new ReceiveCommand(c.getOldId(), c.getNewId(), c.getRefName()); - } - - boolean shouldPushUnbatchedCommit(LogIndex committed, boolean leaderIdle) { - return (leaderIdle || commitSpeed == FAST) && hasAccepted(committed); - } - - void pushCommitAsync(LogIndex committed) { - List<ReceiveCommand> cmds = new ArrayList<>(); - prepareTxnCommitted(cmds, committed); - pushAsync(new ReplicaPushRequest(this, cmds)); - } - - private void prepareTxnCommitted(List<ReceiveCommand> cmds, - ObjectId committed) { - removeStaged(cmds, committed); - cmds.add(new ReceiveCommand( - txnCommitted, committed, - getSystem().getTxnCommitted())); - } - - private void removeStaged(List<ReceiveCommand> cmds, ObjectId committed) { - List<ReceiveCommand> a = staged.remove(committed); - if (a != null) { - delete(cmds, a); - } - if (staged.isEmpty() || !(committed instanceof LogIndex)) { - return; - } - - LogIndex committedIndex = (LogIndex) committed; - Iterator<Map.Entry<ObjectId, List<ReceiveCommand>>> itr = staged - .entrySet().iterator(); - while (itr.hasNext()) { - Map.Entry<ObjectId, List<ReceiveCommand>> e = itr.next(); - if (e.getKey() instanceof LogIndex) { - LogIndex stagedIndex = (LogIndex) e.getKey(); - if (stagedIndex.isBefore(committedIndex)) { - delete(cmds, e.getValue()); - itr.remove(); - } - } - } - } - - private static void delete(List<ReceiveCommand> cmds, - List<ReceiveCommand> createCmds) { - for (ReceiveCommand cmd : createCmds) { - ObjectId id = cmd.getNewId(); - String name = cmd.getRefName(); - cmds.add(new ReceiveCommand(id, ObjectId.zeroId(), name)); - } - } - - /** - * Determine the next push for this replica (if any) and start it. - * <p> - * If the replica has successfully accepted the committed state of the - * leader, this method will push all references to the replica using the - * configured {@link CommitMethod}. - * <p> - * If the replica is {@link State#LAGGING} this method will begin catch up - * by sending a more recent {@code refs/txn/accepted}. - * <p> - * Must be invoked with {@link KetchLeader#lock} held by caller. - */ - private void runNextPushRequest() { - LogIndex committed = leader.getCommitted(); - if (!equals(txnCommitted, committed) - && shouldPushUnbatchedCommit(committed, leader.isIdle())) { - pushCommitAsync(committed); - } - - if (queued.isEmpty() || !running.isEmpty() || waitingForRetry()) { - return; - } - - // Collapse all queued requests into a single request. - Map<String, ReceiveCommand> cmdMap = new HashMap<>(); - for (ReplicaPushRequest req : queued) { - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - ReceiveCommand old = cmdMap.remove(name); - if (old != null) { - cmd = new ReceiveCommand( - old.getOldId(), cmd.getNewId(), - name); - } - cmdMap.put(name, cmd); - } - } - queued.clear(); - waiting.clear(); - - List<ReceiveCommand> next = new ArrayList<>(cmdMap.values()); - for (ReceiveCommand cmd : next) { - running.put(cmd.getRefName(), cmd); - } - startPush(new ReplicaPushRequest(this, next)); - } - - private void pushAsync(ReplicaPushRequest req) { - if (defer(req)) { - // TODO(sop) Collapse during long retry outage. - for (ReceiveCommand cmd : req.getCommands()) { - waiting.put(cmd.getRefName(), cmd); - } - queued.add(req); - } else { - for (ReceiveCommand cmd : req.getCommands()) { - running.put(cmd.getRefName(), cmd); - } - startPush(req); - } - } - - private boolean defer(ReplicaPushRequest req) { - if (waitingForRetry()) { - // Prior communication failure; everything is deferred. - return true; - } - - for (ReceiveCommand nextCmd : req.getCommands()) { - ReceiveCommand priorCmd = waiting.get(nextCmd.getRefName()); - if (priorCmd == null) { - priorCmd = running.get(nextCmd.getRefName()); - } - if (priorCmd != null) { - // Another request pending on same ref; that must go first. - // Verify priorCmd.newId == nextCmd.oldId? - return true; - } - } - return false; - } - - private boolean waitingForRetry() { - Future<?> f = retryFuture; - return f != null && !f.isDone(); - } - - private void retryLater(ReplicaPushRequest req) { - Collection<ReceiveCommand> cmds = req.getCommands(); - for (ReceiveCommand cmd : cmds) { - cmd.setResult(NOT_ATTEMPTED, null); - if (!waiting.containsKey(cmd.getRefName())) { - waiting.put(cmd.getRefName(), cmd); - } - } - queued.add(0, new ReplicaPushRequest(this, cmds)); - - if (!waitingForRetry()) { - long delay = FileUtils - .delay(lastRetryMillis, minRetryMillis, maxRetryMillis); - if (log.isDebugEnabled()) { - log.debug("Retrying {} after {} ms", //$NON-NLS-1$ - describeForLog(), Long.valueOf(delay)); - } - lastRetryMillis = delay; - retryAtMillis = SystemReader.getInstance().getCurrentTime() + delay; - retryFuture = getSystem().getExecutor() - .schedule(new WeakRetryPush(this), delay, MILLISECONDS); - } - } - - /** Weakly holds a retrying replica, allowing it to garbage collect. */ - static class WeakRetryPush extends WeakReference<KetchReplica> - implements Callable<Void> { - WeakRetryPush(KetchReplica r) { - super(r); - } - - @Override - public Void call() throws Exception { - KetchReplica r = get(); - if (r != null) { - r.doRetryPush(); - } - return null; - } - } - - private void doRetryPush() { - leader.lock.lock(); - try { - retryFuture = null; - runNextPushRequest(); - } finally { - leader.lock.unlock(); - } - } - - /** - * Begin executing a single push. - * <p> - * This method must move processing onto another thread. Called with - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held by caller. - * - * @param req - * the request to send to the replica. - */ - protected abstract void startPush(ReplicaPushRequest req); - - /** - * Callback from {@link ReplicaPushRequest} upon success or failure. - * <p> - * Acquires the {@link KetchLeader#lock} and updates the leader's internal - * knowledge about this replica to reflect what has been learned during a - * push to the replica. In some cases of divergence this method may take - * some time to determine how the replica has diverged; to reduce contention - * this is evaluated before acquiring the leader lock. - * - * @param repo - * local repository instance used by the push thread. - * @param req - * push request just attempted. - */ - void afterPush(@Nullable Repository repo, ReplicaPushRequest req) { - ReceiveCommand acceptCmd = null; - ReceiveCommand commitCmd = null; - List<ReceiveCommand> stages = null; - - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - if (name.equals(getSystem().getTxnAccepted())) { - acceptCmd = cmd; - } else if (name.equals(getSystem().getTxnCommitted())) { - commitCmd = cmd; - } else if (cmd.getResult() == OK && cmd.getType() == CREATE - && name.startsWith(getSystem().getTxnStage())) { - if (stages == null) { - stages = new ArrayList<>(); - } - stages.add(cmd); - } - } - - State newState = null; - ObjectId acceptId = readId(req, acceptCmd); - if (repo != null && acceptCmd != null && acceptCmd.getResult() != OK - && req.getException() == null) { - try (LagCheck lag = new LagCheck(this, repo)) { - newState = lag.check(acceptId, acceptCmd); - acceptId = lag.getRemoteId(); - } - } - - leader.lock.lock(); - try { - for (ReceiveCommand cmd : req.getCommands()) { - running.remove(cmd.getRefName()); - } - - Throwable err = req.getException(); - if (err != null) { - state = OFFLINE; - error = err.toString(); - retryLater(req); - leader.onReplicaUpdate(this); - return; - } - - lastRetryMillis = 0; - error = null; - updateView(req, acceptId, commitCmd); - - if (acceptCmd != null && acceptCmd.getResult() == OK) { - state = hasAccepted(leader.getHead()) ? CURRENT : LAGGING; - if (stages != null) { - staged.put(acceptCmd.getNewId(), stages); - } - } else if (newState != null) { - state = newState; - } - - leader.onReplicaUpdate(this); - runNextPushRequest(); - } finally { - leader.lock.unlock(); - } - } - - private void updateView(ReplicaPushRequest req, @Nullable ObjectId acceptId, - ReceiveCommand commitCmd) { - if (acceptId != null) { - txnAccepted = acceptId; - } - - ObjectId committed = readId(req, commitCmd); - if (committed != null) { - txnCommitted = committed; - } else if (acceptId != null && txnCommitted == null) { - // Initialize during first conversation. - Map<String, Ref> adv = req.getRefs(); - if (adv != null) { - Ref refs = adv.get(getSystem().getTxnCommitted()); - txnCommitted = getId(refs); - } - } - } - - @Nullable - private static ObjectId readId(ReplicaPushRequest req, - @Nullable ReceiveCommand cmd) { - if (cmd == null) { - // Ref was not in the command list, do not trust advertisement. - return null; - - } else if (cmd.getResult() == OK) { - // Currently at newId. - return cmd.getNewId(); - } - - Map<String, Ref> refs = req.getRefs(); - return refs != null ? getId(refs.get(cmd.getRefName())) : null; - } - - /** - * Fetch objects from the remote using the calling thread. - * <p> - * Called without {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock}. - * - * @param repo - * local repository to fetch objects into. - * @param req - * the request to fetch from a replica. - * @throws java.io.IOException - * communication with the replica was not possible. - */ - protected abstract void blockingFetch(Repository repo, - ReplicaFetchRequest req) throws IOException; - - /** - * Build a list of commands to commit - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS}. - * - * @param git - * local leader repository to read committed state from. - * @param current - * all known references in the replica's repository. Typically - * this comes from a push advertisement. - * @param committed - * state being pushed to {@code refs/txn/committed}. - * @return commands to update during commit. - * @throws java.io.IOException - * cannot read the committed state. - */ - protected Collection<ReceiveCommand> prepareCommit(Repository git, - Map<String, Ref> current, ObjectId committed) throws IOException { - List<ReceiveCommand> delta = new ArrayList<>(); - Map<String, Ref> remote = new HashMap<>(current); - try (RevWalk rw = new RevWalk(git); - TreeWalk tw = new TreeWalk(rw.getObjectReader())) { - tw.setRecursive(true); - tw.addTree(rw.parseCommit(committed).getTree()); - while (tw.next()) { - if (tw.getRawMode(0) != TYPE_GITLINK - || tw.isPathSuffix(PEEL, 2)) { - // Symbolic references cannot be pushed. - // Caching peeled values is handled remotely. - continue; - } - - // TODO(sop) Do not send certain ref names to replica. - String name = RefTree.refName(tw.getPathString()); - Ref oldRef = remote.remove(name); - ObjectId oldId = getId(oldRef); - ObjectId newId = tw.getObjectId(0); - if (!AnyObjectId.isEqual(oldId, newId)) { - delta.add(new ReceiveCommand(oldId, newId, name)); - } - } - } - - // Delete any extra references not in the committed state. - for (Ref ref : remote.values()) { - if (canDelete(ref)) { - delta.add(new ReceiveCommand( - ref.getObjectId(), ObjectId.zeroId(), - ref.getName())); - } - } - return delta; - } - - boolean canDelete(Ref ref) { - String name = ref.getName(); - if (HEAD.equals(name)) { - return false; - } - if (name.startsWith(getSystem().getTxnNamespace())) { - return false; - } - // TODO(sop) Do not delete precious names from replica. - return true; - } - - @NonNull - static ObjectId getId(@Nullable Ref ref) { - if (ref != null) { - ObjectId id = ref.getObjectId(); - if (id != null) { - return id; - } - } - return ObjectId.zeroId(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java deleted file mode 100644 index 8ad1d6033a..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchConstants.ACCEPTED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.COMMITTED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_SECTION_KETCH; -import static org.eclipse.jgit.internal.ketch.KetchConstants.DEFAULT_TXN_NAMESPACE; -import static org.eclipse.jgit.internal.ketch.KetchConstants.STAGE; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_NAME; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; - -import java.net.URISyntaxException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.URIish; -import org.eclipse.jgit.util.time.MonotonicClock; -import org.eclipse.jgit.util.time.MonotonicSystemClock; -import org.eclipse.jgit.util.time.ProposedTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Ketch system-wide configuration. - * <p> - * This class provides useful defaults for testing and small proof of concepts. - * Full scale installations are expected to subclass and override methods to - * provide consistent configuration across all managed repositories. - * <p> - * Servers should configure their own - * {@link java.util.concurrent.ScheduledExecutorService}. - */ -public class KetchSystem { - private static final Random RNG = new Random(); - - /** - * Get default executor, one thread per available processor. - * - * @return default executor, one thread per available processor. - */ - public static ScheduledExecutorService defaultExecutor() { - return DefaultExecutorHolder.I; - } - - private final ScheduledExecutorService executor; - private final MonotonicClock clock; - private final String txnNamespace; - private final String txnAccepted; - private final String txnCommitted; - private final String txnStage; - - /** - * Create a default system with a thread pool of 1 thread per CPU. - */ - public KetchSystem() { - this(defaultExecutor(), new MonotonicSystemClock(), DEFAULT_TXN_NAMESPACE); - } - - /** - * Create a Ketch system with the provided executor service. - * - * @param executor - * thread pool to run background operations. - * @param clock - * clock to create timestamps. - * @param txnNamespace - * reference namespace for the RefTree graph and associated - * transaction state. Must begin with {@code "refs/"} and end - * with {@code '/'}, for example {@code "refs/txn/"}. - */ - public KetchSystem(ScheduledExecutorService executor, MonotonicClock clock, - String txnNamespace) { - this.executor = executor; - this.clock = clock; - this.txnNamespace = txnNamespace; - this.txnAccepted = txnNamespace + ACCEPTED; - this.txnCommitted = txnNamespace + COMMITTED; - this.txnStage = txnNamespace + STAGE; - } - - /** - * Get executor to perform background operations. - * - * @return executor to perform background operations. - */ - public ScheduledExecutorService getExecutor() { - return executor; - } - - /** - * Get clock to obtain timestamps from. - * - * @return clock to obtain timestamps from. - */ - public MonotonicClock getClock() { - return clock; - } - - /** - * Get how long the leader will wait for the {@link #getClock()}'s - * {@code ProposedTimestamp} used in commits proposed to the RefTree graph - * ({@link #getTxnAccepted()}) - * - * @return how long the leader will wait for the {@link #getClock()}'s - * {@code ProposedTimestamp} used in commits proposed to the RefTree - * graph ({@link #getTxnAccepted()}). Defaults to 5 seconds. - */ - public Duration getMaxWaitForMonotonicClock() { - return Duration.ofSeconds(5); - } - - /** - * Whether elections should require monotonically increasing commit - * timestamps - * - * @return {@code true} if elections should require monotonically increasing - * commit timestamps. This requires a very good - * {@link org.eclipse.jgit.util.time.MonotonicClock}. - */ - public boolean requireMonotonicLeaderElections() { - return false; - } - - /** - * Get the namespace used for the RefTree graph and transaction management. - * - * @return reference namespace such as {@code "refs/txn/"}. - */ - public String getTxnNamespace() { - return txnNamespace; - } - - /** - * Get name of the accepted RefTree graph. - * - * @return name of the accepted RefTree graph. - */ - public String getTxnAccepted() { - return txnAccepted; - } - - /** - * Get name of the committed RefTree graph. - * - * @return name of the committed RefTree graph. - */ - public String getTxnCommitted() { - return txnCommitted; - } - - /** - * Get prefix for staged objects, e.g. {@code "refs/txn/stage/"}. - * - * @return prefix for staged objects, e.g. {@code "refs/txn/stage/"}. - */ - public String getTxnStage() { - return txnStage; - } - - /** - * Create new committer {@code PersonIdent} for ketch system - * - * @param time - * timestamp for the committer. - * @return identity line for the committer header of a RefTreeGraph. - */ - public PersonIdent newCommitter(ProposedTimestamp time) { - String name = "ketch"; //$NON-NLS-1$ - String email = "ketch@system"; //$NON-NLS-1$ - return new PersonIdent(name, email, time); - } - - /** - * Construct a random tag to identify a candidate during leader election. - * <p> - * Multiple processes trying to elect themselves leaders at exactly the same - * time (rounded to seconds) using the same - * {@link #newCommitter(ProposedTimestamp)} identity strings, for the same - * term, may generate the same ObjectId for the election commit and falsely - * assume they have both won. - * <p> - * Candidates add this tag to their election ballot commit to disambiguate - * the election. The tag only needs to be unique for a given triplet of - * {@link #newCommitter(ProposedTimestamp)}, system time (rounded to - * seconds), and term. If every replica in the system uses a unique - * {@code newCommitter} (such as including the host name after the - * {@code "@"} in the email address) the tag could be the empty string. - * <p> - * The default implementation generates a few bytes of random data. - * - * @return unique tag; null or empty string if {@code newCommitter()} is - * sufficiently unique to identify the leader. - */ - @Nullable - public String newLeaderTag() { - int n = RNG.nextInt(1 << (6 * 4)); - return String.format("%06x", Integer.valueOf(n)); //$NON-NLS-1$ - } - - /** - * Construct the KetchLeader instance of a repository. - * - * @param repo - * local repository stored by the leader. - * @return leader instance. - * @throws java.net.URISyntaxException - * a follower configuration contains an unsupported URI. - */ - public KetchLeader createLeader(Repository repo) - throws URISyntaxException { - KetchLeader leader = new KetchLeader(this) { - @Override - protected Repository openRepository() { - repo.incrementOpen(); - return repo; - } - }; - leader.setReplicas(createReplicas(leader, repo)); - return leader; - } - - /** - * Get the collection of replicas for a repository. - * <p> - * The collection of replicas must include the local repository. - * - * @param leader - * the leader driving these replicas. - * @param repo - * repository to get the replicas of. - * @return collection of replicas for the specified repository. - * @throws java.net.URISyntaxException - * a configured URI is invalid. - */ - protected List<KetchReplica> createReplicas(KetchLeader leader, - Repository repo) throws URISyntaxException { - List<KetchReplica> replicas = new ArrayList<>(); - Config cfg = repo.getConfig(); - String localName = getLocalName(cfg); - for (String name : cfg.getSubsections(CONFIG_KEY_REMOTE)) { - if (!hasParticipation(cfg, name)) { - continue; - } - - ReplicaConfig kc = ReplicaConfig.newFromConfig(cfg, name); - if (name.equals(localName)) { - replicas.add(new LocalReplica(leader, name, kc)); - continue; - } - - RemoteConfig rc = new RemoteConfig(cfg, name); - List<URIish> uris = rc.getPushURIs(); - if (uris.isEmpty()) { - uris = rc.getURIs(); - } - for (URIish uri : uris) { - String n = uris.size() == 1 ? name : uri.getHost(); - replicas.add(new RemoteGitReplica(leader, n, uri, kc, rc)); - } - } - return replicas; - } - - private static boolean hasParticipation(Config cfg, String name) { - return cfg.getString(CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE) != null; - } - - private static String getLocalName(Config cfg) { - return cfg.getString(CONFIG_SECTION_KETCH, null, CONFIG_KEY_NAME); - } - - static class DefaultExecutorHolder { - private static final Logger log = LoggerFactory.getLogger(KetchSystem.class); - static final ScheduledExecutorService I = create(); - - private static ScheduledExecutorService create() { - int cores = Runtime.getRuntime().availableProcessors(); - int threads = Math.max(5, cores); - log.info("Using {} threads", Integer.valueOf(threads)); //$NON-NLS-1$ - return Executors.newScheduledThreadPool( - threads, - new ThreadFactory() { - private final AtomicInteger threadCnt = new AtomicInteger(); - - @Override - public Thread newThread(Runnable r) { - int id = threadCnt.incrementAndGet(); - Thread thr = new Thread(r); - thr.setName("KetchExecutor-" + id); //$NON-NLS-1$ - return thr; - } - }); - } - - private DefaultExecutorHolder() { - } - } - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java deleted file mode 100644 index 6f9038bbdf..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.nls.NLS; -import org.eclipse.jgit.nls.TranslationBundle; - -/** - * Translation bundle for the Ketch implementation. - */ -public class KetchText extends TranslationBundle { - /** - * Get an instance of this translation bundle. - * - * @return instance of this translation bundle. - */ - public static KetchText get() { - return NLS.getBundleFor(KetchText.class); - } - - // @formatter:off - /***/ public String accepted; - /***/ public String cannotFetchFromLocalReplica; - /***/ public String failed; - /***/ public String invalidFollowerUri; - /***/ public String leaderFailedToStore; - /***/ public String localReplicaRequired; - /***/ public String mismatchedTxnNamespace; - /***/ public String outsideTxnNamespace; - /***/ public String proposingUpdates; - /***/ public String queuedProposalFailedToApply; - /***/ public String starting; - /***/ public String unsupportedVoterCount; - /***/ public String waitingForQueue; -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java deleted file mode 100644 index 1f8384ff76..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.AHEAD; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.DIVERGENT; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; -import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * A helper to check if a {@link KetchReplica} is ahead or behind the leader. - */ -class LagCheck implements AutoCloseable { - private final KetchReplica replica; - private final Repository repo; - private RevWalk rw; - private ObjectId remoteId; - - LagCheck(KetchReplica replica, Repository repo) { - this.replica = replica; - this.repo = repo; - initRevWalk(); - } - - private void initRevWalk() { - if (rw != null) { - rw.close(); - } - - rw = new RevWalk(repo); - rw.setRetainBody(false); - } - - /** {@inheritDoc} */ - @Override - public void close() { - if (rw != null) { - rw.close(); - rw = null; - } - } - - ObjectId getRemoteId() { - return remoteId; - } - - KetchReplica.State check(ObjectId acceptId, ReceiveCommand acceptCmd) { - remoteId = acceptId; - if (remoteId == null) { - // Nothing advertised by the replica, value is unknown. - return UNKNOWN; - } - - if (AnyObjectId.isEqual(remoteId, ObjectId.zeroId())) { - // Replica does not have the txnAccepted reference. - return LAGGING; - } - - try { - RevCommit remote; - try { - remote = parseRemoteCommit(acceptCmd.getRefName()); - } catch (RefGoneException gone) { - // Replica does not have the txnAccepted reference. - return LAGGING; - } catch (MissingObjectException notFound) { - // Local repository does not know this commit so it cannot - // be including the replica's log. - return DIVERGENT; - } - - RevCommit head = rw.parseCommit(acceptCmd.getNewId()); - if (rw.isMergedInto(remote, head)) { - return LAGGING; - } - - // TODO(sop) Check term to see if my leader was deposed. - if (rw.isMergedInto(head, remote)) { - return AHEAD; - } - return DIVERGENT; - } catch (IOException err) { - KetchReplica.log.error(String.format( - "Cannot compare %s", //$NON-NLS-1$ - acceptCmd.getRefName()), err); - return UNKNOWN; - } - } - - private RevCommit parseRemoteCommit(String refName) - throws IOException, MissingObjectException, RefGoneException { - try { - return rw.parseCommit(remoteId); - } catch (MissingObjectException notLocal) { - // Fall through and try to acquire the object by fetching it. - } - - ReplicaFetchRequest fetch = new ReplicaFetchRequest( - Collections.singleton(refName), - Collections.<ObjectId> emptySet()); - try { - replica.blockingFetch(repo, fetch); - } catch (IOException fetchErr) { - KetchReplica.log.error(String.format( - "Cannot fetch %s (%s) from %s", //$NON-NLS-1$ - remoteId.abbreviate(8).name(), refName, - replica.describeForLog()), fetchErr); - throw new MissingObjectException(remoteId, OBJ_COMMIT); - } - - Map<String, Ref> adv = fetch.getRefs(); - if (adv == null) { - throw new MissingObjectException(remoteId, OBJ_COMMIT); - } - - Ref ref = adv.get(refName); - if (ref == null || ref.getObjectId() == null) { - throw new RefGoneException(); - } - - initRevWalk(); - remoteId = ref.getObjectId(); - return rw.parseCommit(remoteId); - } - - private static class RefGoneException extends Exception { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java deleted file mode 100644 index ce0672c168..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; - -/** - * A snapshot of a leader and its view of the world. - */ -public class LeaderSnapshot { - final List<ReplicaSnapshot> replicas = new ArrayList<>(); - KetchLeader.State state; - long term; - LogIndex headIndex; - LogIndex committedIndex; - boolean idle; - - LeaderSnapshot() { - } - - /** - * Get unmodifiable view of configured replicas. - * - * @return unmodifiable view of configured replicas. - */ - public Collection<ReplicaSnapshot> getReplicas() { - return Collections.unmodifiableList(replicas); - } - - /** - * Get current state of the leader. - * - * @return current state of the leader. - */ - public KetchLeader.State getState() { - return state; - } - - /** - * Whether the leader is not running a round to reach consensus, and has no - * rounds queued. - * - * @return {@code true} if the leader is not running a round to reach - * consensus, and has no rounds queued. - */ - public boolean isIdle() { - return idle; - } - - /** - * Get term of this leader - * - * @return term of this leader. Valid only if {@link #getState()} is - * currently - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER}. - */ - public long getTerm() { - return term; - } - - /** - * Get end of the leader's log - * - * @return end of the leader's log; null if leader hasn't started up enough - * to begin its own election. - */ - @Nullable - public LogIndex getHead() { - return headIndex; - } - - /** - * Get state the leader knows is committed on a majority of participant - * replicas - * - * @return state the leader knows is committed on a majority of participant - * replicas. Null until the leader instance has committed a log - * index within its own term. - */ - @Nullable - public LogIndex getCommitted() { - return committedIndex; - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - s.append(isIdle() ? "IDLE" : "RUNNING"); //$NON-NLS-1$ //$NON-NLS-2$ - s.append(" state ").append(getState()); //$NON-NLS-1$ - if (getTerm() > 0) { - s.append(" term ").append(getTerm()); //$NON-NLS-1$ - } - s.append('\n'); - s.append(String.format( - "%-10s %12s %12s\n", //$NON-NLS-1$ - "Replica", "Accepted", "Committed")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - s.append("------------------------------------\n"); //$NON-NLS-1$ - debug(s, "(leader)", getHead(), getCommitted()); //$NON-NLS-1$ - s.append('\n'); - for (ReplicaSnapshot r : getReplicas()) { - debug(s, r); - s.append('\n'); - } - s.append('\n'); - return s.toString(); - } - - private static void debug(StringBuilder b, ReplicaSnapshot s) { - KetchReplica replica = s.getReplica(); - debug(b, replica.getName(), s.getAccepted(), s.getCommitted()); - b.append(String.format(" %-8s %s", //$NON-NLS-1$ - replica.getParticipation(), s.getState())); - if (s.getState() == OFFLINE) { - String err = s.getErrorMessage(); - if (err != null) { - b.append(" (").append(err).append(')'); //$NON-NLS-1$ - } - } - } - - private static void debug(StringBuilder s, String name, - ObjectId accepted, ObjectId committed) { - s.append(String.format( - "%-10s %-12s %-12s", //$NON-NLS-1$ - name, str(accepted), str(committed))); - } - - static String str(ObjectId c) { - if (c instanceof LogIndex) { - return ((LogIndex) c).describeForLog(); - } else if (c != null) { - return c.abbreviate(8).name(); - } - return "-"; //$NON-NLS-1$ - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java deleted file mode 100644 index b2d59d77da..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.TXN_COMMITTED; -import static org.eclipse.jgit.lib.RefDatabase.ALL; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.MonotonicClock; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** - * Ketch replica running on the same system as the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. - */ -public class LocalReplica extends KetchReplica { - /** - * Configure a local replica. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this replica for debugging. - * @param cfg - * how Ketch should treat the local system. - */ - public LocalReplica(KetchLeader leader, String name, ReplicaConfig cfg) { - super(leader, name, cfg); - } - - /** {@inheritDoc} */ - @Override - protected String describeForLog() { - return String.format("%s (leader)", getName()); //$NON-NLS-1$ - } - - /** - * Initializes local replica by reading accepted and committed references. - * <p> - * Loads accepted and committed references from the reference database of - * the local replica and stores their current ObjectIds in memory. - * - * @param repo - * repository to initialize state from. - * @throws IOException - * cannot read repository state. - */ - void initialize(Repository repo) throws IOException { - RefDatabase refdb = repo.getRefDatabase(); - if (refdb instanceof RefTreeDatabase) { - RefTreeDatabase treeDb = (RefTreeDatabase) refdb; - String txnNamespace = getSystem().getTxnNamespace(); - if (!txnNamespace.equals(treeDb.getTxnNamespace())) { - throw new IOException(MessageFormat.format( - KetchText.get().mismatchedTxnNamespace, - txnNamespace, treeDb.getTxnNamespace())); - } - refdb = treeDb.getBootstrap(); - } - initialize(refdb.exactRef( - getSystem().getTxnAccepted(), - getSystem().getTxnCommitted())); - } - - /** {@inheritDoc} */ - @Override - protected void startPush(ReplicaPushRequest req) { - getSystem().getExecutor().execute(() -> { - MonotonicClock clk = getSystem().getClock(); - try (Repository git = getLeader().openRepository(); - ProposedTimestamp ts = clk.propose()) { - try { - update(git, req, ts); - req.done(git); - } catch (Throwable err) { - req.setException(git, err); - } - } catch (IOException err) { - req.setException(null, err); - } - }); - } - - /** {@inheritDoc} */ - @Override - protected void blockingFetch(Repository repo, ReplicaFetchRequest req) - throws IOException { - throw new IOException(KetchText.get().cannotFetchFromLocalReplica); - } - - private void update(Repository git, ReplicaPushRequest req, - ProposedTimestamp ts) throws IOException { - RefDatabase refdb = git.getRefDatabase(); - CommitMethod method = getCommitMethod(); - - // Local replica probably uses RefTreeDatabase, the request should - // be only for the txnNamespace, so drop to the bootstrap layer. - if (refdb instanceof RefTreeDatabase) { - if (!isOnlyTxnNamespace(req.getCommands())) { - return; - } - - refdb = ((RefTreeDatabase) refdb).getBootstrap(); - method = TXN_COMMITTED; - } - - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addProposedTimestamp(ts); - batch.setRefLogIdent(getSystem().newCommitter(ts)); - batch.setRefLogMessage("ketch", false); //$NON-NLS-1$ - batch.setAllowNonFastForwards(true); - - // RefDirectory updates multiple references sequentially. - // Run everything else first, then accepted (if present), - // then committed (if present). This ensures an earlier - // failure will not update these critical references. - ReceiveCommand accepted = null; - ReceiveCommand committed = null; - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - if (name.equals(getSystem().getTxnAccepted())) { - accepted = cmd; - } else if (name.equals(getSystem().getTxnCommitted())) { - committed = cmd; - } else { - batch.addCommand(cmd); - } - } - if (committed != null && method == ALL_REFS) { - Map<String, Ref> refs = refdb.getRefs(ALL); - batch.addCommand(prepareCommit(git, refs, committed.getNewId())); - } - if (accepted != null) { - batch.addCommand(accepted); - } - if (committed != null) { - batch.addCommand(committed); - } - - try (RevWalk rw = new RevWalk(git)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - - // KetchReplica only cares about accepted and committed in - // advertisement. If they failed, store the current values - // back in the ReplicaPushRequest. - List<String> failed = new ArrayList<>(2); - checkFailed(failed, accepted); - checkFailed(failed, committed); - if (!failed.isEmpty()) { - String[] arr = failed.toArray(new String[0]); - req.setRefs(refdb.exactRef(arr)); - } - } - - private static void checkFailed(List<String> failed, ReceiveCommand cmd) { - if (cmd != null && cmd.getResult() != OK) { - failed.add(cmd.getRefName()); - } - } - - private boolean isOnlyTxnNamespace(Collection<ReceiveCommand> cmdList) { - // Be paranoid and reject non txnNamespace names, this - // is a programming error in Ketch that should not occur. - - String txnNamespace = getSystem().getTxnNamespace(); - for (ReceiveCommand cmd : cmdList) { - if (!cmd.getRefName().startsWith(txnNamespace)) { - cmd.setResult(REJECTED_OTHER_REASON, - MessageFormat.format( - KetchText.get().outsideTxnNamespace, - cmd.getRefName(), txnNamespace)); - ReceiveCommand.abort(cmdList); - return false; - } - } - return true; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java deleted file mode 100644 index ed65c06fae..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; - -/** - * An ObjectId for a commit extended with incrementing log index. - * <p> - * For any two LogIndex instances, {@code A} is an ancestor of {@code C} - * reachable through parent edges in the graph if {@code A.index < C.index}. - * LogIndex provides a performance optimization for Ketch, the same information - * can be obtained from {@link org.eclipse.jgit.revwalk.RevWalk}. - * <p> - * Index values are only valid within a single - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} instance after it has won - * an election. By restricting scope to a single leader new leaders do not need - * to traverse the entire history to determine the next {@code index} for new - * proposals. This differs from Raft, where leader election uses the log index - * and the term number to determine which replica holds a sufficiently - * up-to-date log. Since Ketch uses Git objects for storage of its replicated - * log, it keeps the term number as Raft does but uses standard Git operations - * to imply the log index. - * <p> - * {@link org.eclipse.jgit.internal.ketch.Round#runAsync(AnyObjectId)} bumps the - * index as each new round is constructed. - */ -public class LogIndex extends ObjectId { - static LogIndex unknown(AnyObjectId id) { - return new LogIndex(id, 0); - } - - private final long index; - - private LogIndex(AnyObjectId id, long index) { - super(id); - this.index = index; - } - - LogIndex nextIndex(AnyObjectId id) { - return new LogIndex(id, index + 1); - } - - /** - * Get index provided by the current leader instance. - * - * @return index provided by the current leader instance. - */ - public long getIndex() { - return index; - } - - /** - * Check if this log position committed before another log position. - * <p> - * Only valid for log positions in memory for the current leader. - * - * @param c - * other (more recent) log position. - * @return true if this log position was before {@code c} or equal to c and - * therefore any agreement of {@code c} implies agreement on this - * log position. - */ - boolean isBefore(LogIndex c) { - return index <= c.index; - } - - /** - * Create string suitable for debug logging containing the log index and - * abbreviated ObjectId. - * - * @return string suitable for debug logging containing the log index and - * abbreviated ObjectId. - */ - @SuppressWarnings("boxing") - public String describeForLog() { - return String.format("%5d/%s", index, abbreviate(6).name()); //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @SuppressWarnings("boxing") - @Override - public String toString() { - return String.format("LogId[%5d/%s]", index, name()); //$NON-NLS-1$ - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java deleted file mode 100644 index ca27281a8e..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.Proposal.State.ABORTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.NEW; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.storage.reftree.Command; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.PushCertificate; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** - * A proposal to be applied in a Ketch system. - * <p> - * Pushing to a Ketch leader results in the leader making a proposal. The - * proposal includes the list of reference updates. The leader attempts to send - * the proposal to a quorum of replicas by pushing the proposal to a "staging" - * area under the {@code refs/txn/stage/} namespace. If the proposal succeeds - * then the changes are durable and the leader can commit the proposal. - * <p> - * Proposals are executed by - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#queueProposal(Proposal)}, - * which runs them asynchronously in the background. Proposals are thread-safe - * futures allowing callers to {@link #await()} for results or be notified by - * callback using {@link #addListener(Runnable)}. - */ -public class Proposal { - /** Current state of the proposal. */ - public enum State { - /** Proposal has not yet been given to a {@link KetchLeader}. */ - NEW(false), - - /** - * Proposal was validated and has entered the queue, but a round - * containing this proposal has not started yet. - */ - QUEUED(false), - - /** Round containing the proposal has begun and is in progress. */ - RUNNING(false), - - /** - * Proposal was executed through a round. Individual results from - * {@link Proposal#getCommands()}, {@link Command#getResult()} explain - * the success or failure outcome. - */ - EXECUTED(true), - - /** Proposal was aborted and did not reach consensus. */ - ABORTED(true); - - private final boolean done; - - private State(boolean done) { - this.done = done; - } - - /** @return true if this is a terminal state. */ - public boolean isDone() { - return done; - } - } - - private final List<Command> commands; - private PersonIdent author; - private String message; - private PushCertificate pushCert; - - private List<ProposedTimestamp> timestamps; - private final List<Runnable> listeners = new CopyOnWriteArrayList<>(); - private final AtomicReference<State> state = new AtomicReference<>(NEW); - - /** - * Create a proposal from a list of Ketch commands. - * - * @param cmds - * prepared list of commands. - */ - public Proposal(List<Command> cmds) { - commands = Collections.unmodifiableList(new ArrayList<>(cmds)); - } - - /** - * Create a proposal from a collection of received commands. - * - * @param rw - * walker to assist in preparing commands. - * @param cmds - * list of pending commands. - * @throws org.eclipse.jgit.errors.MissingObjectException - * newId of a command is not found locally. - * @throws java.io.IOException - * local objects cannot be accessed. - */ - public Proposal(RevWalk rw, Collection<ReceiveCommand> cmds) - throws MissingObjectException, IOException { - commands = asCommandList(rw, cmds); - } - - private static List<Command> asCommandList(RevWalk rw, - Collection<ReceiveCommand> cmds) - throws MissingObjectException, IOException { - List<Command> commands = new ArrayList<>(cmds.size()); - for (ReceiveCommand cmd : cmds) { - commands.add(new Command(rw, cmd)); - } - return Collections.unmodifiableList(commands); - } - - /** - * Get commands from this proposal. - * - * @return commands from this proposal. - */ - public Collection<Command> getCommands() { - return commands; - } - - /** - * Get optional author of the proposal. - * - * @return optional author of the proposal. - */ - @Nullable - public PersonIdent getAuthor() { - return author; - } - - /** - * Set the author for the proposal. - * - * @param who - * optional identity of the author of the proposal. - * @return {@code this} - */ - public Proposal setAuthor(@Nullable PersonIdent who) { - author = who; - return this; - } - - /** - * Get optional message for the commit log of the RefTree. - * - * @return optional message for the commit log of the RefTree. - */ - @Nullable - public String getMessage() { - return message; - } - - /** - * Set the message to appear in the commit log of the RefTree. - * - * @param msg - * message text for the commit. - * @return {@code this} - */ - public Proposal setMessage(@Nullable String msg) { - message = msg != null && !msg.isEmpty() ? msg : null; - return this; - } - - /** - * Get optional certificate signing the references. - * - * @return optional certificate signing the references. - */ - @Nullable - public PushCertificate getPushCertificate() { - return pushCert; - } - - /** - * Set the push certificate signing the references. - * - * @param cert - * certificate, may be null. - * @return {@code this} - */ - public Proposal setPushCertificate(@Nullable PushCertificate cert) { - pushCert = cert; - return this; - } - - /** - * Get timestamps that Ketch must block for. - * - * @return timestamps that Ketch must block for. These may have been used as - * commit times inside the objects involved in the proposal. - */ - public List<ProposedTimestamp> getProposedTimestamps() { - if (timestamps != null) { - return timestamps; - } - return Collections.emptyList(); - } - - /** - * Request the proposal to wait for the affected timestamps to resolve. - * - * @param ts - * a {@link org.eclipse.jgit.util.time.ProposedTimestamp} object. - * @return {@code this}. - */ - public Proposal addProposedTimestamp(ProposedTimestamp ts) { - if (timestamps == null) { - timestamps = new ArrayList<>(4); - } - timestamps.add(ts); - return this; - } - - /** - * Add a callback to be invoked when the proposal is done. - * <p> - * A proposal is done when it has entered either - * {@link org.eclipse.jgit.internal.ketch.Proposal.State#EXECUTED} or - * {@link org.eclipse.jgit.internal.ketch.Proposal.State#ABORTED} state. If - * the proposal is already done {@code callback.run()} is immediately - * invoked on the caller's thread. - * - * @param callback - * method to run after the proposal is done. The callback may be - * run on a Ketch system thread and should be completed quickly. - */ - public void addListener(Runnable callback) { - boolean runNow = false; - synchronized (state) { - if (state.get().isDone()) { - runNow = true; - } else { - listeners.add(callback); - } - } - if (runNow) { - callback.run(); - } - } - - /** Set command result as OK. */ - void success() { - for (Command c : commands) { - if (c.getResult() == NOT_ATTEMPTED) { - c.setResult(OK); - } - } - notifyState(EXECUTED); - } - - /** Mark commands as "transaction aborted". */ - void abort() { - Command.abort(commands, null); - notifyState(ABORTED); - } - - /** - * Read the current state of the proposal. - * - * @return read the current state of the proposal. - */ - public State getState() { - return state.get(); - } - - /** - * Whether the proposal was attempted - * - * @return {@code true} if the proposal was attempted. A true value does not - * mean consensus was reached, only that the proposal was considered - * and will not be making any more progress beyond its current - * state. - */ - public boolean isDone() { - return state.get().isDone(); - } - - /** - * Wait for the proposal to be attempted and {@link #isDone()} to be true. - * - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public void await() throws InterruptedException { - synchronized (state) { - while (!state.get().isDone()) { - state.wait(); - } - } - } - - /** - * Wait for the proposal to be attempted and {@link #isDone()} to be true. - * - * @param wait - * how long to wait. - * @param unit - * unit describing the wait time. - * @return true if the proposal is done; false if the method timed out. - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public boolean await(long wait, TimeUnit unit) throws InterruptedException { - synchronized (state) { - if (state.get().isDone()) { - return true; - } - state.wait(unit.toMillis(wait)); - return state.get().isDone(); - } - } - - /** - * Wait for the proposal to exit a state. - * - * @param notIn - * state the proposal should not be in to return. - * @param wait - * how long to wait. - * @param unit - * unit describing the wait time. - * @return true if the proposal exited the state; false on time out. - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public boolean awaitStateChange(State notIn, long wait, TimeUnit unit) - throws InterruptedException { - synchronized (state) { - if (state.get() != notIn) { - return true; - } - state.wait(unit.toMillis(wait)); - return state.get() != notIn; - } - } - - void notifyState(State s) { - synchronized (state) { - state.set(s); - state.notifyAll(); - } - if (s.isDone()) { - for (Runnable callback : listeners) { - callback.run(); - } - listeners.clear(); - } - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - s.append("Ketch Proposal {\n"); //$NON-NLS-1$ - s.append(" ").append(state.get()).append('\n'); //$NON-NLS-1$ - if (author != null) { - s.append(" author ").append(author).append('\n'); //$NON-NLS-1$ - } - if (message != null) { - s.append(" message ").append(message).append('\n'); //$NON-NLS-1$ - } - for (Command c : commands) { - s.append(" "); //$NON-NLS-1$ - format(s, c.getOldRef(), "CREATE"); //$NON-NLS-1$ - s.append(' '); - format(s, c.getNewRef(), "DELETE"); //$NON-NLS-1$ - s.append(' ').append(c.getRefName()); - if (c.getResult() != ReceiveCommand.Result.NOT_ATTEMPTED) { - s.append(' ').append(c.getResult()); // $NON-NLS-1$ - } - s.append('\n'); - } - s.append('}'); - return s.toString(); - } - - private static void format(StringBuilder s, @Nullable Ref r, String n) { - if (r == null) { - s.append(n); - } else if (r.isSymbolic()) { - s.append(r.getTarget().getName()); - } else { - ObjectId id = r.getObjectId(); - if (id != null) { - s.append(id.abbreviate(8).name()); - } - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java deleted file mode 100644 index b73183abd0..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.Proposal.State.RUNNING; - -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.storage.reftree.Command; -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** A {@link Round} that aggregates and sends user {@link Proposal}s. */ -class ProposalRound extends Round { - private final List<Proposal> todo; - private RefTree queuedTree; - - ProposalRound(KetchLeader leader, LogIndex head, List<Proposal> todo, - @Nullable RefTree tree) { - super(leader, head); - this.todo = todo; - - if (tree != null && canCombine(todo)) { - this.queuedTree = tree; - } else { - leader.roundHoldsReferenceToRefTree = false; - } - } - - private static boolean canCombine(List<Proposal> todo) { - Proposal first = todo.get(0); - for (int i = 1; i < todo.size(); i++) { - if (!canCombine(first, todo.get(i))) { - return false; - } - } - return true; - } - - private static boolean canCombine(Proposal a, Proposal b) { - String aMsg = nullToEmpty(a.getMessage()); - String bMsg = nullToEmpty(b.getMessage()); - return aMsg.equals(bMsg) && canCombine(a.getAuthor(), b.getAuthor()); - } - - private static String nullToEmpty(@Nullable String str) { - return str != null ? str : ""; //$NON-NLS-1$ - } - - private static boolean canCombine(@Nullable PersonIdent a, - @Nullable PersonIdent b) { - if (a != null && b != null) { - // Same name and email address. Combine timestamp as the two - // proposals are running concurrently and appear together or - // not at all from the point of view of an outside reader. - return a.getName().equals(b.getName()) - && a.getEmailAddress().equals(b.getEmailAddress()); - } - - // If a and b are null, both will be the system identity. - return a == null && b == null; - } - - @Override - void start() throws IOException { - for (Proposal p : todo) { - p.notifyState(RUNNING); - } - try { - ObjectId id; - try (Repository git = leader.openRepository(); - ProposedTimestamp ts = getSystem().getClock().propose()) { - id = insertProposals(git, ts); - blockUntil(ts); - } - runAsync(id); - } catch (NoOp e) { - for (Proposal p : todo) { - p.success(); - } - leader.lock.lock(); - try { - leader.nextRound(); - } finally { - leader.lock.unlock(); - } - } catch (IOException e) { - abort(); - throw e; - } - } - - private ObjectId insertProposals(Repository git, ProposedTimestamp ts) - throws IOException, NoOp { - ObjectId id; - try (ObjectInserter inserter = git.newObjectInserter()) { - // TODO(sop) Process signed push certificates. - - if (queuedTree != null) { - id = insertSingleProposal(git, ts, inserter); - } else { - id = insertMultiProposal(git, ts, inserter); - } - - stageCommands = makeStageList(git, inserter); - inserter.flush(); - } - return id; - } - - private ObjectId insertSingleProposal(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException, NoOp { - // Fast path: tree is passed in with all proposals applied. - ObjectId treeId = queuedTree.writeTree(inserter); - queuedTree = null; - leader.roundHoldsReferenceToRefTree = false; - - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(acceptedOldIndex); - if (treeId.equals(c.getTree())) { - throw new NoOp(); - } - } - } - - Proposal p = todo.get(0); - CommitBuilder b = new CommitBuilder(); - b.setTreeId(treeId); - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - b.setParentId(acceptedOldIndex); - } - b.setCommitter(leader.getSystem().newCommitter(ts)); - b.setAuthor(p.getAuthor() != null ? p.getAuthor() : b.getCommitter()); - b.setMessage(message(p)); - return inserter.insert(b); - } - - private ObjectId insertMultiProposal(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException, NoOp { - // The tree was not passed in, or there are multiple proposals - // each needing their own commit. Reset the tree and replay each - // proposal in order as individual commits. - ObjectId lastIndex = acceptedOldIndex; - ObjectId oldTreeId; - RefTree tree; - if (ObjectId.zeroId().equals(lastIndex)) { - oldTreeId = ObjectId.zeroId(); - tree = RefTree.newEmptyTree(); - } else { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(lastIndex); - oldTreeId = c.getTree(); - tree = RefTree.read(rw.getObjectReader(), c.getTree()); - } - } - - PersonIdent committer = leader.getSystem().newCommitter(ts); - for (Proposal p : todo) { - if (!tree.apply(p.getCommands())) { - // This should not occur, previously during queuing the - // commands were successfully applied to the pending tree. - // Abort the entire round. - throw new IOException( - KetchText.get().queuedProposalFailedToApply); - } - - ObjectId treeId = tree.writeTree(inserter); - if (treeId.equals(oldTreeId)) { - continue; - } - - CommitBuilder b = new CommitBuilder(); - b.setTreeId(treeId); - if (!ObjectId.zeroId().equals(lastIndex)) { - b.setParentId(lastIndex); - } - b.setAuthor(p.getAuthor() != null ? p.getAuthor() : committer); - b.setCommitter(committer); - b.setMessage(message(p)); - lastIndex = inserter.insert(b); - } - if (lastIndex.equals(acceptedOldIndex)) { - throw new NoOp(); - } - return lastIndex; - } - - private String message(Proposal p) { - StringBuilder m = new StringBuilder(); - String msg = p.getMessage(); - if (msg != null && !msg.isEmpty()) { - m.append(msg); - while (m.length() < 2 || m.charAt(m.length() - 2) != '\n' - || m.charAt(m.length() - 1) != '\n') { - m.append('\n'); - } - } - m.append(KetchConstants.TERM.getName()) - .append(": ") //$NON-NLS-1$ - .append(leader.getTerm()); - return m.toString(); - } - - void abort() { - for (Proposal p : todo) { - p.abort(); - } - } - - @Override - void success() { - for (Proposal p : todo) { - p.success(); - } - } - - private List<ReceiveCommand> makeStageList(Repository git, - ObjectInserter inserter) throws IOException { - // For each branch, collapse consecutive updates to only most recent, - // avoiding sending multiple objects in a rapid fast-forward chain, or - // rewritten content. - Map<String, ObjectId> byRef = new HashMap<>(); - for (Proposal p : todo) { - for (Command c : p.getCommands()) { - Ref n = c.getNewRef(); - if (n != null && !n.isSymbolic()) { - byRef.put(n.getName(), n.getObjectId()); - } - } - } - if (byRef.isEmpty()) { - return Collections.emptyList(); - } - - Set<ObjectId> newObjs = new HashSet<>(byRef.values()); - StageBuilder b = new StageBuilder( - leader.getSystem().getTxnStage(), - acceptedNewIndex); - return b.makeStageList(newObjs, git, inserter); - } - - private void blockUntil(ProposedTimestamp ts) - throws TimeIsUncertainException { - List<ProposedTimestamp> times = todo.stream() - .flatMap(p -> p.getProposedTimestamps().stream()) - .collect(Collectors.toCollection(ArrayList::new)); - times.add(ts); - - try { - Duration maxWait = getSystem().getMaxWaitForMonotonicClock(); - ProposedTimestamp.blockUntil(times, maxWait); - } catch (InterruptedException | TimeoutException e) { - throw new TimeIsUncertainException(e); - } - } - - private static class NoOp extends Exception { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java deleted file mode 100644 index fac93c84b3..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; -import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NODELETE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.NotSupportedException; -import org.eclipse.jgit.errors.TransportException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.FetchConnection; -import org.eclipse.jgit.transport.PushConnection; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.RemoteRefUpdate; -import org.eclipse.jgit.transport.Transport; -import org.eclipse.jgit.transport.URIish; - -/** - * Representation of a Git repository on a remote replica system. - * <p> - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} will contact the replica - * using the Git wire protocol. - * <p> - * The remote replica may be fully Ketch-aware, or a standard Git server. - */ -public class RemoteGitReplica extends KetchReplica { - private final URIish uri; - private final RemoteConfig remoteConfig; - - /** - * Configure a new remote. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this remote for debugging. - * @param uri - * URI to connect to the follower's repository. - * @param cfg - * how Ketch should treat the remote system. - * @param rc - * optional remote configuration describing how to contact the - * peer repository. - */ - public RemoteGitReplica(KetchLeader leader, String name, URIish uri, - ReplicaConfig cfg, @Nullable RemoteConfig rc) { - super(leader, name, cfg); - this.uri = uri; - this.remoteConfig = rc; - } - - /** - * Get URI to contact the remote peer repository. - * - * @return URI to contact the remote peer repository. - */ - public URIish getURI() { - return uri; - } - - /** - * Get optional configuration describing how to contact the peer. - * - * @return optional configuration describing how to contact the peer. - */ - @Nullable - protected RemoteConfig getRemoteConfig() { - return remoteConfig; - } - - /** {@inheritDoc} */ - @Override - protected String describeForLog() { - return String.format("%s @ %s", getName(), getURI()); //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @Override - protected void startPush(ReplicaPushRequest req) { - getSystem().getExecutor().execute(() -> { - try (Repository git = getLeader().openRepository()) { - try { - push(git, req); - req.done(git); - } catch (Throwable err) { - req.setException(git, err); - } - } catch (IOException err) { - req.setException(null, err); - } - }); - } - - private void push(Repository repo, ReplicaPushRequest req) - throws NotSupportedException, TransportException, IOException { - Map<String, Ref> adv; - List<RemoteCommand> cmds = asUpdateList(req.getCommands()); - try (Transport transport = Transport.open(repo, uri)) { - RemoteConfig rc = getRemoteConfig(); - if (rc != null) { - transport.applyConfig(rc); - } - transport.setPushAtomic(true); - adv = push(repo, transport, cmds); - } - for (RemoteCommand c : cmds) { - c.copyStatusToResult(); - } - req.setRefs(adv); - } - - private Map<String, Ref> push(Repository git, Transport transport, - List<RemoteCommand> cmds) throws IOException { - Map<String, RemoteRefUpdate> updates = asUpdateMap(cmds); - try (PushConnection connection = transport.openPush()) { - Map<String, Ref> adv = connection.getRefsMap(); - RemoteRefUpdate accepted = updates.get(getSystem().getTxnAccepted()); - if (accepted != null && !isExpectedValue(adv, accepted)) { - abort(cmds); - return adv; - } - - RemoteRefUpdate committed = updates.get(getSystem().getTxnCommitted()); - if (committed != null && !isExpectedValue(adv, committed)) { - abort(cmds); - return adv; - } - if (committed != null && getCommitMethod() == ALL_REFS) { - prepareCommit(git, cmds, updates, adv, - committed.getNewObjectId()); - } - - connection.push(NullProgressMonitor.INSTANCE, updates); - return adv; - } - } - - private static boolean isExpectedValue(Map<String, Ref> adv, - RemoteRefUpdate u) { - Ref r = adv.get(u.getRemoteName()); - if (!AnyObjectId.isEqual(getId(r), u.getExpectedOldObjectId())) { - ((RemoteCommand) u).cmd.setResult(LOCK_FAILURE); - return false; - } - return true; - } - - private void prepareCommit(Repository git, List<RemoteCommand> cmds, - Map<String, RemoteRefUpdate> updates, Map<String, Ref> adv, - ObjectId committed) throws IOException { - for (ReceiveCommand cmd : prepareCommit(git, adv, committed)) { - RemoteCommand c = new RemoteCommand(cmd); - cmds.add(c); - updates.put(c.getRemoteName(), c); - } - } - - private static List<RemoteCommand> asUpdateList( - Collection<ReceiveCommand> cmds) { - try { - List<RemoteCommand> toPush = new ArrayList<>(cmds.size()); - for (ReceiveCommand cmd : cmds) { - toPush.add(new RemoteCommand(cmd)); - } - return toPush; - } catch (IOException e) { - // Cannot occur as no IO was required to build the command. - throw new IllegalStateException(e); - } - } - - private static Map<String, RemoteRefUpdate> asUpdateMap( - List<RemoteCommand> cmds) { - Map<String, RemoteRefUpdate> m = new LinkedHashMap<>(); - for (RemoteCommand cmd : cmds) { - m.put(cmd.getRemoteName(), cmd); - } - return m; - } - - private static void abort(List<RemoteCommand> cmds) { - List<ReceiveCommand> tmp = new ArrayList<>(cmds.size()); - for (RemoteCommand cmd : cmds) { - tmp.add(cmd.cmd); - } - ReceiveCommand.abort(tmp); - } - - /** {@inheritDoc} */ - @Override - protected void blockingFetch(Repository repo, ReplicaFetchRequest req) - throws NotSupportedException, TransportException { - try (Transport transport = Transport.open(repo, uri)) { - RemoteConfig rc = getRemoteConfig(); - if (rc != null) { - transport.applyConfig(rc); - } - fetch(transport, req); - } - } - - private void fetch(Transport transport, ReplicaFetchRequest req) - throws NotSupportedException, TransportException { - try (FetchConnection conn = transport.openFetch()) { - Map<String, Ref> remoteRefs = conn.getRefsMap(); - req.setRefs(remoteRefs); - - List<Ref> want = new ArrayList<>(); - for (String name : req.getWantRefs()) { - Ref ref = remoteRefs.get(name); - if (ref != null && ref.getObjectId() != null) { - want.add(ref); - } - } - for (ObjectId id : req.getWantObjects()) { - want.add(new ObjectIdRef.Unpeeled(NETWORK, id.name(), id)); - } - - conn.fetch(NullProgressMonitor.INSTANCE, want, - Collections.<ObjectId> emptySet()); - } - } - - static class RemoteCommand extends RemoteRefUpdate { - final ReceiveCommand cmd; - - RemoteCommand(ReceiveCommand cmd) throws IOException { - super(null, null, - cmd.getNewId(), cmd.getRefName(), - true /* force update */, - null /* no local tracking ref */, - cmd.getOldId()); - this.cmd = cmd; - } - - void copyStatusToResult() { - if (cmd.getResult() == NOT_ATTEMPTED) { - switch (getStatus()) { - case OK: - case UP_TO_DATE: - case NON_EXISTING: - cmd.setResult(OK); - break; - - case REJECTED_NODELETE: - cmd.setResult(REJECTED_NODELETE); - break; - - case REJECTED_NONFASTFORWARD: - cmd.setResult(REJECTED_NONFASTFORWARD); - break; - - case REJECTED_OTHER_REASON: - cmd.setResult(REJECTED_OTHER_REASON, getMessage()); - break; - - default: - cmd.setResult(REJECTED_OTHER_REASON, getStatus().name()); - break; - } - } - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java deleted file mode 100644 index 1d323b8490..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_COMMIT; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_SPEED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod; -import org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed; -import org.eclipse.jgit.internal.ketch.KetchReplica.Participation; -import org.eclipse.jgit.lib.Config; - -/** - * Configures a {@link org.eclipse.jgit.internal.ketch.KetchReplica}. - */ -public class ReplicaConfig { - /** - * Read a configuration from a config block. - * - * @param cfg - * configuration to read. - * @param name - * of the replica being configured. - * @return replica configuration for {@code name}. - */ - public static ReplicaConfig newFromConfig(Config cfg, String name) { - return new ReplicaConfig().fromConfig(cfg, name); - } - - private Participation participation = Participation.FULL; - private CommitMethod commitMethod = CommitMethod.ALL_REFS; - private CommitSpeed commitSpeed = CommitSpeed.BATCHED; - private long minRetry = SECONDS.toMillis(5); - private long maxRetry = MINUTES.toMillis(1); - - /** - * Get participation of the replica in the system. - * - * @return participation of the replica in the system. - */ - public Participation getParticipation() { - return participation; - } - - /** - * Get how Ketch should apply committed changes. - * - * @return how Ketch should apply committed changes. - */ - public CommitMethod getCommitMethod() { - return commitMethod; - } - - /** - * Get how quickly should Ketch commit. - * - * @return how quickly should Ketch commit. - */ - public CommitSpeed getCommitSpeed() { - return commitSpeed; - } - - /** - * Returns the minimum wait delay before retrying a failure. - * - * @param unit - * to get retry delay in. - * @return minimum delay before retrying a failure. - */ - public long getMinRetry(TimeUnit unit) { - return unit.convert(minRetry, MILLISECONDS); - } - - /** - * Returns the maximum wait delay before retrying a failure. - * - * @param unit - * to get retry delay in. - * @return maximum delay before retrying a failure. - */ - public long getMaxRetry(TimeUnit unit) { - return unit.convert(maxRetry, MILLISECONDS); - } - - /** - * Update the configuration from a config block. - * - * @param cfg - * configuration to read. - * @param name - * of the replica being configured. - * @return {@code this} - */ - public ReplicaConfig fromConfig(Config cfg, String name) { - participation = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE, - participation); - commitMethod = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_COMMIT, - commitMethod); - commitSpeed = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_SPEED, - commitSpeed); - minRetry = getMillis(cfg, name, "ketch-minRetry", minRetry); //$NON-NLS-1$ - maxRetry = getMillis(cfg, name, "ketch-maxRetry", maxRetry); //$NON-NLS-1$ - return this; - } - - private static long getMillis(Config cfg, String name, String key, - long defaultValue) { - String valStr = cfg.getString(CONFIG_KEY_REMOTE, name, key); - if (valStr == null) { - return defaultValue; - } - - valStr = valStr.trim(); - if (valStr.isEmpty()) { - return defaultValue; - } - - Matcher m = UnitMap.PATTERN.matcher(valStr); - if (!m.matches()) { - return defaultValue; - } - - String digits = m.group(1); - String unitName = m.group(2).trim(); - TimeUnit unit = UnitMap.UNITS.get(unitName); - if (unit == null) { - return defaultValue; - } - - try { - if (digits.indexOf('.') == -1) { - return unit.toMillis(Long.parseLong(digits)); - } - - double val = Double.parseDouble(digits); - return (long) (val * unit.toMillis(1)); - } catch (NumberFormatException nfe) { - return defaultValue; - } - } - - static class UnitMap { - static final Pattern PATTERN = Pattern - .compile("^([1-9][0-9]*(?:\\.[0-9]*)?)\\s*(.*)$"); //$NON-NLS-1$ - - static final Map<String, TimeUnit> UNITS; - - static { - Map<String, TimeUnit> m = new HashMap<>(); - TimeUnit u = MILLISECONDS; - m.put("", u); //$NON-NLS-1$ - m.put("ms", u); //$NON-NLS-1$ - m.put("millis", u); //$NON-NLS-1$ - m.put("millisecond", u); //$NON-NLS-1$ - m.put("milliseconds", u); //$NON-NLS-1$ - - u = SECONDS; - m.put("s", u); //$NON-NLS-1$ - m.put("sec", u); //$NON-NLS-1$ - m.put("secs", u); //$NON-NLS-1$ - m.put("second", u); //$NON-NLS-1$ - m.put("seconds", u); //$NON-NLS-1$ - - u = MINUTES; - m.put("m", u); //$NON-NLS-1$ - m.put("min", u); //$NON-NLS-1$ - m.put("mins", u); //$NON-NLS-1$ - m.put("minute", u); //$NON-NLS-1$ - m.put("minutes", u); //$NON-NLS-1$ - - u = HOURS; - m.put("h", u); //$NON-NLS-1$ - m.put("hr", u); //$NON-NLS-1$ - m.put("hrs", u); //$NON-NLS-1$ - m.put("hour", u); //$NON-NLS-1$ - m.put("hours", u); //$NON-NLS-1$ - - u = DAYS; - m.put("d", u); //$NON-NLS-1$ - m.put("day", u); //$NON-NLS-1$ - m.put("days", u); //$NON-NLS-1$ - - UNITS = Collections.unmodifiableMap(m); - } - - private UnitMap() { - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java deleted file mode 100644 index f50ad62c80..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Map; -import java.util.Set; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; - -/** - * A fetch request to obtain objects from a replica, and its result. - */ -public class ReplicaFetchRequest { - private final Set<String> wantRefs; - private final Set<ObjectId> wantObjects; - private Map<String, Ref> refs; - - /** - * Construct a new fetch request for a replica. - * - * @param wantRefs - * named references to be fetched. - * @param wantObjects - * specific objects to be fetched. - */ - public ReplicaFetchRequest(Set<String> wantRefs, - Set<ObjectId> wantObjects) { - this.wantRefs = wantRefs; - this.wantObjects = wantObjects; - } - - /** - * Get references to be fetched. - * - * @return references to be fetched. - */ - public Set<String> getWantRefs() { - return wantRefs; - } - - /** - * Get objects to be fetched. - * - * @return objects to be fetched. - */ - public Set<ObjectId> getWantObjects() { - return wantObjects; - } - - /** - * Get remote references, usually from the advertisement. - * - * @return remote references, usually from the advertisement. - */ - @Nullable - public Map<String, Ref> getRefs() { - return refs; - } - - /** - * Set references observed from the replica. - * - * @param refs - * references observed from the replica. - */ - public void setRefs(Map<String, Ref> refs) { - this.refs = refs; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java deleted file mode 100644 index 273760bc6e..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Collection; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * A push request sending objects to a replica, and its result. - * <p> - * Implementors of {@link org.eclipse.jgit.internal.ketch.KetchReplica} must - * populate the command result fields, {@link #setRefs(Map)}, and call one of - * {@link #setException(Repository, Throwable)} or {@link #done(Repository)} to - * finish processing. - */ -public class ReplicaPushRequest { - private final KetchReplica replica; - private final Collection<ReceiveCommand> commands; - private Map<String, Ref> refs; - private Throwable exception; - private boolean notified; - - /** - * Construct a new push request for a replica. - * - * @param replica - * the replica being pushed to. - * @param commands - * commands to be executed. - */ - public ReplicaPushRequest(KetchReplica replica, - Collection<ReceiveCommand> commands) { - this.replica = replica; - this.commands = commands; - } - - /** - * Get commands to be executed, and their results. - * - * @return commands to be executed, and their results. - */ - public Collection<ReceiveCommand> getCommands() { - return commands; - } - - /** - * Get remote references, usually from the advertisement. - * - * @return remote references, usually from the advertisement. - */ - @Nullable - public Map<String, Ref> getRefs() { - return refs; - } - - /** - * Set references observed from the replica. - * - * @param refs - * references observed from the replica. - */ - public void setRefs(Map<String, Ref> refs) { - this.refs = refs; - } - - /** - * Get exception thrown, if any. - * - * @return exception thrown, if any. - */ - @Nullable - public Throwable getException() { - return exception; - } - - /** - * Mark the request as crashing with a communication error. - * <p> - * This method may take significant time acquiring the leader lock and - * updating the Ketch state machine with the failure. - * - * @param repo - * local repository reference used by the push attempt. - * @param err - * exception thrown during communication. - */ - public void setException(@Nullable Repository repo, Throwable err) { - if (KetchReplica.log.isErrorEnabled()) { - KetchReplica.log.error(describe("failed"), err); //$NON-NLS-1$ - } - if (!notified) { - notified = true; - exception = err; - replica.afterPush(repo, this); - } - } - - /** - * Mark the request as completed without exception. - * <p> - * This method may take significant time acquiring the leader lock and - * updating the Ketch state machine with results from this replica. - * - * @param repo - * local repository reference used by the push attempt. - */ - public void done(Repository repo) { - if (KetchReplica.log.isDebugEnabled()) { - KetchReplica.log.debug(describe("completed")); //$NON-NLS-1$ - } - if (!notified) { - notified = true; - replica.afterPush(repo, this); - } - } - - private String describe(String heading) { - StringBuilder b = new StringBuilder(); - b.append("push to "); //$NON-NLS-1$ - b.append(replica.describeForLog()); - b.append(' ').append(heading).append(":\n"); //$NON-NLS-1$ - for (ReceiveCommand cmd : commands) { - b.append(String.format( - " %-12s %-12s %s %s", //$NON-NLS-1$ - LeaderSnapshot.str(cmd.getOldId()), - LeaderSnapshot.str(cmd.getNewId()), - cmd.getRefName(), - cmd.getResult())); - if (cmd.getMessage() != null) { - b.append(' ').append(cmd.getMessage()); - } - b.append('\n'); - } - return b.toString(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java deleted file mode 100644 index 05e4ed693a..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Date; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; - -/** - * A snapshot of a replica. - * - * @see LeaderSnapshot - */ -public class ReplicaSnapshot { - final KetchReplica replica; - ObjectId accepted; - ObjectId committed; - KetchReplica.State state; - String error; - long retryAtMillis; - - ReplicaSnapshot(KetchReplica replica) { - this.replica = replica; - } - - /** - * Get the replica this snapshot describes the state of - * - * @return the replica this snapshot describes the state of - */ - public KetchReplica getReplica() { - return replica; - } - - /** - * Get current state of the replica - * - * @return current state of the replica - */ - public KetchReplica.State getState() { - return state; - } - - /** - * Get last known Git commit at {@code refs/txn/accepted} - * - * @return last known Git commit at {@code refs/txn/accepted} - */ - @Nullable - public ObjectId getAccepted() { - return accepted; - } - - /** - * Get last known Git commit at {@code refs/txn/committed} - * - * @return last known Git commit at {@code refs/txn/committed} - */ - @Nullable - public ObjectId getCommitted() { - return committed; - } - - /** - * Get error message - * - * @return if {@link #getState()} == - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.State#OFFLINE} - * an optional human-readable message from the transport system - * explaining the failure. - */ - @Nullable - public String getErrorMessage() { - return error; - } - - /** - * Get when the leader will retry communication with the offline or lagging - * replica - * - * @return time (usually in the future) when the leader will retry - * communication with the offline or lagging replica; null if no - * retry is scheduled or necessary. - */ - @Nullable - public Date getRetryAt() { - return retryAtMillis > 0 ? new Date(retryAtMillis) : null; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java deleted file mode 100644 index 05da5be056..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.io.IOException; -import java.util.List; - -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * One round-trip to all replicas proposing a log entry. - * <p> - * In Raft a log entry represents a state transition at a specific index in the - * replicated log. The leader can only append log entries to the log. - * <p> - * In Ketch a log entry is recorded under the {@code refs/txn} namespace. This - * occurs when: - * <ul> - * <li>a replica wants to establish itself as a new leader by proposing a new - * term (see {@link ElectionRound}) - * <li>an established leader wants to gain consensus on new {@link Proposal}s - * (see {@link ProposalRound}) - * </ul> - */ -abstract class Round { - final KetchLeader leader; - final LogIndex acceptedOldIndex; - LogIndex acceptedNewIndex; - List<ReceiveCommand> stageCommands; - - Round(KetchLeader leader, LogIndex head) { - this.leader = leader; - this.acceptedOldIndex = head; - } - - KetchSystem getSystem() { - return leader.getSystem(); - } - - /** - * Creates a commit for {@code refs/txn/accepted} and calls - * {@link #runAsync(AnyObjectId)} to begin execution of the round across - * the system. - * <p> - * If references are being updated (such as in a {@link ProposalRound}) the - * RefTree may be modified. - * <p> - * Invoked without {@link KetchLeader#lock} to build objects. - * - * @throws IOException - * the round cannot build new objects within the leader's - * repository. The leader may be unable to execute. - */ - abstract void start() throws IOException; - - /** - * Asynchronously distribute the round's new value for - * {@code refs/txn/accepted} to all replicas. - * <p> - * Invoked by {@link #start()} after new commits have been created for the - * log. The method passes {@code newId} to {@link KetchLeader} to be - * distributed to all known replicas. - * - * @param newId - * new value for {@code refs/txn/accepted}. - */ - void runAsync(AnyObjectId newId) { - acceptedNewIndex = acceptedOldIndex.nextIndex(newId); - leader.runAsync(this); - } - - /** - * Notify the round it was accepted by a majority of the system. - * <p> - * Invoked by the leader with {@link KetchLeader#lock} held by the caller. - */ - abstract void success(); -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java deleted file mode 100644 index 40d86e1a85..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.treewalk.EmptyTreeIterator; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.treewalk.filter.TreeFilter; - -/** - * Constructs a set of commands to stage content during a proposal. - */ -public class StageBuilder { - /** - * Acceptable number of references to send in a single stage transaction. - * <p> - * If the number of unique objects exceeds this amount the builder will - * attempt to decrease the reference count by chaining commits.. - */ - private static final int SMALL_BATCH_SIZE = 5; - - /** - * Acceptable number of commits to chain together using parent pointers. - * <p> - * When staging many unique commits the {@link StageBuilder} batches - * together unrelated commits as parents of a temporary commit. After the - * proposal completes the temporary commit is discarded and can be garbage - * collected by all replicas. - */ - private static final int TEMP_PARENT_BATCH_SIZE = 128; - - private static final byte[] PEEL = { ' ', '^' }; - - private final String txnStage; - private final String txnId; - - /** - * Construct a stage builder for a transaction. - * - * @param txnStageNamespace - * namespace for transaction references to build - * {@code "txnStageNamespace/txnId.n"} style names. - * @param txnId - * identifier used to name temporary staging refs. - */ - public StageBuilder(String txnStageNamespace, ObjectId txnId) { - this.txnStage = txnStageNamespace; - this.txnId = txnId.name(); - } - - /** - * Compare two RefTrees and return commands to stage new objects. - * <p> - * This method ignores the lineage between the two RefTrees and does a - * straight diff on the two trees. New objects will be staged. The diff - * strategy is useful to catch-up a lagging replica, without sending every - * intermediate step. This may mean the replica does not have the same - * object set as other replicas if there are rewinds or branch deletes. - * - * @param git - * source repository to read {@code oldTree} and {@code newTree} - * from. - * @param oldTree - * accepted RefTree on the replica ({@code refs/txn/accepted}). - * Use {@link org.eclipse.jgit.lib.ObjectId#zeroId()} if the - * remote does not have any ref tree, e.g. a new replica catching - * up. - * @param newTree - * RefTree being sent to the replica. The trees will be compared. - * @return list of commands to create {@code "refs/txn/stage/..."} - * references on replicas anchoring new objects into the repository - * while a transaction gains consensus. - * @throws java.io.IOException - * {@code git} cannot be accessed to compare {@code oldTree} and - * {@code newTree} to build the object set. - */ - public List<ReceiveCommand> makeStageList(Repository git, ObjectId oldTree, - ObjectId newTree) throws IOException { - try (RevWalk rw = new RevWalk(git); - TreeWalk tw = new TreeWalk(rw.getObjectReader()); - ObjectInserter ins = git.newObjectInserter()) { - if (AnyObjectId.isEqual(oldTree, ObjectId.zeroId())) { - tw.addTree(new EmptyTreeIterator()); - } else { - tw.addTree(rw.parseTree(oldTree)); - } - tw.addTree(rw.parseTree(newTree)); - tw.setFilter(TreeFilter.ANY_DIFF); - tw.setRecursive(true); - - Set<ObjectId> newObjs = new HashSet<>(); - while (tw.next()) { - if (tw.getRawMode(1) == TYPE_GITLINK - && !tw.isPathSuffix(PEEL, 2)) { - newObjs.add(tw.getObjectId(1)); - } - } - - List<ReceiveCommand> cmds = makeStageList(newObjs, git, ins); - ins.flush(); - return cmds; - } - } - - /** - * Construct a set of commands to stage objects on a replica. - * - * @param newObjs - * objects to send to a replica. - * @param git - * local repository to read source objects from. Required to - * perform minification of {@code newObjs}. - * @param inserter - * inserter to write temporary commit objects during minification - * if many new branches are created by {@code newObjs}. - * @return list of commands to create {@code "refs/txn/stage/..."} - * references on replicas anchoring {@code newObjs} into the - * repository while a transaction gains consensus. - * @throws java.io.IOException - * {@code git} cannot be accessed to perform minification of - * {@code newObjs}. - */ - public List<ReceiveCommand> makeStageList(Set<ObjectId> newObjs, - @Nullable Repository git, @Nullable ObjectInserter inserter) - throws IOException { - if (git == null || newObjs.size() <= SMALL_BATCH_SIZE) { - // Without a source repository can only construct unique set. - List<ReceiveCommand> cmds = new ArrayList<>(newObjs.size()); - for (ObjectId id : newObjs) { - stage(cmds, id); - } - return cmds; - } - - List<ReceiveCommand> cmds = new ArrayList<>(); - List<RevCommit> commits = new ArrayList<>(); - reduceObjects(cmds, commits, git, newObjs); - - if (inserter == null || commits.size() <= 1 - || (cmds.size() + commits.size()) <= SMALL_BATCH_SIZE) { - // Without an inserter to aggregate commits, or for a small set of - // commits just send one stage ref per commit. - for (RevCommit c : commits) { - stage(cmds, c.copy()); - } - return cmds; - } - - // 'commits' is sorted most recent to least recent commit. - // Group batches of commits and build a chain. - // TODO(sop) Cluster by restricted graphs to support filtering. - ObjectId tip = null; - for (int end = commits.size(); end > 0;) { - int start = Math.max(0, end - TEMP_PARENT_BATCH_SIZE); - List<RevCommit> batch = commits.subList(start, end); - List<ObjectId> parents = new ArrayList<>(1 + batch.size()); - if (tip != null) { - parents.add(tip); - } - parents.addAll(batch); - - CommitBuilder b = new CommitBuilder(); - b.setTreeId(batch.get(0).getTree()); - b.setParentIds(parents); - b.setAuthor(tmpAuthor(batch)); - b.setCommitter(b.getAuthor()); - tip = inserter.insert(b); - end = start; - } - stage(cmds, tip); - return cmds; - } - - private static PersonIdent tmpAuthor(List<RevCommit> commits) { - // Construct a predictable author using most recent commit time. - int t = 0; - for (int i = 0; i < commits.size();) { - t = Math.max(t, commits.get(i).getCommitTime()); - } - String name = "Ketch Stage"; //$NON-NLS-1$ - String email = "tmp@tmp"; //$NON-NLS-1$ - return new PersonIdent(name, email, t * 1000L, 0); - } - - private void reduceObjects(List<ReceiveCommand> cmds, - List<RevCommit> commits, Repository git, - Set<ObjectId> newObjs) throws IOException { - try (RevWalk rw = new RevWalk(git)) { - rw.setRetainBody(false); - - for (ObjectId id : newObjs) { - RevObject obj = rw.parseAny(id); - if (obj instanceof RevCommit) { - rw.markStart((RevCommit) obj); - } else { - stage(cmds, id); - } - } - - for (RevCommit c; (c = rw.next()) != null;) { - commits.add(c); - rw.markUninteresting(c); - } - } - } - - private void stage(List<ReceiveCommand> cmds, ObjectId id) { - int estLen = txnStage.length() + txnId.length() + 5; - StringBuilder n = new StringBuilder(estLen); - n.append(txnStage).append(txnId).append('.'); - n.append(Integer.toHexString(cmds.size())); - cmds.add(new ReceiveCommand(ObjectId.zeroId(), id, n.toString())); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java deleted file mode 100644 index f665e6a438..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.io.IOException; - -import org.eclipse.jgit.internal.JGitText; - -class TimeIsUncertainException extends IOException { - private static final long serialVersionUID = 1L; - - TimeIsUncertainException() { - super(JGitText.get().timeIsUncertain); - } - - TimeIsUncertainException(Exception e) { - super(JGitText.get().timeIsUncertain, e); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java deleted file mode 100644 index dfe03752ca..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Distributed consensus system built on Git. - */ -package org.eclipse.jgit.internal.ketch; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityChecker.java index 89aef7dc41..d8056490aa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedObjectReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedObjectReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.util.ArrayList; @@ -21,12 +21,16 @@ import java.util.stream.Stream; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; +import org.eclipse.jgit.revwalk.BitmapWalker; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevObject; /** * Checks if all objects are reachable from certain starting points using * bitmaps. */ -class BitmappedObjectReachabilityChecker +public class BitmappedObjectReachabilityChecker implements ObjectReachabilityChecker { private final ObjectWalk walk; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityChecker.java index 0d9c4593bf..37721ad1ea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/BitmappedReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/BitmappedReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.util.ArrayList; @@ -23,12 +23,17 @@ import org.eclipse.jgit.lib.BitmapIndex; import org.eclipse.jgit.lib.BitmapIndex.Bitmap; import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevFlag; +import org.eclipse.jgit.revwalk.RevSort; +import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.RevFilter; /** * Checks the reachability using bitmaps. */ -class BitmappedReachabilityChecker implements ReachabilityChecker { +public class BitmappedReachabilityChecker implements ReachabilityChecker { private final RevWalk walk; @@ -42,7 +47,7 @@ class BitmappedReachabilityChecker implements ReachabilityChecker { * @throws IOException * if the index or the object reader cannot be opened. */ - BitmappedReachabilityChecker(RevWalk walk) + public BitmappedReachabilityChecker(RevWalk walk) throws IOException { this.walk = walk; if (walk.getObjectReader().getBitmapIndex() == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityChecker.java index df5d68a66e..1d1f5fddaf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianObjectReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianObjectReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.io.InvalidObjectException; @@ -17,12 +17,18 @@ import java.util.Optional; import java.util.stream.Stream; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevSort; /** * Checks if all objects are reachable from certain starting points doing a * walk. */ -class PedestrianObjectReachabilityChecker implements ObjectReachabilityChecker { +public class PedestrianObjectReachabilityChecker + implements ObjectReachabilityChecker { private final ObjectWalk walk; /** @@ -31,7 +37,7 @@ class PedestrianObjectReachabilityChecker implements ObjectReachabilityChecker { * @param walk * ObjectWalk instance to reuse. Caller retains ownership. */ - PedestrianObjectReachabilityChecker(ObjectWalk walk) { + public PedestrianObjectReachabilityChecker(ObjectWalk walk) { this.walk = walk; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityChecker.java index 5dc03776c2..a03306b6ee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/PedestrianReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/revwalk/PedestrianReachabilityChecker.java @@ -7,7 +7,7 @@ * * SPDX-License-Identifier: BSD-3-Clause */ -package org.eclipse.jgit.revwalk; +package org.eclipse.jgit.internal.revwalk; import java.io.IOException; import java.util.Collection; @@ -17,12 +17,16 @@ import java.util.stream.Stream; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevSort; +import org.eclipse.jgit.revwalk.RevWalk; /** * Checks the reachability walking the graph from the starters towards the * target. */ -class PedestrianReachabilityChecker implements ReachabilityChecker { +public class PedestrianReachabilityChecker implements ReachabilityChecker { private final boolean topoSort; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index 876cbec161..26d5b5b176 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -13,7 +13,6 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; -import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; @@ -45,7 +44,6 @@ import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; -import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -95,7 +93,6 @@ public class DfsGarbageCollector { private Set<ObjectId> allHeadsAndTags; private Set<ObjectId> allTags; private Set<ObjectId> nonHeads; - private Set<ObjectId> txnHeads; private Set<ObjectId> tagTargets; /** @@ -318,7 +315,6 @@ public class DfsGarbageCollector { allHeadsAndTags = new HashSet<>(); allTags = new HashSet<>(); nonHeads = new HashSet<>(); - txnHeads = new HashSet<>(); tagTargets = new HashSet<>(); for (Ref ref : refsBefore) { if (ref.isSymbolic() || ref.getObjectId() == null) { @@ -328,8 +324,6 @@ public class DfsGarbageCollector { allHeads.add(ref.getObjectId()); } else if (isTag(ref)) { allTags.add(ref.getObjectId()); - } else if (RefTreeNames.isRefTree(refdb, ref.getName())) { - txnHeads.add(ref.getObjectId()); } else { nonHeads.add(ref.getObjectId()); } @@ -355,7 +349,6 @@ public class DfsGarbageCollector { try { packHeads(pm); packRest(pm); - packRefTreeGraph(pm); packGarbage(pm); objdb.commitPack(newPackDesc, toPrune()); rollback = false; @@ -559,19 +552,6 @@ public class DfsGarbageCollector { } } - private void packRefTreeGraph(ProgressMonitor pm) throws IOException { - if (txnHeads.isEmpty()) - return; - - try (PackWriter pw = newPackWriter()) { - for (ObjectIdSet packedObjs : newPackObj) - pw.excludeObjects(packedObjs); - pw.preparePack(pm, txnHeads, NONE); - if (0 < pw.getObjectCount()) - writePack(GC_TXN, pw, pm, 0 /* unknown pack size */); - } - } - private void packGarbage(ProgressMonitor pm) throws IOException { PackConfig cfg = new PackConfig(packConfig); cfg.setReuseDeltas(true); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 4dab3b20c5..46ec87df54 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -105,13 +105,6 @@ public abstract class DfsObjDatabase extends ObjectDatabase { GC_REST, /** - * RefTreeGraph pack was created by Git garbage collection. - * - * @see DfsGarbageCollector - */ - GC_TXN, - - /** * Pack was created by Git garbage collection. * <p> * This pack contains only unreachable garbage that was found during the @@ -133,7 +126,6 @@ public abstract class DfsObjDatabase extends ObjectDatabase { .add(COMPACT) .add(GC) .add(GC_REST) - .add(GC_TXN) .add(UNREACHABLE_GARBAGE) .build(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java index 3f113a3ee3..8e124e3c20 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java @@ -48,7 +48,6 @@ class DfsObjectRepresentation extends StoredObjectRepresentation { switch (pack.getPackDescription().getPackSource()) { case GC: case GC_REST: - case GC_TXN: return true; default: return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java index 0c8755fca3..4f418ab4db 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java @@ -533,7 +533,6 @@ public class DfsPackDescription { switch (s) { case GC: case GC_REST: - case GC_TXN: return true; default: return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index b1e95520cc..96ca690c1c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -607,8 +607,15 @@ public final class DfsPackFile extends BlockBasedFile { private void readFully(long position, byte[] dstbuf, int dstoff, int cnt, DfsReader ctx) throws IOException { - if (ctx.copy(this, position, dstbuf, dstoff, cnt) != cnt) - throw new EOFException(); + while (cnt > 0) { + int copied = ctx.copy(this, position, dstbuf, dstoff, cnt); + if (copied == 0) { + throw new EOFException(); + } + position += copied; + dstoff += copied; + cnt -= copied; + } } ObjectLoader load(DfsReader ctx, long pos) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java index 8a54431d5c..6c3b056efd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.eclipse.jgit.annotations.Nullable; @@ -62,11 +63,12 @@ public class DfsReftableDatabase extends DfsRefDatabase { reftableDatabase = new ReftableDatabase() { @Override public MergedReftable openMergedReftable() throws IOException { - DfsReftableDatabase.this.getLock().lock(); + Lock l = DfsReftableDatabase.this.getLock(); + l.lock(); try { return new MergedReftable(stack().readers()); } finally { - DfsReftableDatabase.this.getLock().unlock(); + l.unlock(); } } }; @@ -176,6 +178,13 @@ public class DfsReftableDatabase extends DfsRefDatabase { /** {@inheritDoc} */ @Override + public List<Ref> getRefsByPrefixWithExclusions(String include, Set<String> excludes) + throws IOException { + return reftableDatabase.getRefsByPrefixWithExclusions(include, excludes); + } + + /** {@inheritDoc} */ + @Override public Set<Ref> getTipsWithSha1(ObjectId id) throws IOException { if (!getReftableConfig().isIndexObjects()) { return super.getTipsWithSha1(id); @@ -207,7 +216,8 @@ public class DfsReftableDatabase extends DfsRefDatabase { @Override void clearCache() { - getLock().lock(); + ReentrantLock l = getLock(); + l.lock(); try { if (ctx != null) { ctx.close(); @@ -219,7 +229,7 @@ public class DfsReftableDatabase extends DfsRefDatabase { stack = null; } } finally { - getLock().unlock(); + l.unlock(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java index 45d9c85c8c..1036535423 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteArrayWindow.java @@ -25,7 +25,7 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream; final class ByteArrayWindow extends ByteWindow { private final byte[] array; - ByteArrayWindow(PackFile pack, long o, byte[] b) { + ByteArrayWindow(Pack pack, long o, byte[] b) { super(pack, o, b.length); array = b; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java index 8703216322..b6877578c9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteBufferWindow.java @@ -27,7 +27,7 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream; final class ByteBufferWindow extends ByteWindow { private final ByteBuffer buffer; - ByteBufferWindow(PackFile pack, long o, ByteBuffer b) { + ByteBufferWindow(Pack pack, long o, ByteBuffer b) { super(pack, o, b.capacity()); buffer = b; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java index 159f31c971..31e7eadd8a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ByteWindow.java @@ -27,7 +27,7 @@ import org.eclipse.jgit.internal.storage.pack.PackOutputStream; * </p> */ abstract class ByteWindow { - protected final PackFile pack; + protected final Pack pack; protected final long start; @@ -37,13 +37,13 @@ abstract class ByteWindow { * Constructor for ByteWindow. * * @param p - * a {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * a {@link org.eclipse.jgit.internal.storage.file.Pack}. * @param s * where the byte window starts in the pack file * @param n * size of the byte window */ - protected ByteWindow(PackFile p, long s, int n) { + protected ByteWindow(Pack p, long s, int n) { pack = p; start = s; end = start + n; @@ -53,8 +53,8 @@ abstract class ByteWindow { return (int) (end - start); } - final boolean contains(PackFile neededFile, long neededPos) { - return pack == neededFile && start <= neededPos && neededPos < end; + final boolean contains(Pack neededPack, long neededPos) { + return pack == neededPack && start <= neededPos && neededPos < end; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java index 9c7a2e7111..7dedeb57ab 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/CachedObjectDirectory.java @@ -239,7 +239,7 @@ class CachedObjectDirectory extends FileObjectDatabase { } @Override - PackFile openPack(File pack) throws IOException { + Pack openPack(File pack) throws IOException { return wrapped.openPack(pack); } @@ -250,7 +250,7 @@ class CachedObjectDirectory extends FileObjectDatabase { } @Override - Collection<PackFile> getPacks() { + Collection<Pack> getPacks() { return wrapped.getPacks(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java index cef5a330f2..69cebadf1e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/DeltaBaseCache.java @@ -49,7 +49,7 @@ class DeltaBaseCache { cache = new Slot[CACHE_SZ]; } - Entry get(PackFile pack, long position) { + Entry get(Pack pack, long position) { Slot e = cache[hash(position)]; if (e == null) return null; @@ -63,7 +63,7 @@ class DeltaBaseCache { return null; } - void store(final PackFile pack, final long position, + void store(final Pack pack, final long position, final byte[] data, final int objectType) { if (data.length > maxByteCount) return; // Too large to cache. @@ -146,7 +146,7 @@ class DeltaBaseCache { Slot lruNext; - PackFile provider; + Pack provider; long position; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java index 11ed10c90a..01dd27d9fb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileObjectDatabase.java @@ -71,7 +71,7 @@ abstract class FileObjectDatabase extends ObjectDatabase { abstract InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException; - abstract PackFile openPack(File pack) throws IOException; + abstract Pack openPack(File pack) throws IOException; - abstract Collection<PackFile> getPacks(); + abstract Collection<Pack> getPacks(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index e613a58062..a80fa837b7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -21,7 +21,9 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -107,12 +109,13 @@ public class FileReftableDatabase extends RefDatabase { * @throws IOException on I/O errors */ public void compactFully() throws IOException { - reftableDatabase.getLock().lock(); + Lock l = reftableDatabase.getLock(); + l.lock(); try { reftableStack.compactFully(); reftableDatabase.clearCache(); } finally { - reftableDatabase.getLock().unlock(); + l.unlock(); } } @@ -179,6 +182,13 @@ public class FileReftableDatabase extends RefDatabase { /** {@inheritDoc} */ @Override + public List<Ref> getRefsByPrefixWithExclusions(String include, Set<String> excludes) + throws IOException { + return reftableDatabase.getRefsByPrefixWithExclusions(include, excludes); + } + + /** {@inheritDoc} */ + @Override public List<Ref> getAdditionalRefs() throws IOException { return Collections.emptyList(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java index bc2039c56b..b5e3927bcc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.StandardCopyOption; +import java.security.SecureRandom; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -65,6 +66,8 @@ public class FileReftableStack implements AutoCloseable { private final Runnable onChange; + private final SecureRandom random = new SecureRandom(); + private final Supplier<Config> configSupplier; // Used for stats & testing. @@ -365,8 +368,9 @@ public class FileReftableStack implements AutoCloseable { } private String filename(long low, long high) { - return String.format("%012x-%012x", //$NON-NLS-1$ - Long.valueOf(low), Long.valueOf(high)); + return String.format("%012x-%012x-%08x", //$NON-NLS-1$ + Long.valueOf(low), Long.valueOf(high), + Integer.valueOf(random.nextInt())); } /** @@ -636,6 +640,9 @@ public class FileReftableStack implements AutoCloseable { @Override public boolean equals(Object other) { + if (other == null) { + return false; + } Segment o = (Segment) other; return o.bytes == bytes && o.log == log && o.start == start && o.end == end; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index fd052cec28..fecced1ae6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -40,7 +40,6 @@ import org.eclipse.jgit.events.IndexChangedEvent; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepository; -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; import org.eclipse.jgit.lib.BaseRepositoryBuilder; import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.ConfigConstants; @@ -182,9 +181,6 @@ public class FileRepository extends Repository { if (StringUtils.equalsIgnoreCase(reftype, ConfigConstants.CONFIG_REF_STORAGE_REFTABLE)) { refs = new FileReftableDatabase(this); - } else if (StringUtils.equalsIgnoreCase(reftype, - ConfigConstants.CONFIG_REFSTORAGE_REFTREE)) { - refs = new RefTreeDatabase(this, new RefDirectory(this)); } else { throw new IOException(JGitText.get().unknownRepositoryFormat); } @@ -247,7 +243,7 @@ public class FileRepository extends Repository { RefUpdate head = updateRef(Constants.HEAD); head.disableRefLog(); - head.link(Constants.R_HEADS + Constants.MASTER); + head.link(Constants.R_HEADS + getInitialBranch()); final boolean fileMode; if (getFS().supportsExecute()) { @@ -640,7 +636,7 @@ public class FileRepository extends Repository { refsHeadsFile.delete(); // RefDirectory wants to create the refs/ directory from scratch, so // remove that too. - refsFile.delete(); + refsFile.delete(); // remove HEAD so its previous invalid value doesn't cause issues. headFile.delete(); @@ -668,7 +664,7 @@ public class FileRepository extends Repository { for (ReflogEntry e : logs) { logWriter.log(r.getName(), e); } - } + } } try (RevWalk rw = new RevWalk(this)) { @@ -731,7 +727,7 @@ public class FileRepository extends Repository { throws IOException { File reftableDir = new File(getDirectory(), Constants.REFTABLE); File headFile = new File(getDirectory(), Constants.HEAD); - if (reftableDir.exists() && reftableDir.listFiles().length > 0) { + if (reftableDir.exists() && FileUtils.hasFiles(reftableDir.toPath())) { throw new IOException(JGitText.get().reftableDirExists); } @@ -763,9 +759,11 @@ public class FileRepository extends Repository { } } else { FileUtils.delete(packedRefs, FileUtils.SKIP_MISSING); - FileUtils.delete(headFile); - FileUtils.delete(logsDir, FileUtils.RECURSIVE); - FileUtils.delete(refsFile, FileUtils.RECURSIVE); + FileUtils.delete(headFile, FileUtils.SKIP_MISSING); + FileUtils.delete(logsDir, + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); + FileUtils.delete(refsFile, + FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); for (String r : additional) { new File(getDirectory(), r).delete(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 54ff7d29c6..8c48cd94b0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -12,8 +12,10 @@ package org.eclipse.jgit.internal.storage.file; import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_FILESTORE_ATTRIBUTES; import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_TIMESTAMP_RESOLUTION; + import java.io.File; import java.io.IOException; +import java.nio.file.NoSuchFileException; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; @@ -221,14 +223,20 @@ public class FileSnapshot { this.file = file; this.lastRead = Instant.now(); this.fileStoreAttributeCache = useConfig - ? FS.getFileStoreAttributes(file.toPath().getParent()) + ? FS.getFileStoreAttributes(file.toPath()) : FALLBACK_FILESTORE_ATTRIBUTES; BasicFileAttributes fileAttributes = null; try { fileAttributes = FS.DETECTED.fileAttributes(file); + } catch (NoSuchFileException e) { + this.lastModified = Instant.EPOCH; + this.size = 0L; + this.fileKey = MISSING_FILEKEY; + return; } catch (IOException e) { - this.lastModified = Instant.ofEpochMilli(file.lastModified()); - this.size = file.length(); + LOG.error(e.getMessage(), e); + this.lastModified = Instant.EPOCH; + this.size = 0L; this.fileKey = MISSING_FILEKEY; return; } @@ -309,9 +317,14 @@ public class FileSnapshot { currLastModified = fileAttributes.lastModifiedTime().toInstant(); currSize = fileAttributes.size(); currFileKey = getFileKey(fileAttributes); + } catch (NoSuchFileException e) { + currLastModified = Instant.EPOCH; + currSize = 0L; + currFileKey = MISSING_FILEKEY; } catch (IOException e) { - currLastModified = Instant.ofEpochMilli(path.lastModified()); - currSize = path.length(); + LOG.error(e.getMessage(), e); + currLastModified = Instant.EPOCH; + currSize = 0L; currFileKey = MISSING_FILEKEY; } sizeChanged = isSizeChanged(currSize); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 1f2fe1057f..75de3be89e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -60,7 +60,6 @@ import org.eclipse.jgit.errors.NoWorkTreeException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; @@ -206,16 +205,16 @@ public class GC { * gc.log. * * @return the collection of - * {@link org.eclipse.jgit.internal.storage.file.PackFile}'s which + * {@link org.eclipse.jgit.internal.storage.file.Pack}'s which * are newly created * @throws java.io.IOException * @throws java.text.ParseException * If the configuration parameter "gc.pruneexpire" couldn't be * parsed */ - // TODO(ms): change signature and return Future<Collection<PackFile>> + // TODO(ms): change signature and return Future<Collection<Pack>> @SuppressWarnings("FutureReturnValueIgnored") - public Collection<PackFile> gc() throws IOException, ParseException { + public Collection<Pack> gc() throws IOException, ParseException { if (!background) { return doGc(); } @@ -225,9 +224,9 @@ public class GC { return Collections.emptyList(); } - Callable<Collection<PackFile>> gcTask = () -> { + Callable<Collection<Pack>> gcTask = () -> { try { - Collection<PackFile> newPacks = doGc(); + Collection<Pack> newPacks = doGc(); if (automatic && tooManyLooseObjects()) { String message = JGitText.get().gcTooManyUnpruned; gcLog.write(message); @@ -259,14 +258,14 @@ public class GC { return (executor != null) ? executor : WorkQueue.getExecutor(); } - private Collection<PackFile> doGc() throws IOException, ParseException { + private Collection<Pack> doGc() throws IOException, ParseException { if (automatic && !needGc()) { return Collections.emptyList(); } pm.start(6 /* tasks */); packRefs(); // TODO: implement reflog_expire(pm, repo); - Collection<PackFile> newPacks = repack(); + Collection<Pack> newPacks = repack(); prune(Collections.emptySet()); // TODO: implement rerere_gc(pm); return newPacks; @@ -282,7 +281,7 @@ public class GC { * @param existing * @throws IOException */ - private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing) + private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, Pack pack, HashSet<ObjectId> existing) throws IOException { for (PackIndex.MutableEntry entry : pack) { ObjectId oid = entry.toObjectId(); @@ -314,10 +313,10 @@ public class GC { * @throws ParseException * @throws IOException */ - private void deleteOldPacks(Collection<PackFile> oldPacks, - Collection<PackFile> newPacks) throws ParseException, IOException { + private void deleteOldPacks(Collection<Pack> oldPacks, + Collection<Pack> newPacks) throws ParseException, IOException { HashSet<ObjectId> ids = new HashSet<>(); - for (PackFile pack : newPacks) { + for (Pack pack : newPacks) { for (PackIndex.MutableEntry entry : pack) { ids.add(entry.toObjectId()); } @@ -330,12 +329,12 @@ public class GC { prunePreserved(); long packExpireDate = getPackExpireDate(); - oldPackLoop: for (PackFile oldPack : oldPacks) { + oldPackLoop: for (Pack oldPack : oldPacks) { checkCancelled(); String oldName = oldPack.getPackName(); // check whether an old pack file is also among the list of new // pack files. Then we must not delete it. - for (PackFile newPack : newPacks) + for (Pack newPack : newPacks) if (oldName.equals(newPack.getPackName())) continue oldPackLoop; @@ -439,7 +438,7 @@ public class GC { */ public void prunePacked() throws IOException { ObjectDirectory objdb = repo.getObjectDatabase(); - Collection<PackFile> packs = objdb.getPacks(); + Collection<Pack> packs = objdb.getPacks(); File objects = repo.getObjectsDirectory(); String[] fanout = objects.list(); @@ -467,7 +466,7 @@ public class GC { continue; } boolean found = false; - for (PackFile p : packs) { + for (Pack p : packs) { checkCancelled(); if (p.hasObject(id)) { found = true; @@ -789,8 +788,8 @@ public class GC { * reflog-entries or during writing to the packfiles * {@link java.io.IOException} occurs */ - public Collection<PackFile> repack() throws IOException { - Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks(); + public Collection<Pack> repack() throws IOException { + Collection<Pack> toBeDeleted = repo.getObjectDatabase().getPacks(); long time = System.currentTimeMillis(); Collection<Ref> refsBefore = getAllRefs(); @@ -802,7 +801,6 @@ public class GC { Set<ObjectId> txnHeads = new HashSet<>(); Set<ObjectId> tagTargets = new HashSet<>(); Set<ObjectId> indexObjects = listNonHEADIndexObjects(); - RefDatabase refdb = repo.getRefDatabase(); for (Ref ref : refsBefore) { checkCancelled(); @@ -814,8 +812,6 @@ public class GC { allHeads.add(ref.getObjectId()); } else if (isTag(ref)) { allTags.add(ref.getObjectId()); - } else if (RefTreeNames.isRefTree(refdb, ref.getName())) { - txnHeads.add(ref.getObjectId()); } else { nonHeads.add(ref.getObjectId()); } @@ -825,10 +821,10 @@ public class GC { } List<ObjectIdSet> excluded = new LinkedList<>(); - for (PackFile f : repo.getObjectDatabase().getPacks()) { + for (Pack p : repo.getObjectDatabase().getPacks()) { checkCancelled(); - if (f.shouldBeKept()) - excluded.add(f.getIndex()); + if (p.shouldBeKept()) + excluded.add(p.getIndex()); } // Don't exclude tags that are also branch tips @@ -846,8 +842,8 @@ public class GC { nonHeads.clear(); } - List<PackFile> ret = new ArrayList<>(2); - PackFile heads = null; + List<Pack> ret = new ArrayList<>(2); + Pack heads = null; if (!allHeadsAndTags.isEmpty()) { heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags, tagTargets, excluded); @@ -857,13 +853,13 @@ public class GC { } } if (!nonHeads.isEmpty()) { - PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, + Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE, tagTargets, excluded); if (rest != null) ret.add(rest); } if (!txnHeads.isEmpty()) { - PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, + Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE, null, excluded); if (txn != null) ret.add(txn); @@ -1133,7 +1129,7 @@ public class GC { } } - private PackFile writePack(@NonNull Set<? extends ObjectId> want, + private Pack writePack(@NonNull Set<? extends ObjectId> want, @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags, Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects) throws IOException { @@ -1360,13 +1356,13 @@ public class GC { */ public RepoStatistics getStatistics() throws IOException { RepoStatistics ret = new RepoStatistics(); - Collection<PackFile> packs = repo.getObjectDatabase().getPacks(); - for (PackFile f : packs) { - ret.numberOfPackedObjects += f.getIndex().getObjectCount(); + Collection<Pack> packs = repo.getObjectDatabase().getPacks(); + for (Pack p : packs) { + ret.numberOfPackedObjects += p.getIndex().getObjectCount(); ret.numberOfPackFiles++; - ret.sizeOfPackedObjects += f.getPackFile().length(); - if (f.getBitmapIndex() != null) - ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount(); + ret.sizeOfPackedObjects += p.getPackFile().length(); + if (p.getBitmapIndex() != null) + ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java index ee4bbc1964..e2fbd7a0b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LargePackedWholeObject.java @@ -30,12 +30,12 @@ class LargePackedWholeObject extends ObjectLoader { private final int headerLength; - private final PackFile pack; + private final Pack pack; private final FileObjectDatabase db; LargePackedWholeObject(int type, long size, long objectOffset, - int headerLength, PackFile pack, FileObjectDatabase db) { + int headerLength, Pack pack, FileObjectDatabase db) { this.type = type; this.size = size; this.objectOffset = objectOffset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java index 9d04062e37..ae5bce6985 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java @@ -25,31 +25,31 @@ class LocalCachedPack extends CachedPack { private final String[] packNames; - private PackFile[] packs; + private Pack[] packs; LocalCachedPack(ObjectDirectory odb, List<String> packNames) { this.odb = odb; this.packNames = packNames.toArray(new String[0]); } - LocalCachedPack(List<PackFile> packs) { + LocalCachedPack(List<Pack> packs) { odb = null; packNames = null; - this.packs = packs.toArray(new PackFile[0]); + this.packs = packs.toArray(new Pack[0]); } /** {@inheritDoc} */ @Override public long getObjectCount() throws IOException { long cnt = 0; - for (PackFile pack : getPacks()) + for (Pack pack : getPacks()) cnt += pack.getObjectCount(); return cnt; } void copyAsIs(PackOutputStream out, WindowCursor wc) throws IOException { - for (PackFile pack : getPacks()) + for (Pack pack : getPacks()) pack.copyPackAsIs(out, wc); } @@ -58,7 +58,7 @@ class LocalCachedPack extends CachedPack { public boolean hasObject(ObjectToPack obj, StoredObjectRepresentation rep) { try { LocalObjectRepresentation local = (LocalObjectRepresentation) rep; - for (PackFile pack : getPacks()) { + for (Pack pack : getPacks()) { if (local.pack == pack) return true; } @@ -68,9 +68,9 @@ class LocalCachedPack extends CachedPack { } } - private PackFile[] getPacks() throws FileNotFoundException { + private Pack[] getPacks() throws FileNotFoundException { if (packs == null) { - PackFile[] p = new PackFile[packNames.length]; + Pack[] p = new Pack[packNames.length]; for (int i = 0; i < packNames.length; i++) p[i] = getPackFile(packNames[i]); packs = p; @@ -78,8 +78,8 @@ class LocalCachedPack extends CachedPack { return packs; } - private PackFile getPackFile(String packName) throws FileNotFoundException { - for (PackFile pack : odb.getPacks()) { + private Pack getPackFile(String packName) throws FileNotFoundException { + for (Pack pack : odb.getPacks()) { if (packName.equals(pack.getPackName())) return pack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java index 3950dde4a5..559718af3a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java @@ -16,40 +16,40 @@ import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation; import org.eclipse.jgit.lib.ObjectId; class LocalObjectRepresentation extends StoredObjectRepresentation { - static LocalObjectRepresentation newWhole(PackFile f, long p, long length) { + static LocalObjectRepresentation newWhole(Pack pack, long offset, long length) { LocalObjectRepresentation r = new LocalObjectRepresentation() { @Override public int getFormat() { return PACK_WHOLE; } }; - r.pack = f; - r.offset = p; + r.pack = pack; + r.offset = offset; r.length = length; return r; } - static LocalObjectRepresentation newDelta(PackFile f, long p, long n, + static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, ObjectId base) { LocalObjectRepresentation r = new Delta(); - r.pack = f; - r.offset = p; - r.length = n; + r.pack = pack; + r.offset = offset; + r.length = length; r.baseId = base; return r; } - static LocalObjectRepresentation newDelta(PackFile f, long p, long n, + static LocalObjectRepresentation newDelta(Pack pack, long offset, long length, long base) { LocalObjectRepresentation r = new Delta(); - r.pack = f; - r.offset = p; - r.length = n; + r.pack = pack; + r.offset = offset; + r.length = length; r.baseOffset = base; return r; } - PackFile pack; + Pack pack; long offset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java index 4a0ac1fd84..ac6cd212d5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectToPack.java @@ -17,7 +17,7 @@ import org.eclipse.jgit.lib.AnyObjectId; /** {@link ObjectToPack} for {@link ObjectDirectory}. */ class LocalObjectToPack extends ObjectToPack { /** Pack to reuse compressed data from, otherwise null. */ - PackFile pack; + Pack pack; /** Offset of the object's header in {@link #pack}. */ long offset; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java new file mode 100644 index 0000000000..e7cb285c34 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2009, Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.StandardCopyOption; +import java.util.Set; + +import org.eclipse.jgit.internal.storage.file.FileObjectDatabase.InsertLooseObjectResult; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Traditional file system based loose objects handler. + * <p> + * This is the loose object representation for a Git object database, where + * objects are stored loose by hashing them into directories by their + * {@link org.eclipse.jgit.lib.ObjectId}. + */ +class LooseObjects { + private static final Logger LOG = LoggerFactory + .getLogger(LooseObjects.class); + + private final File directory; + + private final UnpackedObjectCache unpackedObjectCache; + + /** + * Initialize a reference to an on-disk object directory. + * + * @param dir + * the location of the <code>objects</code> directory. + */ + LooseObjects(File dir) { + directory = dir; + unpackedObjectCache = new UnpackedObjectCache(); + } + + /** + * Getter for the field <code>directory</code>. + * + * @return the location of the <code>objects</code> directory. + */ + File getDirectory() { + return directory; + } + + void create() throws IOException { + FileUtils.mkdirs(directory); + } + + void close() { + unpackedObjectCache.clear(); + } + + /** {@inheritDoc} */ + @Override + public String toString() { + return "LooseObjects[" + directory + "]"; //$NON-NLS-1$ //$NON-NLS-2$ + } + + boolean hasCached(AnyObjectId id) { + return unpackedObjectCache.isUnpacked(id); + } + + /** + * Does the requested object exist as a loose object? + * + * @param objectId + * identity of the object to test for existence of. + * @return {@code true} if the specified object is stored as a loose object. + */ + boolean has(AnyObjectId objectId) { + return fileFor(objectId).exists(); + } + + /** + * Find objects matching the prefix abbreviation. + * + * @param matches + * set to add any located ObjectIds to. This is an output + * parameter. + * @param id + * prefix to search for. + * @param matchLimit + * maximum number of results to return. At most this many + * ObjectIds should be added to matches before returning. + * @return {@code true} if the matches were exhausted before reaching + * {@code maxLimit}. + */ + boolean resolve(Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) { + String fanOut = id.name().substring(0, 2); + String[] entries = new File(directory, fanOut).list(); + if (entries != null) { + for (String e : entries) { + if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2) { + continue; + } + try { + ObjectId entId = ObjectId.fromString(fanOut + e); + if (id.prefixCompare(entId) == 0) { + matches.add(entId); + } + } catch (IllegalArgumentException notId) { + continue; + } + if (matches.size() > matchLimit) { + return false; + } + } + } + return true; + } + + ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException { + File path = fileFor(id); + try (FileInputStream in = new FileInputStream(path)) { + unpackedObjectCache.add(id); + return UnpackedObject.open(in, path, id, curs); + } catch (FileNotFoundException noFile) { + if (path.exists()) { + throw noFile; + } + unpackedObjectCache.remove(id); + return null; + } + } + + long getSize(WindowCursor curs, AnyObjectId id) throws IOException { + File f = fileFor(id); + try (FileInputStream in = new FileInputStream(f)) { + unpackedObjectCache.add(id); + return UnpackedObject.getSize(in, id, curs); + } catch (FileNotFoundException noFile) { + if (f.exists()) { + throw noFile; + } + unpackedObjectCache.remove(id); + return -1; + } + } + + InsertLooseObjectResult insert(File tmp, ObjectId id) throws IOException { + final File dst = fileFor(id); + if (dst.exists()) { + // We want to be extra careful and avoid replacing an object + // that already exists. We can't be sure renameTo() would + // fail on all platforms if dst exists, so we check first. + // + FileUtils.delete(tmp, FileUtils.RETRY); + return InsertLooseObjectResult.EXISTS_LOOSE; + } + + try { + return tryMove(tmp, dst, id); + } catch (NoSuchFileException e) { + // It's possible the directory doesn't exist yet as the object + // directories are always lazily created. Note that we try the + // rename/move first as the directory likely does exist. + // + // Create the directory. + // + FileUtils.mkdir(dst.getParentFile(), true); + } catch (IOException e) { + // Any other IO error is considered a failure. + // + LOG.error(e.getMessage(), e); + FileUtils.delete(tmp, FileUtils.RETRY); + return InsertLooseObjectResult.FAILURE; + } + + try { + return tryMove(tmp, dst, id); + } catch (IOException e) { + // The object failed to be renamed into its proper location and + // it doesn't exist in the repository either. We really don't + // know what went wrong, so fail. + // + LOG.error(e.getMessage(), e); + FileUtils.delete(tmp, FileUtils.RETRY); + return InsertLooseObjectResult.FAILURE; + } + } + + private InsertLooseObjectResult tryMove(File tmp, File dst, ObjectId id) + throws IOException { + Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst), + StandardCopyOption.ATOMIC_MOVE); + dst.setReadOnly(); + unpackedObjectCache.add(id); + return InsertLooseObjectResult.INSERTED; + } + + /** + * Compute the location of a loose object file. + * + * @param objectId + * identity of the object to get the File location for. + * @return {@link java.io.File} location of the specified loose object. + */ + File fileFor(AnyObjectId objectId) { + String n = objectId.name(); + String d = n.substring(0, 2); + String f = n.substring(2); + return new File(new File(getDirectory(), d), f); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index d32182864a..e71a960603 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -16,28 +16,19 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.errors.PackInvalidException; -import org.eclipse.jgit.errors.PackMismatchException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -45,7 +36,6 @@ import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectDatabase; import org.eclipse.jgit.lib.ObjectId; @@ -54,8 +44,6 @@ import org.eclipse.jgit.lib.RepositoryCache; import org.eclipse.jgit.lib.RepositoryCache.FileKey; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Traditional file system based {@link org.eclipse.jgit.lib.ObjectDatabase}. @@ -63,7 +51,7 @@ import org.slf4j.LoggerFactory; * This is the classical object database representation for a Git repository, * where objects are stored loose by hashing them into directories by their * {@link org.eclipse.jgit.lib.ObjectId}, or are stored in compressed containers - * known as {@link org.eclipse.jgit.internal.storage.file.PackFile}s. + * known as {@link org.eclipse.jgit.internal.storage.file.Pack}s. * <p> * Optionally an object database can reference one or more alternates; other * ObjectDatabase instances that are searched in addition to the current @@ -76,12 +64,6 @@ import org.slf4j.LoggerFactory; * considered. */ public class ObjectDirectory extends FileObjectDatabase { - private static final Logger LOG = LoggerFactory - .getLogger(ObjectDirectory.class); - - private static final PackList NO_PACKS = new PackList( - FileSnapshot.DIRTY, new PackFile[0]); - /** Maximum number of candidates offered as resolutions of abbreviation. */ private static final int RESOLVE_ABBREV_LIMIT = 256; @@ -93,7 +75,9 @@ public class ObjectDirectory extends FileObjectDatabase { private final File infoDirectory; - private final File packDirectory; + private final LooseObjects loose; + + private final PackDirectory packed; private final File preservedDirectory; @@ -103,16 +87,12 @@ public class ObjectDirectory extends FileObjectDatabase { private final AtomicReference<AlternateHandle[]> alternates; - private final UnpackedObjectCache unpackedObjectCache; - private final File shallowFile; private FileSnapshot shallowFileSnapshot = FileSnapshot.DIRTY; private Set<ObjectId> shallowCommitsIds; - final AtomicReference<PackList> packList; - /** * Initialize a reference to an on-disk object directory. * @@ -136,11 +116,11 @@ public class ObjectDirectory extends FileObjectDatabase { config = cfg; objects = dir; infoDirectory = new File(objects, "info"); //$NON-NLS-1$ - packDirectory = new File(objects, "pack"); //$NON-NLS-1$ + File packDirectory = new File(objects, "pack"); //$NON-NLS-1$ preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(objects, Constants.INFO_ALTERNATES); - packList = new AtomicReference<>(NO_PACKS); - unpackedObjectCache = new UnpackedObjectCache(); + loose = new LooseObjects(objects); + packed = new PackDirectory(config, packDirectory); this.fs = fs; this.shallowFile = shallowFile; @@ -158,7 +138,7 @@ public class ObjectDirectory extends FileObjectDatabase { /** {@inheritDoc} */ @Override public final File getDirectory() { - return objects; + return loose.getDirectory(); } /** @@ -167,7 +147,7 @@ public class ObjectDirectory extends FileObjectDatabase { * @return the location of the <code>pack</code> directory. */ public final File getPackDirectory() { - return packDirectory; + return packed.getDirectory(); } /** @@ -188,9 +168,9 @@ public class ObjectDirectory extends FileObjectDatabase { /** {@inheritDoc} */ @Override public void create() throws IOException { - FileUtils.mkdirs(objects); + loose.create(); FileUtils.mkdir(infoDirectory); - FileUtils.mkdir(packDirectory); + packed.create(); } /** {@inheritDoc} */ @@ -212,13 +192,9 @@ public class ObjectDirectory extends FileObjectDatabase { /** {@inheritDoc} */ @Override public void close() { - unpackedObjectCache.clear(); + loose.close(); - final PackList packs = packList.get(); - if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { - for (PackFile p : packs.packs) - p.close(); - } + packed.close(); // Fully close all loaded alternates and clear the alternate list. AlternateHandle[] alt = alternates.get(); @@ -230,12 +206,8 @@ public class ObjectDirectory extends FileObjectDatabase { /** {@inheritDoc} */ @Override - public Collection<PackFile> getPacks() { - PackList list = packList.get(); - if (list == NO_PACKS) - list = scanPacks(list); - PackFile[] packs = list.packs; - return Collections.unmodifiableCollection(Arrays.asList(packs)); + public Collection<Pack> getPacks() { + return packed.getPacks(); } /** @@ -244,7 +216,7 @@ public class ObjectDirectory extends FileObjectDatabase { * Add a single existing pack to the list of available pack files. */ @Override - public PackFile openPack(File pack) + public Pack openPack(File pack) throws IOException { final String p = pack.getName(); if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ @@ -263,8 +235,8 @@ public class ObjectDirectory extends FileObjectDatabase { } } - PackFile res = new PackFile(pack, extensions); - insertPack(res); + Pack res = new Pack(pack, extensions); + packed.insert(res); return res; } @@ -277,7 +249,7 @@ public class ObjectDirectory extends FileObjectDatabase { /** {@inheritDoc} */ @Override public boolean has(AnyObjectId objectId) { - return unpackedObjectCache.isUnpacked(objectId) + return loose.hasCached(objectId) || hasPackedInSelfOrAlternate(objectId, null) || hasLooseInSelfOrAlternate(objectId, null); } @@ -300,7 +272,7 @@ public class ObjectDirectory extends FileObjectDatabase { private boolean hasLooseInSelfOrAlternate(AnyObjectId objectId, Set<AlternateHandle.Id> skips) { - if (fileFor(objectId).exists()) { + if (loose.has(objectId)) { return true; } skips = addMe(skips); @@ -315,25 +287,7 @@ public class ObjectDirectory extends FileObjectDatabase { } boolean hasPackedObject(AnyObjectId objectId) { - PackList pList; - do { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - if (p.hasObject(objectId)) - return true; - } catch (IOException e) { - // The hasObject call should have only touched the index, - // so any failure here indicates the index is unreadable - // by this process, and the pack is likewise not readable. - LOG.warn(MessageFormat.format( - JGitText.get().unableToReadPackfile, - p.getPackFile().getAbsolutePath()), e); - removePack(p); - } - } - } while (searchPacksAgain(pList)); - return false; + return packed.has(objectId); } @Override @@ -345,41 +299,11 @@ public class ObjectDirectory extends FileObjectDatabase { private void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, Set<AlternateHandle.Id> skips) throws IOException { - // Go through the packs once. If we didn't find any resolutions - // scan for new packs and check once more. - int oldSize = matches.size(); - PackList pList; - do { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - p.resolve(matches, id, RESOLVE_ABBREV_LIMIT); - p.resetTransientErrorCount(); - } catch (IOException e) { - handlePackError(e, p); - } - if (matches.size() > RESOLVE_ABBREV_LIMIT) - return; - } - } while (matches.size() == oldSize && searchPacksAgain(pList)); - - String fanOut = id.name().substring(0, 2); - String[] entries = new File(getDirectory(), fanOut).list(); - if (entries != null) { - for (String e : entries) { - if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2) - continue; - try { - ObjectId entId = ObjectId.fromString(fanOut + e); - if (id.prefixCompare(entId) == 0) - matches.add(entId); - } catch (IllegalArgumentException notId) { - continue; - } - if (matches.size() > RESOLVE_ABBREV_LIMIT) - return; - } - } + if (!packed.resolve(matches, id, RESOLVE_ABBREV_LIMIT)) + return; + + if (!loose.resolve(matches, id, RESOLVE_ABBREV_LIMIT)) + return; skips = addMe(skips); for (AlternateHandle alt : myAlternates()) { @@ -395,7 +319,7 @@ public class ObjectDirectory extends FileObjectDatabase { @Override ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId) throws IOException { - if (unpackedObjectCache.isUnpacked(objectId)) { + if (loose.hasCached(objectId)) { ObjectLoader ldr = openLooseObject(curs, objectId); if (ldr != null) { return ldr; @@ -446,51 +370,20 @@ public class ObjectDirectory extends FileObjectDatabase { } ObjectLoader openPackedObject(WindowCursor curs, AnyObjectId objectId) { - PackList pList; - do { - SEARCH: for (;;) { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - ObjectLoader ldr = p.get(curs, objectId); - p.resetTransientErrorCount(); - if (ldr != null) - return ldr; - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - if (searchPacksAgain(pList)) - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } - } while (searchPacksAgain(pList)); - return null; + return packed.open(curs, objectId); } @Override ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id) throws IOException { - File path = fileFor(id); - try (FileInputStream in = new FileInputStream(path)) { - unpackedObjectCache.add(id); - return UnpackedObject.open(in, path, id, curs); - } catch (FileNotFoundException noFile) { - if (path.exists()) { - throw noFile; - } - unpackedObjectCache.remove(id); - return null; - } + return loose.open(curs, id); } @Override long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { - if (unpackedObjectCache.isUnpacked(id)) { - long len = getLooseObjectSize(curs, id); + if (loose.hasCached(id)) { + long len = loose.getSize(curs, id); if (0 <= len) { return len; } @@ -504,7 +397,7 @@ public class ObjectDirectory extends FileObjectDatabase { private long getPackedSizeFromSelfOrAlternate(WindowCursor curs, AnyObjectId id, Set<AlternateHandle.Id> skips) { - long len = getPackedObjectSize(curs, id); + long len = packed.getSize(curs, id); if (0 <= len) { return len; } @@ -522,7 +415,7 @@ public class ObjectDirectory extends FileObjectDatabase { private long getLooseSizeFromSelfOrAlternate(WindowCursor curs, AnyObjectId id, Set<AlternateHandle.Id> skips) throws IOException { - long len = getLooseObjectSize(curs, id); + long len = loose.getSize(curs, id); if (0 <= len) { return len; } @@ -538,46 +431,6 @@ public class ObjectDirectory extends FileObjectDatabase { return -1; } - private long getPackedObjectSize(WindowCursor curs, AnyObjectId id) { - PackList pList; - do { - SEARCH: for (;;) { - pList = packList.get(); - for (PackFile p : pList.packs) { - try { - long len = p.getObjectSize(curs, id); - p.resetTransientErrorCount(); - if (0 <= len) - return len; - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - if (searchPacksAgain(pList)) - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } - } while (searchPacksAgain(pList)); - return -1; - } - - private long getLooseObjectSize(WindowCursor curs, AnyObjectId id) - throws IOException { - File f = fileFor(id); - try (FileInputStream in = new FileInputStream(f)) { - unpackedObjectCache.add(id); - return UnpackedObject.getSize(in, id, curs); - } catch (FileNotFoundException noFile) { - if (f.exists()) { - throw noFile; - } - unpackedObjectCache.remove(id); - return -1; - } - } - @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { @@ -586,25 +439,7 @@ public class ObjectDirectory extends FileObjectDatabase { private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs, Set<AlternateHandle.Id> skips) throws IOException { - PackList pList = packList.get(); - SEARCH: for (;;) { - for (PackFile p : pList.packs) { - try { - LocalObjectRepresentation rep = p.representation(curs, otp); - p.resetTransientErrorCount(); - if (rep != null) - packer.select(otp, rep); - } catch (PackMismatchException e) { - // Pack was modified; refresh the entire pack list. - // - pList = scanPacks(pList); - continue SEARCH; - } catch (IOException e) { - handlePackError(e, p); - } - } - break SEARCH; - } + packed.selectRepresentation(packer, otp, curs); skips = addMe(skips); for (AlternateHandle h : myAlternates()) { @@ -614,60 +449,12 @@ public class ObjectDirectory extends FileObjectDatabase { } } - private void handlePackError(IOException e, PackFile p) { - String warnTmpl = null; - int transientErrorCount = 0; - String errTmpl = JGitText.get().exceptionWhileReadingPack; - if ((e instanceof CorruptObjectException) - || (e instanceof PackInvalidException)) { - warnTmpl = JGitText.get().corruptPack; - LOG.warn(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath()), e); - // Assume the pack is corrupted, and remove it from the list. - removePack(p); - } else if (e instanceof FileNotFoundException) { - if (p.getPackFile().exists()) { - errTmpl = JGitText.get().packInaccessible; - transientErrorCount = p.incrementTransientErrorCount(); - } else { - warnTmpl = JGitText.get().packWasDeleted; - removePack(p); - } - } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { - warnTmpl = JGitText.get().packHandleIsStale; - removePack(p); - } else { - transientErrorCount = p.incrementTransientErrorCount(); - } - if (warnTmpl != null) { - LOG.warn(MessageFormat.format(warnTmpl, - p.getPackFile().getAbsolutePath()), e); - } else { - if (doLogExponentialBackoff(transientErrorCount)) { - // Don't remove the pack from the list, as the error may be - // transient. - LOG.error(MessageFormat.format(errTmpl, - p.getPackFile().getAbsolutePath(), - Integer.valueOf(transientErrorCount)), e); - } - } - } - - /** - * @param n - * count of consecutive failures - * @return @{code true} if i is a power of 2 - */ - private boolean doLogExponentialBackoff(int n) { - return (n & (n - 1)) == 0; - } - @Override InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException { // If the object is already in the repository, remove temporary file. // - if (unpackedObjectCache.isUnpacked(id)) { + if (loose.hasCached(id)) { FileUtils.delete(tmp, FileUtils.RETRY); return InsertLooseObjectResult.EXISTS_LOOSE; } @@ -675,71 +462,7 @@ public class ObjectDirectory extends FileObjectDatabase { FileUtils.delete(tmp, FileUtils.RETRY); return InsertLooseObjectResult.EXISTS_PACKED; } - - final File dst = fileFor(id); - if (dst.exists()) { - // We want to be extra careful and avoid replacing an object - // that already exists. We can't be sure renameTo() would - // fail on all platforms if dst exists, so we check first. - // - FileUtils.delete(tmp, FileUtils.RETRY); - return InsertLooseObjectResult.EXISTS_LOOSE; - } - - try { - return tryMove(tmp, dst, id); - } catch (NoSuchFileException e) { - // It's possible the directory doesn't exist yet as the object - // directories are always lazily created. Note that we try the - // rename/move first as the directory likely does exist. - // - // Create the directory. - // - FileUtils.mkdir(dst.getParentFile(), true); - } catch (IOException e) { - // Any other IO error is considered a failure. - // - LOG.error(e.getMessage(), e); - FileUtils.delete(tmp, FileUtils.RETRY); - return InsertLooseObjectResult.FAILURE; - } - - try { - return tryMove(tmp, dst, id); - } catch (IOException e) { - // The object failed to be renamed into its proper location and - // it doesn't exist in the repository either. We really don't - // know what went wrong, so fail. - // - LOG.error(e.getMessage(), e); - FileUtils.delete(tmp, FileUtils.RETRY); - return InsertLooseObjectResult.FAILURE; - } - } - - private InsertLooseObjectResult tryMove(File tmp, File dst, - ObjectId id) - throws IOException { - Files.move(FileUtils.toPath(tmp), FileUtils.toPath(dst), - StandardCopyOption.ATOMIC_MOVE); - dst.setReadOnly(); - unpackedObjectCache.add(id); - return InsertLooseObjectResult.INSERTED; - } - - boolean searchPacksAgain(PackList old) { - // Whether to trust the pack folder's modification time. If set - // to false we will always scan the .git/objects/pack folder to - // check for new pack files. If set to true (default) we use the - // lastmodified attribute of the folder and assume that no new - // pack files can be in this folder if his modification time has - // not changed. - boolean trustFolderStat = config.getBoolean( - ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); - - return ((!trustFolderStat) || old.snapshot.isModified(packDirectory)) - && old != scanPacks(old); + return loose.insert(tmp, id); } @Override @@ -780,182 +503,13 @@ public class ObjectDirectory extends FileObjectDatabase { return shallowCommitsIds; } - private void insertPack(PackFile pf) { - PackList o, n; - do { - o = packList.get(); - - // If the pack in question is already present in the list - // (picked up by a concurrent thread that did a scan?) we - // do not want to insert it a second time. - // - final PackFile[] oldList = o.packs; - final String name = pf.getPackFile().getName(); - for (PackFile p : oldList) { - if (name.equals(p.getPackFile().getName())) - return; - } - - final PackFile[] newList = new PackFile[1 + oldList.length]; - newList[0] = pf; - System.arraycopy(oldList, 0, newList, 1, oldList.length); - n = new PackList(o.snapshot, newList); - } while (!packList.compareAndSet(o, n)); - } - - private void removePack(PackFile deadPack) { - PackList o, n; - do { - o = packList.get(); - - final PackFile[] oldList = o.packs; - final int j = indexOf(oldList, deadPack); - if (j < 0) - break; - - final PackFile[] newList = new PackFile[oldList.length - 1]; - System.arraycopy(oldList, 0, newList, 0, j); - System.arraycopy(oldList, j + 1, newList, j, newList.length - j); - n = new PackList(o.snapshot, newList); - } while (!packList.compareAndSet(o, n)); - deadPack.close(); - } - - private static int indexOf(PackFile[] list, PackFile pack) { - for (int i = 0; i < list.length; i++) { - if (list[i] == pack) - return i; - } - return -1; - } - - private PackList scanPacks(PackList original) { - synchronized (packList) { - PackList o, n; - do { - o = packList.get(); - if (o != original) { - // Another thread did the scan for us, while we - // were blocked on the monitor above. - // - return o; - } - n = scanPacksImpl(o); - if (n == o) - return n; - } while (!packList.compareAndSet(o, n)); - return n; - } - } - - private PackList scanPacksImpl(PackList old) { - final Map<String, PackFile> forReuse = reuseMap(old); - final FileSnapshot snapshot = FileSnapshot.save(packDirectory); - final Set<String> names = listPackDirectory(); - final List<PackFile> list = new ArrayList<>(names.size() >> 2); - boolean foundNew = false; - for (String indexName : names) { - // Must match "pack-[0-9a-f]{40}.idx" to be an index. - // - if (indexName.length() != 49 || !indexName.endsWith(".idx")) //$NON-NLS-1$ - continue; - - final String base = indexName.substring(0, indexName.length() - 3); - int extensions = 0; - for (PackExt ext : PackExt.values()) { - if (names.contains(base + ext.getExtension())) - extensions |= ext.getBit(); - } - - if ((extensions & PACK.getBit()) == 0) { - // Sometimes C Git's HTTP fetch transport leaves a - // .idx file behind and does not download the .pack. - // We have to skip over such useless indexes. - // - continue; - } - - final String packName = base + PACK.getExtension(); - final File packFile = new File(packDirectory, packName); - final PackFile oldPack = forReuse.get(packName); - if (oldPack != null - && !oldPack.getFileSnapshot().isModified(packFile)) { - forReuse.remove(packName); - list.add(oldPack); - continue; - } - - list.add(new PackFile(packFile, extensions)); - foundNew = true; - } - - // If we did not discover any new files, the modification time was not - // changed, and we did not remove any files, then the set of files is - // the same as the set we were given. Instead of building a new object - // return the same collection. - // - if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) { - old.snapshot.setClean(snapshot); - return old; - } - - for (PackFile p : forReuse.values()) { - p.close(); - } - - if (list.isEmpty()) - return new PackList(snapshot, NO_PACKS.packs); - - final PackFile[] r = list.toArray(new PackFile[0]); - Arrays.sort(r, PackFile.SORT); - return new PackList(snapshot, r); - } - - private static Map<String, PackFile> reuseMap(PackList old) { - final Map<String, PackFile> forReuse = new HashMap<>(); - for (PackFile p : old.packs) { - if (p.invalid()) { - // The pack instance is corrupted, and cannot be safely used - // again. Do not include it in our reuse map. - // - p.close(); - continue; - } - - final PackFile prior = forReuse.put(p.getPackFile().getName(), p); - if (prior != null) { - // This should never occur. It should be impossible for us - // to have two pack files with the same name, as all of them - // came out of the same directory. If it does, we promised to - // close any PackFiles we did not reuse, so close the second, - // readers are likely to be actively using the first. - // - forReuse.put(prior.getPackFile().getName(), prior); - p.close(); - } - } - return forReuse; - } - - private Set<String> listPackDirectory() { - final String[] nameList = packDirectory.list(); - if (nameList == null) - return Collections.emptySet(); - final Set<String> nameSet = new HashSet<>(nameList.length << 1); - for (String name : nameList) { - if (name.startsWith("pack-")) //$NON-NLS-1$ - nameSet.add(name); - } - return nameSet; - } - void closeAllPackHandles(File packFile) { // if the packfile already exists (because we are rewriting a // packfile for the same set of objects maybe with different // PackConfig) then make sure we get rid of all handles on the file. // Windows will not allow for rename otherwise. if (packFile.exists()) { - for (PackFile p : getPacks()) { + for (Pack p : packed.getPacks()) { if (packFile.getPath().equals(p.getPackFile().getPath())) { p.close(); break; @@ -1025,29 +579,11 @@ public class ObjectDirectory extends FileObjectDatabase { } /** - * {@inheritDoc} - * <p> * Compute the location of a loose object file. */ @Override public File fileFor(AnyObjectId objectId) { - String n = objectId.name(); - String d = n.substring(0, 2); - String f = n.substring(2); - return new File(new File(getDirectory(), d), f); - } - - static final class PackList { - /** State just before reading the pack directory. */ - final FileSnapshot snapshot; - - /** All known packs, sorted by {@link PackFile#SORT}. */ - final PackFile[] packs; - - PackList(FileSnapshot monitor, PackFile[] packs) { - this.snapshot = monitor; - this.packs = packs; - } + return loose.fileFor(objectId); } static class AlternateHandle { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index e27518690b..04d2ff8ab2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -88,7 +88,7 @@ public class ObjectDirectoryPackParser extends PackParser { private Deflater def; /** The pack that was created, if parsing was successful. */ - private PackFile newPack; + private Pack newPack; private PackConfig pconfig; @@ -129,14 +129,14 @@ public class ObjectDirectoryPackParser extends PackParser { } /** - * Get the imported {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * Get the imported {@link org.eclipse.jgit.internal.storage.file.Pack}. * <p> * This method is supplied only to support testing; applications shouldn't * be using it directly to access the imported data. * * @return the imported PackFile, if parsing was successful. */ - public PackFile getPackFile() { + public Pack getPack() { return newPack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 254c020237..d928633a73 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -69,13 +69,13 @@ import org.slf4j.LoggerFactory; * delta packed format yielding high compression of lots of object where some * objects are similar. */ -public class PackFile implements Iterable<PackIndex.MutableEntry> { - private static final Logger LOG = LoggerFactory.getLogger(PackFile.class); +public class Pack implements Iterable<PackIndex.MutableEntry> { + private static final Logger LOG = LoggerFactory.getLogger(Pack.class); /** * Sorts PackFiles to be most recently created to least recently created. */ - public static final Comparator<PackFile> SORT = (a, b) -> b.packLastModified + public static final Comparator<Pack> SORT = (a, b) -> b.packLastModified .compareTo(a.packLastModified); private final File packFile; @@ -136,7 +136,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { * @param extensions * additional pack file extensions with the same base as the pack */ - public PackFile(File packFile, int extensions) { + public Pack(File packFile, int extensions) { this.packFile = packFile; this.fileSnapshot = PackFileSnapshot.save(packFile); this.packLastModified = fileSnapshot.lastModifiedInstant(); @@ -650,6 +650,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { private void doOpen() throws IOException { if (invalid) { + openFail(true, invalidatingCause); throw new PackInvalidException(packFile, invalidatingCause); } try { @@ -1200,7 +1201,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { @SuppressWarnings("nls") @Override public String toString() { - return "PackFile [packFileName=" + packFile.getName() + ", length=" + return "Pack [packFileName=" + packFile.getName() + ", length=" + packFile.length() + ", packChecksum=" + ObjectId.fromRaw(packChecksum).name() + "]"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java new file mode 100644 index 0000000000..b2ba36bf91 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -0,0 +1,516 @@ +/* + * Copyright (C) 2009, Google Inc. and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.errors.PackInvalidException; +import org.eclipse.jgit.errors.PackMismatchException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.lib.AbbreviatedObjectId; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.util.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Traditional file system packed objects directory handler. + * <p> + * This is the {@link org.eclipse.jgit.internal.storage.file.Pack}s object + * representation for a Git object database, where objects are stored in + * compressed containers known as + * {@link org.eclipse.jgit.internal.storage.file.Pack}s. + */ +class PackDirectory { + private final static Logger LOG = LoggerFactory + .getLogger(PackDirectory.class); + + private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY, + new Pack[0]); + + private final Config config; + + private final File directory; + + private final AtomicReference<PackList> packList; + + /** + * Initialize a reference to an on-disk 'pack' directory. + * + * @param config + * configuration this directory consults for write settings. + * @param directory + * the location of the {@code pack} directory. + */ + PackDirectory(Config config, File directory) { + this.config = config; + this.directory = directory; + packList = new AtomicReference<>(NO_PACKS); + } + + /** + * Getter for the field {@code directory}. + * + * @return the location of the {@code pack} directory. + */ + File getDirectory() { + return directory; + } + + void create() throws IOException { + FileUtils.mkdir(directory); + } + + void close() { + PackList packs = packList.get(); + if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { + for (Pack p : packs.packs) { + p.close(); + } + } + } + + Collection<Pack> getPacks() { + PackList list = packList.get(); + if (list == NO_PACKS) { + list = scanPacks(list); + } + Pack[] packs = list.packs; + return Collections.unmodifiableCollection(Arrays.asList(packs)); + } + + /** {@inheritDoc} */ + @Override + public String toString() { + return "PackDirectory[" + getDirectory() + "]"; //$NON-NLS-1$ //$NON-NLS-2$ + } + + /** + * Does the requested object exist in this PackDirectory? + * + * @param objectId + * identity of the object to test for existence of. + * @return true if the specified object is stored in this PackDirectory. + */ + boolean has(AnyObjectId objectId) { + PackList pList; + do { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + if (p.hasObject(objectId)) { + return true; + } + } catch (IOException e) { + // The hasObject call should have only touched the index, + // so any failure here indicates the index is unreadable + // by this process, and the pack is likewise not readable. + LOG.warn(MessageFormat.format( + JGitText.get().unableToReadPackfile, + p.getPackFile().getAbsolutePath()), e); + remove(p); + } + } + } while (searchPacksAgain(pList)); + return false; + } + + /** + * Find objects matching the prefix abbreviation. + * + * @param matches + * set to add any located ObjectIds to. This is an output + * parameter. + * @param id + * prefix to search for. + * @param matchLimit + * maximum number of results to return. At most this many + * ObjectIds should be added to matches before returning. + * @return {@code true} if the matches were exhausted before reaching + * {@code maxLimit}. + */ + boolean resolve(Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) { + // Go through the packs once. If we didn't find any resolutions + // scan for new packs and check once more. + int oldSize = matches.size(); + PackList pList; + do { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + p.resolve(matches, id, matchLimit); + p.resetTransientErrorCount(); + } catch (IOException e) { + handlePackError(e, p); + } + if (matches.size() > matchLimit) { + return false; + } + } + } while (matches.size() == oldSize && searchPacksAgain(pList)); + return true; + } + + ObjectLoader open(WindowCursor curs, AnyObjectId objectId) { + PackList pList; + do { + SEARCH: for (;;) { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + ObjectLoader ldr = p.get(curs, objectId); + p.resetTransientErrorCount(); + if (ldr != null) + return ldr; + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + if (searchPacksAgain(pList)) { + continue SEARCH; + } + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } while (searchPacksAgain(pList)); + return null; + } + + long getSize(WindowCursor curs, AnyObjectId id) { + PackList pList; + do { + SEARCH: for (;;) { + pList = packList.get(); + for (Pack p : pList.packs) { + try { + long len = p.getObjectSize(curs, id); + p.resetTransientErrorCount(); + if (0 <= len) { + return len; + } + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + if (searchPacksAgain(pList)) { + continue SEARCH; + } + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } while (searchPacksAgain(pList)); + return -1; + } + + void selectRepresentation(PackWriter packer, ObjectToPack otp, + WindowCursor curs) { + PackList pList = packList.get(); + SEARCH: for (;;) { + for (Pack p : pList.packs) { + try { + LocalObjectRepresentation rep = p.representation(curs, otp); + p.resetTransientErrorCount(); + if (rep != null) { + packer.select(otp, rep); + } + } catch (PackMismatchException e) { + // Pack was modified; refresh the entire pack list. + // + pList = scanPacks(pList); + continue SEARCH; + } catch (IOException e) { + handlePackError(e, p); + } + } + break SEARCH; + } + } + + private void handlePackError(IOException e, Pack p) { + String warnTmpl = null; + int transientErrorCount = 0; + String errTmpl = JGitText.get().exceptionWhileReadingPack; + if ((e instanceof CorruptObjectException) + || (e instanceof PackInvalidException)) { + warnTmpl = JGitText.get().corruptPack; + LOG.warn(MessageFormat.format(warnTmpl, + p.getPackFile().getAbsolutePath()), e); + // Assume the pack is corrupted, and remove it from the list. + remove(p); + } else if (e instanceof FileNotFoundException) { + if (p.getPackFile().exists()) { + errTmpl = JGitText.get().packInaccessible; + transientErrorCount = p.incrementTransientErrorCount(); + } else { + warnTmpl = JGitText.get().packWasDeleted; + remove(p); + } + } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { + warnTmpl = JGitText.get().packHandleIsStale; + remove(p); + } else { + transientErrorCount = p.incrementTransientErrorCount(); + } + if (warnTmpl != null) { + LOG.warn(MessageFormat.format(warnTmpl, + p.getPackFile().getAbsolutePath()), e); + } else { + if (doLogExponentialBackoff(transientErrorCount)) { + // Don't remove the pack from the list, as the error may be + // transient. + LOG.error(MessageFormat.format(errTmpl, + p.getPackFile().getAbsolutePath(), + Integer.valueOf(transientErrorCount)), e); + } + } + } + + /** + * @param n + * count of consecutive failures + * @return @{code true} if i is a power of 2 + */ + private boolean doLogExponentialBackoff(int n) { + return (n & (n - 1)) == 0; + } + + boolean searchPacksAgain(PackList old) { + // Whether to trust the pack folder's modification time. If set + // to false we will always scan the .git/objects/pack folder to + // check for new pack files. If set to true (default) we use the + // lastmodified attribute of the folder and assume that no new + // pack files can be in this folder if his modification time has + // not changed. + boolean trustFolderStat = config.getBoolean( + ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + + return ((!trustFolderStat) || old.snapshot.isModified(directory)) + && old != scanPacks(old); + } + + void insert(Pack pack) { + PackList o, n; + do { + o = packList.get(); + + // If the pack in question is already present in the list + // (picked up by a concurrent thread that did a scan?) we + // do not want to insert it a second time. + // + final Pack[] oldList = o.packs; + final String name = pack.getPackFile().getName(); + for (Pack p : oldList) { + if (name.equals(p.getPackFile().getName())) { + return; + } + } + + final Pack[] newList = new Pack[1 + oldList.length]; + newList[0] = pack; + System.arraycopy(oldList, 0, newList, 1, oldList.length); + n = new PackList(o.snapshot, newList); + } while (!packList.compareAndSet(o, n)); + } + + private void remove(Pack deadPack) { + PackList o, n; + do { + o = packList.get(); + + final Pack[] oldList = o.packs; + final int j = indexOf(oldList, deadPack); + if (j < 0) { + break; + } + + final Pack[] newList = new Pack[oldList.length - 1]; + System.arraycopy(oldList, 0, newList, 0, j); + System.arraycopy(oldList, j + 1, newList, j, newList.length - j); + n = new PackList(o.snapshot, newList); + } while (!packList.compareAndSet(o, n)); + deadPack.close(); + } + + private static int indexOf(Pack[] list, Pack pack) { + for (int i = 0; i < list.length; i++) { + if (list[i] == pack) { + return i; + } + } + return -1; + } + + private PackList scanPacks(PackList original) { + synchronized (packList) { + PackList o, n; + do { + o = packList.get(); + if (o != original) { + // Another thread did the scan for us, while we + // were blocked on the monitor above. + // + return o; + } + n = scanPacksImpl(o); + if (n == o) { + return n; + } + } while (!packList.compareAndSet(o, n)); + return n; + } + } + + private PackList scanPacksImpl(PackList old) { + final Map<String, Pack> forReuse = reuseMap(old); + final FileSnapshot snapshot = FileSnapshot.save(directory); + final Set<String> names = listPackDirectory(); + final List<Pack> list = new ArrayList<>(names.size() >> 2); + boolean foundNew = false; + for (String indexName : names) { + // Must match "pack-[0-9a-f]{40}.idx" to be an index. + // + if (indexName.length() != 49 || !indexName.endsWith(".idx")) { //$NON-NLS-1$ + continue; + } + + final String base = indexName.substring(0, indexName.length() - 3); + int extensions = 0; + for (PackExt ext : PackExt.values()) { + if (names.contains(base + ext.getExtension())) { + extensions |= ext.getBit(); + } + } + + if ((extensions & PACK.getBit()) == 0) { + // Sometimes C Git's HTTP fetch transport leaves a + // .idx file behind and does not download the .pack. + // We have to skip over such useless indexes. + // + continue; + } + + final String packName = base + PACK.getExtension(); + final File packFile = new File(directory, packName); + final Pack oldPack = forReuse.get(packName); + if (oldPack != null + && !oldPack.getFileSnapshot().isModified(packFile)) { + forReuse.remove(packName); + list.add(oldPack); + continue; + } + + list.add(new Pack(packFile, extensions)); + foundNew = true; + } + + // If we did not discover any new files, the modification time was not + // changed, and we did not remove any files, then the set of files is + // the same as the set we were given. Instead of building a new object + // return the same collection. + // + if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) { + old.snapshot.setClean(snapshot); + return old; + } + + for (Pack p : forReuse.values()) { + p.close(); + } + + if (list.isEmpty()) { + return new PackList(snapshot, NO_PACKS.packs); + } + + final Pack[] r = list.toArray(new Pack[0]); + Arrays.sort(r, Pack.SORT); + return new PackList(snapshot, r); + } + + private static Map<String, Pack> reuseMap(PackList old) { + final Map<String, Pack> forReuse = new HashMap<>(); + for (Pack p : old.packs) { + if (p.invalid()) { + // The pack instance is corrupted, and cannot be safely used + // again. Do not include it in our reuse map. + // + p.close(); + continue; + } + + final Pack prior = forReuse.put(p.getPackFile().getName(), p); + if (prior != null) { + // This should never occur. It should be impossible for us + // to have two pack files with the same name, as all of them + // came out of the same directory. If it does, we promised to + // close any PackFiles we did not reuse, so close the second, + // readers are likely to be actively using the first. + // + forReuse.put(prior.getPackFile().getName(), prior); + p.close(); + } + } + return forReuse; + } + + private Set<String> listPackDirectory() { + final String[] nameList = directory.list(); + if (nameList == null) { + return Collections.emptySet(); + } + final Set<String> nameSet = new HashSet<>(nameList.length << 1); + for (String name : nameList) { + if (name.startsWith("pack-")) { //$NON-NLS-1$ + nameSet.add(name); + } + } + return nameSet; + } + + static final class PackList { + /** State just before reading the pack directory. */ + final FileSnapshot snapshot; + + /** All known packs, sorted by {@link Pack#SORT}. */ + final Pack[] packs; + + PackList(FileSnapshot monitor, Pack[] packs) { + this.snapshot = monitor; + this.packs = packs; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 31686befc9..942cc96745 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -34,7 +34,7 @@ import org.eclipse.jgit.util.io.SilentFileInputStream; /** * Access path to locate objects by {@link org.eclipse.jgit.lib.ObjectId} in a - * {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * {@link org.eclipse.jgit.internal.storage.file.Pack}. * <p> * Indexes are strictly redundant information in that we can rebuild all of the * data held in the index file from the on disk representation of the pack file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java index 612b12366c..87e0b44d46 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java @@ -25,7 +25,7 @@ import org.eclipse.jgit.util.NB; /** * Creates a table of contents to support random access by - * {@link org.eclipse.jgit.internal.storage.file.PackFile}. + * {@link org.eclipse.jgit.internal.storage.file.Pack}. * <p> * Pack index files (the <code>.idx</code> suffix in a pack file pair) provides * random access to any object in the pack by associating an ObjectId to the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java index a9e0588885..0bceca72ea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInputStream.java @@ -16,11 +16,11 @@ import java.io.InputStream; class PackInputStream extends InputStream { private final WindowCursor wc; - private final PackFile pack; + private final Pack pack; private long pos; - PackInputStream(PackFile pack, long pos, WindowCursor wc) + PackInputStream(Pack pack, long pos, WindowCursor wc) throws IOException { this.pack = pack; this.pos = pos; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java index 2c2f7911aa..482b143e33 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackLock.java @@ -18,7 +18,7 @@ import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; /** - * Keeps track of a {@link org.eclipse.jgit.internal.storage.file.PackFile}'s + * Keeps track of a {@link org.eclipse.jgit.internal.storage.file.Pack}'s * associated <code>.keep</code> file. */ public class PackLock { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java index 4d80a0312a..ee458e27ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java @@ -25,7 +25,7 @@ import org.eclipse.jgit.lib.ObjectId; * </p> * * @see PackIndex - * @see PackFile + * @see Pack */ public class PackReverseIndex { /** Index we were created from, and that has our ObjectId data. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java index 9dbdbc73f0..2c0ade681b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectoryRename.java @@ -51,9 +51,6 @@ class RefDirectoryRename extends RefRename { */ private ObjectId objId; - /** True if HEAD must be moved to the destination reference. */ - private boolean updateHEAD; - /** A reference we backup {@link #objId} into during the rename. */ private RefDirectoryUpdate tmp; @@ -69,7 +66,7 @@ class RefDirectoryRename extends RefRename { return Result.IO_FAILURE; // not supported objId = source.getOldObjectId(); - updateHEAD = needToUpdateHEAD(); + boolean updateHEAD = needToUpdateHEAD(); tmp = refdb.newTemporaryUpdate(); try (RevWalk rw = new RevWalk(refdb.getRepository())) { // First backup the source so its never unreachable. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 3e8cb3a3f2..25653b3ce3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -33,7 +33,7 @@ import org.eclipse.jgit.storage.file.WindowCacheStats; import org.eclipse.jgit.util.Monitoring; /** - * Caches slices of a {@link org.eclipse.jgit.internal.storage.file.PackFile} in + * Caches slices of a {@link org.eclipse.jgit.internal.storage.file.Pack} in * memory for faster read access. * <p> * The WindowCache serves as a Java based "buffer cache", loading segments of a @@ -41,7 +41,7 @@ import org.eclipse.jgit.util.Monitoring; * only tiny slices of a file, the WindowCache tries to smooth out these tiny * reads into larger block-sized IO operations. * <p> - * Whenever a cache miss occurs, {@link #load(PackFile, long)} is invoked by + * Whenever a cache miss occurs, {@link #load(Pack, long)} is invoked by * exactly one thread for the given <code>(PackFile,position)</code> key tuple. * This is ensured by an array of locks, with the tuple hashed to a lock * instance. @@ -80,10 +80,10 @@ import org.eclipse.jgit.util.Monitoring; * <p> * This cache has an implementation rule such that: * <ul> - * <li>{@link #load(PackFile, long)} is invoked by at most one thread at a time + * <li>{@link #load(Pack, long)} is invoked by at most one thread at a time * for a given <code>(PackFile,position)</code> tuple.</li> * <li>For every <code>load()</code> invocation there is exactly one - * {@link #createRef(PackFile, long, ByteWindow)} invocation to wrap a + * {@link #createRef(Pack, long, ByteWindow)} invocation to wrap a * SoftReference or a StrongReference around the cached entity.</li> * <li>For every Reference created by <code>createRef()</code> there will be * exactly one call to {@link #clear(PageRef)} to cleanup any resources associated @@ -91,10 +91,10 @@ import org.eclipse.jgit.util.Monitoring; * </ul> * <p> * Therefore, it is safe to perform resource accounting increments during the - * {@link #load(PackFile, long)} or - * {@link #createRef(PackFile, long, ByteWindow)} methods, and matching + * {@link #load(Pack, long)} or + * {@link #createRef(Pack, long, ByteWindow)} methods, and matching * decrements during {@link #clear(PageRef)}. Implementors may need to override - * {@link #createRef(PackFile, long, ByteWindow)} in order to embed additional + * {@link #createRef(Pack, long, ByteWindow)} in order to embed additional * accounting information into an implementation specific * {@link org.eclipse.jgit.internal.storage.file.WindowCache.PageRef} subclass, as * the cached entity may have already been evicted by the JRE's garbage @@ -170,7 +170,7 @@ public class WindowCache { * @param delta * delta of cached bytes */ - void recordOpenBytes(PackFile pack, int delta); + void recordOpenBytes(Pack pack, int delta); /** * Returns a snapshot of this recorder's stats. Note that this may be an @@ -242,7 +242,7 @@ public class WindowCache { } @Override - public void recordOpenBytes(PackFile pack, int delta) { + public void recordOpenBytes(Pack pack, int delta) { openByteCount.add(delta); String repositoryId = repositoryId(pack); LongAdder la = openByteCountPerRepository @@ -254,9 +254,8 @@ public class WindowCache { } } - private static String repositoryId(PackFile pack) { - // use repository's gitdir since packfile doesn't know its - // repository + private static String repositoryId(Pack pack) { + // use repository's gitdir since Pack doesn't know its repository return pack.getPackFile().getParentFile().getParentFile() .getParent(); } @@ -380,7 +379,7 @@ public class WindowCache { return cache.publishMBeanIfNeeded(); } - static final ByteWindow get(PackFile pack, long offset) + static final ByteWindow get(Pack pack, long offset) throws IOException { final WindowCache c = cache; final ByteWindow r = c.getOrLoad(pack, c.toStart(offset)); @@ -395,7 +394,7 @@ public class WindowCache { return r; } - static final void purge(PackFile pack) { + static final void purge(Pack pack) { cache.removeAll(pack); } @@ -506,7 +505,7 @@ public class WindowCache { return packHash + (int) (off >>> windowSizeShift); } - private ByteWindow load(PackFile pack, long offset) throws IOException { + private ByteWindow load(Pack pack, long offset) throws IOException { long startTime = System.nanoTime(); if (pack.beginWindowCache()) statsRecorder.recordOpenFiles(1); @@ -525,7 +524,7 @@ public class WindowCache { } } - private PageRef<ByteWindow> createRef(PackFile p, long o, ByteWindow v) { + private PageRef<ByteWindow> createRef(Pack p, long o, ByteWindow v) { final PageRef<ByteWindow> ref = useStrongRefs ? new StrongRef(p, o, v, queue) : new SoftRef(p, o, v, (SoftCleanupQueue) queue); @@ -539,7 +538,7 @@ public class WindowCache { close(ref.getPack()); } - private void close(PackFile pack) { + private void close(Pack pack) { if (pack.endWindowCache()) { statsRecorder.recordOpenFiles(-1); } @@ -578,9 +577,9 @@ public class WindowCache { * @return the object reference. * @throws IOException * the object reference was not in the cache and could not be - * obtained by {@link #load(PackFile, long)}. + * obtained by {@link #load(Pack, long)}. */ - private ByteWindow getOrLoad(PackFile pack, long position) + private ByteWindow getOrLoad(Pack pack, long position) throws IOException { final int slot = slot(pack, position); final Entry e1 = table.get(slot); @@ -623,7 +622,7 @@ public class WindowCache { return v; } - private ByteWindow scan(Entry n, PackFile pack, long position) { + private ByteWindow scan(Entry n, Pack pack, long position) { for (; n != null; n = n.next) { final PageRef<ByteWindow> r = n.ref; if (r.getPack() == pack && r.getPosition() == position) { @@ -704,7 +703,7 @@ public class WindowCache { /** * Clear all entries related to a single file. * <p> - * Typically this method is invoked during {@link PackFile#close()}, when we + * Typically this method is invoked during {@link Pack#close()}, when we * know the pack is never going to be useful to us again (for example, it no * longer exists on disk). A concurrent reader loading an entry from this * same pack may cause the pack to become stuck in the cache anyway. @@ -712,7 +711,7 @@ public class WindowCache { * @param pack * the file to purge all entries of. */ - private void removeAll(PackFile pack) { + private void removeAll(Pack pack) { for (int s = 0; s < tableSize; s++) { final Entry e1 = table.get(s); boolean hasDead = false; @@ -733,11 +732,11 @@ public class WindowCache { queue.gc(); } - private int slot(PackFile pack, long position) { + private int slot(Pack pack, long position) { return (hash(pack.hash, position) >>> 1) % tableSize; } - private Lock lock(PackFile pack, long position) { + private Lock lock(Pack pack, long position) { return locks[(hash(pack.hash, position) >>> 1) % locks.length]; } @@ -799,16 +798,20 @@ public class WindowCache { boolean kill(); /** - * Get the packfile the referenced cache page is allocated for + * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} the + * referenced cache page is allocated for * - * @return the packfile the referenced cache page is allocated for + * @return the {@link org.eclipse.jgit.internal.storage.file.Pack} the + * referenced cache page is allocated for */ - PackFile getPack(); + Pack getPack(); /** - * Get the position of the referenced cache page in the packfile + * Get the position of the referenced cache page in the + * {@link org.eclipse.jgit.internal.storage.file.Pack} * - * @return the position of the referenced cache page in the packfile + * @return the position of the referenced cache page in the + * {@link org.eclipse.jgit.internal.storage.file.Pack} */ long getPosition(); @@ -844,7 +847,7 @@ public class WindowCache { /** A soft reference wrapped around a cached object. */ private static class SoftRef extends SoftReference<ByteWindow> implements PageRef<ByteWindow> { - private final PackFile pack; + private final Pack pack; private final long position; @@ -852,7 +855,7 @@ public class WindowCache { private long lastAccess; - protected SoftRef(final PackFile pack, final long position, + protected SoftRef(final Pack pack, final long position, final ByteWindow v, final SoftCleanupQueue queue) { super(v, queue); this.pack = pack; @@ -861,7 +864,7 @@ public class WindowCache { } @Override - public PackFile getPack() { + public Pack getPack() { return pack; } @@ -900,7 +903,7 @@ public class WindowCache { private static class StrongRef implements PageRef<ByteWindow> { private ByteWindow referent; - private final PackFile pack; + private final Pack pack; private final long position; @@ -910,7 +913,7 @@ public class WindowCache { private CleanupQueue queue; - protected StrongRef(final PackFile pack, final long position, + protected StrongRef(final Pack pack, final long position, final ByteWindow v, final CleanupQueue queue) { this.pack = pack; this.position = position; @@ -920,7 +923,7 @@ public class WindowCache { } @Override - public PackFile getPack() { + public Pack getPack() { return pack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 6c975708a3..e7fd7b9e76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -86,7 +86,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { /** {@inheritDoc} */ @Override public BitmapIndex getBitmapIndex() throws IOException { - for (PackFile pack : db.getPacks()) { + for (Pack pack : db.getPacks()) { PackBitmapIndex index = pack.getBitmapIndex(); if (index != null) return new BitmapIndexImpl(index); @@ -98,7 +98,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { @Override public Collection<CachedPack> getCachedPacksAndUpdate( BitmapBuilder needBitmap) throws IOException { - for (PackFile pack : db.getPacks()) { + for (Pack pack : db.getPacks()) { PackBitmapIndex index = pack.getBitmapIndex(); if (needBitmap.removeAllOrNone(index)) return Collections.<CachedPack> singletonList( @@ -218,7 +218,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { * this cursor does not match the provider or id and the proper * window could not be acquired through the provider's cache. */ - int copy(final PackFile pack, long position, final byte[] dstbuf, + int copy(final Pack pack, long position, final byte[] dstbuf, int dstoff, final int cnt) throws IOException { final long length = pack.length; int need = cnt; @@ -239,7 +239,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { ((LocalCachedPack) pack).copyAsIs(out, this); } - void copyPackAsIs(final PackFile pack, final long length, + void copyPackAsIs(final Pack pack, final long length, final PackOutputStream out) throws IOException { long position = 12; long remaining = length - (12 + 20); @@ -275,7 +275,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { * the inflater encountered an invalid chunk of data. Data * stream corruption is likely. */ - int inflate(final PackFile pack, long position, final byte[] dstbuf, + int inflate(final Pack pack, long position, final byte[] dstbuf, boolean headerOnly) throws IOException, DataFormatException { prepareInflater(); pin(pack, position); @@ -293,7 +293,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { } } - ByteArrayWindow quickCopy(PackFile p, long pos, long cnt) + ByteArrayWindow quickCopy(Pack p, long pos, long cnt) throws IOException { pin(p, pos); if (window instanceof ByteArrayWindow @@ -314,7 +314,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { inf.reset(); } - void pin(PackFile pack, long position) + void pin(Pack pack, long position) throws IOException { final ByteWindow w = window; if (w == null || !w.contains(pack, position)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java index a78f4d24da..e210acf058 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.internal.storage.reftable; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.PriorityQueue; @@ -215,6 +216,23 @@ public class MergedReftable extends Reftable { } } + @Override + public void seekPastPrefix(String prefixName) throws IOException { + List<RefQueueEntry> entriesToAdd = new ArrayList<>(); + entriesToAdd.addAll(queue); + if (head != null) { + entriesToAdd.add(head); + } + + head = null; + queue.clear(); + + for(RefQueueEntry entry : entriesToAdd){ + entry.rc.seekPastPrefix(prefixName); + add(entry); + } + } + private RefQueueEntry poll() { RefQueueEntry e = head; if (e != null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java index d96648eb50..5e2c350883 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/RefCursor.java @@ -29,6 +29,19 @@ public abstract class RefCursor implements AutoCloseable { public abstract boolean next() throws IOException; /** + * Seeks forward to the first ref record lexicographically beyond + * {@code prefixName} that doesn't start with {@code prefixName}. If there are + * no more results, skipping some refs won't add new results. E.g if we create a + * RefCursor that returns only results with a specific prefix, skipping that + * prefix won't give results that are not part of the original prefix. + * + * @param prefixName prefix that should be skipped. All previous refs before it + * will be skipped. + * @throws java.io.IOException references cannot be read. + */ + public abstract void seekPastPrefix(String prefixName) throws IOException; + + /** * Get reference at the current position. * * @return reference at the current position. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java index 4747be3544..0c16828617 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableDatabase.java @@ -14,10 +14,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.lib.ObjectId; @@ -266,6 +268,54 @@ public abstract class ReftableDatabase { } /** + * Returns refs whose names start with a given prefix excluding all refs that + * start with one of the given prefixes. + * + * @param include string that names of refs should start with; may be empty. + * @param excludes strings that names of refs can't start with; may be empty. + * @return immutable list of refs whose names start with {@code include} and + * none of the strings in {@code exclude}. + * @throws java.io.IOException the reference space cannot be accessed. + */ + public List<Ref> getRefsByPrefixWithExclusions(String include, Set<String> excludes) throws IOException { + if (excludes.isEmpty()) { + return getRefsByPrefix(include); + } + List<Ref> results = new ArrayList<>(); + lock.lock(); + try { + Reftable table = reader(); + Iterator<String> excludeIterator = + excludes.stream().sorted().collect(Collectors.toList()).iterator(); + String currentExclusion = excludeIterator.hasNext() ? excludeIterator.next() : null; + try (RefCursor rc = RefDatabase.ALL.equals(include) ? table.allRefs() : table.seekRefsWithPrefix(include)) { + while (rc.next()) { + Ref ref = table.resolve(rc.getRef()); + if (ref == null || ref.getObjectId() == null) { + continue; + } + // Skip prefixes that will never see since we are already further than those + // prefixes lexicographically. + while (excludeIterator.hasNext() && !ref.getName().startsWith(currentExclusion) + && ref.getName().compareTo(currentExclusion) > 0) { + currentExclusion = excludeIterator.next(); + } + + if (currentExclusion != null && ref.getName().startsWith(currentExclusion)) { + rc.seekPastPrefix(currentExclusion); + continue; + } + results.add(ref); + } + } + } finally { + lock.unlock(); + } + + return Collections.unmodifiableList(results); + } + + /** * @return whether there is a fast SHA1 to ref map. * @throws IOException in case of I/O problems. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java index 095276f57b..9e2ae91608 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java @@ -509,6 +509,21 @@ public class ReftableReader extends Reftable implements AutoCloseable { } @Override + public void seekPastPrefix(String prefixName) throws IOException { + initRefIndex(); + byte[] key = prefixName.getBytes(UTF_8); + ByteBuffer byteBuffer = ByteBuffer.allocate(key.length + 1); + byteBuffer.put(key); + // Add the representation of the last byte lexicographically. Based on how UTF_8 + // representation works, this byte will be bigger lexicographically than any + // UTF_8 character when translated into bytes, since 0xFF can never be a part of + // a UTF_8 string. + byteBuffer.put((byte) 0xFF); + + block = seek(REF_BLOCK_TYPE, byteBuffer.array(), refIndex, 0, refEnd); + } + + @Override public Ref getRef() { return ref; } @@ -682,6 +697,17 @@ public class ReftableReader extends Reftable implements AutoCloseable { } @Override + /** + * The implementation here would not be efficient complexity-wise since it + * expected that there are a small number of refs that match the same object id. + * In such case it's better to not even use this method (as the caller might + * expect it to be efficient). + */ + public void seekPastPrefix(String prefixName) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public Ref getRef() { return ref; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java deleted file mode 100644 index 5138636089..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import java.io.IOException; - -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; - -/** Update that always rejects with {@code LOCK_FAILURE}. */ -class AlwaysFailUpdate extends RefUpdate { - private final RefTreeDatabase refdb; - - AlwaysFailUpdate(RefTreeDatabase refdb, String name) { - super(new ObjectIdRef.Unpeeled(Ref.Storage.NEW, name, null)); - this.refdb = refdb; - setCheckConflicting(false); - } - - /** {@inheritDoc} */ - @Override - protected RefDatabase getRefDatabase() { - return refdb; - } - - /** {@inheritDoc} */ - @Override - protected Repository getRepository() { - return refdb.getRepository(); - } - - /** {@inheritDoc} */ - @Override - protected boolean tryLock(boolean deref) throws IOException { - return false; - } - - /** {@inheritDoc} */ - @Override - protected void unlock() { - // No locks are held here. - } - - /** {@inheritDoc} */ - @Override - protected Result doUpdate(Result desiredResult) { - return Result.LOCK_FAILURE; - } - - /** {@inheritDoc} */ - @Override - protected Result doDelete(Result desiredResult) { - return Result.LOCK_FAILURE; - } - - /** {@inheritDoc} */ - @Override - protected Result doLink(String target) { - return Result.LOCK_FAILURE; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java deleted file mode 100644 index bb06a9e0cf..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.ReceiveCommand.Result; - -/** - * Command to create, update or delete an entry inside a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - * <p> - * Unlike {@link org.eclipse.jgit.transport.ReceiveCommand} (which can only - * update a reference to an {@link org.eclipse.jgit.lib.ObjectId}), a RefTree - * Command can also create, modify or delete symbolic references to a target - * reference. - * <p> - * RefTree Commands may wrap a {@code ReceiveCommand} to allow callers to - * process an existing ReceiveCommand against a RefTree. - * <p> - * Commands should be passed into - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree#apply(java.util.Collection)} - * for processing. - */ -public class Command { - /** - * Set unprocessed commands as failed due to transaction aborted. - * <p> - * If a command is still - * {@link org.eclipse.jgit.transport.ReceiveCommand.Result#NOT_ATTEMPTED} it - * will be set to - * {@link org.eclipse.jgit.transport.ReceiveCommand.Result#REJECTED_OTHER_REASON}. - * If {@code why} is non-null its contents will be used as the message for - * the first command status. - * - * @param commands - * commands to mark as failed. - * @param why - * optional message to set on the first aborted command. - */ - public static void abort(Iterable<Command> commands, @Nullable String why) { - if (why == null || why.isEmpty()) { - why = JGitText.get().transactionAborted; - } - for (Command c : commands) { - if (c.getResult() == NOT_ATTEMPTED) { - c.setResult(REJECTED_OTHER_REASON, why); - why = JGitText.get().transactionAborted; - } - } - } - - private final Ref oldRef; - private final Ref newRef; - private final ReceiveCommand cmd; - private Result result; - - /** - * Create a command to create, update or delete a reference. - * <p> - * At least one of {@code oldRef} or {@code newRef} must be supplied. - * - * @param oldRef - * expected value. Null if the ref should not exist. - * @param newRef - * desired value, must be peeled if not null and not symbolic. - * Null to delete the ref. - */ - public Command(@Nullable Ref oldRef, @Nullable Ref newRef) { - this.oldRef = oldRef; - this.newRef = newRef; - this.cmd = null; - this.result = NOT_ATTEMPTED; - - if (oldRef == null && newRef == null) { - throw new IllegalArgumentException(); - } - if (newRef != null && !newRef.isPeeled() && !newRef.isSymbolic()) { - throw new IllegalArgumentException(); - } - if (oldRef != null && newRef != null - && !oldRef.getName().equals(newRef.getName())) { - throw new IllegalArgumentException(); - } - } - - /** - * Construct a RefTree command wrapped around a ReceiveCommand. - * - * @param rw - * walk instance to peel the {@code newId}. - * @param cmd - * command received from a push client. - * @throws org.eclipse.jgit.errors.MissingObjectException - * {@code oldId} or {@code newId} is missing. - * @throws java.io.IOException - * {@code oldId} or {@code newId} cannot be peeled. - */ - public Command(RevWalk rw, ReceiveCommand cmd) - throws MissingObjectException, IOException { - this.oldRef = toRef(rw, cmd.getOldId(), cmd.getOldSymref(), - cmd.getRefName(), false); - this.newRef = toRef(rw, cmd.getNewId(), cmd.getNewSymref(), - cmd.getRefName(), true); - this.cmd = cmd; - } - - static Ref toRef(RevWalk rw, ObjectId id, @Nullable String target, - String name, boolean mustExist) - throws MissingObjectException, IOException { - if (target != null) { - return new SymbolicRef(name, - new ObjectIdRef.Unpeeled(NETWORK, target, id)); - } else if (ObjectId.zeroId().equals(id)) { - return null; - } - - try { - RevObject o = rw.parseAny(id); - if (o instanceof RevTag) { - RevObject p = rw.peel(o); - return new ObjectIdRef.PeeledTag(NETWORK, name, id, p.copy()); - } - return new ObjectIdRef.PeeledNonTag(NETWORK, name, id); - } catch (MissingObjectException e) { - if (mustExist) { - throw e; - } - return new ObjectIdRef.Unpeeled(NETWORK, name, id); - } - } - - /** - * Get name of the reference affected by this command. - * - * @return name of the reference affected by this command. - */ - public String getRefName() { - if (cmd != null) { - return cmd.getRefName(); - } else if (newRef != null) { - return newRef.getName(); - } - return oldRef.getName(); - } - - /** - * Set the result of this command. - * - * @param result - * the command result. - */ - public void setResult(Result result) { - setResult(result, null); - } - - /** - * Set the result of this command. - * - * @param result - * the command result. - * @param why - * optional message explaining the result status. - */ - public void setResult(Result result, @Nullable String why) { - if (cmd != null) { - cmd.setResult(result, why); - } else { - this.result = result; - } - } - - /** - * Get result of executing this command. - * - * @return result of executing this command. - */ - public Result getResult() { - return cmd != null ? cmd.getResult() : result; - } - - /** - * Get optional message explaining command failure. - * - * @return optional message explaining command failure. - */ - @Nullable - public String getMessage() { - return cmd != null ? cmd.getMessage() : null; - } - - /** - * Old peeled reference. - * - * @return the old reference; null if the command is creating the reference. - */ - @Nullable - public Ref getOldRef() { - return oldRef; - } - - /** - * New peeled reference. - * - * @return the new reference; null if the command is deleting the reference. - */ - @Nullable - public Ref getNewRef() { - return newRef; - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - append(s, oldRef, "CREATE"); //$NON-NLS-1$ - s.append(' '); - append(s, newRef, "DELETE"); //$NON-NLS-1$ - s.append(' ').append(getRefName()); - s.append(' ').append(getResult()); - if (getMessage() != null) { - s.append(' ').append(getMessage()); - } - return s.toString(); - } - - private static void append(StringBuilder s, Ref r, String nullName) { - if (r == null) { - s.append(nullName); - } else if (r.isSymbolic()) { - s.append(r.getTarget().getName()); - } else { - ObjectId id = r.getObjectId(); - if (id != null) { - s.append(id.name()); - } - } - } - - /** - * Check the entry is consistent with either the old or the new ref. - * - * @param entry - * current entry; null if the entry does not exist. - * @return true if entry matches {@link #getOldRef()} or - * {@link #getNewRef()}; otherwise false. - */ - boolean checkRef(@Nullable DirCacheEntry entry) { - if (entry != null && entry.getRawMode() == 0) { - entry = null; - } - return check(entry, oldRef) || check(entry, newRef); - } - - private static boolean check(@Nullable DirCacheEntry cur, - @Nullable Ref exp) { - if (cur == null) { - // Does not exist, ok if oldRef does not exist. - return exp == null; - } else if (exp == null) { - // Expected to not exist, but currently exists, fail. - return false; - } - - if (exp.isSymbolic()) { - String dst = exp.getTarget().getName(); - return cur.getRawMode() == TYPE_SYMLINK - && cur.getObjectId().equals(symref(dst)); - } - - return cur.getRawMode() == TYPE_GITLINK - && cur.getObjectId().equals(exp.getObjectId()); - } - - static ObjectId symref(String s) { - @SuppressWarnings("resource") - ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); - return fmt.idFor(OBJ_BLOB, encode(s)); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java deleted file mode 100644 index 6f12e9ccb6..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.R_REFS; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.GITLINK; -import static org.eclipse.jgit.lib.FileMode.SYMLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.dircache.DirCache; -import org.eclipse.jgit.dircache.DirCacheBuilder; -import org.eclipse.jgit.dircache.DirCacheEditor; -import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; -import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; -import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.errors.DirCacheNameConflictException; -import org.eclipse.jgit.errors.IncorrectObjectTypeException; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.util.RawParseUtils; - -/** - * Tree of references in the reference graph. - * <p> - * The root corresponds to the {@code "refs/"} subdirectory, for example the - * default reference {@code "refs/heads/master"} is stored at path - * {@code "heads/master"} in a {@code RefTree}. - * <p> - * Normal references are stored as {@link org.eclipse.jgit.lib.FileMode#GITLINK} - * tree entries. The ObjectId in the tree entry is the ObjectId the reference - * refers to. - * <p> - * Symbolic references are stored as - * {@link org.eclipse.jgit.lib.FileMode#SYMLINK} entries, with the blob storing - * the name of the target reference. - * <p> - * Annotated tags also store the peeled object using a {@code GITLINK} entry - * with the suffix <code>" ^"</code> (space carrot), for example - * {@code "tags/v1.0"} stores the annotated tag object, while - * <code>"tags/v1.0 ^"</code> stores the commit the tag annotates. - * <p> - * {@code HEAD} is a special case and stored as {@code "..HEAD"}. - */ -public class RefTree { - /** Suffix applied to GITLINK to indicate its the peeled value of a tag. */ - public static final String PEELED_SUFFIX = " ^"; //$NON-NLS-1$ - static final String ROOT_DOTDOT = ".."; //$NON-NLS-1$ - - /** - * Create an empty reference tree. - * - * @return a new empty reference tree. - */ - public static RefTree newEmptyTree() { - return new RefTree(DirCache.newInCore()); - } - - /** - * Load a reference tree. - * - * @param reader - * reader to scan the reference tree with. - * @param tree - * the tree to read. - * @return the ref tree read from the commit. - * @throws java.io.IOException - * the repository cannot be accessed through the reader. - * @throws org.eclipse.jgit.errors.CorruptObjectException - * a tree object is corrupt and cannot be read. - * @throws org.eclipse.jgit.errors.IncorrectObjectTypeException - * a tree object wasn't actually a tree. - * @throws org.eclipse.jgit.errors.MissingObjectException - * a reference tree object doesn't exist. - */ - public static RefTree read(ObjectReader reader, RevTree tree) - throws MissingObjectException, IncorrectObjectTypeException, - CorruptObjectException, IOException { - return new RefTree(DirCache.read(reader, tree)); - } - - private DirCache contents; - private Map<ObjectId, String> pendingBlobs; - - private RefTree(DirCache dc) { - this.contents = dc; - } - - /** - * Read one reference. - * <p> - * References are always returned peeled - * ({@link org.eclipse.jgit.lib.Ref#isPeeled()} is true). If the reference - * points to an annotated tag, the returned reference will be peeled and - * contain {@link org.eclipse.jgit.lib.Ref#getPeeledObjectId()}. - * <p> - * If the reference is a symbolic reference and the chain depth is less than - * {@link org.eclipse.jgit.lib.RefDatabase#MAX_SYMBOLIC_REF_DEPTH} the - * returned reference is resolved. If the chain depth is longer, the - * symbolic reference is returned without resolving. - * - * @param reader - * to access objects necessary to read the requested reference. - * @param name - * name of the reference to read. - * @return the reference; null if it does not exist. - * @throws java.io.IOException - * cannot read a symbolic reference target. - */ - @Nullable - public Ref exactRef(ObjectReader reader, String name) throws IOException { - Ref r = readRef(reader, name); - if (r == null) { - return null; - } else if (r.isSymbolic()) { - return resolve(reader, r, 0); - } - - DirCacheEntry p = contents.getEntry(peeledPath(name)); - if (p != null && p.getRawMode() == TYPE_GITLINK) { - return new ObjectIdRef.PeeledTag(PACKED, r.getName(), - r.getObjectId(), p.getObjectId()); - } - return r; - } - - private Ref readRef(ObjectReader reader, String name) throws IOException { - DirCacheEntry e = contents.getEntry(refPath(name)); - return e != null ? toRef(reader, e, name) : null; - } - - private Ref toRef(ObjectReader reader, DirCacheEntry e, String name) - throws IOException { - int mode = e.getRawMode(); - if (mode == TYPE_GITLINK) { - ObjectId id = e.getObjectId(); - return new ObjectIdRef.PeeledNonTag(PACKED, name, id); - } - - if (mode == TYPE_SYMLINK) { - ObjectId id = e.getObjectId(); - String n = pendingBlobs != null ? pendingBlobs.get(id) : null; - if (n == null) { - byte[] bin = reader.open(id, OBJ_BLOB).getCachedBytes(); - n = RawParseUtils.decode(bin); - } - Ref dst = new ObjectIdRef.Unpeeled(NEW, n, null); - return new SymbolicRef(name, dst); - } - - return null; // garbage file or something; not a reference. - } - - private Ref resolve(ObjectReader reader, Ref ref, int depth) - throws IOException { - if (ref.isSymbolic() && depth < MAX_SYMBOLIC_REF_DEPTH) { - Ref r = readRef(reader, ref.getTarget().getName()); - if (r == null) { - return ref; - } - Ref dst = resolve(reader, r, depth + 1); - return new SymbolicRef(ref.getName(), dst); - } - return ref; - } - - /** - * Attempt a batch of commands against this RefTree. - * <p> - * The batch is applied atomically, either all commands apply at once, or - * they all reject and the RefTree is left unmodified. - * <p> - * On success (when this method returns {@code true}) the command results - * are left as-is (probably {@code NOT_ATTEMPTED}). Result fields are set - * only when this method returns {@code false} to indicate failure. - * - * @param cmdList - * to apply. All commands should still have result NOT_ATTEMPTED. - * @return true if the commands applied; false if they were rejected. - */ - public boolean apply(Collection<Command> cmdList) { - try { - DirCacheEditor ed = contents.editor(); - for (Command cmd : cmdList) { - if (!isValidRef(cmd)) { - cmd.setResult(REJECTED_OTHER_REASON, - JGitText.get().funnyRefname); - Command.abort(cmdList, null); - return false; - } - apply(ed, cmd); - } - ed.finish(); - return true; - } catch (DirCacheNameConflictException e) { - String r1 = refName(e.getPath1()); - String r2 = refName(e.getPath2()); - for (Command cmd : cmdList) { - if (r1.equals(cmd.getRefName()) - || r2.equals(cmd.getRefName())) { - cmd.setResult(LOCK_FAILURE); - break; - } - } - Command.abort(cmdList, null); - return false; - } catch (LockFailureException e) { - Command.abort(cmdList, null); - return false; - } - } - - private static boolean isValidRef(Command cmd) { - String n = cmd.getRefName(); - return HEAD.equals(n) || Repository.isValidRefName(n); - } - - private void apply(DirCacheEditor ed, Command cmd) { - String path = refPath(cmd.getRefName()); - Ref oldRef = cmd.getOldRef(); - final Ref newRef = cmd.getNewRef(); - - if (newRef == null) { - checkRef(contents.getEntry(path), cmd); - ed.add(new DeletePath(path)); - cleanupPeeledRef(ed, oldRef); - return; - } - - if (newRef.isSymbolic()) { - final String dst = newRef.getTarget().getName(); - ed.add(new PathEdit(path) { - @Override - public void apply(DirCacheEntry ent) { - checkRef(ent, cmd); - ObjectId id = Command.symref(dst); - ent.setFileMode(SYMLINK); - ent.setObjectId(id); - if (pendingBlobs == null) { - pendingBlobs = new HashMap<>(4); - } - pendingBlobs.put(id, dst); - } - }.setReplace(false)); - cleanupPeeledRef(ed, oldRef); - return; - } - - ed.add(new PathEdit(path) { - @Override - public void apply(DirCacheEntry ent) { - checkRef(ent, cmd); - ent.setFileMode(GITLINK); - ent.setObjectId(newRef.getObjectId()); - } - }.setReplace(false)); - - if (newRef.getPeeledObjectId() != null) { - ed.add(new PathEdit(peeledPath(newRef.getName())) { - @Override - public void apply(DirCacheEntry ent) { - ent.setFileMode(GITLINK); - ent.setObjectId(newRef.getPeeledObjectId()); - } - }.setReplace(false)); - } else { - cleanupPeeledRef(ed, oldRef); - } - } - - private static void checkRef(@Nullable DirCacheEntry ent, Command cmd) { - if (!cmd.checkRef(ent)) { - cmd.setResult(LOCK_FAILURE); - throw new LockFailureException(); - } - } - - private static void cleanupPeeledRef(DirCacheEditor ed, Ref ref) { - if (ref != null && !ref.isSymbolic() - && (!ref.isPeeled() || ref.getPeeledObjectId() != null)) { - ed.add(new DeletePath(peeledPath(ref.getName()))); - } - } - - /** - * Convert a path name in a RefTree to the reference name known by Git. - * - * @param path - * name read from the RefTree structure, for example - * {@code "heads/master"}. - * @return reference name for the path, {@code "refs/heads/master"}. - */ - public static String refName(String path) { - if (path.startsWith(ROOT_DOTDOT)) { - return path.substring(2); - } - return R_REFS + path; - } - - static String refPath(String name) { - if (name.startsWith(R_REFS)) { - return name.substring(R_REFS.length()); - } - return ROOT_DOTDOT + name; - } - - private static String peeledPath(String name) { - return refPath(name) + PEELED_SUFFIX; - } - - /** - * Write this reference tree. - * - * @param inserter - * inserter to use when writing trees to the object database. - * Caller is responsible for flushing the inserter before trying - * to read the objects, or exposing them through a reference. - * @return the top level tree. - * @throws java.io.IOException - * a tree could not be written. - */ - public ObjectId writeTree(ObjectInserter inserter) throws IOException { - if (pendingBlobs != null) { - for (String s : pendingBlobs.values()) { - inserter.insert(OBJ_BLOB, encode(s)); - } - pendingBlobs = null; - } - return contents.writeTree(inserter); - } - - /** - * Create a deep copy of this RefTree. - * - * @return a deep copy of this RefTree. - */ - public RefTree copy() { - RefTree r = new RefTree(DirCache.newInCore()); - DirCacheBuilder b = r.contents.builder(); - for (int i = 0; i < contents.getEntryCount(); i++) { - b.add(new DirCacheEntry(contents.getEntry(i))); - } - b.finish(); - if (pendingBlobs != null) { - r.pendingBlobs = new HashMap<>(pendingBlobs); - } - return r; - } - - private static class LockFailureException extends RuntimeException { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java deleted file mode 100644 index b154b95adc..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_TREE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** Batch update a {@link RefTreeDatabase}. */ -class RefTreeBatch extends BatchRefUpdate { - private final RefTreeDatabase refdb; - private Ref src; - private ObjectId parentCommitId; - private ObjectId parentTreeId; - private RefTree tree; - private PersonIdent author; - private ObjectId newCommitId; - - RefTreeBatch(RefTreeDatabase refdb) { - super(refdb); - this.refdb = refdb; - } - - /** {@inheritDoc} */ - @Override - public void execute(RevWalk rw, ProgressMonitor monitor) - throws IOException { - List<Command> todo = new ArrayList<>(getCommands().size()); - for (ReceiveCommand c : getCommands()) { - if (!isAllowNonFastForwards()) { - if (c.getType() == UPDATE) { - c.updateType(rw); - } - if (c.getType() == UPDATE_NONFASTFORWARD) { - c.setResult(REJECTED_NONFASTFORWARD); - if (isAtomic()) { - ReceiveCommand.abort(getCommands()); - return; - } - continue; - } - } - todo.add(new Command(rw, c)); - } - init(rw); - execute(rw, todo); - } - - void init(RevWalk rw) throws IOException { - src = refdb.getBootstrap().exactRef(refdb.getTxnCommitted()); - if (src != null && src.getObjectId() != null) { - RevCommit c = rw.parseCommit(src.getObjectId()); - parentCommitId = c; - parentTreeId = c.getTree(); - tree = RefTree.read(rw.getObjectReader(), c.getTree()); - } else { - parentCommitId = ObjectId.zeroId(); - try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { - parentTreeId = fmt.idFor(OBJ_TREE, new byte[] {}); - } - tree = RefTree.newEmptyTree(); - } - } - - @Nullable - Ref exactRef(ObjectReader reader, String name) throws IOException { - return tree.exactRef(reader, name); - } - - /** - * Execute an update from {@link RefTreeUpdate} or {@link RefTreeRename}. - * - * @param rw - * current RevWalk handling the update or rename. - * @param todo - * commands to execute. Must never be a bootstrap reference name. - * @throws IOException - * the storage system is unable to read or write data. - */ - void execute(RevWalk rw, List<Command> todo) throws IOException { - for (Command c : todo) { - if (c.getResult() != NOT_ATTEMPTED) { - Command.abort(todo, null); - return; - } - if (refdb.conflictsWithBootstrap(c.getRefName())) { - c.setResult(REJECTED_OTHER_REASON, MessageFormat - .format(JGitText.get().invalidRefName, c.getRefName())); - Command.abort(todo, null); - return; - } - } - - if (apply(todo) && newCommitId != null) { - commit(rw, todo); - } - } - - private boolean apply(List<Command> todo) throws IOException { - if (!tree.apply(todo)) { - // apply set rejection information on commands. - return false; - } - - Repository repo = refdb.getRepository(); - try (ObjectInserter ins = repo.newObjectInserter()) { - CommitBuilder b = new CommitBuilder(); - b.setTreeId(tree.writeTree(ins)); - if (parentTreeId.equals(b.getTreeId())) { - for (Command c : todo) { - c.setResult(OK); - } - return true; - } - if (!parentCommitId.equals(ObjectId.zeroId())) { - b.setParentId(parentCommitId); - } - - author = getRefLogIdent(); - if (author == null) { - author = new PersonIdent(repo); - } - b.setAuthor(author); - b.setCommitter(author); - b.setMessage(getRefLogMessage()); - newCommitId = ins.insert(b); - ins.flush(); - } - return true; - } - - private void commit(RevWalk rw, List<Command> todo) throws IOException { - ReceiveCommand commit = new ReceiveCommand( - parentCommitId, newCommitId, - refdb.getTxnCommitted()); - updateBootstrap(rw, commit); - - if (commit.getResult() == OK) { - for (Command c : todo) { - c.setResult(OK); - } - } else { - Command.abort(todo, commit.getResult().name()); - } - } - - private void updateBootstrap(RevWalk rw, ReceiveCommand commit) - throws IOException { - BatchRefUpdate u = refdb.getBootstrap().newBatchUpdate(); - u.setAllowNonFastForwards(true); - u.setPushCertificate(getPushCertificate()); - if (isRefLogDisabled()) { - u.disableRefLog(); - } else { - u.setRefLogIdent(author); - u.setRefLogMessage(getRefLogMessage(), false); - } - u.addCommand(commit); - u.execute(rw, NullProgressMonitor.INSTANCE); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java deleted file mode 100644 index 34b8f2cd36..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Ref.Storage; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefRename; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.RefList; -import org.eclipse.jgit.util.RefMap; - -/** - * Reference database backed by a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - * <p> - * The storage for RefTreeDatabase has two parts. The main part is a native Git - * tree object stored under the {@code refs/txn} namespace. To avoid cycles, - * references to {@code refs/txn} are not stored in that tree object, but - * instead in a "bootstrap" layer, which is a separate - * {@link org.eclipse.jgit.lib.RefDatabase} such as - * {@link org.eclipse.jgit.internal.storage.file.RefDirectory} using local - * reference files inside of {@code $GIT_DIR/refs}. - */ -public class RefTreeDatabase extends RefDatabase { - private final Repository repo; - private final RefDatabase bootstrap; - private final String txnCommitted; - - @Nullable - private final String txnNamespace; - private volatile Scanner.Result refs; - - /** - * Create a RefTreeDb for a repository. - * - * @param repo - * the repository using references in this database. - * @param bootstrap - * bootstrap reference database storing the references that - * anchor the - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - */ - public RefTreeDatabase(Repository repo, RefDatabase bootstrap) { - Config cfg = repo.getConfig(); - String committed = cfg.getString("reftree", null, "committedRef"); //$NON-NLS-1$ //$NON-NLS-2$ - if (committed == null || committed.isEmpty()) { - committed = "refs/txn/committed"; //$NON-NLS-1$ - } - - this.repo = repo; - this.bootstrap = bootstrap; - this.txnNamespace = initNamespace(committed); - this.txnCommitted = committed; - } - - /** - * Create a RefTreeDb for a repository. - * - * @param repo - * the repository using references in this database. - * @param bootstrap - * bootstrap reference database storing the references that - * anchor the - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - * @param txnCommitted - * name of the bootstrap reference holding the committed RefTree. - */ - public RefTreeDatabase(Repository repo, RefDatabase bootstrap, - String txnCommitted) { - this.repo = repo; - this.bootstrap = bootstrap; - this.txnNamespace = initNamespace(txnCommitted); - this.txnCommitted = txnCommitted; - } - - private static String initNamespace(String committed) { - int s = committed.lastIndexOf('/'); - if (s < 0) { - return null; - } - return committed.substring(0, s + 1); // Keep trailing '/'. - } - - Repository getRepository() { - return repo; - } - - /** - * Get the bootstrap reference database - * - * @return the bootstrap reference database, which must be used to access - * {@link #getTxnCommitted()}, {@link #getTxnNamespace()}. - */ - public RefDatabase getBootstrap() { - return bootstrap; - } - - /** - * Get name of bootstrap reference anchoring committed RefTree. - * - * @return name of bootstrap reference anchoring committed RefTree. - */ - public String getTxnCommitted() { - return txnCommitted; - } - - /** - * Get namespace used by bootstrap layer. - * - * @return namespace used by bootstrap layer, e.g. {@code refs/txn/}. Always - * ends in {@code '/'}. - */ - @Nullable - public String getTxnNamespace() { - return txnNamespace; - } - - /** {@inheritDoc} */ - @Override - public void create() throws IOException { - bootstrap.create(); - } - - /** {@inheritDoc} */ - @Override - public boolean performsAtomicTransactions() { - return true; - } - - /** {@inheritDoc} */ - @Override - public void refresh() { - bootstrap.refresh(); - } - - /** {@inheritDoc} */ - @Override - public void close() { - refs = null; - bootstrap.close(); - } - - /** {@inheritDoc} */ - @Override - public Ref exactRef(String name) throws IOException { - if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { - // Pass through names like MERGE_HEAD, ORIG_HEAD, FETCH_HEAD. - return bootstrap.exactRef(name); - } else if (conflictsWithBootstrap(name)) { - return null; - } - - boolean partial = false; - Ref src = bootstrap.exactRef(txnCommitted); - Scanner.Result c = refs; - if (c == null || !c.refTreeId.equals(idOf(src))) { - c = Scanner.scanRefTree(repo, src, prefixOf(name), false); - partial = true; - } - - Ref r = c.all.get(name); - if (r != null && r.isSymbolic()) { - r = c.sym.get(name); - if (partial && r.getObjectId() == null) { - // Attempting exactRef("HEAD") with partial scan will leave - // an unresolved symref as its target e.g. refs/heads/master - // was not read by the partial scan. Scan everything instead. - return getRefs(ALL).get(name); - } - } - return r; - } - - private static String prefixOf(String name) { - int s = name.lastIndexOf('/'); - if (s >= 0) { - return name.substring(0, s); - } - return ""; //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @Override - public Map<String, Ref> getRefs(String prefix) throws IOException { - if (!prefix.isEmpty() && prefix.charAt(prefix.length() - 1) != '/') { - return new HashMap<>(0); - } - - Ref src = bootstrap.exactRef(txnCommitted); - Scanner.Result c = refs; - if (c == null || !c.refTreeId.equals(idOf(src))) { - c = Scanner.scanRefTree(repo, src, prefix, true); - if (prefix.isEmpty()) { - refs = c; - } - } - return new RefMap(prefix, RefList.<Ref> emptyList(), c.all, c.sym); - } - - private static ObjectId idOf(@Nullable Ref src) { - return src != null && src.getObjectId() != null - ? src.getObjectId() - : ObjectId.zeroId(); - } - - /** {@inheritDoc} */ - @Override - public List<Ref> getAdditionalRefs() throws IOException { - Collection<Ref> txnRefs; - if (txnNamespace != null) { - txnRefs = bootstrap.getRefsByPrefix(txnNamespace); - } else { - Ref r = bootstrap.exactRef(txnCommitted); - if (r != null && r.getObjectId() != null) { - txnRefs = Collections.singleton(r); - } else { - txnRefs = Collections.emptyList(); - } - } - - List<Ref> otherRefs = bootstrap.getAdditionalRefs(); - List<Ref> all = new ArrayList<>(txnRefs.size() + otherRefs.size()); - all.addAll(txnRefs); - all.addAll(otherRefs); - return all; - } - - /** {@inheritDoc} */ - @Override - public Ref peel(Ref ref) throws IOException { - Ref i = ref.getLeaf(); - ObjectId id = i.getObjectId(); - if (i.isPeeled() || id == null) { - return ref; - } - try (RevWalk rw = new RevWalk(repo)) { - RevObject obj = rw.parseAny(id); - if (obj instanceof RevTag) { - ObjectId p = rw.peel(obj).copy(); - i = new ObjectIdRef.PeeledTag(PACKED, i.getName(), id, p); - } else { - i = new ObjectIdRef.PeeledNonTag(PACKED, i.getName(), id); - } - } - return recreate(ref, i); - } - - private static Ref recreate(Ref old, Ref leaf) { - if (old.isSymbolic()) { - Ref dst = recreate(old.getTarget(), leaf); - return new SymbolicRef(old.getName(), dst); - } - return leaf; - } - - /** {@inheritDoc} */ - @Override - public boolean isNameConflicting(String name) throws IOException { - return conflictsWithBootstrap(name) - || !getConflictingNames(name).isEmpty(); - } - - /** {@inheritDoc} */ - @Override - public BatchRefUpdate newBatchUpdate() { - return new RefTreeBatch(this); - } - - /** {@inheritDoc} */ - @Override - public RefUpdate newUpdate(String name, boolean detach) throws IOException { - if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { - return bootstrap.newUpdate(name, detach); - } - if (conflictsWithBootstrap(name)) { - return new AlwaysFailUpdate(this, name); - } - - Ref r = exactRef(name); - if (r == null) { - r = new ObjectIdRef.Unpeeled(Storage.NEW, name, null); - } - - boolean detaching = detach && r.isSymbolic(); - if (detaching) { - r = new ObjectIdRef.Unpeeled(LOOSE, name, r.getObjectId()); - } - - RefTreeUpdate u = new RefTreeUpdate(this, r); - if (detaching) { - u.setDetachingSymbolicRef(); - } - return u; - } - - /** {@inheritDoc} */ - @Override - public RefRename newRename(String fromName, String toName) - throws IOException { - RefUpdate from = newUpdate(fromName, true); - RefUpdate to = newUpdate(toName, true); - return new RefTreeRename(this, from, to); - } - - boolean conflictsWithBootstrap(String name) { - if (txnNamespace != null && name.startsWith(txnNamespace)) { - return true; - } else if (txnCommitted.equals(name)) { - return true; - } - - if (name.indexOf('/') < 0 && !HEAD.equals(name)) { - return true; - } - - if (name.length() > txnCommitted.length() - && name.charAt(txnCommitted.length()) == '/' - && name.startsWith(txnCommitted)) { - return true; - } - return false; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java deleted file mode 100644 index eec0da29e2..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import org.eclipse.jgit.lib.RefDatabase; - -/** - * Magic reference name logic for RefTrees. - */ -public class RefTreeNames { - /** - * Suffix used on a {@link RefTreeDatabase#getTxnNamespace()} for user data. - * <p> - * A {@link RefTreeDatabase}'s namespace may include a subspace (e.g. - * {@code "refs/txn/stage/"}) containing commit objects from the usual user - * portion of the repository (e.g. {@code "refs/heads/"}). These should be - * packed by the garbage collector alongside other user content rather than - * with the RefTree. - */ - private static final String STAGE = "stage/"; //$NON-NLS-1$ - - /** - * Determine if the reference is likely to be a RefTree. - * - * @param refdb - * database instance. - * @param ref - * reference name. - * @return {@code true} if the reference is a RefTree. - */ - public static boolean isRefTree(RefDatabase refdb, String ref) { - if (refdb instanceof RefTreeDatabase) { - RefTreeDatabase b = (RefTreeDatabase) refdb; - if (ref.equals(b.getTxnCommitted())) { - return true; - } - - String namespace = b.getTxnNamespace(); - if (namespace != null - && ref.startsWith(namespace) - && !ref.startsWith(namespace + STAGE)) { - return true; - } - } - return false; - } - - private RefTreeNames() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java deleted file mode 100644 index ccf8f75e7d..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.RefUpdate.Result.REJECTED; -import static org.eclipse.jgit.lib.RefUpdate.Result.RENAMED; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefRename; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.RefUpdate.Result; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevWalk; - -/** Single reference rename to {@link RefTreeDatabase}. */ -class RefTreeRename extends RefRename { - private final RefTreeDatabase refdb; - - RefTreeRename(RefTreeDatabase refdb, RefUpdate src, RefUpdate dst) { - super(src, dst); - this.refdb = refdb; - } - - /** {@inheritDoc} */ - @Override - protected Result doRename() throws IOException { - try (RevWalk rw = new RevWalk(refdb.getRepository())) { - RefTreeBatch batch = new RefTreeBatch(refdb); - batch.setRefLogIdent(getRefLogIdent()); - batch.setRefLogMessage(getRefLogMessage(), false); - batch.init(rw); - - Ref head = batch.exactRef(rw.getObjectReader(), HEAD); - Ref oldRef = batch.exactRef(rw.getObjectReader(), source.getName()); - if (oldRef == null) { - return REJECTED; - } - - Ref newRef = asNew(oldRef); - List<Command> mv = new ArrayList<>(3); - mv.add(new Command(oldRef, null)); - mv.add(new Command(null, newRef)); - if (head != null && head.isSymbolic() - && head.getTarget().getName().equals(oldRef.getName())) { - mv.add(new Command( - head, - new SymbolicRef(head.getName(), newRef))); - } - batch.execute(rw, mv); - return RefTreeUpdate.translate(mv.get(1).getResult(), RENAMED); - } - } - - private Ref asNew(Ref src) { - String name = destination.getName(); - if (src.isSymbolic()) { - return new SymbolicRef(name, src.getTarget()); - } - - ObjectId peeled = src.getPeeledObjectId(); - if (peeled != null) { - return new ObjectIdRef.PeeledTag( - src.getStorage(), - name, - src.getObjectId(), - peeled); - } - - return new ObjectIdRef.PeeledNonTag( - src.getStorage(), - name, - src.getObjectId()); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java deleted file mode 100644 index 6e6ccd9986..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; - -import java.io.IOException; -import java.util.Collections; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** Single reference update to {@link RefTreeDatabase}. */ -class RefTreeUpdate extends RefUpdate { - private final RefTreeDatabase refdb; - private RevWalk rw; - private RefTreeBatch batch; - private Ref oldRef; - - RefTreeUpdate(RefTreeDatabase refdb, Ref ref) { - super(ref); - this.refdb = refdb; - setCheckConflicting(false); // Done automatically by doUpdate. - } - - /** {@inheritDoc} */ - @Override - protected RefDatabase getRefDatabase() { - return refdb; - } - - /** {@inheritDoc} */ - @Override - protected Repository getRepository() { - return refdb.getRepository(); - } - - /** {@inheritDoc} */ - @Override - protected boolean tryLock(boolean deref) throws IOException { - rw = new RevWalk(getRepository()); - batch = new RefTreeBatch(refdb); - batch.init(rw); - oldRef = batch.exactRef(rw.getObjectReader(), getName()); - if (oldRef != null && oldRef.getObjectId() != null) { - setOldObjectId(oldRef.getObjectId()); - } else if (oldRef == null && getExpectedOldObjectId() != null) { - setOldObjectId(ObjectId.zeroId()); - } - return true; - } - - /** {@inheritDoc} */ - @Override - protected void unlock() { - batch = null; - if (rw != null) { - rw.close(); - rw = null; - } - } - - /** {@inheritDoc} */ - @Override - protected Result doUpdate(Result desiredResult) throws IOException { - return run(newRef(getName(), getNewObjectId()), desiredResult); - } - - private Ref newRef(String name, ObjectId id) - throws MissingObjectException, IOException { - RevObject o = rw.parseAny(id); - if (o instanceof RevTag) { - RevObject p = rw.peel(o); - return new ObjectIdRef.PeeledTag(LOOSE, name, id, p.copy()); - } - return new ObjectIdRef.PeeledNonTag(LOOSE, name, id); - } - - /** {@inheritDoc} */ - @Override - protected Result doDelete(Result desiredResult) throws IOException { - return run(null, desiredResult); - } - - /** {@inheritDoc} */ - @Override - protected Result doLink(String target) throws IOException { - Ref dst = new ObjectIdRef.Unpeeled(NEW, target, null); - SymbolicRef n = new SymbolicRef(getName(), dst); - Result desiredResult = getRef().getStorage() == NEW - ? Result.NEW - : Result.FORCED; - return run(n, desiredResult); - } - - private Result run(@Nullable Ref newRef, Result desiredResult) - throws IOException { - Command c = new Command(oldRef, newRef); - batch.setRefLogIdent(getRefLogIdent()); - batch.setRefLogMessage(getRefLogMessage(), isRefLogIncludingResult()); - batch.execute(rw, Collections.singletonList(c)); - return translate(c.getResult(), desiredResult); - } - - static Result translate(ReceiveCommand.Result r, Result desiredResult) { - switch (r) { - case OK: - return desiredResult; - - case LOCK_FAILURE: - return Result.LOCK_FAILURE; - - case NOT_ATTEMPTED: - return Result.NOT_ATTEMPTED; - - case REJECTED_MISSING_OBJECT: - return Result.IO_FAILURE; - - case REJECTED_CURRENT_BRANCH: - return Result.REJECTED_CURRENT_BRANCH; - - case REJECTED_OTHER_REASON: - case REJECTED_NOCREATE: - case REJECTED_NODELETE: - case REJECTED_NONFASTFORWARD: - default: - return Result.REJECTED; - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java deleted file mode 100644 index 3f5122911c..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.R_REFS; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_TREE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; - -import java.io.IOException; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.IncorrectObjectTypeException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.treewalk.AbstractTreeIterator; -import org.eclipse.jgit.treewalk.CanonicalTreeParser; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.Paths; -import org.eclipse.jgit.util.RawParseUtils; -import org.eclipse.jgit.util.RefList; - -/** A tree parser that extracts references from a {@link RefTree}. */ -class Scanner { - private static final int MAX_SYMLINK_BYTES = 10 << 10; - private static final byte[] BINARY_R_REFS = encode(R_REFS); - private static final byte[] REFS_DOT_DOT = encode("refs/.."); //$NON-NLS-1$ - - static class Result { - final ObjectId refTreeId; - final RefList<Ref> all; - final RefList<Ref> sym; - - Result(ObjectId id, RefList<Ref> all, RefList<Ref> sym) { - this.refTreeId = id; - this.all = all; - this.sym = sym; - } - } - - /** - * Scan a {@link RefTree} and parse entries into {@link Ref} instances. - * - * @param repo - * source repository containing the commit and tree objects that - * make up the RefTree. - * @param src - * bootstrap reference such as {@code refs/txn/committed} to read - * the reference tree tip from. The current ObjectId will be - * included in {@link Result#refTreeId}. - * @param prefix - * if non-empty a reference prefix to scan only a subdirectory. - * For example {@code prefix = "refs/heads/"} will limit the scan - * to only the {@code "heads"} directory of the RefTree, avoiding - * other directories like {@code "tags"}. Empty string reads all - * entries in the RefTree. - * @param recursive - * if true recurse into subdirectories of the reference tree; - * false to read only one level. Callers may use false during an - * implementation of {@code exactRef(String)} where only one - * reference is needed out of a specific subtree. - * @return sorted list of references after parsing. - * @throws IOException - * tree cannot be accessed from the repository. - */ - static Result scanRefTree(Repository repo, @Nullable Ref src, String prefix, - boolean recursive) throws IOException { - RefList.Builder<Ref> all = new RefList.Builder<>(); - RefList.Builder<Ref> sym = new RefList.Builder<>(); - - ObjectId srcId; - if (src != null && src.getObjectId() != null) { - try (ObjectReader reader = repo.newObjectReader()) { - srcId = src.getObjectId(); - scan(reader, srcId, prefix, recursive, all, sym); - } - } else { - srcId = ObjectId.zeroId(); - } - - RefList<Ref> aList = all.toRefList(); - for (int idx = 0; idx < sym.size();) { - Ref s = sym.get(idx); - Ref r = resolve(s, 0, aList); - if (r != null) { - sym.set(idx++, r); - } else { - // Remove broken symbolic reference, they don't exist. - sym.remove(idx); - int rm = aList.find(s.getName()); - if (0 <= rm) { - aList = aList.remove(rm); - } - } - } - return new Result(srcId, aList, sym.toRefList()); - } - - private static void scan(ObjectReader reader, AnyObjectId srcId, - String prefix, boolean recursive, - RefList.Builder<Ref> all, RefList.Builder<Ref> sym) - throws IncorrectObjectTypeException, IOException { - CanonicalTreeParser p = createParserAtPath(reader, srcId, prefix); - if (p == null) { - return; - } - - while (!p.eof()) { - int mode = p.getEntryRawMode(); - if (mode == TYPE_TREE) { - if (recursive) { - p = p.createSubtreeIterator(reader); - } else { - p = p.next(); - } - continue; - } - - if (!curElementHasPeelSuffix(p)) { - Ref r = toRef(reader, mode, p); - if (r != null) { - all.add(r); - if (r.isSymbolic()) { - sym.add(r); - } - } - } else if (mode == TYPE_GITLINK) { - peel(all, p); - } - p = p.next(); - } - } - - private static CanonicalTreeParser createParserAtPath(ObjectReader reader, - AnyObjectId srcId, String prefix) throws IOException { - ObjectId root = toTree(reader, srcId); - if (prefix.isEmpty()) { - return new CanonicalTreeParser(BINARY_R_REFS, reader, root); - } - - String dir = RefTree.refPath(Paths.stripTrailingSeparator(prefix)); - TreeWalk tw = TreeWalk.forPath(reader, dir, root); - if (tw == null || !tw.isSubtree()) { - return null; - } - - ObjectId id = tw.getObjectId(0); - return new CanonicalTreeParser(encode(prefix), reader, id); - } - - private static Ref resolve(Ref ref, int depth, RefList<Ref> refs) - throws IOException { - if (!ref.isSymbolic()) { - return ref; - } else if (MAX_SYMBOLIC_REF_DEPTH <= depth) { - return null; - } - - Ref r = refs.get(ref.getTarget().getName()); - if (r == null) { - return ref; - } - - Ref dst = resolve(r, depth + 1, refs); - if (dst == null) { - return null; - } - return new SymbolicRef(ref.getName(), dst); - } - - private static RevTree toTree(ObjectReader reader, AnyObjectId id) - throws IOException { - try (RevWalk rw = new RevWalk(reader)) { - return rw.parseTree(id); - } - } - - private static boolean curElementHasPeelSuffix(AbstractTreeIterator itr) { - int n = itr.getEntryPathLength(); - byte[] c = itr.getEntryPathBuffer(); - return n > 2 && c[n - 2] == ' ' && c[n - 1] == '^'; - } - - private static void peel(RefList.Builder<Ref> all, CanonicalTreeParser p) { - String name = refName(p, true); - for (int idx = all.size() - 1; 0 <= idx; idx--) { - Ref r = all.get(idx); - int cmp = r.getName().compareTo(name); - if (cmp == 0) { - all.set(idx, new ObjectIdRef.PeeledTag(PACKED, r.getName(), - r.getObjectId(), p.getEntryObjectId())); - break; - } else if (cmp < 0) { - // Stray peeled name without matching base name; skip entry. - break; - } - } - } - - private static Ref toRef(ObjectReader reader, int mode, - CanonicalTreeParser p) throws IOException { - if (mode == TYPE_GITLINK) { - String name = refName(p, false); - ObjectId id = p.getEntryObjectId(); - return new ObjectIdRef.PeeledNonTag(PACKED, name, id); - - } else if (mode == TYPE_SYMLINK) { - ObjectId id = p.getEntryObjectId(); - byte[] bin = reader.open(id, OBJ_BLOB) - .getCachedBytes(MAX_SYMLINK_BYTES); - String dst = RawParseUtils.decode(bin); - Ref trg = new ObjectIdRef.Unpeeled(NEW, dst, null); - String name = refName(p, false); - return new SymbolicRef(name, trg); - } - return null; - } - - private static String refName(CanonicalTreeParser p, boolean peel) { - byte[] buf = p.getEntryPathBuffer(); - int len = p.getEntryPathLength(); - if (peel) { - len -= 2; - } - int ptr = 0; - if (RawParseUtils.match(buf, ptr, REFS_DOT_DOT) > 0) { - ptr = 7; - } - return RawParseUtils.decode(buf, ptr, len); - } - - private Scanner() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java index e51995f93d..b2242a11ca 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java @@ -28,6 +28,8 @@ import java.util.Collection; import java.util.LinkedList; import java.util.List; +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.RepositoryNotFoundException; import org.eclipse.jgit.internal.JGitText; @@ -38,6 +40,7 @@ import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.SystemReader; /** @@ -107,6 +110,8 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re private File workTree; + private String initialBranch = Constants.MASTER; + /** Directories limiting the search for a Git repository. */ private List<File> ceilingDirectories; @@ -350,6 +355,43 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re } /** + * Set the initial branch of the new repository. If not specified + * ({@code null} or empty), fall back to the default name (currently + * master). + * + * @param branch + * initial branch name of the new repository. If {@code null} or + * empty the configured default branch will be used. + * @return {@code this} + * @throws InvalidRefNameException + * if the branch name is not valid + * + * @since 5.11 + */ + public B setInitialBranch(String branch) throws InvalidRefNameException { + if (StringUtils.isEmptyOrNull(branch)) { + this.initialBranch = Constants.MASTER; + } else { + if (!Repository.isValidRefName(Constants.R_HEADS + branch)) { + throw new InvalidRefNameException(MessageFormat + .format(JGitText.get().branchNameInvalid, branch)); + } + this.initialBranch = branch; + } + return self(); + } + + /** + * Get the initial branch of the new repository. + * + * @return the initial branch of the new repository. + * @since 5.11 + */ + public @NonNull String getInitialBranch() { + return initialBranch; + } + + /** * Read standard Git environment variables and configure from those. * <p> * This method tries to read the standard Git environment variables, such as diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java index 4f93fda49f..1665f051e9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java @@ -1,7 +1,7 @@ /* * Copyright (C) 2007, Dave Watson <dwatson@mimvista.com> - * Copyright (C) 2006-2007, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2006-2007, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2006, 2007, Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2006, 2020, Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -16,14 +16,11 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; -import java.text.MessageFormat; import java.util.List; -import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.util.References; /** @@ -37,7 +34,7 @@ import org.eclipse.jgit.util.References; * and obtain a {@link org.eclipse.jgit.revwalk.RevCommit} instance by calling * {@link org.eclipse.jgit.revwalk.RevWalk#parseCommit(AnyObjectId)}. */ -public class CommitBuilder { +public class CommitBuilder extends ObjectBuilder { private static final ObjectId[] EMPTY_OBJECTID_LIST = new ObjectId[0]; private static final byte[] htree = Constants.encodeASCII("tree"); //$NON-NLS-1$ @@ -50,28 +47,17 @@ public class CommitBuilder { private static final byte[] hgpgsig = Constants.encodeASCII("gpgsig"); //$NON-NLS-1$ - private static final byte[] hencoding = Constants.encodeASCII("encoding"); //$NON-NLS-1$ - private ObjectId treeId; private ObjectId[] parentIds; - private PersonIdent author; - private PersonIdent committer; - private GpgSignature gpgSignature; - - private String message; - - private Charset encoding; - /** * Initialize an empty commit. */ public CommitBuilder() { parentIds = EMPTY_OBJECTID_LIST; - encoding = UTF_8; } /** @@ -98,8 +84,9 @@ public class CommitBuilder { * * @return the author of this commit (who wrote it). */ + @Override public PersonIdent getAuthor() { - return author; + return super.getAuthor(); } /** @@ -108,8 +95,9 @@ public class CommitBuilder { * @param newAuthor * the new author. Should not be null. */ + @Override public void setAuthor(PersonIdent newAuthor) { - author = newAuthor; + super.setAuthor(newAuthor); } /** @@ -132,38 +120,6 @@ public class CommitBuilder { } /** - * Set the GPG signature of this commit. - * <p> - * Note, the signature set here will change the payload of the commit, i.e. - * the output of {@link #build()} will include the signature. Thus, the - * typical flow will be: - * <ol> - * <li>call {@link #build()} without a signature set to obtain payload</li> - * <li>create {@link GpgSignature} from payload</li> - * <li>set {@link GpgSignature}</li> - * </ol> - * </p> - * - * @param newSignature - * the signature to set or <code>null</code> to unset - * @since 5.3 - */ - public void setGpgSignature(GpgSignature newSignature) { - gpgSignature = newSignature; - } - - /** - * Get the GPG signature of this commit. - * - * @return the GPG signature of this commit, maybe <code>null</code> if the - * commit is not to be signed - * @since 5.3 - */ - public GpgSignature getGpgSignature() { - return gpgSignature; - } - - /** * Get the ancestors of this commit. * * @return the ancestors of this commit. Never null. @@ -239,25 +195,6 @@ public class CommitBuilder { } /** - * Get the complete commit message. - * - * @return the complete commit message. - */ - public String getMessage() { - return message; - } - - /** - * Set the commit message. - * - * @param newMessage - * the commit message. Should not be null. - */ - public void setMessage(String newMessage) { - message = newMessage; - } - - /** * Set the encoding for the commit information. * * @param encodingName @@ -267,37 +204,10 @@ public class CommitBuilder { */ @Deprecated public void setEncoding(String encodingName) { - encoding = Charset.forName(encodingName); - } - - /** - * Set the encoding for the commit information. - * - * @param enc - * the encoding to use. - */ - public void setEncoding(Charset enc) { - encoding = enc; + setEncoding(Charset.forName(encodingName)); } - /** - * Get the encoding that should be used for the commit message text. - * - * @return the encoding that should be used for the commit message text. - */ - public Charset getEncoding() { - return encoding; - } - - /** - * Format this builder's state as a commit object. - * - * @return this object in the canonical commit format, suitable for storage - * in a repository. - * @throws java.io.UnsupportedEncodingException - * the encoding specified by {@link #getEncoding()} is not - * supported by this Java runtime. - */ + @Override public byte[] build() throws UnsupportedEncodingException { ByteArrayOutputStream os = new ByteArrayOutputStream(); OutputStreamWriter w = new OutputStreamWriter(os, getEncoding()); @@ -326,19 +236,16 @@ public class CommitBuilder { w.flush(); os.write('\n'); - if (getGpgSignature() != null) { + GpgSignature signature = getGpgSignature(); + if (signature != null) { os.write(hgpgsig); os.write(' '); - writeGpgSignatureString(getGpgSignature().toExternalString(), os); + writeMultiLineHeader(signature.toExternalString(), os, + true); os.write('\n'); } - if (!References.isSameObject(getEncoding(), UTF_8)) { - os.write(hencoding); - os.write(' '); - os.write(Constants.encodeASCII(getEncoding().name())); - os.write('\n'); - } + writeEncoding(getEncoding(), os); os.write('\n'); @@ -356,58 +263,6 @@ public class CommitBuilder { } /** - * Writes signature to output as per <a href= - * "https://github.com/git/git/blob/master/Documentation/technical/signature-format.txt#L66,L89">gpgsig - * header</a>. - * <p> - * CRLF and CR will be sanitized to LF and signature will have a hanging - * indent of one space starting with line two. A trailing line break is - * <em>not</em> written; the caller is supposed to terminate the GPG - * signature header by writing a single newline. - * </p> - * - * @param in - * signature string with line breaks - * @param out - * output stream - * @throws IOException - * thrown by the output stream - * @throws IllegalArgumentException - * if the signature string contains non 7-bit ASCII chars - */ - static void writeGpgSignatureString(String in, OutputStream out) - throws IOException, IllegalArgumentException { - int length = in.length(); - for (int i = 0; i < length; ++i) { - char ch = in.charAt(i); - switch (ch) { - case '\r': - if (i + 1 < length && in.charAt(i + 1) == '\n') { - ++i; - } - if (i + 1 < length) { - out.write('\n'); - out.write(' '); - } - break; - case '\n': - if (i + 1 < length) { - out.write('\n'); - out.write(' '); - } - break; - default: - // sanity check - if (ch > 127) - throw new IllegalArgumentException(MessageFormat - .format(JGitText.get().notASCIIString, in)); - out.write(ch); - break; - } - } - } - - /** * Format this builder's state as a commit object. * * @return this object in the canonical commit format, suitable for storage @@ -439,7 +294,7 @@ public class CommitBuilder { } r.append("author "); - r.append(author != null ? author.toString() : "NOT_SET"); + r.append(getAuthor() != null ? getAuthor().toString() : "NOT_SET"); r.append("\n"); r.append("committer "); @@ -447,17 +302,20 @@ public class CommitBuilder { r.append("\n"); r.append("gpgSignature "); - r.append(gpgSignature != null ? gpgSignature.toString() : "NOT_SET"); + GpgSignature signature = getGpgSignature(); + r.append(signature != null ? signature.toString() + : "NOT_SET"); r.append("\n"); - if (encoding != null && !References.isSameObject(encoding, UTF_8)) { + Charset encoding = getEncoding(); + if (!References.isSameObject(encoding, UTF_8)) { r.append("encoding "); r.append(encoding.name()); r.append("\n"); } r.append("\n"); - r.append(message != null ? message : ""); + r.append(getMessage() != null ? getMessage() : ""); r.append("}"); return r.toString(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 4fcf8e2dcd..03c1ef904c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -93,13 +93,27 @@ public final class ConfigConstants { public static final String CONFIG_GPG_SECTION = "gpg"; /** + * The "protocol" section + * @since 5.9 + */ + public static final String CONFIG_PROTOCOL_SECTION = "protocol"; + + /** * The "format" key * @since 5.2 */ public static final String CONFIG_KEY_FORMAT = "format"; /** + * The "program" key + * + * @since 5.11 + */ + public static final String CONFIG_KEY_PROGRAM = "program"; + + /** * The "signingKey" key + * * @since 5.2 */ public static final String CONFIG_KEY_SIGNINGKEY = "signingKey"; @@ -111,13 +125,29 @@ public final class ConfigConstants { public static final String CONFIG_COMMIT_SECTION = "commit"; /** + * The "tag" section + * + * @since 5.11 + */ + public static final String CONFIG_TAG_SECTION = "tag"; + + /** * The "gpgSign" key + * * @since 5.2 */ public static final String CONFIG_KEY_GPGSIGN = "gpgSign"; /** + * The "forceSignAnnotated" key + * + * @since 5.11 + */ + public static final String CONFIG_KEY_FORCE_SIGN_ANNOTATED = "forceSignAnnotated"; + + /** * The "hooksPath" key. + * * @since 5.6 */ public static final String CONFIG_KEY_HOOKS_PATH = "hooksPath"; @@ -532,12 +562,6 @@ public final class ConfigConstants { public static final String CONFIG_REF_STORAGE_REFTABLE = "reftable"; /** - * The "reftree" refStorage format - * @since 5.7 - */ - public static final String CONFIG_REFSTORAGE_REFTREE = "reftree"; - - /** * The "jmx" section * @since 5.1.13 */ @@ -685,10 +709,23 @@ public final class ConfigConstants { public static final String CONFIG_INDEX_SECTION = "index"; /** - * The "index.version" key + * The "version" key * * @since 5.9 */ public static final String CONFIG_KEY_VERSION = "version"; + /** + * The "init" section + * + * @since 5.11 + */ + public static final String CONFIG_INIT_SECTION = "init"; + + /** + * The "defaultBranch" key + * + * @since 5.11 + */ + public static final String CONFIG_KEY_DEFAULT_BRANCH = "defaultbranch"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index c1527bc47b..427a235f3b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, Salesforce. and others + * Copyright (C) 2018, 2021 Salesforce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -43,16 +43,65 @@ public class GpgConfig { } } - private final Config config; + private final GpgFormat keyFormat; + + private final String signingKey; + + private final String program; + + private final boolean signCommits; + + private final boolean signAllTags; + + private final boolean forceAnnotated; + + /** + * Create a {@link GpgConfig} with the given parameters and default + * {@code true} for signing commits and {@code false} for tags. + * + * @param keySpec + * to use + * @param format + * to use + * @param gpgProgram + * to use + * @since 5.11 + */ + public GpgConfig(String keySpec, GpgFormat format, String gpgProgram) { + keyFormat = format; + signingKey = keySpec; + program = gpgProgram; + signCommits = true; + signAllTags = false; + forceAnnotated = false; + } /** - * Create a new GPG config, which will read configuration from config. + * Create a new GPG config that reads the configuration from config. * * @param config * the config to read from */ public GpgConfig(Config config) { - this.config = config; + keyFormat = config.getEnum(GpgFormat.values(), + ConfigConstants.CONFIG_GPG_SECTION, null, + ConfigConstants.CONFIG_KEY_FORMAT, GpgFormat.OPENPGP); + signingKey = config.getString(ConfigConstants.CONFIG_USER_SECTION, null, + ConfigConstants.CONFIG_KEY_SIGNINGKEY); + + String exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, + keyFormat.toConfigValue(), ConfigConstants.CONFIG_KEY_PROGRAM); + if (exe == null) { + exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, null, + ConfigConstants.CONFIG_KEY_PROGRAM); + } + program = exe; + signCommits = config.getBoolean(ConfigConstants.CONFIG_COMMIT_SECTION, + ConfigConstants.CONFIG_KEY_GPGSIGN, false); + signAllTags = config.getBoolean(ConfigConstants.CONFIG_TAG_SECTION, + ConfigConstants.CONFIG_KEY_GPGSIGN, false); + forceAnnotated = config.getBoolean(ConfigConstants.CONFIG_TAG_SECTION, + ConfigConstants.CONFIG_KEY_FORCE_SIGN_ANNOTATED, false); } /** @@ -61,9 +110,19 @@ public class GpgConfig { * @return the {@link org.eclipse.jgit.lib.GpgConfig.GpgFormat} */ public GpgFormat getKeyFormat() { - return config.getEnum(GpgFormat.values(), - ConfigConstants.CONFIG_GPG_SECTION, null, - ConfigConstants.CONFIG_KEY_FORMAT, GpgFormat.OPENPGP); + return keyFormat; + } + + /** + * Retrieves the value of the configured GPG program to use, as defined by + * gpg.openpgp.program, gpg.x509.program (depending on the defined + * {@link #getKeyFormat() format}), or gpg.program. + * + * @return the program string configured, or {@code null} if none + * @since 5.11 + */ + public String getProgram() { + return program; } /** @@ -72,8 +131,7 @@ public class GpgConfig { * @return the value of user.signingKey (may be <code>null</code>) */ public String getSigningKey() { - return config.getString(ConfigConstants.CONFIG_USER_SECTION, null, - ConfigConstants.CONFIG_KEY_SIGNINGKEY); + return signingKey; } /** @@ -82,7 +140,29 @@ public class GpgConfig { * @return the value of commit.gpgSign (defaults to <code>false</code>) */ public boolean isSignCommits() { - return config.getBoolean(ConfigConstants.CONFIG_COMMIT_SECTION, - ConfigConstants.CONFIG_KEY_GPGSIGN, false); + return signCommits; + } + + /** + * Retrieves the value of git config {@code tag.gpgSign}. + * + * @return the value of {@code tag.gpgSign}; by default {@code false} + * + * @since 5.11 + */ + public boolean isSignAllTags() { + return signAllTags; + } + + /** + * Retrieves the value of git config {@code tag.forceSignAnnotated}. + * + * @return the value of {@code tag.forceSignAnnotated}; by default + * {@code false} + * + * @since 5.11 + */ + public boolean isSignAnnotated() { + return forceAnnotated; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java new file mode 100644 index 0000000000..074f46567b --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java @@ -0,0 +1,95 @@ +/* + * Copyright (C) 2020 Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.CanceledException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; +import org.eclipse.jgit.transport.CredentialsProvider; + +/** + * Creates GPG signatures for Git objects. + * + * @since 5.11 + */ +public interface GpgObjectSigner { + + /** + * Signs the specified object. + * + * <p> + * Implementors should obtain the payload for signing from the specified + * object via {@link ObjectBuilder#build()} and create a proper + * {@link GpgSignature}. The generated signature must be set on the + * specified {@code object} (see + * {@link ObjectBuilder#setGpgSignature(GpgSignature)}). + * </p> + * <p> + * Any existing signature on the object must be discarded prior obtaining + * the payload via {@link ObjectBuilder#build()}. + * </p> + * + * @param object + * the object to sign (must not be {@code null} and must be + * complete to allow proper calculation of payload) + * @param gpgSigningKey + * the signing key to locate (passed as is to the GPG signing + * tool as is; eg., value of <code>user.signingkey</code>) + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @param config + * GPG settings from the git config + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + void signObject(@NonNull ObjectBuilder object, + @Nullable String gpgSigningKey, @NonNull PersonIdent committer, + CredentialsProvider credentialsProvider, GpgConfig config) + throws CanceledException, UnsupportedSigningFormatException; + + /** + * Indicates if a signing key is available for the specified committer + * and/or signing key. + * + * @param gpgSigningKey + * the signing key to locate (passed as is to the GPG signing + * tool as is; eg., value of <code>user.signingkey</code>) + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @param config + * GPG settings from the git config + * @return <code>true</code> if a signing key is available, + * <code>false</code> otherwise + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + public abstract boolean canLocateSigningKey(@Nullable String gpgSigningKey, + @NonNull PersonIdent committer, + CredentialsProvider credentialsProvider, GpgConfig config) + throws CanceledException, UnsupportedSigningFormatException; + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java new file mode 100644 index 0000000000..a7a39c998f --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.util.Date; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.revwalk.RevObject; + +/** + * A {@code GpgVerifier} can verify GPG signatures on git commits and tags. + * + * @since 5.11 + */ +public interface GpgSignatureVerifier { + + /** + * Verifies the signature on a signed commit or tag. + * + * @param object + * to verify + * @param config + * the {@link GpgConfig} to use + * @return a {@link SignatureVerification} describing the outcome of the + * verification, or {@code null} if the object was not signed + * @throws IOException + * if an error occurs getting a public key + * @throws org.eclipse.jgit.api.errors.JGitInternalException + * if signature verification fails + */ + @Nullable + SignatureVerification verifySignature(@NonNull RevObject object, + @NonNull GpgConfig config) throws IOException; + + + /** + * Verifies a given signature for given data. + * + * @param data + * the signature is for + * @param signatureData + * the ASCII-armored signature + * @return a {@link SignatureVerification} describing the outcome + * @throws IOException + * if the signature cannot be parsed + * @throws JGitInternalException + * if signature verification fails + */ + public SignatureVerification verify(byte[] data, byte[] signatureData) + throws IOException; + + /** + * Retrieves the name of this verifier. This should be a short string + * identifying the engine that verified the signature, like "gpg" if GPG is + * used, or "bc" for a BouncyCastle implementation. + * + * @return the name + */ + @NonNull + String getName(); + + /** + * A {@link GpgSignatureVerifier} may cache public keys to speed up + * verifying signatures on multiple objects. This clears this cache, if any. + */ + void clear(); + + /** + * A {@code SignatureVerification} returns data about a (positively or + * negatively) verified signature. + */ + interface SignatureVerification { + + // Data about the signature. + + @NonNull + Date getCreationDate(); + + // Data from the signature used to find a public key. + + /** + * Obtains the signer as stored in the signature, if known. + * + * @return the signer, or {@code null} if unknown + */ + String getSigner(); + + /** + * Obtains the short or long fingerprint of the public key as stored in + * the signature, if known. + * + * @return the fingerprint, or {@code null} if unknown + */ + String getKeyFingerprint(); + + // Some information about the found public key. + + /** + * Obtains the OpenPGP user ID associated with the key. + * + * @return the user id, or {@code null} if unknown + */ + String getKeyUser(); + + /** + * Tells whether the public key used for this signature verification was + * expired when the signature was created. + * + * @return {@code true} if the key was expired already, {@code false} + * otherwise + */ + boolean isExpired(); + + /** + * Obtains the trust level of the public key used to verify the + * signature. + * + * @return the trust level + */ + @NonNull + TrustLevel getTrustLevel(); + + // The verification result. + + /** + * Tells whether the signature verification was successful. + * + * @return {@code true} if the signature was verified successfully; + * {@code false} if not. + */ + boolean getVerified(); + + /** + * Obtains a human-readable message giving additional information about + * the outcome of the verification. + * + * @return the message, or {@code null} if none set. + */ + String getMessage(); + } + + /** + * The owner's trust in a public key. + */ + enum TrustLevel { + UNKNOWN, NEVER, MARGINAL, FULL, ULTIMATE + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java new file mode 100644 index 0000000000..4b1dbedeb1 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.util.Iterator; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A {@code GpgSignatureVerifierFactory} creates {@link GpgSignatureVerifier} instances. + * + * @since 5.11 + */ +public abstract class GpgSignatureVerifierFactory { + + private static final Logger LOG = LoggerFactory + .getLogger(GpgSignatureVerifierFactory.class); + + private static volatile GpgSignatureVerifierFactory defaultFactory = loadDefault(); + + private static GpgSignatureVerifierFactory loadDefault() { + try { + ServiceLoader<GpgSignatureVerifierFactory> loader = ServiceLoader + .load(GpgSignatureVerifierFactory.class); + Iterator<GpgSignatureVerifierFactory> iter = loader.iterator(); + if (iter.hasNext()) { + return iter.next(); + } + } catch (ServiceConfigurationError e) { + LOG.error(e.getMessage(), e); + } + return null; + } + + /** + * Retrieves the default factory. + * + * @return the default factory or {@code null} if none set + */ + public static GpgSignatureVerifierFactory getDefault() { + return defaultFactory; + } + + /** + * Sets the default factory. + * + * @param factory + * the new default factory + */ + public static void setDefault(GpgSignatureVerifierFactory factory) { + defaultFactory = factory; + } + + /** + * Creates a new {@link GpgSignatureVerifier}. + * + * @return the new {@link GpgSignatureVerifier} + */ + public abstract GpgSignatureVerifier getVerifier(); + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectBuilder.java new file mode 100644 index 0000000000..4b7054f72b --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectBuilder.java @@ -0,0 +1,225 @@ +/* + * Copyright (C) 2020, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Objects; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.util.References; + +/** + * Common base class for {@link CommitBuilder} and {@link TagBuilder}. + * + * @since 5.11 + */ +public abstract class ObjectBuilder { + + /** Byte representation of "encoding". */ + private static final byte[] hencoding = Constants.encodeASCII("encoding"); //$NON-NLS-1$ + + private PersonIdent author; + + private GpgSignature gpgSignature; + + private String message; + + private Charset encoding = StandardCharsets.UTF_8; + + /** + * Retrieves the author of this object. + * + * @return the author of this object, or {@code null} if not set yet + */ + protected PersonIdent getAuthor() { + return author; + } + + /** + * Sets the author (name, email address, and date) of this object. + * + * @param newAuthor + * the new author, must be non-{@code null} + */ + protected void setAuthor(PersonIdent newAuthor) { + author = Objects.requireNonNull(newAuthor); + } + + /** + * Sets the GPG signature of this object. + * <p> + * Note, the signature set here will change the payload of the object, i.e. + * the output of {@link #build()} will include the signature. Thus, the + * typical flow will be: + * <ol> + * <li>call {@link #build()} without a signature set to obtain payload</li> + * <li>create {@link GpgSignature} from payload</li> + * <li>set {@link GpgSignature}</li> + * </ol> + * </p> + * + * @param gpgSignature + * the signature to set or {@code null} to unset + * @since 5.3 + */ + public void setGpgSignature(@Nullable GpgSignature gpgSignature) { + this.gpgSignature = gpgSignature; + } + + /** + * Retrieves the GPG signature of this object. + * + * @return the GPG signature of this object, or {@code null} if the object + * is not signed + * @since 5.3 + */ + @Nullable + public GpgSignature getGpgSignature() { + return gpgSignature; + } + + /** + * Retrieves the complete message of the object. + * + * @return the complete message; can be {@code null}. + */ + @Nullable + public String getMessage() { + return message; + } + + /** + * Sets the message (commit message, or message of an annotated tag). + * + * @param message + * the message. + */ + public void setMessage(@Nullable String message) { + this.message = message; + } + + /** + * Retrieves the encoding that should be used for the message text. + * + * @return the encoding that should be used for the message text. + */ + @NonNull + public Charset getEncoding() { + return encoding; + } + + /** + * Sets the encoding for the object message. + * + * @param encoding + * the encoding to use. + */ + public void setEncoding(@NonNull Charset encoding) { + this.encoding = encoding; + } + + /** + * Format this builder's state as a git object. + * + * @return this object in the canonical git format, suitable for storage in + * a repository. + * @throws java.io.UnsupportedEncodingException + * the encoding specified by {@link #getEncoding()} is not + * supported by this Java runtime. + */ + @NonNull + public abstract byte[] build() throws UnsupportedEncodingException; + + /** + * Writes signature to output as per <a href= + * "https://github.com/git/git/blob/master/Documentation/technical/signature-format.txt#L66,L89">gpgsig + * header</a>. + * <p> + * CRLF and CR will be sanitized to LF and signature will have a hanging + * indent of one space starting with line two. A trailing line break is + * <em>not</em> written; the caller is supposed to terminate the GPG + * signature header by writing a single newline. + * </p> + * + * @param in + * signature string with line breaks + * @param out + * output stream + * @param enforceAscii + * whether to throw {@link IllegalArgumentException} if non-ASCII + * characters are encountered + * @throws IOException + * thrown by the output stream + * @throws IllegalArgumentException + * if the signature string contains non 7-bit ASCII chars and + * {@code enforceAscii == true} + */ + static void writeMultiLineHeader(@NonNull String in, + @NonNull OutputStream out, boolean enforceAscii) + throws IOException, IllegalArgumentException { + int length = in.length(); + for (int i = 0; i < length; ++i) { + char ch = in.charAt(i); + switch (ch) { + case '\r': + if (i + 1 < length && in.charAt(i + 1) == '\n') { + ++i; + } + if (i + 1 < length) { + out.write('\n'); + out.write(' '); + } + break; + case '\n': + if (i + 1 < length) { + out.write('\n'); + out.write(' '); + } + break; + default: + // sanity check + if (ch > 127 && enforceAscii) + throw new IllegalArgumentException(MessageFormat + .format(JGitText.get().notASCIIString, in)); + out.write(ch); + break; + } + } + } + + /** + * Writes an "encoding" header. + * + * @param encoding + * to write + * @param out + * to write to + * @throws IOException + * if writing fails + */ + static void writeEncoding(@NonNull Charset encoding, + @NonNull OutputStream out) throws IOException { + if (!References.isSameObject(encoding, UTF_8)) { + out.write(hencoding); + out.write(' '); + out.write(Constants.encodeASCII(encoding.name())); + out.write('\n'); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java index 6bb6ae590a..718ed89142 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java @@ -17,9 +17,18 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.revwalk.BitmappedObjectReachabilityChecker; +import org.eclipse.jgit.internal.revwalk.BitmappedReachabilityChecker; +import org.eclipse.jgit.internal.revwalk.PedestrianObjectReachabilityChecker; +import org.eclipse.jgit.internal.revwalk.PedestrianReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectReachabilityChecker; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.ReachabilityChecker; +import org.eclipse.jgit.revwalk.RevWalk; /** * Reads an {@link org.eclipse.jgit.lib.ObjectDatabase} for a single thread. @@ -408,6 +417,54 @@ public abstract class ObjectReader implements AutoCloseable { } /** + * Create a reachability checker that will use bitmaps if possible. + * + * @param rw + * revwalk for use by the reachability checker + * @return the most efficient reachability checker for this repository. + * @throws IOException + * if it cannot open any of the underlying indices. + * + * @since 5.11 + */ + @NonNull + public ReachabilityChecker createReachabilityChecker(RevWalk rw) + throws IOException { + if (getBitmapIndex() != null) { + return new BitmappedReachabilityChecker(rw); + } + + return new PedestrianReachabilityChecker(true, rw); + } + + /** + * Create an object reachability checker that will use bitmaps if possible. + * + * This reachability checker accepts any object as target. For checks + * exclusively between commits, use + * {@link #createReachabilityChecker(RevWalk)}. + * + * @param ow + * objectwalk for use by the reachability checker + * @return the most efficient object reachability checker for this + * repository. + * + * @throws IOException + * if it cannot open any of the underlying indices. + * + * @since 5.11 + */ + @NonNull + public ObjectReachabilityChecker createObjectReachabilityChecker( + ObjectWalk ow) throws IOException { + if (getBitmapIndex() != null) { + return new BitmappedObjectReachabilityChecker(ow); + } + + return new PedestrianObjectReachabilityChecker(ow); + } + + /** * Get the {@link org.eclipse.jgit.lib.ObjectInserter} from which this * reader was created using {@code inserter.newReader()} * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 6832c9cd80..7b7bdebac8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -21,6 +21,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; @@ -414,6 +417,31 @@ public abstract class RefDatabase { } /** + * Returns refs whose names start with a given prefix excluding all refs that + * start with one of the given prefixes. + * + * <p> + * The default implementation is not efficient. Implementors of {@link RefDatabase} + * should override this method directly if a better implementation is possible. + * + * @param include string that names of refs should start with; may be empty. + * @param excludes strings that names of refs can't start with; may be empty. + * @return immutable list of refs whose names start with {@code prefix} and none + * of the strings in {@code exclude}. + * @throws java.io.IOException the reference space cannot be accessed. + * @since 5.11 + */ + @NonNull + public List<Ref> getRefsByPrefixWithExclusions(String include, Set<String> excludes) + throws IOException { + Stream<Ref> refs = getRefs(include).values().stream(); + for(String exclude: excludes) { + refs = refs.filter(r -> !r.getName().startsWith(exclude)); + } + return Collections.unmodifiableList(refs.collect(Collectors.toList())); + } + + /** * Returns refs whose names start with one of the given prefixes. * <p> * The default implementation uses {@link #getRefsByPrefix(String)}. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index a7a832c1aa..1e8a6c9175 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -127,6 +127,8 @@ public abstract class Repository implements AutoCloseable { /** If not bare, the index file caching the working file states. */ private final File indexFile; + private final String initialBranch; + /** * Initialize a new repository instance. * @@ -138,6 +140,7 @@ public abstract class Repository implements AutoCloseable { fs = options.getFS(); workTree = options.getWorkTree(); indexFile = options.getIndexFile(); + initialBranch = options.getInitialBranch(); } /** @@ -1034,6 +1037,16 @@ public abstract class Repository implements AutoCloseable { } /** + * Get the initial branch name of a new repository + * + * @return the initial branch name of a new repository + * @since 5.11 + */ + protected @NonNull String getInitialBranch() { + return initialBranch; + } + + /** * Objects known to exist but not expressed by {@link #getAllRefs()}. * <p> * When a repository borrows objects from another repository, it can diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java index d36ccd5bea..41f291b57b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java @@ -14,6 +14,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -193,7 +194,7 @@ public class RepositoryCache { cache.configureEviction(repositoryCacheConfig); } - private final ConcurrentHashMap<Key, Repository> cacheMap; + private final Map<Key, Repository> cacheMap; private final Lock[] openLocks; @@ -201,6 +202,8 @@ public class RepositoryCache { private volatile long expireAfter; + private final Object schedulerLock = new Lock(); + private RepositoryCache() { cacheMap = new ConcurrentHashMap<>(); openLocks = new Lock[4]; @@ -214,7 +217,7 @@ public class RepositoryCache { RepositoryCacheConfig repositoryCacheConfig) { expireAfter = repositoryCacheConfig.getExpireAfter(); ScheduledThreadPoolExecutor scheduler = WorkQueue.getExecutor(); - synchronized (scheduler) { + synchronized (schedulerLock) { if (cleanupTask != null) { cleanupTask.cancel(false); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java index 71f01150c9..facb4a54be 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2006-2008, Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2006, 2008, Robin Rosenberg <robin.rosenberg@dewire.com> * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> - * Copyright (C) 2010, Chris Aniszczyk <caniszczyk@gmail.com> and others + * Copyright (C) 2010, 2020, Chris Aniszczyk <caniszczyk@gmail.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -17,8 +17,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.Charset; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.util.References; /** * Mutable builder to construct an annotated tag recording a project state. @@ -30,17 +35,22 @@ import org.eclipse.jgit.revwalk.RevObject; * and obtain a {@link org.eclipse.jgit.revwalk.RevTag} instance by calling * {@link org.eclipse.jgit.revwalk.RevWalk#parseTag(AnyObjectId)}. */ -public class TagBuilder { +public class TagBuilder extends ObjectBuilder { + + private static final byte[] hobject = Constants.encodeASCII("object"); //$NON-NLS-1$ + + private static final byte[] htype = Constants.encodeASCII("type"); //$NON-NLS-1$ + + private static final byte[] htag = Constants.encodeASCII("tag"); //$NON-NLS-1$ + + private static final byte[] htagger = Constants.encodeASCII("tagger"); //$NON-NLS-1$ + private ObjectId object; private int type = Constants.OBJ_BAD; private String tag; - private PersonIdent tagger; - - private String message; - /** * Get the type of object this tag refers to. * @@ -109,7 +119,7 @@ public class TagBuilder { * @return creator of this tag. May be null. */ public PersonIdent getTagger() { - return tagger; + return getAuthor(); } /** @@ -119,26 +129,7 @@ public class TagBuilder { * the creator. May be null. */ public void setTagger(PersonIdent taggerIdent) { - tagger = taggerIdent; - } - - /** - * Get the complete commit message. - * - * @return the complete commit message. - */ - public String getMessage() { - return message; - } - - /** - * Set the tag's message. - * - * @param newMessage - * the tag's message. - */ - public void setMessage(String newMessage) { - message = newMessage; + setAuthor(taggerIdent); } /** @@ -147,31 +138,65 @@ public class TagBuilder { * @return this object in the canonical annotated tag format, suitable for * storage in a repository. */ - public byte[] build() { + @Override + public byte[] build() throws UnsupportedEncodingException { ByteArrayOutputStream os = new ByteArrayOutputStream(); try (OutputStreamWriter w = new OutputStreamWriter(os, - UTF_8)) { - w.write("object "); //$NON-NLS-1$ - getObjectId().copyTo(w); - w.write('\n'); + getEncoding())) { - w.write("type "); //$NON-NLS-1$ - w.write(Constants.typeString(getObjectType())); - w.write("\n"); //$NON-NLS-1$ + os.write(hobject); + os.write(' '); + getObjectId().copyTo(os); + os.write('\n'); - w.write("tag "); //$NON-NLS-1$ + os.write(htype); + os.write(' '); + os.write(Constants + .encodeASCII(Constants.typeString(getObjectType()))); + os.write('\n'); + + os.write(htag); + os.write(' '); w.write(getTag()); - w.write("\n"); //$NON-NLS-1$ + w.flush(); + os.write('\n'); if (getTagger() != null) { - w.write("tagger "); //$NON-NLS-1$ + os.write(htagger); + os.write(' '); w.write(getTagger().toExternalString()); - w.write('\n'); + w.flush(); + os.write('\n'); + } + + writeEncoding(getEncoding(), os); + + os.write('\n'); + String msg = getMessage(); + if (msg != null) { + w.write(msg); + w.flush(); } - w.write('\n'); - if (getMessage() != null) - w.write(getMessage()); + GpgSignature signature = getGpgSignature(); + if (signature != null) { + if (msg != null && !msg.isEmpty() && !msg.endsWith("\n")) { //$NON-NLS-1$ + // If signed, the message *must* end with a linefeed + // character, otherwise signature verification will fail. + // (The signature will have been computed over the payload + // containing the message without LF, but will be verified + // against a payload with the LF.) The signature must start + // on a new line. + throw new JGitInternalException( + JGitText.get().signedTagMessageNoLf); + } + String externalForm = signature.toExternalString(); + w.write(externalForm); + w.flush(); + if (!externalForm.endsWith("\n")) { //$NON-NLS-1$ + os.write('\n'); + } + } } catch (IOException err) { // This should never occur, the only way to get it above is // for the ByteArrayOutputStream to throw, but it doesn't. @@ -185,10 +210,17 @@ public class TagBuilder { * Format this builder's state as an annotated tag object. * * @return this object in the canonical annotated tag format, suitable for - * storage in a repository. + * storage in a repository, or {@code null} if the tag cannot be + * encoded + * @deprecated since 5.11; use {@link #build()} instead */ + @Deprecated public byte[] toByteArray() { - return build(); + try { + return build(); + } catch (UnsupportedEncodingException e) { + return null; + } } /** {@inheritDoc} */ @@ -211,14 +243,23 @@ public class TagBuilder { r.append(tag != null ? tag : "NOT_SET"); r.append("\n"); - if (tagger != null) { + if (getTagger() != null) { r.append("tagger "); - r.append(tagger); + r.append(getTagger()); + r.append("\n"); + } + + Charset encoding = getEncoding(); + if (!References.isSameObject(encoding, UTF_8)) { + r.append("encoding "); + r.append(encoding.name()); r.append("\n"); } r.append("\n"); - r.append(message != null ? message : ""); + r.append(getMessage() != null ? getMessage() : ""); + GpgSignature signature = getGpgSignature(); + r.append(signature != null ? signature.toExternalString() : ""); r.append("}"); return r.toString(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index 6c217fdf25..4bfb38d286 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -703,18 +703,21 @@ public class ResolveMerger extends ThreeWayMerger { // conflict between ours and theirs. file/folder conflicts between // base/index/workingTree and something else are not relevant or // detected later - if (nonTree(modeO) && !nonTree(modeT)) { - if (nonTree(modeB)) - add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); - add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); - unmergedPaths.add(tw.getPathString()); - enterSubtree = false; - return true; - } - if (nonTree(modeT) && !nonTree(modeO)) { + if (nonTree(modeO) != nonTree(modeT)) { + if (ignoreConflicts) { + // In case of merge failures, ignore this path instead of reporting unmerged, so + // a caller can use virtual commit. This will not result in files with conflict + // markers in the index/working tree. The actual diff on the path will be + // computed directly on children. + enterSubtree = false; + return true; + } if (nonTree(modeB)) add(tw.getRawPath(), base, DirCacheEntry.STAGE_1, EPOCH, 0); - add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, 0); + if (nonTree(modeO)) + add(tw.getRawPath(), ours, DirCacheEntry.STAGE_2, EPOCH, 0); + if (nonTree(modeT)) + add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_3, EPOCH, 0); unmergedPaths.add(tw.getPathString()); enterSubtree = false; return true; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java b/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java index d7dd3bee52..881873de6f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/nls/NLS.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.nls; import java.util.Locale; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.eclipse.jgit.errors.TranslationBundleLoadingException; @@ -110,7 +111,8 @@ public class NLS { } private final Locale locale; - private final ConcurrentHashMap<Class, TranslationBundle> map = new ConcurrentHashMap<>(); + + private final Map<Class, TranslationBundle> map = new ConcurrentHashMap<>(); private NLS(Locale locale) { this.locale = locale; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java index b875be9270..0cabf07057 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/DateRevQueue.java @@ -93,7 +93,7 @@ public class DateRevQueue extends AbstractRevQueue { head = n; } else { Entry p = q.next; - while (p != null && p.commit.commitTime > when) { + while (p != null && p.commit.commitTime >= when) { q = p; p = q.next; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java index 4c7a6f556e..e6f9580bf7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java @@ -172,14 +172,14 @@ public class ObjectWalk extends RevWalk { * when the index fails to load. * * @since 5.8 + * @deprecated use + * {@code ObjectReader#createObjectReachabilityChecker(ObjectWalk)} + * instead. */ - public ObjectReachabilityChecker createObjectReachabilityChecker() + @Deprecated + public final ObjectReachabilityChecker createObjectReachabilityChecker() throws IOException { - if (reader.getBitmapIndex() != null) { - return new BitmappedObjectReachabilityChecker(this); - } - - return new PedestrianObjectReachabilityChecker(this); + return reader.createObjectReachabilityChecker(this); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java index cac257199f..b9d145008b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2009, Google Inc. + * Copyright (C) 2008, 2009, Google Inc. * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2021, Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -18,7 +18,9 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; +import java.util.Arrays; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; @@ -35,6 +37,10 @@ import org.eclipse.jgit.util.StringUtils; * An annotated tag. */ public class RevTag extends RevObject { + + private static final byte[] hSignature = Constants + .encodeASCII("-----BEGIN PGP SIGNATURE-----"); //$NON-NLS-1$ + /** * Parse an annotated tag from its canonical format. * @@ -171,6 +177,71 @@ public class RevTag extends RevObject { return RawParseUtils.parsePersonIdent(raw, nameB); } + private static int nextStart(byte[] prefix, byte[] buffer, int from) { + int stop = buffer.length - prefix.length + 1; + int ptr = from; + if (ptr > 0) { + ptr = RawParseUtils.nextLF(buffer, ptr - 1); + } + while (ptr < stop) { + int lineStart = ptr; + boolean found = true; + for (byte element : prefix) { + if (element != buffer[ptr++]) { + found = false; + break; + } + } + if (found) { + return lineStart; + } + do { + ptr = RawParseUtils.nextLF(buffer, ptr); + } while (ptr < stop && buffer[ptr] == '\n'); + } + return -1; + } + + private int getSignatureStart() { + byte[] raw = buffer; + int msgB = RawParseUtils.tagMessage(raw, 0); + if (msgB < 0) { + return msgB; + } + // Find the last signature start and return the rest + int start = nextStart(hSignature, raw, msgB); + if (start < 0) { + return start; + } + int next = RawParseUtils.nextLF(raw, start); + while (next < raw.length) { + int newStart = nextStart(hSignature, raw, next); + if (newStart < 0) { + break; + } + start = newStart; + next = RawParseUtils.nextLF(raw, start); + } + return start; + } + + /** + * Parse the GPG signature from the raw buffer. + * + * @return contents of the GPG signature; {@code null} if the tag was not + * signed. + * @since 5.11 + */ + @Nullable + public final byte[] getRawGpgSignature() { + byte[] raw = buffer; + int start = getSignatureStart(); + if (start < 0) { + return null; + } + return Arrays.copyOfRange(raw, start, raw.length); + } + /** * Parse the complete tag message and decode it to a string. * <p> @@ -187,7 +258,12 @@ public class RevTag extends RevObject { if (msgB < 0) { return ""; //$NON-NLS-1$ } - return RawParseUtils.decode(guessEncoding(), raw, msgB, raw.length); + int signatureStart = getSignatureStart(); + int end = signatureStart < 0 ? raw.length : signatureStart; + if (end == msgB) { + return ""; //$NON-NLS-1$ + } + return RawParseUtils.decode(guessEncoding(), raw, msgB, end); } /** @@ -213,6 +289,16 @@ public class RevTag extends RevObject { } int msgE = RawParseUtils.endOfParagraph(raw, msgB); + int signatureStart = getSignatureStart(); + if (signatureStart >= msgB && msgE > signatureStart) { + msgE = signatureStart; + if (msgE > msgB) { + msgE--; + } + if (msgB == msgE) { + return ""; //$NON-NLS-1$ + } + } String str = RawParseUtils.decode(guessEncoding(), raw, msgB, msgE); if (RevCommit.hasLF(raw, msgB, msgE)) { str = StringUtils.replaceLineBreaksWithSpace(str); @@ -258,6 +344,22 @@ public class RevTag extends RevObject { } /** + * Obtain the raw unparsed tag body (<b>NOTE - THIS IS NOT A COPY</b>). + * <p> + * This method is exposed only to provide very fast, efficient access to + * this tag's message buffer. Applications relying on this buffer should be + * very careful to ensure they do not modify its contents during their use + * of it. + * + * @return the raw unparsed tag body. This is <b>NOT A COPY</b>. Do not + * alter the returned array. + * @since 5.11 + */ + public final byte[] getRawBuffer() { + return buffer; + } + + /** * Discard the message buffer to reduce memory usage. * <p> * After discarding the memory usage of the {@code RevTag} is reduced to diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index 6b62fcdf6d..631d861c0d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -236,13 +236,13 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { * if it cannot open any of the underlying indices. * * @since 5.4 + * @deprecated use {@code ObjectReader#createReachabilityChecker(RevWalk)} + * instead. */ - public ReachabilityChecker createReachabilityChecker() throws IOException { - if (reader.getBitmapIndex() != null) { - return new BitmappedReachabilityChecker(this); - } - - return new PedestrianReachabilityChecker(true, this); + @Deprecated + public final ReachabilityChecker createReachabilityChecker() + throws IOException { + return reader.createReachabilityChecker(this); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java index 3a36398629..3826bf7401 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -13,7 +13,12 @@ package org.eclipse.jgit.transport; +import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_LS_REFS; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_1; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2; import java.io.EOFException; import java.io.IOException; @@ -22,23 +27,29 @@ import java.io.OutputStream; import java.text.MessageFormat; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import java.util.Set; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.errors.RemoteRepositoryException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.SymbolicRef; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.io.InterruptTimer; import org.eclipse.jgit.util.io.TimeoutInputStream; import org.eclipse.jgit.util.io.TimeoutOutputStream; @@ -92,17 +103,27 @@ abstract class BasePackConnection extends BaseConnection { protected boolean statelessRPC; /** Capability tokens advertised by the remote side. */ - private final Set<String> remoteCapablities = new HashSet<>(); + private final Map<String, String> remoteCapabilities = new HashMap<>(); /** Extra objects the remote has, but which aren't offered as refs. */ protected final Set<ObjectId> additionalHaves = new HashSet<>(); + private TransferConfig.ProtocolVersion protocol = TransferConfig.ProtocolVersion.V0; + BasePackConnection(PackTransport packTransport) { transport = (Transport) packTransport; local = transport.local; uri = transport.uri; } + TransferConfig.ProtocolVersion getProtocolVersion() { + return protocol; + } + + void setProtocolVersion(@NonNull TransferConfig.ProtocolVersion protocol) { + this.protocol = protocol; + } + /** * Configure this connection with the directional pipes. * @@ -147,12 +168,15 @@ abstract class BasePackConnection extends BaseConnection { * {@link #close()} and the exception is wrapped (if necessary) and thrown * as a {@link org.eclipse.jgit.errors.TransportException}. * + * @return {@code true} if the refs were read; {@code false} otherwise + * indicating that {@link #lsRefs} must be called + * * @throws org.eclipse.jgit.errors.TransportException * the reference list could not be scanned. */ - protected void readAdvertisedRefs() throws TransportException { + protected boolean readAdvertisedRefs() throws TransportException { try { - readAdvertisedRefsImpl(); + return readAdvertisedRefsImpl(); } catch (TransportException err) { close(); throw err; @@ -162,35 +186,79 @@ abstract class BasePackConnection extends BaseConnection { } } - private void readAdvertisedRefsImpl() throws IOException { - final LinkedHashMap<String, Ref> avail = new LinkedHashMap<>(); - for (;;) { - String line; - - try { - line = pckIn.readString(); - } catch (EOFException eof) { - if (avail.isEmpty()) - throw noRepository(); - throw eof; - } - if (PacketLineIn.isEnd(line)) - break; + private String readLine() throws IOException { + String line = pckIn.readString(); + if (PacketLineIn.isEnd(line)) { + return null; + } + if (line.startsWith("ERR ")) { //$NON-NLS-1$ + // This is a customized remote service error. + // Users should be informed about it. + throw new RemoteRepositoryException(uri, line.substring(4)); + } + return line; + } - if (line.startsWith("ERR ")) { //$NON-NLS-1$ - // This is a customized remote service error. - // Users should be informed about it. - throw new RemoteRepositoryException(uri, line.substring(4)); - } + private boolean readAdvertisedRefsImpl() throws IOException { + final Map<String, Ref> avail = new LinkedHashMap<>(); + final Map<String, String> symRefs = new LinkedHashMap<>(); + for (boolean first = true;; first = false) { + String line; - if (avail.isEmpty()) { + if (first) { + boolean isV1 = false; + try { + line = readLine(); + } catch (EOFException e) { + TransportException noRepo = noRepository(); + noRepo.initCause(e); + throw noRepo; + } + if (line != null && VERSION_1.equals(line)) { + // Same as V0, except for this extra line. We shouldn't get + // it since we never request V1. + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + isV1 = true; + line = readLine(); + } + if (line == null) { + break; + } final int nul = line.indexOf('\0'); if (nul >= 0) { - // The first line (if any) may contain "hidden" - // capability values after a NUL byte. - remoteCapablities.addAll( - Arrays.asList(line.substring(nul + 1).split(" "))); //$NON-NLS-1$ + // Protocol V0: The first line (if any) may contain + // "hidden" capability values after a NUL byte. + for (String capability : line.substring(nul + 1) + .split(" ")) { //$NON-NLS-1$ + if (capability.startsWith(CAPABILITY_SYMREF_PREFIX)) { + String[] parts = capability + .substring( + CAPABILITY_SYMREF_PREFIX.length()) + .split(":", 2); //$NON-NLS-1$ + if (parts.length == 2) { + symRefs.put(parts[0], parts[1]); + } + } else { + addCapability(capability); + } + } line = line.substring(0, nul); + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + } else if (!isV1 && VERSION_2.equals(line)) { + // Protocol V2: remaining lines are capabilities as + // key=value pairs + setProtocolVersion(TransferConfig.ProtocolVersion.V2); + readCapabilitiesV2(); + // Break out here so that stateless RPC transports get a + // chance to set up the output stream. + return false; + } else { + setProtocolVersion(TransferConfig.ProtocolVersion.V0); + } + } else { + line = readLine(); + if (line == null) { + break; } } @@ -199,73 +267,214 @@ abstract class BasePackConnection extends BaseConnection { throw invalidRefAdvertisementLine(line); } String name = line.substring(41, line.length()); - if (avail.isEmpty() && name.equals("capabilities^{}")) { //$NON-NLS-1$ - // special line from git-receive-pack to show + if (first && name.equals("capabilities^{}")) { //$NON-NLS-1$ + // special line from git-receive-pack (protocol V0) to show // capabilities when there are no refs to advertise continue; } - final ObjectId id; - try { - id = ObjectId.fromString(line.substring(0, 40)); - } catch (InvalidObjectIdException e) { - PackProtocolException ppe = invalidRefAdvertisementLine(line); - ppe.initCause(e); - throw ppe; - } + final ObjectId id = toId(line, line.substring(0, 40)); if (name.equals(".have")) { //$NON-NLS-1$ additionalHaves.add(id); - } else if (name.endsWith("^{}")) { //$NON-NLS-1$ - name = name.substring(0, name.length() - 3); - final Ref prior = avail.get(name); - if (prior == null) - throw new PackProtocolException(uri, MessageFormat.format( - JGitText.get().advertisementCameBefore, name, name)); - - if (prior.getPeeledObjectId() != null) - throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$ - - avail.put(name, new ObjectIdRef.PeeledTag( - Ref.Storage.NETWORK, name, prior.getObjectId(), id)); } else { - final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag( - Ref.Storage.NETWORK, name, id)); - if (prior != null) - throw duplicateAdvertisement(name); + processLineV1(name, id, avail); } } - updateWithSymRefs(avail, extractSymRefsFromCapabilities(remoteCapablities)); + updateWithSymRefs(avail, symRefs); available(avail); + return true; } /** - * Finds values in the given capabilities of the form: - * - * <pre> - * symref=<em>source</em>:<em>target</em> - * </pre> + * Issue a protocol V2 ls-refs command and read its response. * - * And returns a Map of source->target entries. - * - * @param capabilities - * the capabilities lines - * @return a Map of the symref entries from capabilities - * @throws NullPointerException - * if capabilities, or any entry in it, is null + * @param refSpecs + * to produce ref prefixes from if the server supports git + * protocol V2 + * @param additionalPatterns + * to use for ref prefixes if the server supports git protocol V2 + * @throws TransportException + * if the command could not be run or its output not be read */ - static Map<String, String> extractSymRefsFromCapabilities(Collection<String> capabilities) { + protected void lsRefs(Collection<RefSpec> refSpecs, + String... additionalPatterns) throws TransportException { + try { + lsRefsImpl(refSpecs, additionalPatterns); + } catch (TransportException err) { + close(); + throw err; + } catch (IOException | RuntimeException err) { + close(); + throw new TransportException(err.getMessage(), err); + } + } + + private void lsRefsImpl(Collection<RefSpec> refSpecs, + String... additionalPatterns) throws IOException { + pckOut.writeString("command=" + COMMAND_LS_REFS); //$NON-NLS-1$ + // Add the user-agent + String agent = UserAgent.get(); + if (agent != null && isCapableOf(OPTION_AGENT)) { + pckOut.writeString(OPTION_AGENT + '=' + agent); + } + pckOut.writeDelim(); + pckOut.writeString("peel"); //$NON-NLS-1$ + pckOut.writeString("symrefs"); //$NON-NLS-1$ + for (String refPrefix : getRefPrefixes(refSpecs, additionalPatterns)) { + pckOut.writeString("ref-prefix " + refPrefix); //$NON-NLS-1$ + } + pckOut.end(); + final Map<String, Ref> avail = new LinkedHashMap<>(); final Map<String, String> symRefs = new LinkedHashMap<>(); - for (String option : capabilities) { - if (option.startsWith(CAPABILITY_SYMREF_PREFIX)) { - String[] symRef = option - .substring(CAPABILITY_SYMREF_PREFIX.length()) - .split(":", 2); //$NON-NLS-1$ - if (symRef.length == 2) { - symRefs.put(symRef[0], symRef[1]); + for (;;) { + String line = readLine(); + if (line == null) { + break; + } + // Expecting to get a line in the form "sha1 refname" + if (line.length() < 41 || line.charAt(40) != ' ') { + throw invalidRefAdvertisementLine(line); + } + String name = line.substring(41, line.length()); + final ObjectId id = toId(line, line.substring(0, 40)); + if (name.equals(".have")) { //$NON-NLS-1$ + additionalHaves.add(id); + } else { + processLineV2(line, id, name, avail, symRefs); + } + } + updateWithSymRefs(avail, symRefs); + available(avail); + } + + private Collection<String> getRefPrefixes(Collection<RefSpec> refSpecs, + String... additionalPatterns) { + if (refSpecs.isEmpty() && (additionalPatterns == null + || additionalPatterns.length == 0)) { + return Collections.emptyList(); + } + Set<String> patterns = new HashSet<>(); + if (additionalPatterns != null) { + Arrays.stream(additionalPatterns).filter(Objects::nonNull) + .forEach(patterns::add); + } + for (RefSpec spec : refSpecs) { + // TODO: for now we only do protocol V2 for fetch. For push + // RefSpecs, the logic would need to be different. (At the + // minimum, take spec.getDestination().) + String src = spec.getSource(); + if (ObjectId.isId(src)) { + continue; + } + if (spec.isWildcard()) { + patterns.add(src.substring(0, src.indexOf('*'))); + } else { + patterns.add(src); + patterns.add(Constants.R_REFS + src); + patterns.add(Constants.R_HEADS + src); + patterns.add(Constants.R_TAGS + src); + } + } + return patterns; + } + + private void readCapabilitiesV2() throws IOException { + // In git protocol V2, capabilities are different. If it's a key-value + // pair, the key may be a command name, and the value a space-separated + // list of capabilities for that command. We still store it in the same + // map as for protocol v0/v1. Protocol v2 code has to account for this. + for (;;) { + String line = readLine(); + if (line == null) { + break; + } + addCapability(line); + } + } + + private void addCapability(String capability) { + String parts[] = capability.split("=", 2); //$NON-NLS-1$ + if (parts.length == 2) { + remoteCapabilities.put(parts[0], parts[1]); + } + remoteCapabilities.put(capability, null); + } + + private ObjectId toId(String line, String value) + throws PackProtocolException { + try { + return ObjectId.fromString(value); + } catch (InvalidObjectIdException e) { + PackProtocolException ppe = invalidRefAdvertisementLine(line); + ppe.initCause(e); + throw ppe; + } + } + + private void processLineV1(String name, ObjectId id, Map<String, Ref> avail) + throws IOException { + if (name.endsWith("^{}")) { //$NON-NLS-1$ + name = name.substring(0, name.length() - 3); + final Ref prior = avail.get(name); + if (prior == null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().advertisementCameBefore, name, name)); + } + if (prior.getPeeledObjectId() != null) { + throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$ + } + avail.put(name, new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, + prior.getObjectId(), id)); + } else { + final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag( + Ref.Storage.NETWORK, name, id)); + if (prior != null) { + throw duplicateAdvertisement(name); + } + } + } + + private void processLineV2(String line, ObjectId id, String rest, + Map<String, Ref> avail, Map<String, String> symRefs) + throws IOException { + String[] parts = rest.split(" "); //$NON-NLS-1$ + String name = parts[0]; + // Two attributes possible, symref-target or peeled + String symRefTarget = null; + String peeled = null; + for (int i = 1; i < parts.length; i++) { + if (parts[i].startsWith(REF_ATTR_SYMREF_TARGET)) { + if (symRefTarget != null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().duplicateRefAttribute, line)); + } + symRefTarget = parts[i] + .substring(REF_ATTR_SYMREF_TARGET.length()); + } else if (parts[i].startsWith(REF_ATTR_PEELED)) { + if (peeled != null) { + throw new PackProtocolException(uri, MessageFormat.format( + JGitText.get().duplicateRefAttribute, line)); } + peeled = parts[i].substring(REF_ATTR_PEELED.length()); } + if (peeled != null && symRefTarget != null) { + break; + } + } + Ref idRef; + if (peeled != null) { + idRef = new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, id, + toId(line, peeled)); + } else { + idRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, name, id); + } + Ref prior = avail.put(name, idRef); + if (prior != null) { + throw duplicateAdvertisement(name); + } + if (!StringUtils.isEmptyOrNull(symRefTarget)) { + symRefs.put(name, symRefTarget); } - return symRefs; } /** @@ -334,6 +543,22 @@ abstract class BasePackConnection extends BaseConnection { } } } + // If HEAD is still in the symRefs map here, the real ref was not + // reported, but we know it must point to the object reported for HEAD. + // So fill it in in the refMap. + String headRefName = symRefs.get(Constants.HEAD); + if (headRefName != null && !refMap.containsKey(headRefName)) { + Ref headRef = refMap.get(Constants.HEAD); + if (headRef != null) { + ObjectId headObj = headRef.getObjectId(); + headRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, + headRefName, headObj); + refMap.put(headRefName, headRef); + headRef = new SymbolicRef(Constants.HEAD, headRef); + refMap.put(Constants.HEAD, headRef); + symRefs.remove(Constants.HEAD); + } + } } /** @@ -357,7 +582,7 @@ abstract class BasePackConnection extends BaseConnection { * @return whether this option is supported */ protected boolean isCapableOf(String option) { - return remoteCapablities.contains(option); + return remoteCapabilities.containsKey(option); } /** @@ -378,6 +603,17 @@ abstract class BasePackConnection extends BaseConnection { } /** + * Return a capability value. + * + * @param option + * to get + * @return the value stored, if any. + */ + protected String getCapability(String option) { + return remoteCapabilities.get(option); + } + + /** * Add user agent capability * * @param b @@ -385,7 +621,7 @@ abstract class BasePackConnection extends BaseConnection { */ protected void addUserAgentCapability(StringBuilder b) { String a = UserAgent.get(); - if (a != null && UserAgent.hasAgent(remoteCapablities)) { + if (a != null && remoteCapabilities.get(OPTION_AGENT) != null) { b.append(' ').append(OPTION_AGENT).append('=').append(a); } } @@ -393,7 +629,8 @@ abstract class BasePackConnection extends BaseConnection { /** {@inheritDoc} */ @Override public String getPeerUserAgent() { - return UserAgent.getAgent(remoteCapablities, super.getPeerUserAgent()); + String agent = remoteCapabilities.get(OPTION_AGENT); + return agent != null ? agent : super.getPeerUserAgent(); } private PackProtocolException duplicateAdvertisement(String name) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index a2fb51f46d..d344deac26 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -16,18 +16,21 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Set; import org.eclipse.jgit.errors.PackProtocolException; +import org.eclipse.jgit.errors.RemoteRepositoryException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackLock; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -44,6 +47,7 @@ import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck; import org.eclipse.jgit.transport.PacketLineIn.AckNackResult; +import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.TemporaryBuffer; /** @@ -207,7 +211,10 @@ public abstract class BasePackFetchConnection extends BasePackConnection private int maxHaves; - /** RPC state, if {@link BasePackConnection#statelessRPC} is true. */ + /** + * RPC state, if {@link BasePackConnection#statelessRPC} is true or protocol + * V2 is used. + */ private TemporaryBuffer.Heap state; private PacketLineOut pckState; @@ -321,6 +328,13 @@ public abstract class BasePackFetchConnection extends BasePackConnection return Collections.<PackLock> emptyList(); } + private void clearState() { + walk.dispose(); + reachableCommits = null; + state = null; + pckState = null; + } + /** * Execute common ancestor negotiation and fetch the objects. * @@ -349,18 +363,34 @@ public abstract class BasePackFetchConnection extends BasePackConnection markRefsAdvertised(); markReachable(have, maxTimeWanted(want)); + if (TransferConfig.ProtocolVersion.V2 + .equals(getProtocolVersion())) { + // Protocol V2 always is a "stateless" protocol, even over a + // bidirectional pipe: the server serves one "fetch" request and + // then forgets anything it has learned, so the next fetch + // request has to re-send all wants and previously determined + // common objects as "have"s again. + state = new TemporaryBuffer.Heap(Integer.MAX_VALUE); + pckState = new PacketLineOut(state); + try { + doFetchV2(monitor, want, outputStream); + } finally { + clearState(); + } + return; + } + // Protocol V0/1 if (statelessRPC) { state = new TemporaryBuffer.Heap(Integer.MAX_VALUE); pckState = new PacketLineOut(state); } - - if (sendWants(want)) { + PacketLineOut output = statelessRPC ? pckState : pckOut; + if (sendWants(want, output)) { + output.end(); + outNeedsEnd = false; negotiate(monitor); - walk.dispose(); - reachableCommits = null; - state = null; - pckState = null; + clearState(); receivePack(monitor, outputStream); } @@ -373,6 +403,185 @@ public abstract class BasePackFetchConnection extends BasePackConnection } } + private void doFetchV2(ProgressMonitor monitor, Collection<Ref> want, + OutputStream outputStream) throws IOException, CancelledException { + sideband = true; + negotiateBegin(); + + pckState.writeString("command=" + GitProtocolConstants.COMMAND_FETCH); //$NON-NLS-1$ + // Capabilities are sent as command arguments in protocol V2 + String agent = UserAgent.get(); + if (agent != null && isCapableOf(GitProtocolConstants.OPTION_AGENT)) { + pckState.writeString( + GitProtocolConstants.OPTION_AGENT + '=' + agent); + } + Set<String> capabilities = new HashSet<>(); + String advertised = getCapability(GitProtocolConstants.COMMAND_FETCH); + if (!StringUtils.isEmptyOrNull(advertised)) { + capabilities.addAll(Arrays.asList(advertised.split("\\s+"))); //$NON-NLS-1$ + } + // Arguments + pckState.writeDelim(); + for (String capability : getCapabilitiesV2(capabilities)) { + pckState.writeString(capability); + } + if (!sendWants(want, pckState)) { + // We already have everything we wanted. + return; + } + // If we send something, we always close it properly ourselves. + outNeedsEnd = false; + + FetchStateV2 fetchState = new FetchStateV2(); + boolean sentDone = false; + for (;;) { + // The "state" buffer contains the full fetch request with all + // common objects found so far. + state.writeTo(out, monitor); + sentDone = sendNextHaveBatch(fetchState, pckOut, monitor); + if (sentDone) { + break; + } + if (readAcknowledgments(fetchState, pckIn, monitor)) { + // We got a "ready": next should be a patch file. + break; + } + // Note: C git reads and requires here (and after a packfile) a + // "0002" packet in stateless RPC transports (https). This "response + // end" packet is even mentioned in the protocol V2 technical + // documentation. However, it is not actually part of the public + // protocol; it occurs only in an internal protocol wrapper in the C + // git implementation. + } + clearState(); + String line = pckIn.readString(); + // If we sent a done, we may have an error reply here. + if (sentDone && line.startsWith("ERR ")) { //$NON-NLS-1$ + throw new RemoteRepositoryException(uri, line.substring(4)); + } + // "shallow-info", "wanted-refs", and "packfile-uris" would have to be + // handled here in that order. + if (!GitProtocolConstants.SECTION_PACKFILE.equals(line)) { + throw new PackProtocolException( + MessageFormat.format(JGitText.get().expectedGot, + GitProtocolConstants.SECTION_PACKFILE, line)); + } + receivePack(monitor, outputStream); + } + + /** + * Sends the next batch of "have"s and terminates the {@code output}. + * + * @param fetchState + * is updated with information about the number of items written, + * and whether to expect a packfile next + * @param output + * to write to + * @param monitor + * for progress reporting and cancellation + * @return {@code true} if a "done" was written and we should thus expect a + * packfile next + * @throws IOException + * on errors + * @throws CancelledException + * on cancellation + */ + private boolean sendNextHaveBatch(FetchStateV2 fetchState, + PacketLineOut output, ProgressMonitor monitor) + throws IOException, CancelledException { + long n = 0; + while (n < fetchState.havesToSend) { + final RevCommit c = walk.next(); + if (c == null) { + break; + } + output.writeString("have " + c.getId().name() + '\n'); //$NON-NLS-1$ + n++; + if (n % 10 == 0 && monitor.isCancelled()) { + throw new CancelledException(); + } + } + fetchState.havesTotal += n; + if (n == 0 + || (fetchState.hadAcks + && fetchState.havesWithoutAck > MAX_HAVES) + || fetchState.havesTotal > maxHaves) { + output.writeString("done\n"); //$NON-NLS-1$ + output.end(); + return true; + } + // Increment only after the test above. Of course we have no ACKs yet + // for the newly added "have"s, so it makes no sense to count them + // against the MAX_HAVES limit. + fetchState.havesWithoutAck += n; + output.end(); + fetchState.incHavesToSend(statelessRPC); + return false; + } + + /** + * Reads and processes acknowledgments, adding ACKed objects as "have"s to + * the global state {@link TemporaryBuffer}. + * + * @param fetchState + * to update + * @param input + * to read from + * @param monitor + * for progress reporting and cancellation + * @return {@code true} if a "ready" was received and a packfile is expected + * next + * @throws IOException + * on errors + * @throws CancelledException + * on cancellation + */ + private boolean readAcknowledgments(FetchStateV2 fetchState, + PacketLineIn input, ProgressMonitor monitor) + throws IOException, CancelledException { + String line = input.readString(); + if (!GitProtocolConstants.SECTION_ACKNOWLEDGMENTS.equals(line)) { + throw new PackProtocolException(MessageFormat.format( + JGitText.get().expectedGot, + GitProtocolConstants.SECTION_ACKNOWLEDGMENTS, line)); + } + MutableObjectId returnedId = new MutableObjectId(); + line = input.readString(); + boolean gotReady = false; + long n = 0; + while (!PacketLineIn.isEnd(line) && !PacketLineIn.isDelimiter(line)) { + AckNackResult ack = PacketLineIn.parseACKv2(line, returnedId); + // If we got a "ready", we just skip the remaining lines after + // having checked them for being valid. (Normally, the "ready" + // should be the last line anyway.) + if (!gotReady) { + if (ack == AckNackResult.ACK_COMMON) { + // markCommon appends the object to the "state" + markCommon(walk.parseAny(returnedId), ack, true); + fetchState.havesWithoutAck = 0; + fetchState.hadAcks = true; + } else if (ack == AckNackResult.ACK_READY) { + gotReady = true; + } + } + n++; + if (n % 10 == 0 && monitor.isCancelled()) { + throw new CancelledException(); + } + line = input.readString(); + } + if (gotReady) { + if (!PacketLineIn.isDelimiter(line)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().expectedGot, "0001", line)); //$NON-NLS-1$ + } + } else if (!PacketLineIn.isEnd(line)) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().expectedGot, "0000", line)); //$NON-NLS-1$ + } + return gotReady; + } + /** {@inheritDoc} */ @Override public void close() { @@ -456,8 +665,8 @@ public abstract class BasePackFetchConnection extends BasePackConnection } } - private boolean sendWants(Collection<Ref> want) throws IOException { - final PacketLineOut p = statelessRPC ? pckState : pckOut; + private boolean sendWants(Collection<Ref> want, PacketLineOut p) + throws IOException { boolean first = true; for (Ref r : want) { ObjectId objectId = r.getObjectId(); @@ -479,10 +688,11 @@ public abstract class BasePackFetchConnection extends BasePackConnection final StringBuilder line = new StringBuilder(46); line.append("want "); //$NON-NLS-1$ line.append(objectId.name()); - if (first) { + if (first && TransferConfig.ProtocolVersion.V0 + .equals(getProtocolVersion())) { line.append(enableCapabilities()); - first = false; } + first = false; line.append('\n'); p.writeString(line.toString()); } @@ -492,11 +702,34 @@ public abstract class BasePackFetchConnection extends BasePackConnection if (!filterSpec.isNoOp()) { p.writeString(filterSpec.filterLine()); } - p.end(); - outNeedsEnd = false; return true; } + private Set<String> getCapabilitiesV2(Set<String> advertisedCapabilities) + throws TransportException { + Set<String> capabilities = new LinkedHashSet<>(); + // Protocol V2 is implicitly capable of all these. + if (noProgress) { + capabilities.add(OPTION_NO_PROGRESS); + } + if (includeTags) { + capabilities.add(OPTION_INCLUDE_TAG); + } + if (allowOfsDelta) { + capabilities.add(OPTION_OFS_DELTA); + } + if (thinPack) { + capabilities.add(OPTION_THIN_PACK); + } + if (!filterSpec.isNoOp() + && !advertisedCapabilities.contains(OPTION_FILTER)) { + throw new PackProtocolException(uri, + JGitText.get().filterRequiresCapability); + } + // The FilterSpec will be added later in sendWants(). + return capabilities; + } + private String enableCapabilities() throws TransportException { final StringBuilder line = new StringBuilder(); if (noProgress) @@ -622,7 +855,7 @@ public abstract class BasePackFetchConnection extends BasePackConnection // we need to continue to talk about other parts of // our local history. // - markCommon(walk.parseAny(ackId), anr); + markCommon(walk.parseAny(ackId), anr, statelessRPC); receivedAck = true; receivedContinue = true; havesSinceLastContinue = 0; @@ -757,16 +990,10 @@ public abstract class BasePackFetchConnection extends BasePackConnection } } - private void markCommon(RevObject obj, AckNackResult anr) + private void markCommon(RevObject obj, AckNackResult anr, boolean useState) throws IOException { - if (statelessRPC && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) { - StringBuilder s; - - s = new StringBuilder(6 + Constants.OBJECT_ID_STRING_LENGTH); - s.append("have "); //$NON-NLS-1$ - s.append(obj.name()); - s.append('\n'); - pckState.writeString(s.toString()); + if (useState && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) { + pckState.writeString("have " + obj.name() + '\n'); //$NON-NLS-1$ obj.add(STATE); } obj.add(COMMON); @@ -806,4 +1033,31 @@ public abstract class BasePackFetchConnection extends BasePackConnection private static class CancelledException extends Exception { private static final long serialVersionUID = 1L; } + + private static class FetchStateV2 { + + long havesToSend = 32; + + long havesTotal; + + // Set to true if we got at least one ACK in protocol V2. + boolean hadAcks; + + // Counts haves without ACK. Use as cutoff for negotiation only once + // hadAcks == true. + long havesWithoutAck; + + void incHavesToSend(boolean statelessRPC) { + if (statelessRPC) { + // Increase this quicker since connection setup costs accumulate + if (havesToSend < 16384) { + havesToSend *= 2; + } else { + havesToSend = havesToSend * 11 / 10; + } + } else { + havesToSend += 32; + } + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java index 0f1892a97e..34bad6e029 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchProcess.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -48,6 +48,7 @@ import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.util.StringUtils; class FetchProcess { /** Transport we will fetch over. */ @@ -79,7 +80,8 @@ class FetchProcess { toFetch = f; } - void execute(ProgressMonitor monitor, FetchResult result) + void execute(ProgressMonitor monitor, FetchResult result, + String initialBranch) throws NotSupportedException, TransportException { askFor.clear(); localUpdates.clear(); @@ -87,24 +89,64 @@ class FetchProcess { packLocks.clear(); localRefs = null; + Throwable e1 = null; try { - executeImp(monitor, result); + executeImp(monitor, result, initialBranch); + } catch (NotSupportedException | TransportException err) { + e1 = err; + throw err; } finally { try { - for (PackLock lock : packLocks) - lock.unlock(); + for (PackLock lock : packLocks) { + lock.unlock(); + } } catch (IOException e) { + if (e1 != null) { + e.addSuppressed(e1); + } throw new TransportException(e.getMessage(), e); } } } + private boolean isInitialBranchMissing(Map<String, Ref> refsMap, + String initialBranch) { + if (StringUtils.isEmptyOrNull(initialBranch) || refsMap.isEmpty()) { + return false; + } + if (refsMap.containsKey(initialBranch) + || refsMap.containsKey(Constants.R_HEADS + initialBranch) + || refsMap.containsKey(Constants.R_TAGS + initialBranch)) { + return false; + } + return true; + } + private void executeImp(final ProgressMonitor monitor, - final FetchResult result) throws NotSupportedException, - TransportException { - conn = transport.openFetch(); + final FetchResult result, String initialBranch) + throws NotSupportedException, TransportException { + final TagOpt tagopt = transport.getTagOpt(); + String getTags = (tagopt == TagOpt.NO_TAGS) ? null : Constants.R_TAGS; + String getHead = null; + try { + // If we don't have a HEAD yet, we're cloning and need to get the + // upstream HEAD, too. + Ref head = transport.local.exactRef(Constants.HEAD); + ObjectId id = head != null ? head.getObjectId() : null; + if (id == null || id.equals(ObjectId.zeroId())) { + getHead = Constants.HEAD; + } + } catch (IOException e) { + // Ignore + } + conn = transport.openFetch(toFetch, getTags, getHead); try { - result.setAdvertisedRefs(transport.getURI(), conn.getRefsMap()); + Map<String, Ref> refsMap = conn.getRefsMap(); + if (isInitialBranchMissing(refsMap, initialBranch)) { + throw new TransportException(MessageFormat.format( + JGitText.get().remoteBranchNotFound, initialBranch)); + } + result.setAdvertisedRefs(transport.getURI(), refsMap); result.peerUserAgent = conn.getPeerUserAgent(); final Set<Ref> matched = new HashSet<>(); for (RefSpec spec : toFetch) { @@ -119,7 +161,6 @@ class FetchProcess { } Collection<Ref> additionalTags = Collections.<Ref> emptyList(); - final TagOpt tagopt = transport.getTagOpt(); if (tagopt == TagOpt.AUTO_FOLLOW) additionalTags = expandAutoFollowTags(); else if (tagopt == TagOpt.FETCH_TAGS) @@ -253,7 +294,17 @@ class FetchProcess { if (conn != null) return; - conn = transport.openFetch(); + // Build prefixes + Set<String> prefixes = new HashSet<>(); + for (Ref toGet : askFor.values()) { + String src = toGet.getName(); + prefixes.add(src); + prefixes.add(Constants.R_REFS + src); + prefixes.add(Constants.R_HEADS + src); + prefixes.add(Constants.R_TAGS + src); + } + conn = transport.openFetch(Collections.emptyList(), + prefixes.toArray(new String[0])); // Since we opened a new connection we cannot be certain // that the system we connected to has the same exact set diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java index 35e2978bc4..36fce7a3f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2013, Google Inc. + * Copyright (C) 2008, 2013 Google Inc. * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -247,6 +247,74 @@ public final class GitProtocolConstants { */ public static final String COMMAND_FETCH = "fetch"; //$NON-NLS-1$ + /** + * HTTP header to set by clients to request a specific git protocol version + * in the HTTP transport. + * + * @since 5.11 + */ + public static final String PROTOCOL_HEADER = "Git-Protocol"; //$NON-NLS-1$ + + /** + * Environment variable to set by clients to request a specific git protocol + * in the file:// and ssh:// transports. + * + * @since 5.11 + */ + public static final String PROTOCOL_ENVIRONMENT_VARIABLE = "GIT_PROTOCOL"; //$NON-NLS-1$ + + /** + * Protocol V2 ref advertisement attribute containing the peeled object id + * for annotated tags. + * + * @since 5.11 + */ + public static final String REF_ATTR_PEELED = "peeled:"; //$NON-NLS-1$ + + /** + * Protocol V2 ref advertisement attribute containing the name of the ref + * for symbolic refs. + * + * @since 5.11 + */ + public static final String REF_ATTR_SYMREF_TARGET = "symref-target:"; //$NON-NLS-1$ + + /** + * Protocol V2 acknowledgments section header. + * + * @since 5.11 + */ + public static final String SECTION_ACKNOWLEDGMENTS = "acknowledgments"; //$NON-NLS-1$ + + /** + * Protocol V2 packfile section header. + * + * @since 5.11 + */ + public static final String SECTION_PACKFILE = "packfile"; //$NON-NLS-1$ + + /** + * Protocol announcement for protocol version 1. This is the same as V0, + * except for this initial line. + * + * @since 5.11 + */ + public static final String VERSION_1 = "version 1"; //$NON-NLS-1$ + + /** + * Protocol announcement for protocol version 2. + * + * @since 5.11 + */ + public static final String VERSION_2 = "version 2"; //$NON-NLS-1$ + + /** + * Protocol request for protocol version 2. + * + * @since 5.11 + */ + public static final String VERSION_2_REQUEST = "version=2"; //$NON-NLS-1$ + enum MultiAck { OFF, CONTINUE, DETAILED; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java index 49c8b587d7..febeb3ce9c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpTransport.java @@ -26,7 +26,7 @@ public abstract class HttpTransport extends Transport { * * @since 3.3 */ - protected static HttpConnectionFactory connectionFactory = new JDKHttpConnectionFactory(); + protected static volatile HttpConnectionFactory connectionFactory = new JDKHttpConnectionFactory(); /** * Get the {@link org.eclipse.jgit.transport.http.HttpConnectionFactory} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java index 350311ecc8..68c5b348ad 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java @@ -103,6 +103,38 @@ public class PacketLineIn { this.limit = limit; } + /** + * Parses a ACK/NAK line in protocol V2. + * + * @param line + * to parse + * @param returnedId + * in case of {@link AckNackResult#ACK_COMMON ACK_COMMON} + * @return one of {@link AckNackResult#NAK NAK}, + * {@link AckNackResult#ACK_COMMON ACK_COMMON}, or + * {@link AckNackResult#ACK_READY ACK_READY} + * @throws IOException + * on protocol or transport errors + */ + static AckNackResult parseACKv2(String line, MutableObjectId returnedId) + throws IOException { + if ("NAK".equals(line)) { //$NON-NLS-1$ + return AckNackResult.NAK; + } + if (line.startsWith("ACK ") && line.length() == 44) { //$NON-NLS-1$ + returnedId.fromString(line.substring(4, 44)); + return AckNackResult.ACK_COMMON; + } + if ("ready".equals(line)) { //$NON-NLS-1$ + return AckNackResult.ACK_READY; + } + if (line.startsWith("ERR ")) { //$NON-NLS-1$ + throw new PackProtocolException(line.substring(4)); + } + throw new PackProtocolException( + MessageFormat.format(JGitText.get().expectedACKNAKGot, line)); + } + AckNackResult readACK(MutableObjectId returnedId) throws IOException { final String line = readString(); if (line.length() == 0) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java index 6fc2042e1f..77f0a7a516 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java @@ -1,7 +1,7 @@ /* - * Copyright (C) 2008-2010, Google Inc. - * Copyright (C) 2008-2009, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2010 Google Inc. + * Copyright (C) 2008, 2009 Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -33,12 +33,15 @@ import org.slf4j.LoggerFactory; * against the underlying OutputStream. */ public class PacketLineOut { + private static final Logger log = LoggerFactory.getLogger(PacketLineOut.class); private final OutputStream out; private final byte[] lenbuffer; + private final boolean logEnabled; + private boolean flushOnEnd; private boolean usingSideband; @@ -50,9 +53,24 @@ public class PacketLineOut { * stream. */ public PacketLineOut(OutputStream outputStream) { + this(outputStream, true); + } + + /** + * Create a new packet line writer that potentially doesn't log. + * + * @param outputStream + * stream. + * @param enableLogging + * {@code false} to suppress all logging; {@code true} to log + * normally + * @since 5.11 + */ + public PacketLineOut(OutputStream outputStream, boolean enableLogging) { out = outputStream; lenbuffer = new byte[5]; flushOnEnd = true; + logEnabled = enableLogging; } /** @@ -139,9 +157,15 @@ public class PacketLineOut { out.write(lenbuffer, 0, 4); } out.write(buf, pos, len); - if (log.isDebugEnabled()) { - String s = RawParseUtils.decode(UTF_8, buf, pos, len); - log.debug("git> " + s); //$NON-NLS-1$ + if (logEnabled && log.isDebugEnabled()) { + // Escape a trailing \n to avoid empty lines in the log. + if (len > 0 && buf[pos + len - 1] == '\n') { + log.debug( + "git> " + RawParseUtils.decode(UTF_8, buf, pos, len - 1) //$NON-NLS-1$ + + "\\n"); //$NON-NLS-1$ + } else { + log.debug("git> " + RawParseUtils.decode(UTF_8, buf, pos, len)); //$NON-NLS-1$ + } } } @@ -156,7 +180,9 @@ public class PacketLineOut { public void writeDelim() throws IOException { formatLength(1); out.write(lenbuffer, 0, 4); - log.debug("git> 0001"); //$NON-NLS-1$ + if (logEnabled && log.isDebugEnabled()) { + log.debug("git> 0001"); //$NON-NLS-1$ + } } /** @@ -175,9 +201,12 @@ public class PacketLineOut { public void end() throws IOException { formatLength(0); out.write(lenbuffer, 0, 4); - log.debug("git> 0000"); //$NON-NLS-1$ - if (flushOnEnd) + if (logEnabled && log.isDebugEnabled()) { + log.debug("git> 0000"); //$NON-NLS-1$ + } + if (flushOnEnd) { flush(); + } } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java index 3adebba03c..c525e66848 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2010, Google Inc. and others + * Copyright (C) 2008, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -13,6 +13,8 @@ package org.eclipse.jgit.transport; import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SYMREF; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED; +import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET; import java.io.IOException; import java.nio.ByteBuffer; @@ -287,7 +289,8 @@ public abstract class RefAdvertiser { if (useProtocolV2) { String symrefPart = symrefs.containsKey(ref.getName()) - ? (" symref-target:" + symrefs.get(ref.getName())) //$NON-NLS-1$ + ? (' ' + REF_ATTR_SYMREF_TARGET + + symrefs.get(ref.getName())) : ""; //$NON-NLS-1$ String peelPart = ""; //$NON-NLS-1$ if (derefTags) { @@ -296,7 +299,8 @@ public abstract class RefAdvertiser { } ObjectId peeledObjectId = ref.getPeeledObjectId(); if (peeledObjectId != null) { - peelPart = " peeled:" + peeledObjectId.getName(); //$NON-NLS-1$ + peelPart = ' ' + REF_ATTR_PEELED + + peeledObjectId.getName(); } } writeOne(objectId.getName() + " " + ref.getName() + symrefPart //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java new file mode 100644 index 0000000000..23f670ae25 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RemoteSession2.java @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2020, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.transport; + +import java.io.IOException; +import java.util.Map; + +/** + * A {@link RemoteSession} that supports passing environment variables to + * commands. + * + * @since 5.11 + */ +public interface RemoteSession2 extends RemoteSession { + + /** + * Creates a new remote {@link Process} to execute the given command. The + * returned process's streams exist and are connected, and execution of the + * process is already started. + * + * @param commandName + * command to execute + * @param environment + * environment variables to pass on + * @param timeout + * timeout value, in seconds, for creating the remote process + * @return a new remote process, already started + * @throws java.io.IOException + * may be thrown in several cases. For example, on problems + * opening input or output streams or on problems connecting or + * communicating with the remote host. For the latter two cases, + * a TransportException may be thrown (a subclass of + * java.io.IOException). + */ + Process exec(String commandName, Map<String, String> environment, + int timeout) throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java index cc577fa11e..83ffd4123a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2009, Google Inc. and others + * Copyright (C) 2008, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -21,6 +21,7 @@ import java.util.Map; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.storage.file.LazyObjectIdSetFile; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Config.SectionParser; import org.eclipse.jgit.lib.ObjectChecker; import org.eclipse.jgit.lib.ObjectIdSet; @@ -60,11 +61,19 @@ public class TransferConfig { } /** - * A git configuration variable for which versions of the Git protocol to prefer. - * Used in protocol.version. + * A git configuration variable for which versions of the Git protocol to + * prefer. Used in protocol.version. + * + * @since 5.9 */ - enum ProtocolVersion { + public enum ProtocolVersion { + /** + * Git wire protocol version 0 (the default). + */ V0("0"), //$NON-NLS-1$ + /** + * Git wire protocol version 2. + */ V2("2"); //$NON-NLS-1$ final String name; @@ -73,6 +82,15 @@ public class TransferConfig { this.name = name; } + /** + * Returns version number + * + * @return string version + */ + public String version() { + return name; + } + @Nullable static ProtocolVersion parse(@Nullable String name) { if (name == null) { @@ -83,6 +101,9 @@ public class TransferConfig { return v; } } + if ("1".equals(name)) { //$NON-NLS-1$ + return V0; + } return null; } } @@ -177,7 +198,9 @@ public class TransferConfig { "uploadpack", "allowreachablesha1inwant", false); allowFilter = rc.getBoolean( "uploadpack", "allowfilter", false); - protocolVersion = ProtocolVersion.parse(rc.getString("protocol", null, "version")); + protocolVersion = ProtocolVersion.parse(rc + .getString(ConfigConstants.CONFIG_PROTOCOL_SECTION, null, + ConfigConstants.CONFIG_KEY_VERSION)); hideRefs = rc.getStringList("uploadpack", null, "hiderefs"); allowSidebandAll = rc.getBoolean( "uploadpack", "allowsidebandall", false); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index 2ddd0a6128..5b781ac25f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2009, Google Inc. + * Copyright (C) 2008, 2009 Google Inc. * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -39,6 +39,7 @@ import java.util.Vector; import java.util.concurrent.CopyOnWriteArrayList; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.AbortedByHookException; import org.eclipse.jgit.errors.NotSupportedException; import org.eclipse.jgit.errors.TransportException; @@ -774,6 +775,10 @@ public abstract class Transport implements AutoCloseable { private PrintStream hookOutRedirect; private PrePushHook prePush; + + @Nullable + TransferConfig.ProtocolVersion protocol; + /** * Create a new transport instance. * @@ -789,6 +794,7 @@ public abstract class Transport implements AutoCloseable { final TransferConfig tc = local.getConfig().get(TransferConfig.KEY); this.local = local; this.uri = uri; + this.protocol = tc.protocolVersion; this.objectChecker = tc.newObjectChecker(); this.credentialsProvider = CredentialsProvider.getDefault(); prePush = Hooks.prePush(local, hookOutRedirect); @@ -1225,9 +1231,52 @@ public abstract class Transport implements AutoCloseable { * the remote connection could not be established or object * copying (if necessary) failed or update specification was * incorrect. + * @since 5.11 + */ + public FetchResult fetch(final ProgressMonitor monitor, + Collection<RefSpec> toFetch) + throws NotSupportedException, TransportException { + return fetch(monitor, toFetch, null); + } + + /** + * Fetch objects and refs from the remote repository to the local one. + * <p> + * This is a utility function providing standard fetch behavior. Local + * tracking refs associated with the remote repository are automatically + * updated if this transport was created from a + * {@link org.eclipse.jgit.transport.RemoteConfig} with fetch RefSpecs + * defined. + * + * @param monitor + * progress monitor to inform the user about our processing + * activity. Must not be null. Use + * {@link org.eclipse.jgit.lib.NullProgressMonitor} if progress + * updates are not interesting or necessary. + * @param toFetch + * specification of refs to fetch locally. May be null or the + * empty collection to use the specifications from the + * RemoteConfig. Source for each RefSpec can't be null. + * @param branch + * the initial branch to check out when cloning the repository. + * Can be specified as ref name (<code>refs/heads/master</code>), + * branch name (<code>master</code>) or tag name + * (<code>v1.2.3</code>). The default is to use the branch + * pointed to by the cloned repository's HEAD and can be + * requested by passing {@code null} or <code>HEAD</code>. + * @return information describing the tracking refs updated. + * @throws org.eclipse.jgit.errors.NotSupportedException + * this transport implementation does not support fetching + * objects. + * @throws org.eclipse.jgit.errors.TransportException + * the remote connection could not be established or object + * copying (if necessary) failed or update specification was + * incorrect. + * @since 5.11 */ public FetchResult fetch(final ProgressMonitor monitor, - Collection<RefSpec> toFetch) throws NotSupportedException, + Collection<RefSpec> toFetch, String branch) + throws NotSupportedException, TransportException { if (toFetch == null || toFetch.isEmpty()) { // If the caller did not ask for anything use the defaults. @@ -1257,7 +1306,7 @@ public abstract class Transport implements AutoCloseable { } final FetchResult result = new FetchResult(); - new FetchProcess(this, toFetch).execute(monitor, result); + new FetchProcess(this, toFetch).execute(monitor, result, branch); local.autoGC(monitor); @@ -1453,6 +1502,43 @@ public abstract class Transport implements AutoCloseable { TransportException; /** + * Begins a new connection for fetching from the remote repository. + * <p> + * If the transport has no local repository, the fetch connection can only + * be used for reading remote refs. + * </p> + * <p> + * If the server supports git protocol V2, the {@link RefSpec}s and the + * additional patterns, if any, are used to restrict the server's ref + * advertisement to matching refs only. + * </p> + * <p> + * Transports that want to support git protocol V2 <em>must</em> override + * this; the default implementation ignores its arguments and calls + * {@link #openFetch()}. + * </p> + * + * @param refSpecs + * that will be fetched via + * {@link FetchConnection#fetch(ProgressMonitor, Collection, java.util.Set, OutputStream)} later + * @param additionalPatterns + * that will be set as ref prefixes if the server supports git + * protocol V2; {@code null} values are ignored + * + * @return a fresh connection to fetch from the remote repository. + * @throws org.eclipse.jgit.errors.NotSupportedException + * the implementation does not support fetching. + * @throws org.eclipse.jgit.errors.TransportException + * the remote connection could not be established. + * @since 5.11 + */ + public FetchConnection openFetch(Collection<RefSpec> refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return openFetch(); + } + + /** * Begins a new connection for pushing into the remote repository. * * @return a fresh connection to push into the remote repository. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java index 820ec1a67a..a1914b6182 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java @@ -1,7 +1,7 @@ /* * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -22,6 +22,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Socket; import java.net.UnknownHostException; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.Set; @@ -94,6 +95,13 @@ class TransportGitAnon extends TcpTransport implements PackTransport { return new TcpFetchConnection(); } + @Override + public FetchConnection openFetch(Collection<RefSpec> refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return new TcpFetchConnection(refSpecs, additionalPatterns); + } + /** {@inheritDoc} */ @Override public PushConnection openPush() throws TransportException { @@ -113,7 +121,6 @@ class TransportGitAnon extends TcpTransport implements PackTransport { final Socket s = new Socket(); try { final InetAddress host = InetAddress.getByName(uri.getHost()); - s.bind(null); s.connect(new InetSocketAddress(host, port), tms); } catch (IOException c) { try { @@ -130,7 +137,8 @@ class TransportGitAnon extends TcpTransport implements PackTransport { return s; } - void service(String name, PacketLineOut pckOut) + void service(String name, PacketLineOut pckOut, + TransferConfig.ProtocolVersion gitProtocol) throws IOException { final StringBuilder cmd = new StringBuilder(); cmd.append(name); @@ -144,6 +152,11 @@ class TransportGitAnon extends TcpTransport implements PackTransport { cmd.append(uri.getPort()); } cmd.append('\0'); + if (TransferConfig.ProtocolVersion.V2.equals(gitProtocol)) { + cmd.append('\0'); + cmd.append(GitProtocolConstants.VERSION_2_REQUEST); + cmd.append('\0'); + } pckOut.writeString(cmd.toString()); pckOut.flush(); } @@ -152,6 +165,11 @@ class TransportGitAnon extends TcpTransport implements PackTransport { private Socket sock; TcpFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + TcpFetchConnection(Collection<RefSpec> refSpecs, + String... additionalPatterns) throws TransportException { super(TransportGitAnon.this); sock = openConnection(); try { @@ -162,13 +180,19 @@ class TransportGitAnon extends TcpTransport implements PackTransport { sOut = new BufferedOutputStream(sOut); init(sIn, sOut); - service("git-upload-pack", pckOut); //$NON-NLS-1$ + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + service("git-upload-pack", pckOut, gitProtocol); //$NON-NLS-1$ } catch (IOException err) { close(); throw new TransportException(uri, JGitText.get().remoteHungUpUnexpectedly, err); } - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } @Override @@ -201,7 +225,7 @@ class TransportGitAnon extends TcpTransport implements PackTransport { sOut = new BufferedOutputStream(sOut); init(sIn, sOut); - service("git-receive-pack", pckOut); //$NON-NLS-1$ + service("git-receive-pack", pckOut, null); //$NON-NLS-1$ } catch (IOException err) { close(); throw new TransportException(uri, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java index b9cb2484d8..19ed4fbcc1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -19,11 +19,13 @@ import java.io.InputStream; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Set; import org.eclipse.jgit.errors.NoRemoteRepositoryException; @@ -144,6 +146,13 @@ public class TransportGitSsh extends SshTransport implements PackTransport { return new SshFetchConnection(); } + @Override + public FetchConnection openFetch(Collection<RefSpec> refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { + return new SshFetchConnection(refSpecs, additionalPatterns); + } + /** {@inheritDoc} */ @Override public PushConnection openPush() throws TransportException { @@ -196,29 +205,38 @@ public class TransportGitSsh extends SshTransport implements PackTransport { return SystemReader.getInstance().getenv("GIT_SSH") != null; //$NON-NLS-1$ } - private class ExtSession implements RemoteSession { + private class ExtSession implements RemoteSession2 { + @Override public Process exec(String command, int timeout) throws TransportException { + return exec(command, null, timeout); + } + + @Override + public Process exec(String command, Map<String, String> environment, + int timeout) throws TransportException { String ssh = SystemReader.getInstance().getenv("GIT_SSH"); //$NON-NLS-1$ boolean putty = ssh.toLowerCase(Locale.ROOT).contains("plink"); //$NON-NLS-1$ List<String> args = new ArrayList<>(); args.add(ssh); - if (putty - && !ssh.toLowerCase(Locale.ROOT).contains("tortoiseplink")) //$NON-NLS-1$ + if (putty && !ssh.toLowerCase(Locale.ROOT) + .contains("tortoiseplink")) {//$NON-NLS-1$ args.add("-batch"); //$NON-NLS-1$ + } if (0 < getURI().getPort()) { args.add(putty ? "-P" : "-p"); //$NON-NLS-1$ //$NON-NLS-2$ args.add(String.valueOf(getURI().getPort())); } - if (getURI().getUser() != null) + if (getURI().getUser() != null) { args.add(getURI().getUser() + "@" + getURI().getHost()); //$NON-NLS-1$ - else + } else { args.add(getURI().getHost()); + } args.add(command); - ProcessBuilder pb = createProcess(args); + ProcessBuilder pb = createProcess(args, environment); try { return pb.start(); } catch (IOException err) { @@ -226,9 +244,13 @@ public class TransportGitSsh extends SshTransport implements PackTransport { } } - private ProcessBuilder createProcess(List<String> args) { + private ProcessBuilder createProcess(List<String> args, + Map<String, String> environment) { ProcessBuilder pb = new ProcessBuilder(); pb.command(args); + if (environment != null) { + pb.environment().putAll(environment); + } File directory = local != null ? local.getDirectory() : null; if (directory != null) { pb.environment().put(Constants.GIT_DIR_KEY, @@ -249,10 +271,31 @@ public class TransportGitSsh extends SshTransport implements PackTransport { private StreamCopyThread errorThread; SshFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + SshFetchConnection(Collection<RefSpec> refSpecs, + String... additionalPatterns) throws TransportException { super(TransportGitSsh.this); try { - process = getSession().exec(commandFor(getOptionUploadPack()), - getTimeout()); + RemoteSession session = getSession(); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + if (session instanceof RemoteSession2 + && TransferConfig.ProtocolVersion.V2 + .equals(gitProtocol)) { + process = ((RemoteSession2) session).exec( + commandFor(getOptionUploadPack()), Collections + .singletonMap( + GitProtocolConstants.PROTOCOL_ENVIRONMENT_VARIABLE, + GitProtocolConstants.VERSION_2_REQUEST), + getTimeout()); + } else { + process = session.exec(commandFor(getOptionUploadPack()), + getTimeout()); + } final MessageWriter msg = new MessageWriter(); setMessageWriter(msg); @@ -272,7 +315,9 @@ public class TransportGitSsh extends SshTransport implements PackTransport { } try { - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } catch (NoRemoteRepositoryException notFound) { final String msgs = getMessages(); checkExecFailure(process.exitValue(), getOptionUploadPack(), diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 6768387e65..2e5d18dc15 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -1,8 +1,8 @@ /* - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> * Copyright (C) 2013, Matthias Sohn <matthias.sohn@sap.com> - * Copyright (C) 2017, Thomas Wolf <thomas.wolf@paranor.ch> and others + * Copyright (C) 2017, 2020 Thomas Wolf <thomas.wolf@paranor.ch> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -33,14 +33,15 @@ import static org.eclipse.jgit.util.HttpSupport.HDR_WWW_AUTHENTICATE; import static org.eclipse.jgit.util.HttpSupport.METHOD_GET; import static org.eclipse.jgit.util.HttpSupport.METHOD_POST; +import java.io.BufferedInputStream; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.InterruptedIOException; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.net.HttpCookie; import java.net.MalformedURLException; import java.net.Proxy; @@ -49,10 +50,12 @@ import java.net.SocketException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.net.URLDecoder; import java.nio.charset.StandardCharsets; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.GeneralSecurityException; import java.security.cert.CertPathBuilderException; import java.security.cert.CertPathValidatorException; import java.security.cert.CertificateException; @@ -75,6 +78,7 @@ import java.util.zip.GZIPOutputStream; import javax.net.ssl.SSLHandshakeException; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.NotSupportedException; @@ -95,6 +99,8 @@ import org.eclipse.jgit.lib.SymbolicRef; import org.eclipse.jgit.transport.HttpAuthMethod.Type; import org.eclipse.jgit.transport.HttpConfig.HttpRedirectMode; import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.transport.http.HttpConnectionFactory; +import org.eclipse.jgit.transport.http.HttpConnectionFactory2; import org.eclipse.jgit.util.HttpSupport; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; @@ -132,6 +138,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport, private static final String SVC_RECEIVE_PACK = "git-receive-pack"; //$NON-NLS-1$ + private static final byte[] VERSION = "version" //$NON-NLS-1$ + .getBytes(StandardCharsets.US_ASCII); + /** * Accept-Encoding header in the HTTP request * (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html). @@ -257,6 +266,12 @@ public class TransportHttp extends HttpTransport implements WalkTransport, private boolean sslFailure = false; + private HttpConnectionFactory factory; + + private HttpConnectionFactory2.GitSession gitSession; + + private boolean factoryUsed; + /** * All stored cookies bound to this repo (independent of the baseUrl) */ @@ -279,6 +294,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, sslVerify = http.isSslVerify(); cookieFile = getCookieFileFromConfig(http); relevantCookies = filterCookies(cookieFile, baseUrl); + factory = HttpTransport.getConnectionFactory(); } private URL toURL(URIish urish) throws MalformedURLException { @@ -321,6 +337,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, sslVerify = http.isSslVerify(); cookieFile = getCookieFileFromConfig(http); relevantCookies = filterCookies(cookieFile, baseUrl); + factory = HttpTransport.getConnectionFactory(); } /** @@ -339,11 +356,15 @@ public class TransportHttp extends HttpTransport implements WalkTransport, @SuppressWarnings("resource") // Closed by caller private FetchConnection getConnection(HttpConnection c, InputStream in, - String service) throws IOException { + String service, Collection<RefSpec> refSpecs, + String... additionalPatterns) throws IOException { BaseConnection f; if (isSmartHttp(c, service)) { - readSmartHeaders(in, service); - f = new SmartHttpFetchConnection(in); + InputStream withMark = in.markSupported() ? in + : new BufferedInputStream(in); + readSmartHeaders(withMark, service); + f = new SmartHttpFetchConnection(withMark, refSpecs, + additionalPatterns); } else { // Assume this server doesn't support smart HTTP fetch // and fall back on dumb object walking. @@ -353,15 +374,98 @@ public class TransportHttp extends HttpTransport implements WalkTransport, return (FetchConnection) f; } + /** + * Sets the {@link HttpConnectionFactory} to be used by this + * {@link TransportHttp} instance. + * <p> + * If no factory is set explicitly, the {@link TransportHttp} instance uses + * the {@link HttpTransport#getConnectionFactory() globally defined + * factory}. + * </p> + * + * @param customFactory + * the {@link HttpConnectionFactory} to use + * @throws IllegalStateException + * if an HTTP/HTTPS connection has already been opened on this + * {@link TransportHttp} instance + * @since 5.11 + */ + public void setHttpConnectionFactory( + @NonNull HttpConnectionFactory customFactory) { + if (factoryUsed) { + throw new IllegalStateException(JGitText.get().httpFactoryInUse); + } + factory = customFactory; + } + + /** + * Retrieves the {@link HttpConnectionFactory} used by this + * {@link TransportHttp} instance. + * + * @return the {@link HttpConnectionFactory} + * @since 5.11 + */ + @NonNull + public HttpConnectionFactory getHttpConnectionFactory() { + return factory; + } + + /** + * Sets preemptive Basic HTTP authentication. If the given {@code username} + * or {@code password} is empty or {@code null}, no preemptive + * authentication will be done. If {@code username} and {@code password} are + * set, they will override authority information from the URI + * ("user:password@"). + * <p> + * If the connection encounters redirects, the pre-authentication will be + * cleared if the redirect goes to a different host. + * </p> + * + * @param username + * to use + * @param password + * to use + * @throws IllegalStateException + * if an HTTP/HTTPS connection has already been opened on this + * {@link TransportHttp} instance + * @since 5.11 + */ + public void setPreemptiveBasicAuthentication(String username, + String password) { + if (factoryUsed) { + throw new IllegalStateException(JGitText.get().httpPreAuthTooLate); + } + if (StringUtils.isEmptyOrNull(username) + || StringUtils.isEmptyOrNull(password)) { + authMethod = authFromUri(currentUri); + } else { + HttpAuthMethod basic = HttpAuthMethod.Type.BASIC.method(null); + basic.authorize(username, password); + authMethod = basic; + } + } + /** {@inheritDoc} */ @Override public FetchConnection openFetch() throws TransportException, NotSupportedException { + return openFetch(Collections.emptyList()); + } + + @Override + public FetchConnection openFetch(Collection<RefSpec> refSpecs, + String... additionalPatterns) + throws NotSupportedException, TransportException { final String service = SVC_UPLOAD_PACK; try { - final HttpConnection c = connect(service); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + HttpConnection c = connect(service, gitProtocol); try (InputStream in = openInputStream(c)) { - return getConnection(c, in, service); + return getConnection(c, in, service, refSpecs, + additionalPatterns); } } catch (NotSupportedException | TransportException err) { throw err; @@ -456,8 +560,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport, private PushConnection smartPush(String service, HttpConnection c, InputStream in) throws IOException, TransportException { - readSmartHeaders(in, service); - SmartHttpPushConnection p = new SmartHttpPushConnection(in); + BufferedInputStream inBuf = new BufferedInputStream(in); + readSmartHeaders(inBuf, service); + SmartHttpPushConnection p = new SmartHttpPushConnection(inBuf); p.setPeerUserAgent(c.getHeaderField(HttpSupport.HDR_SERVER)); return p; } @@ -465,7 +570,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport, /** {@inheritDoc} */ @Override public void close() { - // No explicit connections are maintained. + if (gitSession != null) { + gitSession.close(); + gitSession = null; + } } /** @@ -492,9 +600,40 @@ public class TransportHttp extends HttpTransport implements WalkTransport, return new NoRemoteRepositoryException(u, text); } + private HttpAuthMethod authFromUri(URIish u) { + String user = u.getUser(); + String pass = u.getPass(); + if (user != null && pass != null) { + try { + // User/password are _not_ application/x-www-form-urlencoded. In + // particular the "+" sign would be replaced by a space. + user = URLDecoder.decode(user.replace("+", "%2B"), //$NON-NLS-1$ //$NON-NLS-2$ + StandardCharsets.UTF_8.name()); + pass = URLDecoder.decode(pass.replace("+", "%2B"), //$NON-NLS-1$ //$NON-NLS-2$ + StandardCharsets.UTF_8.name()); + HttpAuthMethod basic = HttpAuthMethod.Type.BASIC.method(null); + basic.authorize(user, pass); + return basic; + } catch (IllegalArgumentException + | UnsupportedEncodingException e) { + LOG.warn(JGitText.get().httpUserInfoDecodeError, u); + } + } + return HttpAuthMethod.Type.NONE.method(null); + } + private HttpConnection connect(String service) throws TransportException, NotSupportedException { + return connect(service, null); + } + + private HttpConnection connect(String service, + TransferConfig.ProtocolVersion protocolVersion) + throws TransportException, NotSupportedException { URL u = getServiceURL(service); + if (HttpAuthMethod.Type.NONE.equals(authMethod.getType())) { + authMethod = authFromUri(currentUri); + } int authAttempts = 1; int redirects = 0; Collection<Type> ignoreTypes = null; @@ -507,6 +646,11 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } else { conn.setRequestProperty(HDR_ACCEPT, "*/*"); //$NON-NLS-1$ } + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + conn.setRequestProperty( + GitProtocolConstants.PROTOCOL_HEADER, + GitProtocolConstants.VERSION_2_REQUEST); + } final int status = HttpSupport.response(conn); processResponseCookies(conn); switch (status) { @@ -796,7 +940,13 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } try { URI redirectTo = new URI(location); + // Reset authentication if the redirect has user/password info or + // if the host is different. + boolean resetAuth = !StringUtils + .isEmptyOrNull(redirectTo.getUserInfo()); + String currentHost = currentUrl.getHost(); redirectTo = currentUrl.toURI().resolve(redirectTo); + resetAuth = resetAuth || !currentHost.equals(redirectTo.getHost()); String redirected = redirectTo.toASCIIString(); if (!isValidRedirect(baseUrl, redirected, checkFor)) { throw new TransportException(uri, @@ -805,6 +955,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } redirected = redirected.substring(0, redirected.indexOf(checkFor)); URIish result = new URIish(redirected); + if (resetAuth) { + authMethod = HttpAuthMethod.Type.NONE.method(null); + } if (LOG.isInfoEnabled()) { LOG.info(MessageFormat.format(JGitText.get().redirectHttp, uri.setPass(null), @@ -885,9 +1038,20 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } final Proxy proxy = HttpSupport.proxyFor(proxySelector, u); - HttpConnection conn = connectionFactory.create(u, proxy); + factoryUsed = true; + HttpConnection conn = factory.create(u, proxy); - if (!sslVerify && "https".equals(u.getProtocol())) { //$NON-NLS-1$ + if (gitSession == null && (factory instanceof HttpConnectionFactory2)) { + gitSession = ((HttpConnectionFactory2) factory).newSession(); + } + if (gitSession != null) { + try { + gitSession.configure(conn, sslVerify); + } catch (GeneralSecurityException e) { + throw new IOException(e.getMessage(), e); + } + } else if (!sslVerify && "https".equals(u.getProtocol())) { //$NON-NLS-1$ + // Backwards compatibility HttpSupport.disableSslVerify(conn); } @@ -1148,20 +1312,37 @@ public class TransportHttp extends HttpTransport implements WalkTransport, private void readSmartHeaders(InputStream in, String service) throws IOException { - // A smart reply will have a '#' after the first 4 bytes, but - // a dumb reply cannot contain a '#' until after byte 41. Do a + // A smart protocol V0 reply will have a '#' after the first 4 bytes, + // but a dumb reply cannot contain a '#' until after byte 41. Do a // quick check to make sure its a smart reply before we parse // as a pkt-line stream. // - final byte[] magic = new byte[5]; + // There appears to be a confusion about this in protocol V2. Github + // sends the # service line as a git (not http) header also when + // protocol V2 is used. Gitlab also does so. JGit's UploadPack doesn't, + // and thus Gerrit also does not. + final byte[] magic = new byte[14]; + if (!in.markSupported()) { + throw new TransportException(uri, + JGitText.get().inputStreamMustSupportMark); + } + in.mark(14); IO.readFully(in, magic, 0, magic.length); + // Did we get 000dversion 2 or similar? (Canonical is 000eversion 2\n, + // but JGit and thus Gerrit omits the \n.) + if (Arrays.equals(Arrays.copyOfRange(magic, 4, 11), VERSION) + && magic[12] >= '1' && magic[12] <= '9') { + // It's a smart server doing version 1 or greater, but not sending + // the # service line header. Don't consume the version line. + in.reset(); + return; + } if (magic[4] != '#') { throw new TransportException(uri, MessageFormat.format( JGitText.get().expectedPktLineWithService, RawParseUtils.decode(magic))); } - - final PacketLineIn pckIn = new PacketLineIn(new UnionInputStream( - new ByteArrayInputStream(magic), in)); + in.reset(); + final PacketLineIn pckIn = new PacketLineIn(in); final String exp = "# service=" + service; //$NON-NLS-1$ final String act = pckIn.readString(); if (!exp.equals(act)) { @@ -1327,12 +1508,24 @@ public class TransportHttp extends HttpTransport implements WalkTransport, SmartHttpFetchConnection(InputStream advertisement) throws TransportException { + this(advertisement, Collections.emptyList()); + } + + SmartHttpFetchConnection(InputStream advertisement, + Collection<RefSpec> refSpecs, String... additionalPatterns) + throws TransportException { super(TransportHttp.this); statelessRPC = true; init(advertisement, DisabledOutputStream.INSTANCE); outNeedsEnd = false; - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + // Must be protocol V2 + LongPollService service = new LongPollService(SVC_UPLOAD_PACK, + getProtocolVersion()); + init(service.getInputStream(), service.getOutputStream()); + lsRefs(refSpecs, additionalPatterns); + } } @Override @@ -1340,7 +1533,8 @@ public class TransportHttp extends HttpTransport implements WalkTransport, final Collection<Ref> want, final Set<ObjectId> have, final OutputStream outputStream) throws TransportException { try { - svc = new MultiRequestService(SVC_UPLOAD_PACK); + svc = new MultiRequestService(SVC_UPLOAD_PACK, + getProtocolVersion()); init(svc.getInputStream(), svc.getOutputStream()); super.doFetch(monitor, want, have, outputStream); } finally { @@ -1369,7 +1563,8 @@ public class TransportHttp extends HttpTransport implements WalkTransport, protected void doPush(final ProgressMonitor monitor, final Map<String, RemoteRefUpdate> refUpdates, OutputStream outputStream) throws TransportException { - final Service svc = new MultiRequestService(SVC_RECEIVE_PACK); + final Service svc = new MultiRequestService(SVC_RECEIVE_PACK, + getProtocolVersion()); init(svc.getInputStream(), svc.getOutputStream()); super.doPush(monitor, refUpdates, outputStream); } @@ -1389,10 +1584,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport, protected final HttpExecuteStream execute; + protected final TransferConfig.ProtocolVersion protocolVersion; + final UnionInputStream in; - Service(String serviceName) { + Service(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { this.serviceName = serviceName; + this.protocolVersion = protocolVersion; this.requestType = "application/x-" + serviceName + "-request"; //$NON-NLS-1$ //$NON-NLS-2$ this.responseType = "application/x-" + serviceName + "-result"; //$NON-NLS-1$ //$NON-NLS-2$ @@ -1408,6 +1607,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport, conn.setDoOutput(true); conn.setRequestProperty(HDR_CONTENT_TYPE, requestType); conn.setRequestProperty(HDR_ACCEPT, responseType); + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + conn.setRequestProperty(GitProtocolConstants.PROTOCOL_HEADER, + GitProtocolConstants.VERSION_2_REQUEST); + } } void sendRequest() throws IOException { @@ -1663,8 +1866,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport, class MultiRequestService extends Service { boolean finalRequest; - MultiRequestService(String serviceName) { - super(serviceName); + MultiRequestService(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { + super(serviceName, protocolVersion); } /** Keep opening send-receive pairs to the given URI. */ @@ -1701,11 +1905,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport, /** Service for maintaining a single long-poll connection. */ class LongPollService extends Service { - /** - * @param serviceName - */ - LongPollService(String serviceName) { - super(serviceName); + + LongPollService(String serviceName, + TransferConfig.ProtocolVersion protocolVersion) { + super(serviceName, protocolVersion); } /** Only open one send-receive request. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java index 403f98d869..77d1419ea2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java @@ -1,9 +1,9 @@ /* * Copyright (C) 2007, Dave Watson <dwatson@mimvista.com> - * Copyright (C) 2008-2010, Google Inc. + * Copyright (C) 2008, 2010 Google Inc. * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com> * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com> - * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -153,11 +154,17 @@ class TransportLocal extends Transport implements PackTransport { /** {@inheritDoc} */ @Override public FetchConnection openFetch() throws TransportException { + return openFetch(Collections.emptyList()); + } + + @Override + public FetchConnection openFetch(Collection<RefSpec> refSpecs, + String... additionalPatterns) throws TransportException { final String up = getOptionUploadPack(); if (!"git-upload-pack".equals(up) //$NON-NLS-1$ - && !"git upload-pack".equals(up)) //$NON-NLS-1$ - return new ForkLocalFetchConnection(); - + && !"git upload-pack".equals(up)) {//$NON-NLS-1$ + return new ForkLocalFetchConnection(refSpecs, additionalPatterns); + } UploadPackFactory<Void> upf = (Void req, Repository db) -> createUploadPack(db); return new InternalFetchConnection<>(this, upf, null, openRepo()); @@ -193,6 +200,23 @@ class TransportLocal extends Transport implements PackTransport { */ protected Process spawn(String cmd) throws TransportException { + return spawn(cmd, null); + } + + /** + * Spawn process + * + * @param cmd + * command + * @param protocolVersion + * to use + * @return a {@link java.lang.Process} object. + * @throws org.eclipse.jgit.errors.TransportException + * if any. + */ + private Process spawn(String cmd, + TransferConfig.ProtocolVersion protocolVersion) + throws TransportException { try { String[] args = { "." }; //$NON-NLS-1$ ProcessBuilder proc = local.getFS().runInShell(cmd, args); @@ -208,7 +232,10 @@ class TransportLocal extends Transport implements PackTransport { env.remove("GIT_GRAFT_FILE"); //$NON-NLS-1$ env.remove("GIT_INDEX_FILE"); //$NON-NLS-1$ env.remove("GIT_NO_REPLACE_OBJECTS"); //$NON-NLS-1$ - + if (TransferConfig.ProtocolVersion.V2.equals(protocolVersion)) { + env.put(GitProtocolConstants.PROTOCOL_ENVIRONMENT_VARIABLE, + GitProtocolConstants.VERSION_2_REQUEST); + } return proc.start(); } catch (IOException err) { throw new TransportException(uri, err.getMessage(), err); @@ -221,12 +248,21 @@ class TransportLocal extends Transport implements PackTransport { private Thread errorReaderThread; ForkLocalFetchConnection() throws TransportException { + this(Collections.emptyList()); + } + + ForkLocalFetchConnection(Collection<RefSpec> refSpecs, + String... additionalPatterns) throws TransportException { super(TransportLocal.this); final MessageWriter msg = new MessageWriter(); setMessageWriter(msg); - uploadPack = spawn(getOptionUploadPack()); + TransferConfig.ProtocolVersion gitProtocol = protocol; + if (gitProtocol == null) { + gitProtocol = TransferConfig.ProtocolVersion.V2; + } + uploadPack = spawn(getOptionUploadPack(), gitProtocol); final InputStream upErr = uploadPack.getErrorStream(); errorReaderThread = new StreamCopyThread(upErr, msg.getRawStream()); @@ -239,7 +275,9 @@ class TransportLocal extends Transport implements PackTransport { upOut = new BufferedOutputStream(upOut); init(upIn, upOut); - readAdvertisedRefs(); + if (!readAdvertisedRefs()) { + lsRefs(refSpecs, additionalPatterns); + } } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 1242ef1b4a..7f1ddaab2e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2010, Google Inc. and others + * Copyright (C) 2008, 2020 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -33,6 +33,7 @@ import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDEBAND_AL import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND_64K; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_THIN_PACK; +import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2_REQUEST; import static org.eclipse.jgit.util.RefMap.toRefMap; import java.io.ByteArrayOutputStream; @@ -709,7 +710,7 @@ public class UploadPack { * @since 5.0 */ public void setExtraParameters(Collection<String> params) { - this.clientRequestedV2 = params.contains("version=2"); //$NON-NLS-1$ + this.clientRequestedV2 = params.contains(VERSION_2_REQUEST); } /** @@ -722,7 +723,8 @@ public class UploadPack { } private boolean useProtocolV2() { - return ProtocolVersion.V2.equals(transferConfig.protocolVersion) + return (transferConfig.protocolVersion == null + || ProtocolVersion.V2.equals(transferConfig.protocolVersion)) && clientRequestedV2; } @@ -1191,17 +1193,18 @@ public class UploadPack { if (req.wasDoneReceived()) { processHaveLines(req.getPeerHas(), ObjectId.zeroId(), - new PacketLineOut(NullOutputStream.INSTANCE), + new PacketLineOut(NullOutputStream.INSTANCE, false), accumulator); } else { - pckOut.writeString("acknowledgments\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_ACKNOWLEDGMENTS + '\n'); for (ObjectId id : req.getPeerHas()) { if (walk.getObjectReader().has(id)) { pckOut.writeString("ACK " + id.getName() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ } } processHaveLines(req.getPeerHas(), ObjectId.zeroId(), - new PacketLineOut(NullOutputStream.INSTANCE), + new PacketLineOut(NullOutputStream.INSTANCE, false), accumulator); if (okToGiveUp()) { pckOut.writeString("ready\n"); //$NON-NLS-1$ @@ -1243,7 +1246,8 @@ public class UploadPack { if (!pckOut.isUsingSideband()) { // sendPack will write "packfile\n" for us if sideband-all is used. // But sideband-all is not used, so we have to write it ourselves. - pckOut.writeString("packfile\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_PACKFILE + '\n'); } accumulator.timeNegotiating = Duration @@ -1955,8 +1959,8 @@ public class UploadPack { .map(objId -> objectIdToRevObject(objWalk, objId)) .filter(Objects::nonNull); // Ignore missing tips - ObjectReachabilityChecker reachabilityChecker = objWalk - .createObjectReachabilityChecker(); + ObjectReachabilityChecker reachabilityChecker = reader + .createObjectReachabilityChecker(objWalk); Optional<RevObject> unreachable = reachabilityChecker .areAllReachable(wantsAsObjs, startersAsObjs); if (unreachable.isPresent()) { @@ -1967,8 +1971,8 @@ public class UploadPack { } // All wants are commits, we can use ReachabilityChecker - ReachabilityChecker reachabilityChecker = walk - .createReachabilityChecker(); + ReachabilityChecker reachabilityChecker = reader + .createReachabilityChecker(walk); Stream<RevCommit> reachableCommits = importantRefsFirst(visibleRefs) .map(UploadPack::refToObjectId) @@ -2327,7 +2331,8 @@ public class UploadPack { // for us if provided a PackfileUriConfig. In this case, we // are not providing a PackfileUriConfig, so we have to // write this line ourselves. - pckOut.writeString("packfile\n"); //$NON-NLS-1$ + pckOut.writeString( + GitProtocolConstants.SECTION_PACKFILE + '\n'); } } pw.writePack(pm, NullProgressMonitor.INSTANCE, packOut); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java index c44838ab89..a6b20451dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java @@ -510,6 +510,7 @@ class WalkFetchConnection extends BaseFetchConnection { // and attach it to the local repository so we can use // all of the contained objects. // + Throwable e1 = null; try { pack.downloadPack(monitor); } catch (IOException err) { @@ -519,6 +520,7 @@ class WalkFetchConnection extends BaseFetchConnection { // an alternate. // recordError(id, err); + e1 = err; continue; } finally { // If the pack was good its in the local repository @@ -531,6 +533,9 @@ class WalkFetchConnection extends BaseFetchConnection { if (pack.tmpIdx != null) FileUtils.delete(pack.tmpIdx); } catch (IOException e) { + if (e1 != null) { + e.addSuppressed(e1); + } throw new TransportException(e.getMessage(), e); } packItr.remove(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnectionFactory2.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnectionFactory2.java new file mode 100644 index 0000000000..88abc60162 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnectionFactory2.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2020 Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.transport.http; + +import java.io.IOException; +import java.security.GeneralSecurityException; + +import org.eclipse.jgit.annotations.NonNull; + +/** + * A {@link HttpConnectionFactory} that supports client-side sessions that can + * maintain state and configure connections. + * + * @since 5.11 + */ +public interface HttpConnectionFactory2 extends HttpConnectionFactory { + + /** + * Creates a new {@link GitSession} instance that can be used with + * connections created by this {@link HttpConnectionFactory} instance. + * + * @return a new {@link GitSession} + */ + @NonNull + GitSession newSession(); + + /** + * A {@code GitSession} groups the multiple HTTP connections + * {@link org.eclipse.jgit.transport.TransportHttp TransportHttp} uses for + * the requests it makes during a git fetch or push. A {@code GitSession} + * can maintain client-side HTTPS state and can configure individual + * connections. + */ + interface GitSession { + + /** + * Configure a just created {@link HttpConnection}. + * + * @param connection + * to configure; created by the same + * {@link HttpConnectionFactory} instance + * @param sslVerify + * whether SSL is to be verified + * @return the configured {@connection} + * @throws IOException + * if the connection cannot be configured + * @throws GeneralSecurityException + * if the connection cannot be configured + */ + @NonNull + HttpConnection configure(@NonNull HttpConnection connection, + boolean sslVerify) throws IOException, GeneralSecurityException; + + /** + * Closes the {@link GitSession}, releasing any internal state. + */ + void close(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java index b39f1579be..1b5d1b3c43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/JDKHttpConnectionFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Christian Halstrick <christian.halstrick@sap.com> and others + * Copyright (C) 2013, 2020 Christian Halstrick <christian.halstrick@sap.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -12,6 +12,18 @@ package org.eclipse.jgit.transport.http; import java.io.IOException; import java.net.Proxy; import java.net.URL; +import java.security.GeneralSecurityException; +import java.text.MessageFormat; + +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.transport.http.DelegatingSSLSocketFactory; +import org.eclipse.jgit.util.HttpSupport; /** * A factory returning instances of @@ -19,17 +31,70 @@ import java.net.URL; * * @since 3.3 */ -public class JDKHttpConnectionFactory implements HttpConnectionFactory { - /** {@inheritDoc} */ +public class JDKHttpConnectionFactory implements HttpConnectionFactory2 { + @Override public HttpConnection create(URL url) throws IOException { return new JDKHttpConnection(url); } - /** {@inheritDoc} */ @Override public HttpConnection create(URL url, Proxy proxy) throws IOException { return new JDKHttpConnection(url, proxy); } + + @Override + public GitSession newSession() { + return new JdkConnectionSession(); + } + + private static class JdkConnectionSession implements GitSession { + + private SSLContext securityContext; + + private SSLSocketFactory socketFactory; + + @Override + public JDKHttpConnection configure(HttpConnection connection, + boolean sslVerify) throws GeneralSecurityException { + if (!(connection instanceof JDKHttpConnection)) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().httpWrongConnectionType, + JDKHttpConnection.class.getName(), + connection.getClass().getName())); + } + JDKHttpConnection conn = (JDKHttpConnection) connection; + String scheme = conn.getURL().getProtocol(); + if (!"https".equals(scheme) || sslVerify) { //$NON-NLS-1$ + // sslVerify == true: use the JDK defaults + return conn; + } + if (securityContext == null) { + securityContext = SSLContext.getInstance("TLS"); //$NON-NLS-1$ + TrustManager[] trustAllCerts = { + new NoCheckX509TrustManager() }; + securityContext.init(null, trustAllCerts, null); + socketFactory = new DelegatingSSLSocketFactory( + securityContext.getSocketFactory()) { + + @Override + protected void configure(SSLSocket socket) { + HttpSupport.configureTLS(socket); + } + }; + } + conn.setHostnameVerifier((name, session) -> true); + ((HttpsURLConnection) conn.wrappedUrlConnection) + .setSSLSocketFactory(socketFactory); + return conn; + } + + @Override + public void close() { + securityContext = null; + socketFactory = null; + } + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/NoCheckX509TrustManager.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/NoCheckX509TrustManager.java new file mode 100644 index 0000000000..5cd4fb4c82 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/NoCheckX509TrustManager.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2016, 2020 JGit contributors + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + * + * Contributors: + * Saša Živkov 2016 - initial API + * Thomas Wolf 2020 - extracted from HttpSupport + */ +package org.eclipse.jgit.transport.http; + +import java.security.cert.X509Certificate; + +import javax.net.ssl.X509TrustManager; + +/** + * A {@link X509TrustManager} that doesn't verify anything. Can be used for + * skipping SSL checks. + * + * @since 5.11 + */ +public class NoCheckX509TrustManager implements X509TrustManager { + + @Override + public X509Certificate[] getAcceptedIssuers() { + return null; + } + + @Override + public void checkClientTrusted(X509Certificate[] certs, + String authType) { + // no check + } + + @Override + public void checkServerTrusted(X509Certificate[] certs, + String authType) { + // no check + } +}
\ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 72278dc9c3..55b7d6279a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -2,7 +2,7 @@ * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> * Copyright (C) 2010, Christian Halstrick <christian.halstrick@sap.com> * Copyright (C) 2010, Matthias Sohn <matthias.sohn@sap.com> - * Copyright (C) 2012-2020, Robin Rosenberg and others + * Copyright (C) 2012-2021, Robin Rosenberg and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -63,6 +63,7 @@ import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.treewalk.TreeWalk.OperationType; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.ExecutionResult; +import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.Holder; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.Paths; @@ -799,7 +800,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { if (Constants.DOT_GIT.equals(name)) continue; if (Constants.DOT_GIT_IGNORE.equals(name)) - ignoreNode = new PerDirectoryIgnoreNode(e); + ignoreNode = new PerDirectoryIgnoreNode( + TreeWalk.pathOf(path, 0, pathOffset) + + Constants.DOT_GIT_IGNORE, + e); if (Constants.DOT_GIT_ATTRIBUTES.equals(name)) attributesNode = new PerDirectoryAttributesNode(e); if (i != o) @@ -983,8 +987,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return true; } else if (ObjectId.zeroId().compareTo(idBuffer, idOffset) == 0) { - return new File(repository.getWorkTree(), - entry.getPathString()).list().length > 0; + Path p = repository.getWorkTree().toPath() + .resolve(entry.getPathString()); + return FileUtils.hasFiles(p); } return false; } else if (mode == FileMode.SYMLINK.getBits()) @@ -1272,17 +1277,20 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { /** Magic type indicating we know rules exist, but they aren't loaded. */ private static class PerDirectoryIgnoreNode extends IgnoreNode { - final Entry entry; + protected final Entry entry; + + private final String name; - PerDirectoryIgnoreNode(Entry entry) { + PerDirectoryIgnoreNode(String name, Entry entry) { super(Collections.<FastIgnoreRule> emptyList()); + this.name = name; this.entry = entry; } IgnoreNode load() throws IOException { IgnoreNode r = new IgnoreNode(); try (InputStream in = entry.openInputStream()) { - r.parse(in); + r.parse(name, in); } return r.getRules().isEmpty() ? null : r; } @@ -1293,7 +1301,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { final Repository repository; RootIgnoreNode(Entry entry, Repository repository) { - super(entry); + super(entry != null ? entry.getName() : null, entry); this.repository = repository; } @@ -1327,7 +1335,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { throws FileNotFoundException, IOException { if (FS.DETECTED.exists(exclude)) { try (FileInputStream in = new FileInputStream(exclude)) { - r.parse(in); + r.parse(exclude.getAbsolutePath(), in); } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index d8cab358e7..0946f645fb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -22,7 +22,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; -import java.io.PrintStream; import java.io.Writer; import java.nio.charset.Charset; import java.nio.file.AccessDeniedException; @@ -337,6 +336,9 @@ public abstract class FS { try { path = path.toAbsolutePath(); Path dir = Files.isDirectory(path) ? path : path.getParent(); + if (dir == null) { + return FALLBACK_FILESTORE_ATTRIBUTES; + } FileStoreAttributes cached = attrCacheByPath.get(dir); if (cached != null) { return cached; @@ -511,7 +513,10 @@ public abstract class FS { } private static void write(Path p, String body) throws IOException { - FileUtils.mkdirs(p.getParent().toFile(), true); + Path parent = p.getParent(); + if (parent != null) { + FileUtils.mkdirs(parent.toFile(), true); + } try (Writer w = new OutputStreamWriter(Files.newOutputStream(p), UTF_8)) { w.write(body); @@ -1867,18 +1872,18 @@ public abstract class FS { * @throws org.eclipse.jgit.api.errors.JGitInternalException * if we fail to run the hook somehow. Causes may include an * interrupted process or I/O errors. - * @since 4.0 + * @since 5.11 */ public ProcessResult runHookIfPresent(Repository repository, final String hookName, - String[] args, PrintStream outRedirect, PrintStream errRedirect, + String[] args, OutputStream outRedirect, OutputStream errRedirect, String stdinArgs) throws JGitInternalException { return new ProcessResult(Status.NOT_SUPPORTED); } /** * See - * {@link #runHookIfPresent(Repository, String, String[], PrintStream, PrintStream, String)} + * {@link #runHookIfPresent(Repository, String, String[], OutputStream, OutputStream, String)} * . Should only be called by FS supporting shell scripts execution. * * @param repository @@ -1903,11 +1908,11 @@ public abstract class FS { * @throws org.eclipse.jgit.api.errors.JGitInternalException * if we fail to run the hook somehow. Causes may include an * interrupted process or I/O errors. - * @since 4.0 + * @since 5.11 */ protected ProcessResult internalRunHookIfPresent(Repository repository, - final String hookName, String[] args, PrintStream outRedirect, - PrintStream errRedirect, String stdinArgs) + final String hookName, String[] args, OutputStream outRedirect, + OutputStream errRedirect, String stdinArgs) throws JGitInternalException { File hookFile = findHook(repository, hookName); if (hookFile == null || hookName == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java index fb63dc02bb..946d81c73a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java @@ -16,7 +16,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; -import java.io.PrintStream; +import java.io.OutputStream; import java.nio.charset.Charset; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileStore; @@ -268,7 +268,7 @@ public class FS_POSIX extends FS { /** {@inheritDoc} */ @Override public ProcessResult runHookIfPresent(Repository repository, String hookName, - String[] args, PrintStream outRedirect, PrintStream errRedirect, + String[] args, OutputStream outRedirect, OutputStream errRedirect, String stdinArgs) throws JGitInternalException { return internalRunHookIfPresent(repository, hookName, args, outRedirect, errRedirect, stdinArgs); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java index d53bff78ea..add5498175 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java @@ -13,7 +13,7 @@ package org.eclipse.jgit.util; import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; -import java.io.PrintStream; +import java.io.OutputStream; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.ArrayList; @@ -139,7 +139,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { /** {@inheritDoc} */ @Override public ProcessResult runHookIfPresent(Repository repository, String hookName, - String[] args, PrintStream outRedirect, PrintStream errRedirect, + String[] args, OutputStream outRedirect, OutputStream errRedirect, String stdinArgs) throws JGitInternalException { return internalRunHookIfPresent(repository, hookName, args, outRedirect, errRedirect, stdinArgs); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index aa39a44642..b9dd9baa61 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Locale; import java.util.Random; import java.util.regex.Pattern; +import java.util.stream.Stream; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; @@ -802,6 +803,23 @@ public class FileUtils { } /** + * Whether the path is a directory with files in it. + * + * @param dir + * directory path + * @return {@code true} if the given directory path contains files + * @throws IOException + * on any I/O errors accessing the path + * + * @since 5.11 + */ + public static boolean hasFiles(Path dir) throws IOException { + try (Stream<Path> stream = Files.list(dir)) { + return stream.findAny().isPresent(); + } + } + + /** * Whether the given file can be executed. * * @param file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java index 04b3eab504..23a73faf8c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java @@ -24,21 +24,18 @@ import java.net.URL; import java.net.URLEncoder; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; import java.text.MessageFormat; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; import java.util.Set; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocket; import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.transport.http.NoCheckX509TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -301,40 +298,13 @@ public class HttpSupport { */ public static void disableSslVerify(HttpConnection conn) throws IOException { - final TrustManager[] trustAllCerts = new TrustManager[] { - new DummyX509TrustManager() }; + TrustManager[] trustAllCerts = { + new NoCheckX509TrustManager() }; try { conn.configure(null, trustAllCerts, null); - conn.setHostnameVerifier(new DummyHostnameVerifier()); + conn.setHostnameVerifier((name, session) -> true); } catch (KeyManagementException | NoSuchAlgorithmException e) { - throw new IOException(e.getMessage()); - } - } - - private static class DummyX509TrustManager implements X509TrustManager { - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - } - - @Override - public void checkClientTrusted(X509Certificate[] certs, - String authType) { - // no check - } - - @Override - public void checkServerTrusted(X509Certificate[] certs, - String authType) { - // no check - } - } - - private static class DummyHostnameVerifier implements HostnameVerifier { - @Override - public boolean verify(String hostname, SSLSession session) { - // always accept - return true; + throw new IOException(e.getMessage(), e); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java index 680d90392d..6d5694e435 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/IO.java @@ -246,7 +246,7 @@ public class IO { * buffer that must be fully populated, [off, off+len). * @param off * position within the buffer to start writing to. - * @return number of bytes in buffer or stream, whichever is shortest + * @return number of bytes read * @throws java.io.IOException * there was an error reading from the stream. */ @@ -254,8 +254,8 @@ public class IO { throws IOException { int r; int len = 0; - while ((r = fd.read(dst, off, dst.length - off)) >= 0 - && len < dst.length) { + while (off < dst.length + && (r = fd.read(dst, off, dst.length - off)) >= 0) { off += r; len += r; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java new file mode 100644 index 0000000000..cf06172c17 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import java.text.MessageFormat; +import java.util.Locale; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.GpgSignatureVerifier.TrustLevel; +import org.eclipse.jgit.lib.PersonIdent; + +/** + * Utilities for signature verification. + * + * @since 5.11 + */ +public final class SignatureUtils { + + private SignatureUtils() { + // No instantiation + } + + /** + * Writes information about a signature verification to a string. + * + * @param verification + * to show + * @param creator + * of the object verified; used for time zone information + * @param formatter + * to use for dates + * @return a textual representation of the {@link SignatureVerification}, + * using LF as line separator + */ + public static String toString(SignatureVerification verification, + PersonIdent creator, GitDateFormatter formatter) { + StringBuilder result = new StringBuilder(); + // Use the creator's timezone for the signature date + PersonIdent dateId = new PersonIdent(creator, + verification.getCreationDate()); + result.append(MessageFormat.format(JGitText.get().verifySignatureMade, + formatter.formatDate(dateId))); + result.append('\n'); + result.append(MessageFormat.format( + JGitText.get().verifySignatureKey, + verification.getKeyFingerprint().toUpperCase(Locale.ROOT))); + result.append('\n'); + if (!StringUtils.isEmptyOrNull(verification.getSigner())) { + result.append( + MessageFormat.format(JGitText.get().verifySignatureIssuer, + verification.getSigner())); + result.append('\n'); + } + String msg; + if (verification.getVerified()) { + if (verification.isExpired()) { + msg = JGitText.get().verifySignatureExpired; + } else { + msg = JGitText.get().verifySignatureGood; + } + } else { + msg = JGitText.get().verifySignatureBad; + } + result.append(MessageFormat.format(msg, verification.getKeyUser())); + if (!TrustLevel.UNKNOWN.equals(verification.getTrustLevel())) { + result.append(' ' + MessageFormat + .format(JGitText.get().verifySignatureTrust, verification + .getTrustLevel().name().toLowerCase(Locale.ROOT))); + } + result.append('\n'); + msg = verification.getMessage(); + if (!StringUtils.isEmptyOrNull(msg)) { + result.append(msg); + result.append('\n'); + } + return result.toString(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 447f417e7e..54fd539f67 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -56,9 +56,9 @@ public abstract class SystemReader { private static final SystemReader DEFAULT; - private static Boolean isMacOS; + private static volatile Boolean isMacOS; - private static Boolean isWindows; + private static volatile Boolean isWindows; static { SystemReader r = new Default(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java index 1f0fedda6b..562eb05dd9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java @@ -230,11 +230,16 @@ public abstract class TemporaryBuffer extends OutputStream { if (Integer.MAX_VALUE < len) throw new OutOfMemoryError( JGitText.get().lengthExceedsMaximumArraySize); - final byte[] out = new byte[(int) len]; + int length = (int) len; + final byte[] out = new byte[length]; int outPtr = 0; for (Block b : blocks) { - System.arraycopy(b.buffer, 0, out, outPtr, b.count); - outPtr += b.count; + int toCopy = Math.min(length - outPtr, b.count); + System.arraycopy(b.buffer, 0, out, outPtr, toCopy); + outPtr += toCopy; + if (outPtr == length) { + break; + } } return out; } @@ -461,6 +466,30 @@ public abstract class TemporaryBuffer extends OutputStream { } @Override + public byte[] toByteArray(int limit) throws IOException { + if (onDiskFile == null) { + return super.toByteArray(limit); + } + final long len = Math.min(length(), limit); + if (Integer.MAX_VALUE < len) { + throw new OutOfMemoryError( + JGitText.get().lengthExceedsMaximumArraySize); + } + final byte[] out = new byte[(int) len]; + try (FileInputStream in = new FileInputStream(onDiskFile)) { + int read = 0; + int chunk; + while ((chunk = in.read(out, read, out.length - read)) >= 0) { + read += chunk; + if (read == out.length) { + break; + } + } + } + return out; + } + + @Override public void writeTo(OutputStream os, ProgressMonitor pm) throws IOException { if (onDiskFile == null) { |