diff options
Diffstat (limited to 'org.eclipse.jgit')
111 files changed, 2384 insertions, 901 deletions
diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 089510ac51..f96336d596 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -3,8 +3,8 @@ <resource path="META-INF/MANIFEST.MF"> <filter id="924844039"> <message_arguments> - <message_argument value="5.0.4"/> - <message_argument value="5.0.0"/> + <message_argument value="5.1.3"/> + <message_argument value="5.1.0"/> </message_arguments> </filter> </resource> @@ -24,22 +24,4 @@ </message_arguments> </filter> </resource> - <resource path="src/org/eclipse/jgit/util/FS.java" type="org.eclipse.jgit.util.FS"> - <filter id="1141899266"> - <message_arguments> - <message_argument value="4.7"/> - <message_argument value="5.0"/> - <message_argument value="createNewFileAtomic(File)"/> - </message_arguments> - </filter> - </resource> - <resource path="src/org/eclipse/jgit/util/FS.java" type="org.eclipse.jgit.util.FS$LockToken"> - <filter id="1141899266"> - <message_arguments> - <message_argument value="4.7"/> - <message_argument value="5.0"/> - <message_argument value="LockToken"/> - </message_arguments> - </filter> - </resource> </component> diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index a8114d9527..c7fe72ca17 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -3,12 +3,12 @@ Bundle-ManifestVersion: 2 Bundle-Name: %plugin_name Automatic-Module-Name: org.eclipse.jgit Bundle-SymbolicName: org.eclipse.jgit -Bundle-Version: 5.0.4.qualifier +Bundle-Version: 5.1.4.qualifier Bundle-Localization: plugin Bundle-Vendor: %provider_name Bundle-ActivationPolicy: lazy -Export-Package: org.eclipse.jgit.annotations;version="5.0.4", - org.eclipse.jgit.api;version="5.0.4"; +Export-Package: org.eclipse.jgit.annotations;version="5.1.4", + org.eclipse.jgit.api;version="5.1.4"; uses:="org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.diff, @@ -22,52 +22,52 @@ Export-Package: org.eclipse.jgit.annotations;version="5.0.4", org.eclipse.jgit.submodule, org.eclipse.jgit.transport, org.eclipse.jgit.merge", - org.eclipse.jgit.api.errors;version="5.0.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.errors", - org.eclipse.jgit.attributes;version="5.0.4", - org.eclipse.jgit.blame;version="5.0.4"; + org.eclipse.jgit.api.errors;version="5.1.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.errors", + org.eclipse.jgit.attributes;version="5.1.4", + org.eclipse.jgit.blame;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.diff", - org.eclipse.jgit.diff;version="5.0.4"; + org.eclipse.jgit.diff;version="5.1.4"; uses:="org.eclipse.jgit.patch, org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.util", - org.eclipse.jgit.dircache;version="5.0.4"; + org.eclipse.jgit.dircache;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.util, org.eclipse.jgit.events, org.eclipse.jgit.attributes", - org.eclipse.jgit.errors;version="5.0.4"; + org.eclipse.jgit.errors;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.internal.storage.pack, org.eclipse.jgit.transport, org.eclipse.jgit.dircache", - org.eclipse.jgit.events;version="5.0.4";uses:="org.eclipse.jgit.lib", - org.eclipse.jgit.fnmatch;version="5.0.4", - org.eclipse.jgit.gitrepo;version="5.0.4"; + org.eclipse.jgit.events;version="5.1.4";uses:="org.eclipse.jgit.lib", + org.eclipse.jgit.fnmatch;version="5.1.4", + org.eclipse.jgit.gitrepo;version="5.1.4"; uses:="org.eclipse.jgit.api, org.eclipse.jgit.lib, org.eclipse.jgit.revwalk, org.xml.sax.helpers, org.xml.sax", - org.eclipse.jgit.gitrepo.internal;version="5.0.4";x-internal:=true, - org.eclipse.jgit.hooks;version="5.0.4";uses:="org.eclipse.jgit.lib", - org.eclipse.jgit.ignore;version="5.0.4", - org.eclipse.jgit.ignore.internal;version="5.0.4";x-friends:="org.eclipse.jgit.test", - org.eclipse.jgit.internal;version="5.0.4";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test", - org.eclipse.jgit.internal.fsck;version="5.0.4";x-friends:="org.eclipse.jgit.test", - org.eclipse.jgit.internal.ketch;version="5.0.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.dfs;version="5.0.4"; + org.eclipse.jgit.gitrepo.internal;version="5.1.4";x-internal:=true, + org.eclipse.jgit.hooks;version="5.1.4";uses:="org.eclipse.jgit.lib", + org.eclipse.jgit.ignore;version="5.1.4", + org.eclipse.jgit.ignore.internal;version="5.1.4";x-friends:="org.eclipse.jgit.test", + org.eclipse.jgit.internal;version="5.1.4";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test", + org.eclipse.jgit.internal.fsck;version="5.1.4";x-friends:="org.eclipse.jgit.test", + org.eclipse.jgit.internal.ketch;version="5.1.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", + org.eclipse.jgit.internal.storage.dfs;version="5.1.4"; x-friends:="org.eclipse.jgit.test, org.eclipse.jgit.http.server, org.eclipse.jgit.http.test, org.eclipse.jgit.lfs.test", - org.eclipse.jgit.internal.storage.file;version="5.0.4"; + org.eclipse.jgit.internal.storage.file;version="5.1.4"; x-friends:="org.eclipse.jgit.test, org.eclipse.jgit.junit, org.eclipse.jgit.junit.http, @@ -75,12 +75,12 @@ Export-Package: org.eclipse.jgit.annotations;version="5.0.4", org.eclipse.jgit.lfs, org.eclipse.jgit.pgm, org.eclipse.jgit.pgm.test", - org.eclipse.jgit.internal.storage.io;version="5.0.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.pack;version="5.0.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.reftable;version="5.0.4"; + org.eclipse.jgit.internal.storage.io;version="5.1.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", + org.eclipse.jgit.internal.storage.pack;version="5.1.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", + org.eclipse.jgit.internal.storage.reftable;version="5.1.4"; x-friends:="org.eclipse.jgit.http.test,org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.reftree;version="5.0.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.lib;version="5.0.4"; + org.eclipse.jgit.internal.storage.reftree;version="5.1.4";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", + org.eclipse.jgit.lib;version="5.1.4"; uses:="org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.util, @@ -90,33 +90,33 @@ Export-Package: org.eclipse.jgit.annotations;version="5.0.4", org.eclipse.jgit.treewalk, org.eclipse.jgit.transport, org.eclipse.jgit.submodule", - org.eclipse.jgit.lib.internal;version="5.0.4";x-internal:=true, - org.eclipse.jgit.merge;version="5.0.4"; + org.eclipse.jgit.lib.internal;version="5.1.4";x-internal:=true, + org.eclipse.jgit.merge;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.revwalk, org.eclipse.jgit.diff, org.eclipse.jgit.dircache, org.eclipse.jgit.api", - org.eclipse.jgit.nls;version="5.0.4", - org.eclipse.jgit.notes;version="5.0.4"; + org.eclipse.jgit.nls;version="5.1.4", + org.eclipse.jgit.notes;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.revwalk, org.eclipse.jgit.merge", - org.eclipse.jgit.patch;version="5.0.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.diff", - org.eclipse.jgit.revplot;version="5.0.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.revwalk", - org.eclipse.jgit.revwalk;version="5.0.4"; + org.eclipse.jgit.patch;version="5.1.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.diff", + org.eclipse.jgit.revplot;version="5.1.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.revwalk", + org.eclipse.jgit.revwalk;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.diff, org.eclipse.jgit.revwalk.filter", - org.eclipse.jgit.revwalk.filter;version="5.0.4";uses:="org.eclipse.jgit.revwalk,org.eclipse.jgit.lib,org.eclipse.jgit.util", - org.eclipse.jgit.storage.file;version="5.0.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.util", - org.eclipse.jgit.storage.pack;version="5.0.4";uses:="org.eclipse.jgit.lib", - org.eclipse.jgit.submodule;version="5.0.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.treewalk.filter,org.eclipse.jgit.treewalk", - org.eclipse.jgit.transport;version="5.0.4"; + org.eclipse.jgit.revwalk.filter;version="5.1.4";uses:="org.eclipse.jgit.revwalk,org.eclipse.jgit.lib,org.eclipse.jgit.util", + org.eclipse.jgit.storage.file;version="5.1.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.util", + org.eclipse.jgit.storage.pack;version="5.1.4";uses:="org.eclipse.jgit.lib", + org.eclipse.jgit.submodule;version="5.1.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.treewalk.filter,org.eclipse.jgit.treewalk", + org.eclipse.jgit.transport;version="5.1.4"; uses:="org.eclipse.jgit.transport.resolver, org.eclipse.jgit.revwalk, org.eclipse.jgit.internal.storage.pack, @@ -128,24 +128,24 @@ Export-Package: org.eclipse.jgit.annotations;version="5.0.4", org.eclipse.jgit.transport.http, org.eclipse.jgit.errors, org.eclipse.jgit.storage.pack", - org.eclipse.jgit.transport.http;version="5.0.4";uses:="javax.net.ssl", - org.eclipse.jgit.transport.resolver;version="5.0.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.transport", - org.eclipse.jgit.treewalk;version="5.0.4"; + org.eclipse.jgit.transport.http;version="5.1.4";uses:="javax.net.ssl", + org.eclipse.jgit.transport.resolver;version="5.1.4";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.transport", + org.eclipse.jgit.treewalk;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.revwalk, org.eclipse.jgit.attributes, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.util, org.eclipse.jgit.dircache", - org.eclipse.jgit.treewalk.filter;version="5.0.4";uses:="org.eclipse.jgit.treewalk", - org.eclipse.jgit.util;version="5.0.4"; + org.eclipse.jgit.treewalk.filter;version="5.1.4";uses:="org.eclipse.jgit.treewalk", + org.eclipse.jgit.util;version="5.1.4"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.transport.http, org.eclipse.jgit.storage.file, org.ietf.jgss", - org.eclipse.jgit.util.io;version="5.0.4", - org.eclipse.jgit.util.sha1;version="5.0.4", - org.eclipse.jgit.util.time;version="5.0.4" + org.eclipse.jgit.util.io;version="5.1.4", + org.eclipse.jgit.util.sha1;version="5.1.4", + org.eclipse.jgit.util.time;version="5.1.4" Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", com.jcraft.jsch;version="[0.1.37,0.2.0)", diff --git a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF index b2d694a88c..b9ff58c634 100644 --- a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF @@ -3,5 +3,5 @@ Bundle-ManifestVersion: 2 Bundle-Name: org.eclipse.jgit - Sources Bundle-SymbolicName: org.eclipse.jgit.source Bundle-Vendor: Eclipse.org - JGit -Bundle-Version: 5.0.4.qualifier -Eclipse-SourceBundle: org.eclipse.jgit;version="5.0.4.qualifier";roots="." +Bundle-Version: 5.1.4.qualifier +Eclipse-SourceBundle: org.eclipse.jgit;version="5.1.4.qualifier";roots="." diff --git a/org.eclipse.jgit/pom.xml b/org.eclipse.jgit/pom.xml index 2b56d0ae1f..b57214571d 100644 --- a/org.eclipse.jgit/pom.xml +++ b/org.eclipse.jgit/pom.xml @@ -53,7 +53,7 @@ <parent> <groupId>org.eclipse.jgit</groupId> <artifactId>org.eclipse.jgit-parent</artifactId> - <version>5.0.4-SNAPSHOT</version> + <version>5.1.4-SNAPSHOT</version> </parent> <artifactId>org.eclipse.jgit</artifactId> @@ -84,10 +84,6 @@ <artifactId>JavaEWAH</artifactId> </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> <dependency> <groupId>org.slf4j</groupId> diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index bbea8a3d24..b0c952cd4c 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -390,6 +390,7 @@ invalidPathPeriodAtEndWindows=Invalid path (period at end is ignored by Windows) invalidPathSpaceAtEndWindows=Invalid path (space at end is ignored by Windows): {0} invalidPathReservedOnWindows=Invalid path (''{0}'' is reserved on Windows): {1} invalidRedirectLocation=Invalid redirect location {0} -> {1} +invalidRefAdvertisementLine=Invalid ref advertisement line: ''{1}'' invalidReflogRevision=Invalid reflog revision: {0} invalidRefName=Invalid ref name: {0} invalidReftableBlock=Invalid reftable block @@ -582,6 +583,7 @@ remoteNameCantBeNull=Remote name can't be null. renameBranchFailedBecauseTag=Can not rename as Ref {0} is a tag renameBranchFailedUnknownReason=Rename failed with unknown reason renameBranchUnexpectedResult=Unexpected rename result {0} +renameCancelled=Rename detection was cancelled renameFileFailed=Could not rename file {0} to {1} renamesAlreadyFound=Renames have already been found. renamesBreakingModifies=Breaking apart modified file pairs @@ -765,8 +767,8 @@ updatingRefFailed=Updating the ref {0} to {1} failed. ReturnCode from RefUpdate. upstreamBranchName=branch ''{0}'' of {1} uriNotConfigured=Submodule URI not configured uriNotFound={0} not found +uriNotFoundWithMessage={0} not found: {1} URINotSupported=URI not supported: {0} -URLNotFound={0} not found userConfigFileInvalid=User config file {0} invalid {1} walkFailure=Walk failure. wantNotValid=want {0} not valid diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java index 0d9fe41afb..43085dc775 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java @@ -136,7 +136,7 @@ public class CleanCommand extends GitCommand<Set<String>> { } catch (IOException e) { throw new JGitInternalException(e.getMessage(), e); } finally { - if (!files.isEmpty()) { + if (!dryRun && !files.isEmpty()) { repo.fireEvent(new WorkingTreeModifiedEvent(null, files)); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 98c16b8931..1783c4193e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -275,9 +275,7 @@ public class RebaseCommand extends GitCommand<RebaseResult> { throw new JGitInternalException(ioe.getMessage(), ioe); } case PROCESS_STEPS: - // fall through case SKIP: - // fall through case CONTINUE: String upstreamCommitId = rebaseState.readFile(ONTO); try { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java index 9d9626f5a4..244a15686f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java @@ -77,6 +77,8 @@ import org.eclipse.jgit.treewalk.filter.TreeFilter; public class SubmoduleAddCommand extends TransportCommand<SubmoduleAddCommand, Repository> { + private String name; + private String path; private String uri; @@ -94,6 +96,18 @@ public class SubmoduleAddCommand extends } /** + * Set the submodule name + * + * @param name + * @return this command + * @since 5.1 + */ + public SubmoduleAddCommand setName(String name) { + this.name = name; + return this; + } + + /** * Set repository-relative path of submodule * * @param path @@ -161,9 +175,28 @@ public class SubmoduleAddCommand extends throw new IllegalArgumentException(JGitText.get().pathNotConfigured); if (uri == null || uri.length() == 0) throw new IllegalArgumentException(JGitText.get().uriNotConfigured); + if (name == null || name.length() == 0) { + // Use the path as the default. + name = path; + } + if (name.contains("/../") || name.contains("\\..\\") //$NON-NLS-1$ //$NON-NLS-2$ + || name.startsWith("../") || name.startsWith("..\\") //$NON-NLS-1$ //$NON-NLS-2$ + || name.endsWith("/..") || name.endsWith("\\..")) { //$NON-NLS-1$ //$NON-NLS-2$ + // Submodule names are used to store the submodule repositories + // under $GIT_DIR/modules. Having ".." in submodule names makes a + // vulnerability (CVE-2018-11235 + // https://bugs.eclipse.org/bugs/show_bug.cgi?id=535027#c0) + // Reject the names with them. The callers need to make sure the + // names free from these. We don't automatically replace these + // characters or canonicalize by regarding the name as a file path. + // Since Path class is platform dependent, we manually check '/' and + // '\\' patterns here. + throw new IllegalArgumentException(MessageFormat + .format(JGitText.get().invalidNameContainsDotDot, name)); + } try { - SubmoduleValidator.assertValidSubmoduleName(path); + SubmoduleValidator.assertValidSubmoduleName(name); SubmoduleValidator.assertValidSubmodulePath(path); SubmoduleValidator.assertValidSubmoduleUri(uri); } catch (SubmoduleValidator.SubmoduleValidationException e) { @@ -202,7 +235,7 @@ public class SubmoduleAddCommand extends // Save submodule URL to parent repository's config StoredConfig config = repo.getConfig(); - config.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, + config.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, name, ConfigConstants.CONFIG_KEY_URL, resolvedUri); try { config.save(); @@ -216,9 +249,9 @@ public class SubmoduleAddCommand extends try { modulesConfig.load(); modulesConfig.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, - path, ConfigConstants.CONFIG_KEY_PATH, path); + name, ConfigConstants.CONFIG_KEY_PATH, path); modulesConfig.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, - path, ConfigConstants.CONFIG_KEY_URL, uri); + name, ConfigConstants.CONFIG_KEY_URL, uri); modulesConfig.save(); } catch (IOException e) { throw new JGitInternalException(e.getMessage(), e); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleDeinitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleDeinitCommand.java index 569a8e3596..5a0528b0f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleDeinitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleDeinitCommand.java @@ -227,11 +227,11 @@ public class SubmoduleDeinitCommand return SubmoduleDeinitStatus.DIRTY; } - Repository submoduleRepo = w.getRepository(); - - Status status = Git.wrap(submoduleRepo).status().call(); - return status.isClean() ? SubmoduleDeinitStatus.SUCCESS - : SubmoduleDeinitStatus.DIRTY; + try (Repository submoduleRepo = w.getRepository()) { + Status status = Git.wrap(submoduleRepo).status().call(); + return status.isClean() ? SubmoduleDeinitStatus.SUCCESS + : SubmoduleDeinitStatus.DIRTY; + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesNode.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesNode.java index 62bf9f2737..bcd72311dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesNode.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesNode.java @@ -42,6 +42,8 @@ */ package org.eclipse.jgit.attributes; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -50,8 +52,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.eclipse.jgit.lib.Constants; - /** * Represents a bundle of attributes inherited from a base directory. * @@ -115,7 +115,7 @@ public class AttributesNode { } private static BufferedReader asReader(InputStream in) { - return new BufferedReader(new InputStreamReader(in, Constants.CHARSET)); + return new BufferedReader(new InputStreamReader(in, UTF_8)); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java index 1ad7a3055a..9cec645679 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java @@ -539,7 +539,7 @@ public class BlameGenerator implements AutoCloseable { n.beginResult(revPool); outCandidate = n; outRegion = n.regionList; - return true; + return outRegion != null; } private boolean reverseResult(Candidate parent, Candidate source) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffConfig.java index 21dca6b78c..ca37a10c5a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffConfig.java @@ -82,7 +82,7 @@ public class DiffConfig { renameDetectionType = parseRenameDetectionType(rc.getString( ConfigConstants.CONFIG_DIFF_SECTION, null, ConfigConstants.CONFIG_KEY_RENAMES)); renameLimit = rc.getInt(ConfigConstants.CONFIG_DIFF_SECTION, - ConfigConstants.CONFIG_KEY_RENAMELIMIT, 200); + ConfigConstants.CONFIG_KEY_RENAMELIMIT, 400); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java index 7aaa50030f..e7ad0bc40d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java @@ -67,6 +67,7 @@ import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.AmbiguousObjectException; import org.eclipse.jgit.errors.BinaryBlobException; +import org.eclipse.jgit.errors.CancelledException; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; @@ -577,7 +578,14 @@ public class DiffFormatter implements AutoCloseable { throws IOException { renameDetector.reset(); renameDetector.addAll(files); - return renameDetector.compute(reader, progressMonitor); + try { + return renameDetector.compute(reader, progressMonitor); + } catch (CancelledException e) { + // TODO: consider propagating once bug 536323 is tackled + // (making DiffEntry.scan() and DiffFormatter.scan() and + // format() cancellable). + return Collections.emptyList(); + } } private boolean isAdd(List<DiffEntry> files) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java index 7bb217d04d..772fbb5ffe 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RenameDetector.java @@ -57,6 +57,7 @@ import java.util.List; import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.diff.SimilarityIndex.TableFullException; +import org.eclipse.jgit.errors.CancelledException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.FileMode; @@ -332,8 +333,13 @@ public class RenameDetector { * representing all files that have been changed. * @throws java.io.IOException * file contents cannot be read from the repository. + * @throws CancelledException + * if rename detection was cancelled */ - public List<DiffEntry> compute(ProgressMonitor pm) throws IOException { + // TODO(ms): use org.eclipse.jgit.api.errors.CanceledException in next major + // version + public List<DiffEntry> compute(ProgressMonitor pm) + throws IOException, CancelledException { if (!done) { try { return compute(objectReader, pm); @@ -355,9 +361,13 @@ public class RenameDetector { * representing all files that have been changed. * @throws java.io.IOException * file contents cannot be read from the repository. + * @throws CancelledException + * if rename detection was cancelled */ + // TODO(ms): use org.eclipse.jgit.api.errors.CanceledException in next major + // version public List<DiffEntry> compute(ObjectReader reader, ProgressMonitor pm) - throws IOException { + throws IOException, CancelledException { final ContentSource cs = ContentSource.create(reader); return compute(new ContentSource.Pair(cs, cs), pm); } @@ -373,9 +383,13 @@ public class RenameDetector { * representing all files that have been changed. * @throws java.io.IOException * file contents cannot be read from the repository. + * @throws CancelledException + * if rename detection was cancelled */ + // TODO(ms): use org.eclipse.jgit.api.errors.CanceledException in next major + // version public List<DiffEntry> compute(ContentSource.Pair reader, ProgressMonitor pm) - throws IOException { + throws IOException, CancelledException { if (!done) { done = true; @@ -415,8 +429,15 @@ public class RenameDetector { done = false; } + private void advanceOrCancel(ProgressMonitor pm) throws CancelledException { + if (pm.isCancelled()) { + throw new CancelledException(JGitText.get().renameCancelled); + } + pm.update(1); + } + private void breakModifies(ContentSource.Pair reader, ProgressMonitor pm) - throws IOException { + throws IOException, CancelledException { ArrayList<DiffEntry> newEntries = new ArrayList<>(entries.size()); pm.beginTask(JGitText.get().renamesBreakingModifies, entries.size()); @@ -437,13 +458,13 @@ public class RenameDetector { } else { newEntries.add(e); } - pm.update(1); + advanceOrCancel(pm); } entries = newEntries; } - private void rejoinModifies(ProgressMonitor pm) { + private void rejoinModifies(ProgressMonitor pm) throws CancelledException { HashMap<String, DiffEntry> nameMap = new HashMap<>(); ArrayList<DiffEntry> newAdded = new ArrayList<>(added.size()); @@ -452,7 +473,7 @@ public class RenameDetector { for (DiffEntry src : deleted) { nameMap.put(src.oldPath, src); - pm.update(1); + advanceOrCancel(pm); } for (DiffEntry dst : added) { @@ -468,7 +489,7 @@ public class RenameDetector { } else { newAdded.add(dst); } - pm.update(1); + advanceOrCancel(pm); } added = newAdded; @@ -498,7 +519,7 @@ public class RenameDetector { private void findContentRenames(ContentSource.Pair reader, ProgressMonitor pm) - throws IOException { + throws IOException, CancelledException { int cnt = Math.max(added.size(), deleted.size()); if (getRenameLimit() == 0 || cnt <= getRenameLimit()) { SimilarityRenameDetector d; @@ -516,7 +537,8 @@ public class RenameDetector { } @SuppressWarnings("unchecked") - private void findExactRenames(ProgressMonitor pm) { + private void findExactRenames(ProgressMonitor pm) + throws CancelledException { pm.beginTask(JGitText.get().renamesFindingExact, // added.size() + added.size() + deleted.size() + added.size() * deleted.size()); @@ -562,7 +584,7 @@ public class RenameDetector { } else { left.add(a); } - pm.update(1); + advanceOrCancel(pm); } for (List<DiffEntry> adds : nonUniqueAdds) { @@ -604,6 +626,10 @@ public class RenameDetector { int score = SimilarityRenameDetector.nameScore(addedName, deletedName); matrix[mNext] = SimilarityRenameDetector.encode(score, delIdx, addIdx); mNext++; + if (pm.isCancelled()) { + throw new CancelledException( + JGitText.get().renameCancelled); + } } } @@ -617,7 +643,7 @@ public class RenameDetector { DiffEntry a = adds.get(addIdx); if (a == null) { - pm.update(1); + advanceOrCancel(pm); continue; // was already matched earlier } @@ -635,11 +661,12 @@ public class RenameDetector { entries.add(DiffEntry.pair(type, d, a, 100)); adds.set(addIdx, null); // Claim the destination was matched. - pm.update(1); + advanceOrCancel(pm); } } else { left.addAll(adds); } + advanceOrCancel(pm); } added = left; @@ -692,7 +719,8 @@ public class RenameDetector { @SuppressWarnings("unchecked") private HashMap<AbbreviatedObjectId, Object> populateMap( - List<DiffEntry> diffEntries, ProgressMonitor pm) { + List<DiffEntry> diffEntries, ProgressMonitor pm) + throws CancelledException { HashMap<AbbreviatedObjectId, Object> map = new HashMap<>(); for (DiffEntry de : diffEntries) { Object old = map.put(id(de), de); @@ -706,7 +734,7 @@ public class RenameDetector { ((List<DiffEntry>) old).add(de); map.put(id(de), old); } - pm.update(1); + advanceOrCancel(pm); } return map; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java index 653658be3c..d8a05c34ea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java @@ -54,6 +54,7 @@ import java.util.List; import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.diff.SimilarityIndex.TableFullException; +import org.eclipse.jgit.errors.CancelledException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -128,7 +129,7 @@ class SimilarityRenameDetector { renameScore = score; } - void compute(ProgressMonitor pm) throws IOException { + void compute(ProgressMonitor pm) throws IOException, CancelledException { if (pm == null) pm = NullProgressMonitor.INSTANCE; @@ -142,6 +143,11 @@ class SimilarityRenameDetector { // we have looked at everything that is above our minimum score. // for (--mNext; mNext >= 0; mNext--) { + if (pm.isCancelled()) { + // TODO(ms): use org.eclipse.jgit.api.errors.CanceledException + // in next major version + throw new CancelledException(JGitText.get().renameCancelled); + } long ent = matrix[mNext]; int sIdx = srcFile(ent); int dIdx = dstFile(ent); @@ -209,7 +215,8 @@ class SimilarityRenameDetector { return r; } - private int buildMatrix(ProgressMonitor pm) throws IOException { + private int buildMatrix(ProgressMonitor pm) + throws IOException, CancelledException { // Allocate for the worst-case scenario where every pair has a // score that we need to consider. We might not need that many. // @@ -234,6 +241,14 @@ class SimilarityRenameDetector { SimilarityIndex s = null; for (int dstIdx = 0; dstIdx < dsts.size(); dstIdx++) { + if (pm.isCancelled()) { + // TODO(ms): use + // org.eclipse.jgit.api.errors.CanceledException in next + // major version + throw new CancelledException( + JGitText.get().renameCancelled); + } + DiffEntry dstEnt = dsts.get(dstIdx); if (!isFile(dstEnt.newMode)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 0b03eb1521..ca1e3ab275 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -513,7 +513,7 @@ public class DirCacheCheckout { if (!conflicts.isEmpty()) { if (failOnConflict) - throw new CheckoutConflictException(conflicts.toArray(new String[conflicts.size()])); + throw new CheckoutConflictException(conflicts.toArray(new String[0])); else cleanUpConflicts(); } @@ -1366,7 +1366,11 @@ public class DirCacheCheckout { * object reader to use for checkout * @throws java.io.IOException * @since 3.6 + * @deprecated since 5.1, use + * {@link #checkoutEntry(Repository, DirCacheEntry, ObjectReader, boolean, CheckoutMetadata)} + * instead */ + @Deprecated public static void checkoutEntry(Repository repo, DirCacheEntry entry, ObjectReader or) throws IOException { checkoutEntry(repo, entry, or, false, null); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java index fee9f51000..6b1d4f4d8a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java @@ -46,6 +46,8 @@ package org.eclipse.jgit.dircache; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.IOException; @@ -772,7 +774,7 @@ public class DirCacheEntry { } static String toString(byte[] path) { - return Constants.CHARSET.decode(ByteBuffer.wrap(path)).toString(); + return UTF_8.decode(ByteBuffer.wrap(path)).toString(); } static int getMaximumInfoLength(boolean extended) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheTree.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheTree.java index b605f3ca8e..11a3474a35 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheTree.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheTree.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.dircache; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.FileMode.TREE; import static org.eclipse.jgit.lib.TreeFormatter.entrySize; @@ -276,7 +277,7 @@ public class DirCacheTree { */ public String getNameString() { final ByteBuffer bb = ByteBuffer.wrap(encodedName); - return Constants.CHARSET.decode(bb).toString(); + return UTF_8.decode(bb).toString(); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index 929ffac114..26e783ddd7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -252,6 +252,9 @@ public class ManifestParser extends DefaultHandler { RepoText.get().errorIncludeFile, path), e); } } + } else if ("remove-project".equals(qName)) { //$NON-NLS-1$ + String name = attributes.getValue("name"); //$NON-NLS-1$ + projects.removeIf((p) -> p.getName().equals(name)); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 80fd3cf1a4..45a239da0e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -55,8 +55,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.StringJoiner; +import java.util.TreeMap; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.Git; @@ -115,16 +115,15 @@ public class RepoCommand extends GitCommand<RevCommit> { private String groupsParam; private String branch; private String targetBranch = Constants.HEAD; - private boolean recordRemoteBranch = false; - private boolean recordSubmoduleLabels = false; - private boolean recordShallowSubmodules = false; + private boolean recordRemoteBranch = true; + private boolean recordSubmoduleLabels = true; + private boolean recordShallowSubmodules = true; private PersonIdent author; private RemoteReader callback; private InputStream inputStream; private IncludedFileReader includedReader; private boolean ignoreRemoteFailures = false; - private List<RepoProject> bareProjects; private ProgressMonitor monitor; /** @@ -519,37 +518,33 @@ public class RepoCommand extends GitCommand<RevCommit> { } if (repo.isBare()) { - bareProjects = new ArrayList<>(); if (author == null) author = new PersonIdent(repo); if (callback == null) callback = new DefaultRemoteReader(); - for (RepoProject proj : filteredProjects) { - addSubmoduleBare(proj.getUrl(), proj.getPath(), - proj.getRevision(), proj.getCopyFiles(), - proj.getLinkFiles(), proj.getGroups(), - proj.getRecommendShallow()); - } + List<RepoProject> renamedProjects = renameProjects(filteredProjects); + DirCache index = DirCache.newInCore(); DirCacheBuilder builder = index.builder(); ObjectInserter inserter = repo.newObjectInserter(); try (RevWalk rw = new RevWalk(repo)) { Config cfg = new Config(); StringBuilder attributes = new StringBuilder(); - for (RepoProject proj : bareProjects) { + for (RepoProject proj : renamedProjects) { + String name = proj.getName(); String path = proj.getPath(); - String nameUri = proj.getName(); + String url = proj.getUrl(); ObjectId objectId; if (ObjectId.isId(proj.getRevision())) { objectId = ObjectId.fromString(proj.getRevision()); } else { - objectId = callback.sha1(nameUri, proj.getRevision()); + objectId = callback.sha1(url, proj.getRevision()); if (objectId == null && !ignoreRemoteFailures) { - throw new RemoteUnavailableException(nameUri); + throw new RemoteUnavailableException(url); } if (recordRemoteBranch) { // can be branch or tag - cfg.setString("submodule", path, "branch", //$NON-NLS-1$ //$NON-NLS-2$ + cfg.setString("submodule", name, "branch", //$NON-NLS-1$ //$NON-NLS-2$ proj.getRevision()); } @@ -559,7 +554,7 @@ public class RepoCommand extends GitCommand<RevCommit> { // depth in the 'clone-depth' field, while // git core only uses a binary 'shallow = true/false' // hint, we'll map any depth to 'shallow = true' - cfg.setBoolean("submodule", path, "shallow", //$NON-NLS-1$ //$NON-NLS-2$ + cfg.setBoolean("submodule", name, "shallow", //$NON-NLS-1$ //$NON-NLS-2$ true); } } @@ -575,12 +570,13 @@ public class RepoCommand extends GitCommand<RevCommit> { attributes.append(rec.toString()); } - URI submodUrl = URI.create(nameUri); + URI submodUrl = URI.create(url); if (targetUri != null) { submodUrl = relativize(targetUri, submodUrl); } - cfg.setString("submodule", path, "path", path); //$NON-NLS-1$ //$NON-NLS-2$ - cfg.setString("submodule", path, "url", submodUrl.toString()); //$NON-NLS-1$ //$NON-NLS-2$ + cfg.setString("submodule", name, "path", path); //$NON-NLS-1$ //$NON-NLS-2$ + cfg.setString("submodule", name, "url", //$NON-NLS-1$ //$NON-NLS-2$ + submodUrl.toString()); // create gitlink if (objectId != null) { @@ -591,7 +587,7 @@ public class RepoCommand extends GitCommand<RevCommit> { for (CopyFile copyfile : proj.getCopyFiles()) { byte[] src = callback.readFile( - nameUri, proj.getRevision(), copyfile.src); + url, proj.getRevision(), copyfile.src); objectId = inserter.insert(Constants.OBJ_BLOB, src); dcEntry = new DirCacheEntry(copyfile.dest); dcEntry.setObjectId(objectId); @@ -691,7 +687,7 @@ public class RepoCommand extends GitCommand<RevCommit> { } else { try (Git git = new Git(repo)) { for (RepoProject proj : filteredProjects) { - addSubmodule(proj.getUrl(), proj.getPath(), + addSubmodule(proj.getName(), proj.getUrl(), proj.getPath(), proj.getRevision(), proj.getCopyFiles(), proj.getLinkFiles(), git); } @@ -703,9 +699,9 @@ public class RepoCommand extends GitCommand<RevCommit> { } } - private void addSubmodule(String url, String path, String revision, - List<CopyFile> copyfiles, List<LinkFile> linkfiles, Git git) - throws GitAPIException, IOException { + private void addSubmodule(String name, String url, String path, + String revision, List<CopyFile> copyfiles, List<LinkFile> linkfiles, + Git git) throws GitAPIException, IOException { assert (!repo.isBare()); assert (git != null); if (!linkfiles.isEmpty()) { @@ -713,7 +709,8 @@ public class RepoCommand extends GitCommand<RevCommit> { JGitText.get().nonBareLinkFilesNotSupported); } - SubmoduleAddCommand add = git.submoduleAdd().setPath(path).setURI(url); + SubmoduleAddCommand add = git.submoduleAdd().setName(name).setPath(path) + .setURI(url); if (monitor != null) add.setProgressMonitor(monitor); @@ -731,16 +728,42 @@ public class RepoCommand extends GitCommand<RevCommit> { } } - private void addSubmoduleBare(String url, String path, String revision, - List<CopyFile> copyfiles, List<LinkFile> linkfiles, - Set<String> groups, String recommendShallow) { - assert (repo.isBare()); - assert (bareProjects != null); - RepoProject proj = new RepoProject(url, path, revision, null, groups, - recommendShallow); - proj.addCopyFiles(copyfiles); - proj.addLinkFiles(linkfiles); - bareProjects.add(proj); + /** + * Rename the projects if there's a conflict when converted to submodules. + * + * @param projects + * parsed projects + * @return projects that are renamed if necessary + */ + private List<RepoProject> renameProjects(List<RepoProject> projects) { + Map<String, List<RepoProject>> m = new TreeMap<>(); + for (RepoProject proj : projects) { + List<RepoProject> l = m.get(proj.getName()); + if (l == null) { + l = new ArrayList<>(); + m.put(proj.getName(), l); + } + l.add(proj); + } + + List<RepoProject> ret = new ArrayList<>(); + for (List<RepoProject> ps : m.values()) { + boolean nameConflict = ps.size() != 1; + for (RepoProject proj : ps) { + String name = proj.getName(); + if (nameConflict) { + name += SLASH + proj.getPath(); + } + RepoProject p = new RepoProject(name, + proj.getPath(), proj.getRevision(), null, + proj.getGroups(), proj.getRecommendShallow()); + p.setUrl(proj.getUrl()); + p.addCopyFiles(proj.getCopyFiles()); + p.addLinkFiles(proj.getLinkFiles()); + ret.add(p); + } + } + return ret; } /* diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java index 8a61d1b0b1..ad43e2ca83 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/hooks/GitHook.java @@ -42,7 +42,7 @@ */ package org.eclipse.jgit.hooks; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -158,7 +158,7 @@ abstract class GitHook<T> implements Callable<T> { PrintStream hookErrRedirect = null; try { hookErrRedirect = new PrintStream(errorByteArray, false, - CHARSET.name()); + UTF_8.name()); } catch (UnsupportedEncodingException e) { // UTF-8 is guaranteed to be available } @@ -167,7 +167,7 @@ abstract class GitHook<T> implements Callable<T> { hookErrRedirect, getStdinArgs()); if (result.isExecutedWithError()) { throw new AbortedByHookException( - new String(errorByteArray.toByteArray(), CHARSET), + new String(errorByteArray.toByteArray(), UTF_8), getHookName(), result.getExitCode()); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java index d570fde36f..864f8bfc02 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/IgnoreNode.java @@ -42,6 +42,8 @@ */ package org.eclipse.jgit.ignore; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -50,8 +52,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.eclipse.jgit.lib.Constants; - /** * Represents a bundle of ignore rules inherited from a base directory. * @@ -121,7 +121,7 @@ public class IgnoreNode { } private static BufferedReader asReader(InputStream in) { - return new BufferedReader(new InputStreamReader(in, Constants.CHARSET)); + return new BufferedReader(new InputStreamReader(in, UTF_8)); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 18f4c85d88..6e99ca739e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -451,6 +451,7 @@ public class JGitText extends TranslationBundle { /***/ public String invalidPathSpaceAtEndWindows; /***/ public String invalidPathReservedOnWindows; /***/ public String invalidRedirectLocation; + /***/ public String invalidRefAdvertisementLine; /***/ public String invalidReflogRevision; /***/ public String invalidRefName; /***/ public String invalidReftableBlock; @@ -643,6 +644,7 @@ public class JGitText extends TranslationBundle { /***/ public String renameBranchFailedBecauseTag; /***/ public String renameBranchFailedUnknownReason; /***/ public String renameBranchUnexpectedResult; + /***/ public String renameCancelled; /***/ public String renameFileFailed; /***/ public String renamesAlreadyFound; /***/ public String renamesBreakingModifies; @@ -826,8 +828,8 @@ public class JGitText extends TranslationBundle { /***/ public String upstreamBranchName; /***/ public String uriNotConfigured; /***/ public String uriNotFound; + /***/ public String uriNotFoundWithMessage; /***/ public String URINotSupported; - /***/ public String URLNotFound; /***/ public String userConfigFileInvalid; /***/ public String walkFailure; /***/ public String wantNotValid; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java index bc5ba3926b..fa32b267df 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java @@ -274,8 +274,8 @@ public abstract class KetchLeader { lock.lock(); try { - voters = v.toArray(new KetchReplica[v.size()]); - followers = f.toArray(new KetchReplica[f.size()]); + voters = v.toArray(new KetchReplica[0]); + followers = f.toArray(new KetchReplica[0]); self = me; } finally { lock.unlock(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java index 7ddde6337f..6f1f5c5c2c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java @@ -209,7 +209,7 @@ public class LocalReplica extends KetchReplica { checkFailed(failed, accepted); checkFailed(failed, committed); if (!failed.isEmpty()) { - String[] arr = failed.toArray(new String[failed.size()]); + String[] arr = failed.toArray(new String[0]); req.setRefs(refdb.exactRef(arr)); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsConfig.java index 3e2963af97..2f796a9d40 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsConfig.java @@ -48,7 +48,13 @@ import java.io.IOException; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.StoredConfig; -final class DfsConfig extends StoredConfig { +/** + * Config implementation used by DFS repositories. + * <p> + * The current implementation acts as if there is no persistent storage: loading + * simply clears the config, and saving does nothing. + */ +public final class DfsConfig extends StoredConfig { /** {@inheritDoc} */ @Override public void load() throws IOException, ConfigInvalidException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsFsck.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsFsck.java index 985393c5e2..3f96d0919b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsFsck.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsFsck.java @@ -146,7 +146,7 @@ public class DfsFsck { throws IOException { pm.beginTask(JGitText.get().countingObjects, ProgressMonitor.UNKNOWN); try (ObjectWalk ow = new ObjectWalk(repo)) { - for (Ref r : repo.getAllRefs().values()) { + for (Ref r : repo.getRefDatabase().getRefs()) { ObjectId objectId = r.getObjectId(); if (objectId == null) { // skip unborn branch diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index ca54ee22ea..09d59376a0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -43,13 +43,17 @@ package org.eclipse.jgit.internal.storage.dfs; +import static java.util.stream.Collectors.joining; + import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -86,10 +90,16 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } }; - /** Sources for a pack file. */ + /** + * Sources for a pack file. + * <p> + * <strong>Note:</strong> When sorting packs by source, do not use the default + * comparator based on {@link Enum#compareTo}. Prefer {@link + * #DEFAULT_COMPARATOR} or your own {@link ComparatorBuilder}. + */ public static enum PackSource { /** The pack is created by ObjectInserter due to local activity. */ - INSERT(0), + INSERT, /** * The pack is created by PackParser due to a network event. @@ -100,7 +110,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { * storage layout preferred by this version. Received packs are likely * to be either compacted or garbage collected in the future. */ - RECEIVE(0), + RECEIVE, /** * The pack was created by compacting multiple packs together. @@ -111,7 +121,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { * * @see DfsPackCompactor */ - COMPACT(1), + COMPACT, /** * Pack was created by Git garbage collection by this implementation. @@ -122,17 +132,17 @@ public abstract class DfsObjDatabase extends ObjectDatabase { * * @see DfsGarbageCollector */ - GC(2), + GC, /** Created from non-heads by {@link DfsGarbageCollector}. */ - GC_REST(3), + GC_REST, /** * RefTreeGraph pack was created by Git garbage collection. * * @see DfsGarbageCollector */ - GC_TXN(4), + GC_TXN, /** * Pack was created by Git garbage collection. @@ -141,12 +151,86 @@ public abstract class DfsObjDatabase extends ObjectDatabase { * last GC pass. It is retained in a new pack until it is safe to prune * these objects from the repository. */ - UNREACHABLE_GARBAGE(5); + UNREACHABLE_GARBAGE; - final int category; + /** + * Default comparator for sources. + * <p> + * Sorts generally newer, smaller types such as {@code INSERT} and {@code + * RECEIVE} earlier; older, larger types such as {@code GC} later; and + * {@code UNREACHABLE_GARBAGE} at the end. + */ + public static final Comparator<PackSource> DEFAULT_COMPARATOR = + new ComparatorBuilder() + .add(INSERT, RECEIVE) + .add(COMPACT) + .add(GC) + .add(GC_REST) + .add(GC_TXN) + .add(UNREACHABLE_GARBAGE) + .build(); + + /** + * Builder for describing {@link PackSource} ordering where some values are + * explicitly considered equal to others. + */ + public static class ComparatorBuilder { + private final Map<PackSource, Integer> ranks = new HashMap<>(); + private int counter; + + /** + * Add a collection of sources that should sort as equal. + * <p> + * Sources in the input will sort after sources listed in previous calls + * to this method. + * + * @param sources + * sources in this equivalence class. + * @return this. + */ + public ComparatorBuilder add(PackSource... sources) { + for (PackSource s : sources) { + ranks.put(s, Integer.valueOf(counter)); + } + counter++; + return this; + } - PackSource(int category) { - this.category = category; + /** + * Build the comparator. + * + * @return new comparator instance. + * @throws IllegalArgumentException + * not all {@link PackSource} instances were explicitly assigned + * an equivalence class. + */ + public Comparator<PackSource> build() { + return new PackSourceComparator(ranks); + } + } + + private static class PackSourceComparator implements Comparator<PackSource> { + private final Map<PackSource, Integer> ranks; + + private PackSourceComparator(Map<PackSource, Integer> ranks) { + if (!ranks.keySet().equals( + new HashSet<>(Arrays.asList(PackSource.values())))) { + throw new IllegalArgumentException(); + } + this.ranks = new HashMap<>(ranks); + } + + @Override + public int compare(PackSource a, PackSource b) { + return ranks.get(a).compareTo(ranks.get(b)); + } + + @Override + public String toString() { + return Arrays.stream(PackSource.values()) + .map(s -> s + "=" + ranks.get(s)) //$NON-NLS-1$ + .collect(joining(", ", getClass().getSimpleName() + "{", "}")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + } } } @@ -156,6 +240,8 @@ public abstract class DfsObjDatabase extends ObjectDatabase { private DfsReaderOptions readerOptions; + private Comparator<DfsPackDescription> packComparator; + /** * Initialize an object database for our repository. * @@ -169,6 +255,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { this.repository = repository; this.packList = new AtomicReference<>(NO_PACKS); this.readerOptions = options; + this.packComparator = DfsPackDescription.objectLookupComparator(); } /** @@ -180,6 +267,21 @@ public abstract class DfsObjDatabase extends ObjectDatabase { return readerOptions; } + /** + * Set the comparator used when searching for objects across packs. + * <p> + * An optimal comparator will find more objects without having to load large + * idx files from storage only to find that they don't contain the object. + * See {@link DfsPackDescription#objectLookupComparator()} for the default + * heuristics. + * + * @param packComparator + * comparator. + */ + public void setPackComparator(Comparator<DfsPackDescription> packComparator) { + this.packComparator = packComparator; + } + /** {@inheritDoc} */ @Override public DfsReader newReader() { @@ -523,7 +625,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { Map<DfsPackDescription, DfsReftable> reftables = reftableMap(old); List<DfsPackDescription> scanned = listPacks(); - Collections.sort(scanned); + Collections.sort(scanned, packComparator); List<DfsPackFile> newPacks = new ArrayList<>(scanned.size()); List<DfsReftable> newReftables = new ArrayList<>(scanned.size()); @@ -584,30 +686,9 @@ public abstract class DfsObjDatabase extends ObjectDatabase { * @return comparator to sort {@link DfsReftable} by priority. */ protected Comparator<DfsReftable> reftableComparator() { - return (fa, fb) -> { - DfsPackDescription a = fa.getPackDescription(); - DfsPackDescription b = fb.getPackDescription(); - - // GC, COMPACT reftables first by higher category. - int c = category(b) - category(a); - if (c != 0) { - return c; - } - - // Lower maxUpdateIndex first. - c = Long.signum(a.getMaxUpdateIndex() - b.getMaxUpdateIndex()); - if (c != 0) { - return c; - } - - // Older reftable first. - return Long.signum(a.getLastModified() - b.getLastModified()); - }; - } - - static int category(DfsPackDescription d) { - PackSource s = d.getPackSource(); - return s != null ? s.category : 0; + return Comparator.comparing( + DfsReftable::getPackDescription, + DfsPackDescription.reftableComparator()); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index b43b9b1780..127ee6bf11 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -404,12 +404,11 @@ public class DfsPackCompactor { // Sort packs by description ordering, this places newer packs before // older packs, allowing the PackWriter to be handed newer objects // first and older objects last. - Collections.sort(srcPacks, new Comparator<DfsPackFile>() { - @Override - public int compare(DfsPackFile a, DfsPackFile b) { - return a.getPackDescription().compareTo(b.getPackDescription()); - } - }); + Collections.sort( + srcPacks, + Comparator.comparing( + DfsPackFile::getPackDescription, + DfsPackDescription.objectLookupComparator())); rw = new RevWalk(ctx); added = rw.newFlag("ADDED"); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java index 45eb7b0e1a..5a1ac02e25 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java @@ -47,7 +47,9 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import java.util.Arrays; +import java.util.Comparator; +import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; @@ -61,7 +63,107 @@ import org.eclipse.jgit.storage.pack.PackStatistics; * Instances of this class are cached with the DfsPackFile, and should not be * modified once initialized and presented to the JGit DFS library. */ -public class DfsPackDescription implements Comparable<DfsPackDescription> { +public class DfsPackDescription { + /** + * Comparator for packs when looking up objects in indexes. + * <p> + * This comparator tries to position packs in the order readers should examine + * them when looking for objects by SHA-1. The default tries to sort packs + * with more recent modification dates before older packs, and packs with + * fewer objects before packs with more objects. + * <p> + * Uses {@link PackSource#DEFAULT_COMPARATOR} for the portion of comparison + * where packs are sorted by source. + * + * @return comparator. + */ + public static Comparator<DfsPackDescription> objectLookupComparator() { + return objectLookupComparator(PackSource.DEFAULT_COMPARATOR); + } + + /** + * Comparator for packs when looking up objects in indexes. + * <p> + * This comparator tries to position packs in the order readers should examine + * them when looking for objects by SHA-1. The default tries to sort packs + * with more recent modification dates before older packs, and packs with + * fewer objects before packs with more objects. + * + * @param packSourceComparator + * comparator for the {@link PackSource}, used as the first step in + * comparison. + * @return comparator. + */ + public static Comparator<DfsPackDescription> objectLookupComparator( + Comparator<PackSource> packSourceComparator) { + return Comparator.comparing( + DfsPackDescription::getPackSource, packSourceComparator) + .thenComparing((a, b) -> { + PackSource as = a.getPackSource(); + PackSource bs = b.getPackSource(); + + // Tie break GC type packs by smallest first. There should be at most + // one of each source, but when multiple exist concurrent GCs may have + // run. Preferring the smaller file selects higher quality delta + // compression, placing less demand on the DfsBlockCache. + if (as == bs && isGC(as)) { + int cmp = Long.signum(a.getFileSize(PACK) - b.getFileSize(PACK)); + if (cmp != 0) { + return cmp; + } + } + + // Newer packs should sort first. + int cmp = Long.signum(b.getLastModified() - a.getLastModified()); + if (cmp != 0) { + return cmp; + } + + // Break ties on smaller index. Readers may get lucky and find + // the object they care about in the smaller index. This also pushes + // big historical packs to the end of the list, due to more objects. + return Long.signum(a.getObjectCount() - b.getObjectCount()); + }); + } + + static Comparator<DfsPackDescription> reftableComparator() { + return (a, b) -> { + // GC, COMPACT reftables first by reversing default order. + int c = PackSource.DEFAULT_COMPARATOR.reversed() + .compare(a.getPackSource(), b.getPackSource()); + if (c != 0) { + return c; + } + + // Lower maxUpdateIndex first. + c = Long.signum(a.getMaxUpdateIndex() - b.getMaxUpdateIndex()); + if (c != 0) { + return c; + } + + // Older reftable first. + return Long.signum(a.getLastModified() - b.getLastModified()); + }; + } + + static Comparator<DfsPackDescription> reuseComparator() { + return (a, b) -> { + PackSource as = a.getPackSource(); + PackSource bs = b.getPackSource(); + + if (as == bs && DfsPackDescription.isGC(as)) { + // Push smaller GC files last; these likely have higher quality + // delta compression and the contained representation should be + // favored over other files. + return Long.signum(b.getFileSize(PACK) - a.getFileSize(PACK)); + } + + // DfsPackDescription.compareTo already did a reasonable sort. + // Rely on Arrays.sort being stable, leaving equal elements. + return 0; + }; + } + private final DfsRepositoryDescription repoDesc; private final String packName; private PackSource packSource; @@ -93,11 +195,15 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> { * name of the pack file. Must end with ".pack". * @param repoDesc * description of the repo containing the pack file. + * @param packSource + * the source of the pack. */ - public DfsPackDescription(DfsRepositoryDescription repoDesc, String name) { + public DfsPackDescription(DfsRepositoryDescription repoDesc, String name, + @NonNull PackSource packSource) { this.repoDesc = repoDesc; int dot = name.lastIndexOf('.'); this.packName = (dot < 0) ? name : name.substring(0, dot); + this.packSource = packSource; int extCnt = PackExt.values().length; sizeMap = new long[extCnt]; @@ -162,6 +268,7 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> { * * @return the source of the pack. */ + @NonNull public PackSource getPackSource() { return packSource; } @@ -173,7 +280,7 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> { * the source of the pack. * @return {@code this} */ - public DfsPackDescription setPackSource(PackSource source) { + public DfsPackDescription setPackSource(@NonNull PackSource source) { packSource = source; return this; } @@ -455,49 +562,6 @@ public class DfsPackDescription implements Comparable<DfsPackDescription> { return false; } - /** - * {@inheritDoc} - * <p> - * Sort packs according to the optimal lookup ordering. - * <p> - * This method tries to position packs in the order readers should examine - * them when looking for objects by SHA-1. The default tries to sort packs - * with more recent modification dates before older packs, and packs with - * fewer objects before packs with more objects. - */ - @Override - public int compareTo(DfsPackDescription b) { - // Cluster by PackSource, pushing UNREACHABLE_GARBAGE to the end. - PackSource as = getPackSource(); - PackSource bs = b.getPackSource(); - if (as != null && bs != null) { - int cmp = as.category - bs.category; - if (cmp != 0) - return cmp; - } - - // Tie break GC type packs by smallest first. There should be at most - // one of each source, but when multiple exist concurrent GCs may have - // run. Preferring the smaller file selects higher quality delta - // compression, placing less demand on the DfsBlockCache. - if (as != null && as == bs && isGC(as)) { - int cmp = Long.signum(getFileSize(PACK) - b.getFileSize(PACK)); - if (cmp != 0) { - return cmp; - } - } - - // Newer packs should sort first. - int cmp = Long.signum(b.getLastModified() - getLastModified()); - if (cmp != 0) - return cmp; - - // Break ties on smaller index. Readers may get lucky and find - // the object they care about in the smaller index. This also pushes - // big historical packs to the end of the list, due to more objects. - return Long.signum(getObjectCount() - b.getObjectCount()); - } - static boolean isGC(PackSource s) { switch (s) { case GC: diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 197114bd6b..d04709f6c2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -45,7 +45,6 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; -import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; import java.io.IOException; @@ -66,7 +65,6 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackList; -import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; import org.eclipse.jgit.internal.storage.file.PackIndex; @@ -611,26 +609,9 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } } - private static final Comparator<DfsPackFile> PACK_SORT_FOR_REUSE = new Comparator<DfsPackFile>() { - @Override - public int compare(DfsPackFile af, DfsPackFile bf) { - DfsPackDescription ad = af.getPackDescription(); - DfsPackDescription bd = bf.getPackDescription(); - PackSource as = ad.getPackSource(); - PackSource bs = bd.getPackSource(); - - if (as != null && as == bs && DfsPackDescription.isGC(as)) { - // Push smaller GC files last; these likely have higher quality - // delta compression and the contained representation should be - // favored over other files. - return Long.signum(bd.getFileSize(PACK) - ad.getFileSize(PACK)); - } - - // DfsPackDescription.compareTo already did a reasonable sort. - // Rely on Arrays.sort being stable, leaving equal elements. - return 0; - } - }; + private static final Comparator<DfsPackFile> PACK_SORT_FOR_REUSE = + Comparator.comparing( + DfsPackFile::getPackDescription, DfsPackDescription.reuseComparator()); private List<DfsPackFile> sortPacksForSelectRepresentation() throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java index 40cfb71dde..70816307f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java @@ -45,6 +45,9 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantLock; @@ -196,7 +199,7 @@ public class DfsReftableDatabase extends DfsRefDatabase { } // Cannot be the container of an existing reference. - return table.hasRef(refName + '/'); + return table.hasRefsWithPrefix(refName + '/'); } finally { lock.unlock(); } @@ -238,7 +241,8 @@ public class DfsReftableDatabase extends DfsRefDatabase { try { Reftable table = reader(); try (RefCursor rc = ALL.equals(prefix) ? table.allRefs() - : table.seekRef(prefix)) { + : (prefix.endsWith("/") ? table.seekRefsWithPrefix(prefix) //$NON-NLS-1$ + : table.seekRef(prefix))) { while (rc.next()) { Ref ref = table.resolve(rc.getRef()); if (ref != null && ref.getObjectId() != null) { @@ -256,6 +260,29 @@ public class DfsReftableDatabase extends DfsRefDatabase { /** {@inheritDoc} */ @Override + public List<Ref> getRefsByPrefix(String prefix) throws IOException { + List<Ref> all = new ArrayList<>(); + lock.lock(); + try { + Reftable table = reader(); + try (RefCursor rc = ALL.equals(prefix) ? table.allRefs() + : table.seekRefsWithPrefix(prefix)) { + while (rc.next()) { + Ref ref = table.resolve(rc.getRef()); + if (ref != null && ref.getObjectId() != null) { + all.add(ref); + } + } + } + } finally { + lock.unlock(); + } + + return Collections.unmodifiableList(all); + } + + /** {@inheritDoc} */ + @Override public Ref peel(Ref ref) throws IOException { Ref oldLeaf = ref.getLeaf(); if (oldLeaf.isPeeled() || oldLeaf.getObjectId() == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsStreamKey.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsStreamKey.java index c11f696708..8793d83126 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsStreamKey.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsStreamKey.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.dfs; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.util.Arrays; @@ -67,7 +67,7 @@ public abstract class DfsStreamKey { */ public static DfsStreamKey of(DfsRepositoryDescription repo, String name, @Nullable PackExt ext) { - return new ByteArrayDfsStreamKey(repo, name.getBytes(CHARSET), ext); + return new ByteArrayDfsStreamKey(repo, name.getBytes(UTF_8), ext); } final int hash; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java index 662c3fef81..5b6894da9c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.lib.RefDatabase; @@ -118,10 +119,10 @@ public class InMemoryRepository extends DfsRepository { @Override protected DfsPackDescription newPack(PackSource source) { int id = packId.incrementAndGet(); - DfsPackDescription desc = new MemPack( + return new MemPack( "pack-" + id + "-" + source.name(), //$NON-NLS-1$ //$NON-NLS-2$ - getRepository().getDescription()); - return desc.setPackSource(source); + getRepository().getDescription(), + source); } @Override @@ -169,8 +170,8 @@ public class InMemoryRepository extends DfsRepository { private static class MemPack extends DfsPackDescription { final byte[][] fileMap = new byte[PackExt.values().length][]; - MemPack(String name, DfsRepositoryDescription repoDesc) { - super(repoDesc, name); + MemPack(String name, DfsRepositoryDescription repoDesc, PackSource source) { + super(repoDesc, name, source); } void put(PackExt ext, byte[] data) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java index e0c056a450..47ac4ec72e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ReftableBatchRefUpdate.java @@ -70,7 +70,6 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.io.BlockSource; import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.internal.storage.reftable.RefCursor; import org.eclipse.jgit.internal.storage.reftable.Reftable; import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; @@ -240,11 +239,7 @@ public class ReftableBatchRefUpdate extends BatchRefUpdate { private boolean checkExpected(Reftable table, List<ReceiveCommand> pending) throws IOException { for (ReceiveCommand cmd : pending) { - Ref ref; - try (RefCursor rc = table.seekRef(cmd.getRefName())) { - ref = rc.next() ? rc.getRef() : null; - } - if (!matchOld(cmd, ref)) { + if (!matchOld(cmd, table.exactRef(cmd.getRefName()))) { cmd.setResult(LOCK_FAILURE); if (isAtomic()) { ReceiveCommand.abort(pending); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java index 407061fa16..3884180e19 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java @@ -104,6 +104,7 @@ abstract class BasePackBitmapIndex extends PackBitmapIndex { r = xb.xorBitmap.bitmapContainer; if (r instanceof EWAHCompressedBitmap) { out = out.xor((EWAHCompressedBitmap) r); + out.trim(); bitmapContainer = out; return out; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index d02888a872..5ced68646f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -480,7 +480,7 @@ public class FileRepository extends Repository { /** * {@inheritDoc} * <p> - * Objects known to exist but not expressed by {@link #getAllRefs()}. + * Objects known to exist but not expressed by {@code #getAllRefs()}. * <p> * When a repository borrows objects from another repository, it can * advertise that it safely has that other repository's references, without @@ -493,12 +493,12 @@ public class FileRepository extends Repository { } /** - * Objects known to exist but not expressed by {@link #getAllRefs()}. + * Objects known to exist but not expressed by {@code #getAllRefs()}. * <p> * When a repository borrows objects from another repository, it can * advertise that it safely has that other repository's references, without - * exposing any other details about the other repository. This may help - * a client trying to push changes avoid pushing more than it needs to. + * exposing any other details about the other repository. This may help a + * client trying to push changes avoid pushing more than it needs to. * * @param skips * Set of AlternateHandle Ids already seen diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 0ecfb2868c..7bfec3fd91 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -245,6 +245,7 @@ public class GC { * parsed */ // TODO(ms): in 5.0 change signature and return Future<Collection<PackFile>> + @SuppressWarnings("FutureReturnValueIgnored") public Collection<PackFile> gc() throws IOException, ParseException { if (!background) { return doGc(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java index 0e587ce827..82458c1acf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.IOException; @@ -171,6 +171,6 @@ class GcLog { if (content.length() > 0) { nonEmpty = true; } - lock.write(content.getBytes(CHARSET)); + lock.write(content.getBytes(UTF_8)); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LazyObjectIdSetFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LazyObjectIdSetFile.java index c82d52e79c..3d0e9c7189 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LazyObjectIdSetFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LazyObjectIdSetFile.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.BufferedReader; import java.io.File; @@ -86,7 +86,7 @@ public class LazyObjectIdSetFile implements ObjectIdSet { private ObjectIdOwnerMap<Entry> load() { ObjectIdOwnerMap<Entry> r = new ObjectIdOwnerMap<>(); try (FileInputStream fin = new FileInputStream(src); - Reader rin = new InputStreamReader(fin, CHARSET); + Reader rin = new InputStreamReader(fin, UTF_8); BufferedReader br = new BufferedReader(rin)) { MutableObjectId id = new MutableObjectId(); for (String line; (line = br.readLine()) != null;) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java index 289d89d698..e931c1f9e7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java @@ -62,13 +62,13 @@ class LocalCachedPack extends CachedPack { LocalCachedPack(ObjectDirectory odb, List<String> packNames) { this.odb = odb; - this.packNames = packNames.toArray(new String[packNames.size()]); + this.packNames = packNames.toArray(new String[0]); } LocalCachedPack(List<PackFile> packs) { odb = null; packNames = null; - this.packs = packs.toArray(new PackFile[packs.size()]); + this.packs = packs.toArray(new PackFile[0]); } /** {@inheritDoc} */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 51c5702fc2..24723d850c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -938,7 +938,7 @@ public class ObjectDirectory extends FileObjectDatabase { if (list.isEmpty()) return new PackList(snapshot, NO_PACKS.packs); - final PackFile[] r = list.toArray(new PackFile[list.size()]); + final PackFile[] r = list.toArray(new PackFile[0]); Arrays.sort(r, PackFile.SORT); return new PackList(snapshot, r); } @@ -1030,7 +1030,7 @@ public class ObjectDirectory extends FileObjectDatabase { l.add(openAlternate(line)); } } - return l.toArray(new AlternateHandle[l.size()]); + return l.toArray(new AlternateHandle[0]); } private static BufferedReader open(File f) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java index 70eb10ea8f..eff7958748 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java @@ -351,6 +351,7 @@ public class PackBitmapIndexBuilder extends BasePackBitmapIndex { PositionEntry entry = positionEntries.get(item); if (entry == null) throw new IllegalStateException(); + bestBitmap.trim(); return new StoredEntry(entry.namePosition, bestBitmap, bestXorOffset, item.getFlags()); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java index c04c90f22f..70695880d3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java @@ -200,6 +200,7 @@ public class PackBitmapIndexRemapper extends PackBitmapIndex for (IntIterator i = oldBitmap.getBitmap().intIterator(); i.hasNext();) inflated.set(prevToNewMapping[i.next()]); bitmap = inflated.toEWAHCompressedBitmap(); + bitmap.trim(); convertedBitmaps.add( new StoredBitmap(objectId, bitmap, null, oldBitmap.getFlags())); return bitmap; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 6a18df86e2..de7e4b3f25 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -46,7 +46,7 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.HEAD; import static org.eclipse.jgit.lib.Constants.LOGS; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; @@ -945,7 +945,7 @@ public class RefDirectory extends RefDatabase { try (BufferedReader br = new BufferedReader(new InputStreamReader( new DigestInputStream(new FileInputStream(packedRefsFile), digest), - CHARSET))) { + UTF_8))) { try { return new PackedRefList(parsePackedRefs(br), snapshot, ObjectId.fromRaw(digest.digest())); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BaseSearch.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BaseSearch.java index d231ccb997..f42b3a5463 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BaseSearch.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/BaseSearch.java @@ -91,7 +91,7 @@ class BaseSearch { List<ObjectToPack> edges, ObjectReader or) { progress = countingMonitor; reader = or; - baseTrees = bases.toArray(new ObjectId[bases.size()]); + baseTrees = bases.toArray(new ObjectId[0]); objectsMap = objects; edgeObjects = edges; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 36d6f0aebc..24af8a73ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -1970,7 +1970,7 @@ public class PackWriter implements AutoCloseable { byte[] pathBuf = walker.getPathBuffer(); int pathLen = walker.getPathLength(); bases.addBase(o.getType(), pathBuf, pathLen, pathHash); - filterAndAddObject(o, o.getType(), pathHash); + filterAndAddObject(o, o.getType(), pathHash, want); countingMonitor.update(1); } } else { @@ -1980,7 +1980,7 @@ public class PackWriter implements AutoCloseable { continue; if (exclude(o)) continue; - filterAndAddObject(o, o.getType(), walker.getPathHashCode()); + filterAndAddObject(o, o.getType(), walker.getPathHashCode(), want); countingMonitor.update(1); } } @@ -2013,7 +2013,7 @@ public class PackWriter implements AutoCloseable { needBitmap.remove(objectId); continue; } - filterAndAddObject(objectId, obj.getType(), 0); + filterAndAddObject(objectId, obj.getType(), 0, want); } if (thin) @@ -2075,12 +2075,14 @@ public class PackWriter implements AutoCloseable { // Adds the given object as an object to be packed, first performing // filtering on blobs at or exceeding a given size. private void filterAndAddObject(@NonNull AnyObjectId src, int type, - int pathHashCode) throws IOException { + int pathHashCode, @NonNull Set<? extends AnyObjectId> want) + throws IOException { // Check if this object needs to be rejected, doing the cheaper // checks first. boolean reject = filterBlobLimit >= 0 && type == OBJ_BLOB && + !want.contains(src) && reader.getObjectSize(src, OBJ_BLOB) > filterBlobLimit; if (!reject) { addObject(src, type, pathHashCode); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java index 38d3458cf9..99db74956c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java @@ -91,11 +91,10 @@ class PackWriterBitmapPreparer { private static final int DAY_IN_SECONDS = 24 * 60 * 60; - private static final Comparator<BitmapBuilderEntry> ORDER_BY_CARDINALITY = new Comparator<BitmapBuilderEntry>() { + private static final Comparator<RevCommit> ORDER_BY_REVERSE_TIMESTAMP = new Comparator<RevCommit>() { @Override - public int compare(BitmapBuilderEntry a, BitmapBuilderEntry b) { - return Integer.signum(a.getBuilder().cardinality() - - b.getBuilder().cardinality()); + public int compare(RevCommit a, RevCommit b) { + return Integer.signum(b.getCommitTime() - a.getCommitTime()); } }; @@ -164,154 +163,177 @@ class PackWriterBitmapPreparer { * the cache hits for clients that are close to HEAD, which is the * majority of calculations performed. */ - pm.beginTask(JGitText.get().selectingCommits, ProgressMonitor.UNKNOWN); - RevWalk rw = new RevWalk(reader); - rw.setRetainBody(false); - CommitSelectionHelper selectionHelper = setupTipCommitBitmaps(rw, - expectedCommitCount, excludeFromBitmapSelection); - pm.endTask(); - - int totCommits = selectionHelper.getCommitCount(); - BlockList<BitmapCommit> selections = new BlockList<>( - totCommits / recentCommitSpan + 1); - for (BitmapCommit reuse : selectionHelper.reusedCommits) { - selections.add(reuse); - } - - if (totCommits == 0) { - for (AnyObjectId id : selectionHelper.peeledWants) { - selections.add(new BitmapCommit(id, false, 0)); + try (RevWalk rw = new RevWalk(reader); + RevWalk rw2 = new RevWalk(reader)) { + pm.beginTask(JGitText.get().selectingCommits, + ProgressMonitor.UNKNOWN); + rw.setRetainBody(false); + CommitSelectionHelper selectionHelper = captureOldAndNewCommits(rw, + expectedCommitCount, excludeFromBitmapSelection); + pm.endTask(); + + // Add reused bitmaps from the previous GC pack's bitmap indices. + // Currently they are always fully reused, even if their spans don't + // match this run's PackConfig values. + int newCommits = selectionHelper.getCommitCount(); + BlockList<BitmapCommit> selections = new BlockList<>( + selectionHelper.reusedCommits.size() + + newCommits / recentCommitSpan + 1); + for (BitmapCommit reuse : selectionHelper.reusedCommits) { + selections.add(reuse); } - return selections; - } - pm.beginTask(JGitText.get().selectingCommits, totCommits); - int totalWants = selectionHelper.peeledWants.size(); - - for (BitmapBuilderEntry entry : selectionHelper.tipCommitBitmaps) { - BitmapBuilder bitmap = entry.getBuilder(); - int cardinality = bitmap.cardinality(); - - // Within this branch, keep ordered lists of commits representing - // chains in its history, where each chain is a "sub-branch". - // Ordering commits by these chains makes for fewer differences - // between consecutive selected commits, which in turn provides - // better compression/on the run-length encoding of the XORs between - // them. - List<List<BitmapCommit>> chains = - new ArrayList<>(); - - // Mark the current branch as inactive if its tip commit isn't - // recent and there are an excessive number of branches, to - // prevent memory bloat of computing too many bitmaps for stale - // branches. - boolean isActiveBranch = true; - if (totalWants > excessiveBranchCount - && !isRecentCommit(entry.getCommit())) { - isActiveBranch = false; + if (newCommits == 0) { + for (AnyObjectId id : selectionHelper.newWants) { + selections.add(new BitmapCommit(id, false, 0)); + } + return selections; } - // Insert bitmaps at the offsets suggested by the - // nextSelectionDistance() heuristic. Only reuse bitmaps created - // for more distant commits. - int index = -1; - int nextIn = nextSpan(cardinality); - int nextFlg = nextIn == distantCommitSpan - ? PackBitmapIndex.FLAG_REUSE : 0; - - // For the current branch, iterate through all commits from oldest - // to newest. - for (RevCommit c : selectionHelper) { - // Optimization: if we have found all the commits for this - // branch, stop searching - int distanceFromTip = cardinality - index - 1; - if (distanceFromTip == 0) { - break; + pm.beginTask(JGitText.get().selectingCommits, newCommits); + int totalWants = want.size(); + BitmapBuilder seen = commitBitmapIndex.newBitmapBuilder(); + seen.or(selectionHelper.reusedCommitsBitmap); + rw2.setRetainBody(false); + rw2.setRevFilter(new NotInBitmapFilter(seen)); + + // For each branch, do a revwalk to enumerate its commits. Exclude + // both reused commits and any commits seen in a previous branch. + // Then iterate through all new commits from oldest to newest, + // selecting well-spaced commits in this branch. + for (RevCommit rc : selectionHelper.newWantsByNewest) { + BitmapBuilder tipBitmap = commitBitmapIndex.newBitmapBuilder(); + rw2.markStart((RevCommit) rw2.peel(rw2.parseAny(rc))); + RevCommit rc2; + while ((rc2 = rw2.next()) != null) { + tipBitmap.addObject(rc2, Constants.OBJ_COMMIT); } - - // Ignore commits that are not in this branch - if (!bitmap.contains(c)) { - continue; + int cardinality = tipBitmap.cardinality(); + seen.or(tipBitmap); + + // Within this branch, keep ordered lists of commits + // representing chains in its history, where each chain is a + // "sub-branch". Ordering commits by these chains makes for + // fewer differences between consecutive selected commits, which + // in turn provides better compression/on the run-length + // encoding of the XORs between them. + List<List<BitmapCommit>> chains = new ArrayList<>(); + + // Mark the current branch as inactive if its tip commit isn't + // recent and there are an excessive number of branches, to + // prevent memory bloat of computing too many bitmaps for stale + // branches. + boolean isActiveBranch = true; + if (totalWants > excessiveBranchCount && !isRecentCommit(rc)) { + isActiveBranch = false; } - index++; - nextIn--; - pm.update(1); - - // Always pick the items in wants, prefer merge commits. - if (selectionHelper.peeledWants.remove(c)) { - if (nextIn > 0) { - nextFlg = 0; + // Insert bitmaps at the offsets suggested by the + // nextSelectionDistance() heuristic. Only reuse bitmaps created + // for more distant commits. + int index = -1; + int nextIn = nextSpan(cardinality); + int nextFlg = nextIn == distantCommitSpan + ? PackBitmapIndex.FLAG_REUSE + : 0; + + // For the current branch, iterate through all commits from + // oldest to newest. + for (RevCommit c : selectionHelper) { + // Optimization: if we have found all the commits for this + // branch, stop searching + int distanceFromTip = cardinality - index - 1; + if (distanceFromTip == 0) { + break; } - } else { - boolean stillInSpan = nextIn >= 0; - boolean isMergeCommit = c.getParentCount() > 1; - // Force selection if: - // a) we have exhausted the window looking for merges - // b) we are in the top commits of an active branch - // c) we are at a branch tip - boolean mustPick = (nextIn <= -recentCommitSpan) - || (isActiveBranch - && (distanceFromTip <= contiguousCommitCount)) - || (distanceFromTip == 1); // most recent commit - if (!mustPick && (stillInSpan || !isMergeCommit)) { + + // Ignore commits that are not in this branch + if (!tipBitmap.contains(c)) { continue; } - } - // This commit is selected. - // Calculate where to look for the next one. - int flags = nextFlg; - nextIn = nextSpan(distanceFromTip); - nextFlg = nextIn == distantCommitSpan - ? PackBitmapIndex.FLAG_REUSE : 0; - - BitmapBuilder fullBitmap = commitBitmapIndex.newBitmapBuilder(); - rw.reset(); - rw.markStart(c); - rw.setRevFilter(new AddUnseenToBitmapFilter( - selectionHelper.reusedCommitsBitmap, fullBitmap)); - - while (rw.next() != null) { - // The RevFilter adds the reachable commits from this - // selected commit to fullBitmap. - } + index++; + nextIn--; + pm.update(1); - // Sort the commits by independent chains in this branch's - // history, yielding better compression when building bitmaps. - List<BitmapCommit> longestAncestorChain = null; - for (List<BitmapCommit> chain : chains) { - BitmapCommit mostRecentCommit = chain.get(chain.size() - 1); - if (fullBitmap.contains(mostRecentCommit)) { - if (longestAncestorChain == null - || longestAncestorChain.size() < chain.size()) { - longestAncestorChain = chain; + // Always pick the items in wants, prefer merge commits. + if (selectionHelper.newWants.remove(c)) { + if (nextIn > 0) { + nextFlg = 0; + } + } else { + boolean stillInSpan = nextIn >= 0; + boolean isMergeCommit = c.getParentCount() > 1; + // Force selection if: + // a) we have exhausted the window looking for merges + // b) we are in the top commits of an active branch + // c) we are at a branch tip + boolean mustPick = (nextIn <= -recentCommitSpan) + || (isActiveBranch + && (distanceFromTip <= contiguousCommitCount)) + || (distanceFromTip == 1); // most recent commit + if (!mustPick && (stillInSpan || !isMergeCommit)) { + continue; } } + + // This commit is selected. + // Calculate where to look for the next one. + int flags = nextFlg; + nextIn = nextSpan(distanceFromTip); + nextFlg = nextIn == distantCommitSpan + ? PackBitmapIndex.FLAG_REUSE + : 0; + + // Create the commit bitmap for the current commit + BitmapBuilder bitmap = commitBitmapIndex.newBitmapBuilder(); + rw.reset(); + rw.markStart(c); + rw.setRevFilter(new AddUnseenToBitmapFilter( + selectionHelper.reusedCommitsBitmap, bitmap)); + while (rw.next() != null) { + // The filter adds the reachable commits to bitmap. + } + + // Sort the commits by independent chains in this branch's + // history, yielding better compression when building + // bitmaps. + List<BitmapCommit> longestAncestorChain = null; + for (List<BitmapCommit> chain : chains) { + BitmapCommit mostRecentCommit = chain + .get(chain.size() - 1); + if (bitmap.contains(mostRecentCommit)) { + if (longestAncestorChain == null + || longestAncestorChain.size() < chain + .size()) { + longestAncestorChain = chain; + } + } + } + + if (longestAncestorChain == null) { + longestAncestorChain = new ArrayList<>(); + chains.add(longestAncestorChain); + } + longestAncestorChain.add(new BitmapCommit(c, + !longestAncestorChain.isEmpty(), flags)); + writeBitmaps.addBitmap(c, bitmap, 0); } - if (longestAncestorChain == null) { - longestAncestorChain = new ArrayList<>(); - chains.add(longestAncestorChain); + for (List<BitmapCommit> chain : chains) { + selections.addAll(chain); } - longestAncestorChain.add(new BitmapCommit( - c, !longestAncestorChain.isEmpty(), flags)); - writeBitmaps.addBitmap(c, fullBitmap, 0); } + writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps. - for (List<BitmapCommit> chain : chains) { - selections.addAll(chain); + // Add the remaining peeledWant + for (AnyObjectId remainingWant : selectionHelper.newWants) { + selections.add(new BitmapCommit(remainingWant, false, 0)); } - } - writeBitmaps.clearBitmaps(); // Remove the temporary commit bitmaps. - // Add the remaining peeledWant - for (AnyObjectId remainingWant : selectionHelper.peeledWants) { - selections.add(new BitmapCommit(remainingWant, false, 0)); + pm.endTask(); + return selections; } - - pm.endTask(); - return selections; } private boolean isRecentCommit(RevCommit revCommit) { @@ -358,9 +380,8 @@ class PackWriterBitmapPreparer { } /** - * For each of the {@code want}s, which represent the tip commit of each - * branch, set up an initial {@link BitmapBuilder}. Reuse previously built - * bitmaps if possible. + * Records which of the {@code wants} can be found in the previous GC pack's + * bitmap indices and which are new. * * @param rw * a {@link RevWalk} to find reachable objects in this repository @@ -369,8 +390,9 @@ class PackWriterBitmapPreparer { * unreachable garbage. * @param excludeFromBitmapSelection * commits that should be excluded from bitmap selection - * @return a {@link CommitSelectionHelper} containing bitmaps for the tip - * commits + * @return a {@link CommitSelectionHelper} capturing which commits are + * covered by a previous pack's bitmaps and which new commits need + * bitmap coverage * @throws IncorrectObjectTypeException * if any of the processed objects is not a commit * @throws IOException @@ -378,11 +400,12 @@ class PackWriterBitmapPreparer { * @throws MissingObjectException * if an expected object is missing */ - private CommitSelectionHelper setupTipCommitBitmaps(RevWalk rw, + private CommitSelectionHelper captureOldAndNewCommits(RevWalk rw, int expectedCommitCount, Set<? extends ObjectId> excludeFromBitmapSelection) throws IncorrectObjectTypeException, IOException, MissingObjectException { + // Track bitmaps and commits from the previous GC pack bitmap indices. BitmapBuilder reuse = commitBitmapIndex.newBitmapBuilder(); List<BitmapCommit> reuseCommits = new ArrayList<>(); for (PackBitmapIndexRemapper.Entry entry : bitmapRemapper) { @@ -404,11 +427,10 @@ class PackWriterBitmapPreparer { } } - // Add branch tips that are not represented in old bitmap indices. Set - // up the RevWalk to walk the new commits not in the old packs. - List<BitmapBuilderEntry> tipCommitBitmaps = new ArrayList<>( - want.size()); - Set<RevCommit> peeledWant = new HashSet<>(want.size()); + // Add branch tips that are not represented in a previous pack's bitmap + // indices. Set up a RevWalk to find new commits not in the old packs. + List<RevCommit> newWantsByNewest = new ArrayList<>(want.size()); + Set<RevCommit> newWants = new HashSet<>(want.size()); for (AnyObjectId objectId : want) { RevObject ro = rw.peel(rw.parseAny(objectId)); if (!(ro instanceof RevCommit) || reuse.contains(ro) @@ -417,59 +439,26 @@ class PackWriterBitmapPreparer { } RevCommit rc = (RevCommit) ro; - peeledWant.add(rc); rw.markStart(rc); - - BitmapBuilder bitmap = commitBitmapIndex.newBitmapBuilder(); - bitmap.addObject(rc, Constants.OBJ_COMMIT); - tipCommitBitmaps.add(new BitmapBuilderEntry(rc, bitmap)); + newWants.add(rc); + newWantsByNewest.add(rc); } - // Create a list of commits in reverse order (older to newer). - // For each branch that contains the commit, mark its parents as being - // in the bitmap. + // Create a list of commits in reverse order (older to newer) that are + // not in the previous bitmap indices and are reachable. rw.setRevFilter(new NotInBitmapFilter(reuse)); RevCommit[] commits = new RevCommit[expectedCommitCount]; int pos = commits.length; RevCommit rc; while ((rc = rw.next()) != null && pos > 0) { commits[--pos] = rc; - for (BitmapBuilderEntry entry : tipCommitBitmaps) { - BitmapBuilder bitmap = entry.getBuilder(); - if (!bitmap.contains(rc)) { - continue; - } - for (RevCommit c : rc.getParents()) { - if (reuse.contains(c)) { - continue; - } - bitmap.addObject(c, Constants.OBJ_COMMIT); - } - } pm.update(1); } - // Sort the tip commit bitmaps. Find the one containing the most - // commits, remove those commits from the remaining bitmaps, resort and - // repeat. - List<BitmapBuilderEntry> orderedTipCommitBitmaps = new ArrayList<>( - tipCommitBitmaps.size()); - while (!tipCommitBitmaps.isEmpty()) { - BitmapBuilderEntry largest = - Collections.max(tipCommitBitmaps, ORDER_BY_CARDINALITY); - tipCommitBitmaps.remove(largest); - orderedTipCommitBitmaps.add(largest); - - // Update the remaining paths, by removing the objects from - // the path that was just added. - for (int i = tipCommitBitmaps.size() - 1; i >= 0; i--) { - tipCommitBitmaps.get(i).getBuilder() - .andNot(largest.getBuilder()); - } - } - - return new CommitSelectionHelper(peeledWant, commits, pos, - orderedTipCommitBitmaps, reuse, reuseCommits); + // Sort the new wants by reverse commit time. + Collections.sort(newWantsByNewest, ORDER_BY_REVERSE_TIMESTAMP); + return new CommitSelectionHelper(newWants, commits, pos, + newWantsByNewest, reuse, reuseCommits); } /*- @@ -537,54 +526,36 @@ class PackWriterBitmapPreparer { } /** - * A POJO representing a Pair<RevCommit, BitmapBuidler>. - */ - private static final class BitmapBuilderEntry { - private final RevCommit commit; - private final BitmapBuilder builder; - - BitmapBuilderEntry(RevCommit commit, BitmapBuilder builder) { - this.commit = commit; - this.builder = builder; - } - - RevCommit getCommit() { - return commit; - } - - BitmapBuilder getBuilder() { - return builder; - } - } - - /** * Container for state used in the first phase of selecting commits, which - * walks all of the reachable commits via the branch tips ( - * {@code peeledWants}), stores them in {@code commitsByOldest}, and sets up - * bitmaps for each branch tip ({@code tipCommitBitmaps}). - * {@code commitsByOldest} is initialized with an expected size of all - * commits, but may be smaller if some commits are unreachable, in which - * case {@code commitStartPos} will contain a positive offset to the root - * commit. + * walks all of the reachable commits via the branch tips that are not + * covered by a previous pack's bitmaps ({@code newWants}) and stores them + * in {@code newCommitsByOldest}. {@code newCommitsByOldest} is initialized + * with an expected size of all commits, but may be smaller if some commits + * are unreachable and/or some commits are covered by a previous pack's + * bitmaps. {@code commitStartPos} will contain a positive offset to either + * the root commit or the oldest commit not covered by previous bitmaps. */ private static final class CommitSelectionHelper implements Iterable<RevCommit> { - final Set<? extends ObjectId> peeledWants; - final List<BitmapBuilderEntry> tipCommitBitmaps; + final Set<? extends ObjectId> newWants; + final List<RevCommit> newWantsByNewest; final BitmapBuilder reusedCommitsBitmap; - final Iterable<BitmapCommit> reusedCommits; - final RevCommit[] commitsByOldest; - final int commitStartPos; - CommitSelectionHelper(Set<? extends ObjectId> peeledWant, + final List<BitmapCommit> reusedCommits; + + final RevCommit[] newCommitsByOldest; + + final int newCommitStartPos; + + CommitSelectionHelper(Set<? extends ObjectId> newWants, RevCommit[] commitsByOldest, int commitStartPos, - List<BitmapBuilderEntry> bitmapEntries, + List<RevCommit> newWantsByNewest, BitmapBuilder reusedCommitsBitmap, - Iterable<BitmapCommit> reuse) { - this.peeledWants = peeledWant; - this.commitsByOldest = commitsByOldest; - this.commitStartPos = commitStartPos; - this.tipCommitBitmaps = bitmapEntries; + List<BitmapCommit> reuse) { + this.newWants = newWants; + this.newCommitsByOldest = commitsByOldest; + this.newCommitStartPos = commitStartPos; + this.newWantsByNewest = newWantsByNewest; this.reusedCommitsBitmap = reusedCommitsBitmap; this.reusedCommits = reuse; } @@ -594,16 +565,16 @@ class PackWriterBitmapPreparer { // Member variables referenced by this iterator will have synthetic // accessors generated for them if they are made private. return new Iterator<RevCommit>() { - int pos = commitStartPos; + int pos = newCommitStartPos; @Override public boolean hasNext() { - return pos < commitsByOldest.length; + return pos < newCommitsByOldest.length; } @Override public RevCommit next() { - return commitsByOldest[pos++]; + return newCommitsByOldest[pos++]; } @Override @@ -614,7 +585,7 @@ class PackWriterBitmapPreparer { } int getCommitCount() { - return commitsByOldest.length - commitStartPos; + return newCommitsByOldest.length - newCommitStartPos; } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java index 942d72fe23..ce2ba4a2e1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.reftable; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.internal.storage.reftable.BlockWriter.compare; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_BLOCK_TYPE; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN; @@ -138,7 +138,7 @@ class BlockReader { if (blockType == LOG_BLOCK_TYPE) { len -= 9; } - return RawParseUtils.decode(CHARSET, nameBuf, 0, len); + return RawParseUtils.decode(UTF_8, nameBuf, 0, len); } boolean match(byte[] match, boolean matchIsPrefix) { @@ -171,7 +171,7 @@ class BlockReader { } Ref readRef() throws IOException { - String name = RawParseUtils.decode(CHARSET, nameBuf, 0, nameLen); + String name = RawParseUtils.decode(UTF_8, nameBuf, 0, nameLen); switch (valueType & VALUE_TYPE_MASK) { case VALUE_NONE: // delete return newRef(name); @@ -266,7 +266,7 @@ class BlockReader { private String readValueString() { int len = readVarint32(); int end = ptr + len; - String s = RawParseUtils.decode(CHARSET, buf, ptr, end); + String s = RawParseUtils.decode(UTF_8, buf, ptr, end); ptr = end; return s; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockWriter.java index 3d8fbf4996..b3173e838c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockWriter.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.reftable; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE; @@ -440,7 +440,7 @@ class BlockWriter { } private static byte[] nameUtf8(Ref ref) { - return ref.getName().getBytes(CHARSET); + return ref.getName().getBytes(UTF_8); } } @@ -559,13 +559,13 @@ class BlockWriter { this.newId = newId; this.timeSecs = who.getWhen().getTime() / 1000L; this.tz = (short) who.getTimeZoneOffset(); - this.name = who.getName().getBytes(CHARSET); - this.email = who.getEmailAddress().getBytes(CHARSET); - this.msg = message.getBytes(CHARSET); + this.name = who.getName().getBytes(UTF_8); + this.email = who.getEmailAddress().getBytes(UTF_8); + this.msg = message.getBytes(UTF_8); } static byte[] key(String ref, long index) { - byte[] name = ref.getBytes(CHARSET); + byte[] name = ref.getBytes(UTF_8); byte[] key = Arrays.copyOf(name, name.length + 1 + 8); NB.encodeInt64(key, key.length - 8, reverseUpdateIndex(index)); return key; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java index ef686a3008..17894b1664 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/MergedReftable.java @@ -113,6 +113,16 @@ public class MergedReftable extends Reftable { /** {@inheritDoc} */ @Override + public RefCursor seekRefsWithPrefix(String prefix) throws IOException { + MergedRefCursor m = new MergedRefCursor(); + for (int i = 0; i < tables.length; i++) { + m.add(new RefQueueEntry(tables[i].seekRefsWithPrefix(prefix), i)); + } + return m; + } + + /** {@inheritDoc} */ + @Override public RefCursor byObjectId(AnyObjectId name) throws IOException { MergedRefCursor m = new MergedRefCursor(); for (int i = 0; i < tables.length; i++) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java index 510c1a14e0..a1087e2023 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/Reftable.java @@ -108,18 +108,14 @@ public abstract class Reftable implements AutoCloseable { public abstract RefCursor allRefs() throws IOException; /** - * Seek either to a reference, or a reference subtree. + * Seek to a reference. * <p> - * If {@code refName} ends with {@code "/"} the method will seek to the - * subtree of all references starting with {@code refName} as a prefix. If - * no references start with this prefix, an empty cursor is returned. - * <p> - * Otherwise exactly {@code refName} will be looked for. If present, the + * This method will seek to the reference {@code refName}. If present, the * returned cursor will iterate exactly one entry. If not found, an empty * cursor is returned. * * @param refName - * reference name or subtree to find. + * reference name. * @return cursor to iterate; empty cursor if no references match. * @throws java.io.IOException * if references cannot be read. @@ -127,6 +123,21 @@ public abstract class Reftable implements AutoCloseable { public abstract RefCursor seekRef(String refName) throws IOException; /** + * Seek references with prefix. + * <p> + * The method will seek all the references starting with {@code prefix} as a + * prefix. If no references start with this prefix, an empty cursor is + * returned. + * + * @param prefix + * prefix to find. + * @return cursor to iterate; empty cursor if no references match. + * @throws java.io.IOException + * if references cannot be read. + */ + public abstract RefCursor seekRefsWithPrefix(String prefix) throws IOException; + + /** * Match references pointing to a specific object. * * @param id @@ -191,17 +202,11 @@ public abstract class Reftable implements AutoCloseable { } /** - * Test if a reference or reference subtree exists. - * <p> - * If {@code refName} ends with {@code "/"}, the method tests if any - * reference starts with {@code refName} as a prefix. - * <p> - * Otherwise, the method checks if {@code refName} exists. + * Test if a reference exists. * * @param refName * reference name or subtree to find. - * @return {@code true} if the reference exists, or at least one reference - * exists in the subtree. + * @return {@code true} if the reference exists. * @throws java.io.IOException * if references cannot be read. */ @@ -212,6 +217,21 @@ public abstract class Reftable implements AutoCloseable { } /** + * Test if any reference starts with {@code prefix} as a prefix. + * + * @param prefix + * prefix to find. + * @return {@code true} if at least one reference exists with prefix. + * @throws java.io.IOException + * if references cannot be read. + */ + public boolean hasRefsWithPrefix(String prefix) throws IOException { + try (RefCursor rc = seekRefsWithPrefix(prefix)) { + return rc.next(); + } + } + + /** * Test if any reference directly refers to the object. * * @param id diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java index 44bbb16219..1fc43c9fac 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.reftable; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE; @@ -160,7 +160,7 @@ class ReftableOutputStream extends OutputStream { } void writeVarintString(String s) { - writeVarintString(s.getBytes(CHARSET)); + writeVarintString(s.getBytes(UTF_8)); } void writeVarintString(byte[] msg) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java index 5356952b5d..81b30e4cb9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.internal.storage.reftable; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.internal.storage.reftable.BlockReader.decodeBlockLen; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_BLOCK_TYPE; import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_FOOTER_LEN; @@ -182,10 +182,19 @@ public class ReftableReader extends Reftable { public RefCursor seekRef(String refName) throws IOException { initRefIndex(); - byte[] key = refName.getBytes(CHARSET); - boolean prefix = key[key.length - 1] == '/'; + byte[] key = refName.getBytes(UTF_8); + RefCursorImpl i = new RefCursorImpl(refEnd, key, false); + i.block = seek(REF_BLOCK_TYPE, key, refIndex, 0, refEnd); + return i; + } + + /** {@inheritDoc} */ + @Override + public RefCursor seekRefsWithPrefix(String prefix) throws IOException { + initRefIndex(); - RefCursorImpl i = new RefCursorImpl(refEnd, key, prefix); + byte[] key = prefix.getBytes(UTF_8); + RefCursorImpl i = new RefCursorImpl(refEnd, key, true); i.block = seek(REF_BLOCK_TYPE, key, refIndex, 0, refEnd); return i; } @@ -223,7 +232,7 @@ public class ReftableReader extends Reftable { initLogIndex(); if (logPosition > 0) { byte[] key = LogEntry.key(refName, updateIndex); - byte[] match = refName.getBytes(CHARSET); + byte[] match = refName.getBytes(UTF_8); LogCursorImpl i = new LogCursorImpl(logEnd, match); i.block = seek(LOG_BLOCK_TYPE, key, logIndex, logPosition, logEnd); return i; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java index 2c7c6cb060..d1cf1cd9ae 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java @@ -283,7 +283,7 @@ public class BaseRepositoryBuilder<B extends BaseRepositoryBuilder, R extends Re final List<File> alts = alternateObjectDirectories; if (alts == null) return null; - return alts.toArray(new File[alts.size()]); + return alts.toArray(new File[0]); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobBasedConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobBasedConfig.java index e008be3a3c..59154b78bf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobBasedConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BlobBasedConfig.java @@ -45,7 +45,7 @@ package org.eclipse.jgit.lib; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.FileNotFoundException; import java.io.IOException; @@ -83,7 +83,7 @@ public class BlobBasedConfig extends Config { super(base); final String decoded; if (isUtf8(blob)) { - decoded = RawParseUtils.decode(CHARSET, blob, 3, blob.length); + decoded = RawParseUtils.decode(UTF_8, blob, 3, blob.length); } else { decoded = RawParseUtils.decode(blob); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java index 59a13f6550..c30833d0a6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java @@ -45,6 +45,8 @@ package org.eclipse.jgit.lib; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; @@ -93,7 +95,7 @@ public class CommitBuilder { */ public CommitBuilder() { parentIds = EMPTY_OBJECTID_LIST; - encoding = Constants.CHARSET; + encoding = UTF_8; } /** @@ -314,7 +316,7 @@ public class CommitBuilder { w.flush(); os.write('\n'); - if (getEncoding() != Constants.CHARSET) { + if (getEncoding() != UTF_8) { os.write(hencoding); os.write(' '); os.write(Constants.encodeASCII(getEncoding().name())); @@ -375,7 +377,7 @@ public class CommitBuilder { r.append(committer != null ? committer.toString() : "NOT_SET"); r.append("\n"); - if (encoding != null && encoding != Constants.CHARSET) { + if (encoding != null && encoding != UTF_8) { r.append("encoding "); r.append(encoding.name()); r.append("\n"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java index 0e01cca99b..b666f21d0b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java @@ -51,7 +51,7 @@ package org.eclipse.jgit.lib; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.text.MessageFormat; import java.util.ArrayList; @@ -1168,7 +1168,7 @@ public class Config { String decoded; if (isUtf8(bytes)) { - decoded = RawParseUtils.decode(CHARSET, bytes, 3, bytes.length); + decoded = RawParseUtils.decode(UTF_8, bytes, 3, bytes.length); } else { decoded = RawParseUtils.decode(bytes); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 5a790350b1..d4a0280da6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -49,7 +49,7 @@ package org.eclipse.jgit.lib; * configuration keys */ @SuppressWarnings("nls") -public class ConfigConstants { +public final class ConfigConstants { /** The "core" section */ public static final String CONFIG_CORE_SECTION = "core"; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 9023bd8610..ed0055416b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -227,7 +227,13 @@ public final class Constants { */ public static final byte[] PACK_SIGNATURE = { 'P', 'A', 'C', 'K' }; - /** Native character encoding for commit messages, file names... */ + /** + * Native character encoding for commit messages, file names... + * + * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} directly + * instead. + **/ + @Deprecated public static final Charset CHARSET; /** Native character encoding for commit messages, file names... */ @@ -638,7 +644,7 @@ public final class Constants { * @see #CHARACTER_ENCODING */ public static byte[] encode(String str) { - final ByteBuffer bb = Constants.CHARSET.encode(str); + final ByteBuffer bb = UTF_8.encode(str); final int len = bb.limit(); if (bb.hasArray() && bb.arrayOffset() == 0) { final byte[] arr = bb.array(); @@ -655,7 +661,7 @@ public final class Constants { if (OBJECT_ID_LENGTH != newMessageDigest().getDigestLength()) throw new LinkageError(JGitText.get().incorrectOBJECT_ID_LENGTH); CHARSET = UTF_8; - CHARACTER_ENCODING = CHARSET.name(); + CHARACTER_ENCODING = UTF_8.name(); } /** name of the file containing the commit msg for a merge commit */ @@ -687,11 +693,23 @@ public final class Constants { */ public static final String COMMIT_EDITMSG = "COMMIT_EDITMSG"; - /** objectid for the empty blob */ + /** + * Well-known object ID for the empty blob. + * + * @since 0.9.1 + */ public static final ObjectId EMPTY_BLOB_ID = ObjectId .fromString("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"); /** + * Well-known object ID for the empty tree. + * + * @since 5.1 + */ + public static final ObjectId EMPTY_TREE_ID = ObjectId + .fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"); + + /** * Suffix of lock file name * * @since 4.7 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java index 38716055b5..06b4b227c8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.lib; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.BufferedOutputStream; import java.io.File; @@ -183,7 +183,7 @@ public class RebaseTodoFile { switch (tokenCount) { case 0: String actionToken = new String(buf, tokenBegin, - nextSpace - tokenBegin - 1, CHARSET); + nextSpace - tokenBegin - 1, UTF_8); tokenBegin = nextSpace; action = RebaseTodoLine.Action.parse(actionToken); if (action == null) @@ -192,7 +192,7 @@ public class RebaseTodoFile { case 1: nextSpace = RawParseUtils.next(buf, tokenBegin, ' '); String commitToken = new String(buf, tokenBegin, - nextSpace - tokenBegin - 1, CHARSET); + nextSpace - tokenBegin - 1, UTF_8); tokenBegin = nextSpace; commit = AbbreviatedObjectId.fromString(commitToken); break; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index 29cc19c43c..d73c05e243 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -1094,7 +1094,9 @@ public abstract class Repository implements AutoCloseable { * not point to any object yet. * * @return mutable map of all known refs (heads, tags, remotes). + * @deprecated use {@code getRefDatabase().getRefs()} instead. */ + @Deprecated @NonNull public Map<String, Ref> getAllRefs() { try { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java index bd03165805..7669e95acc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java @@ -45,6 +45,8 @@ package org.eclipse.jgit.lib; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; @@ -181,7 +183,7 @@ public class TagBuilder { public byte[] build() { ByteArrayOutputStream os = new ByteArrayOutputStream(); try (OutputStreamWriter w = new OutputStreamWriter(os, - Constants.CHARSET)) { + UTF_8)) { w.write("object "); //$NON-NLS-1$ getObjectId().copyTo(w); w.write('\n'); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TextProgressMonitor.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TextProgressMonitor.java index 2f759e53ca..936ce3dcc8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TextProgressMonitor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TextProgressMonitor.java @@ -44,7 +44,7 @@ package org.eclipse.jgit.lib; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.io.OutputStreamWriter; @@ -63,7 +63,7 @@ public class TextProgressMonitor extends BatchingProgressMonitor { * Initialize a new progress monitor. */ public TextProgressMonitor() { - this(new PrintWriter(new OutputStreamWriter(System.err, CHARSET))); + this(new PrintWriter(new OutputStreamWriter(System.err, UTF_8))); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index da6a3da67d..f60c95f647 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -78,6 +78,7 @@ import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheBuildIterator; import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheCheckout; +import org.eclipse.jgit.dircache.DirCacheCheckout.CheckoutMetadata; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.errors.BinaryBlobException; import org.eclipse.jgit.errors.CorruptObjectException; @@ -297,6 +298,12 @@ public class ResolveMerger extends ThreeWayMerger { */ private int inCoreLimit; + /** + * Keeps {@link CheckoutMetadata} for {@link #checkout()} and + * {@link #cleanUp()}. + */ + private Map<String, CheckoutMetadata> checkoutMetadata; + private static MergeAlgorithm getMergeAlgorithm(Config config) { SupportedAlgorithm diffAlg = config.getEnum( CONFIG_DIFF_SECTION, null, CONFIG_KEY_ALGORITHM, @@ -313,6 +320,8 @@ public class ResolveMerger extends ThreeWayMerger { return new String[] { "BASE", "OURS", "THEIRS" }; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ } + private static final Attributes NO_ATTRIBUTES = new Attributes(); + /** * Constructor for ResolveMerger. * @@ -369,15 +378,20 @@ public class ResolveMerger extends ThreeWayMerger { /** {@inheritDoc} */ @Override protected boolean mergeImpl() throws IOException { - if (implicitDirCache) + if (implicitDirCache) { dircache = nonNullRepo().lockDirCache(); - + } + if (!inCore) { + checkoutMetadata = new HashMap<>(); + } try { return mergeTrees(mergeBase(), sourceTrees[0], sourceTrees[1], false); } finally { - if (implicitDirCache) + checkoutMetadata = null; + if (implicitDirCache) { dircache.unlock(); + } } } @@ -400,7 +414,8 @@ public class ResolveMerger extends ThreeWayMerger { if (cacheEntry.getFileMode() == FileMode.GITLINK) { new File(nonNullRepo().getWorkTree(), entry.getKey()).mkdirs(); } else { - DirCacheCheckout.checkoutEntry(db, cacheEntry, reader); + DirCacheCheckout.checkoutEntry(db, cacheEntry, reader, false, + checkoutMetadata.get(entry.getKey())); modifiedFiles.add(entry.getKey()); } } @@ -428,10 +443,12 @@ public class ResolveMerger extends ThreeWayMerger { DirCache dc = nonNullRepo().readDirCache(); Iterator<String> mpathsIt=modifiedFiles.iterator(); while(mpathsIt.hasNext()) { - String mpath=mpathsIt.next(); + String mpath = mpathsIt.next(); DirCacheEntry entry = dc.getEntry(mpath); - if (entry != null) - DirCacheCheckout.checkoutEntry(db, entry, reader); + if (entry != null) { + DirCacheCheckout.checkoutEntry(db, entry, reader, false, + checkoutMetadata.get(mpath)); + } mpathsIt.remove(); } } @@ -481,6 +498,71 @@ public class ResolveMerger extends ThreeWayMerger { } /** + * Remembers the {@link CheckoutMetadata} for the given path; it may be + * needed in {@link #checkout()} or in {@link #cleanUp()}. + * + * @param path + * of the current node + * @param attributes + * for the current node + * @throws IOException + * if the smudge filter cannot be determined + * @since 5.1 + */ + protected void addCheckoutMetadata(String path, Attributes attributes) + throws IOException { + if (checkoutMetadata != null) { + EolStreamType eol = EolStreamTypeUtil.detectStreamType( + OperationType.CHECKOUT_OP, workingTreeOptions, attributes); + CheckoutMetadata data = new CheckoutMetadata(eol, + tw.getFilterCommand(Constants.ATTR_FILTER_TYPE_SMUDGE)); + checkoutMetadata.put(path, data); + } + } + + /** + * Adds a {@link DirCacheEntry} for direct checkout and remembers its + * {@link CheckoutMetadata}. + * + * @param path + * of the entry + * @param entry + * to add + * @param attributes + * for the current entry + * @throws IOException + * if the {@link CheckoutMetadata} cannot be determined + * @since 5.1 + */ + protected void addToCheckout(String path, DirCacheEntry entry, + Attributes attributes) throws IOException { + toBeCheckedOut.put(path, entry); + addCheckoutMetadata(path, attributes); + } + + /** + * Remember a path for deletion, and remember its {@link CheckoutMetadata} + * in case it has to be restored in {@link #cleanUp()}. + * + * @param path + * of the entry + * @param isFile + * whether it is a file + * @param attributes + * for the entry + * @throws IOException + * if the {@link CheckoutMetadata} cannot be determined + * @since 5.1 + */ + protected void addDeletion(String path, boolean isFile, + Attributes attributes) throws IOException { + toBeDeleted.add(path); + if (isFile) { + addCheckoutMetadata(path, attributes); + } + } + + /** * Processes one path and tries to merge taking git attributes in account. * This method will do all trivial (not content) merges and will also detect * if a merge will fail. The merge will fail when one of the following is @@ -586,7 +668,7 @@ public class ResolveMerger extends ThreeWayMerger { // This will happen later. Set these values to 0 for know. DirCacheEntry e = add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_0, 0, 0); - toBeCheckedOut.put(tw.getPathString(), e); + addToCheckout(tw.getPathString(), e, attributes); } return true; } else { @@ -627,18 +709,23 @@ public class ResolveMerger extends ThreeWayMerger { // This will happen later. Set these values to 0 for know. DirCacheEntry e = add(tw.getRawPath(), theirs, DirCacheEntry.STAGE_0, 0, 0); - if (e != null) - toBeCheckedOut.put(tw.getPathString(), e); + if (e != null) { + addToCheckout(tw.getPathString(), e, attributes); + } return true; } else { // we want THEIRS ... but THEIRS contains a folder or the - // deletion of the path. Delete what's in the workingtree (the - // workingtree is clean) but do not complain if the file is - // already deleted locally. This complements the test in - // isWorktreeDirty() for the same case. - if (tw.getTreeCount() > T_FILE && tw.getRawMode(T_FILE) == 0) + // deletion of the path. Delete what's in the working tree, + // which we know to be clean. + if (tw.getTreeCount() > T_FILE && tw.getRawMode(T_FILE) == 0) { + // Not present in working tree, so nothing to delete return true; - toBeDeleted.add(tw.getPathString()); + } + if (modeT != 0 && modeT == modeB) { + // Base, ours, and theirs all contain a folder: don't delete + return true; + } + addDeletion(tw.getPathString(), nonTree(modeO), attributes); return true; } } @@ -722,9 +809,12 @@ public class ResolveMerger extends ThreeWayMerger { result.setContainsConflicts(false); } updateIndex(base, ours, theirs, result, attributes); - if (result.containsConflicts() && !ignoreConflicts) - unmergedPaths.add(tw.getPathString()); - modifiedFiles.add(tw.getPathString()); + String currentPath = tw.getPathString(); + if (result.containsConflicts() && !ignoreConflicts) { + unmergedPaths.add(currentPath); + } + modifiedFiles.add(currentPath); + addCheckoutMetadata(currentPath, attributes); } else if (modeO != modeT) { // OURS or THEIRS has been deleted if (((modeO != 0 && !tw.idEqual(T_BASE, T_OURS)) || (modeT != 0 && !tw @@ -743,8 +833,9 @@ public class ResolveMerger extends ThreeWayMerger { if (isWorktreeDirty(work, ourDce)) return false; if (nonTree(modeT)) { - if (e != null) - toBeCheckedOut.put(tw.getPathString(), e); + if (e != null) { + addToCheckout(tw.getPathString(), e, attributes); + } } } @@ -1245,7 +1336,8 @@ public class ResolveMerger extends ThreeWayMerger { hasWorkingTreeIterator ? treeWalk.getTree(T_FILE, WorkingTreeIterator.class) : null, ignoreConflicts, hasAttributeNodeProvider - ? treeWalk.getAttributes() : new Attributes())) { + ? treeWalk.getAttributes() + : NO_ATTRIBUTES)) { cleanUp(); return false; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java index d0a5216e1e..1f4beb017f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.patch; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.encodeASCII; import static org.eclipse.jgit.util.RawParseUtils.decode; import static org.eclipse.jgit.util.RawParseUtils.decodeNoFallback; @@ -63,7 +64,6 @@ import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.EditList; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AbbreviatedObjectId; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.util.QuotedString; import org.eclipse.jgit.util.RawParseUtils; @@ -198,7 +198,7 @@ public class FileHeader extends DiffEntry { * Convert the patch script for this file into a string. * <p> * The default character encoding - * ({@link org.eclipse.jgit.lib.Constants#CHARSET}) is assumed for both the + * ({@link java.nio.charset.StandardCharsets#UTF_8}) is assumed for both the * old and new files. * * @return the patch script, as a Unicode string. @@ -240,8 +240,9 @@ public class FileHeader extends DiffEntry { if (trySimpleConversion(charsetGuess)) { Charset cs = charsetGuess != null ? charsetGuess[0] : null; - if (cs == null) - cs = Constants.CHARSET; + if (cs == null) { + cs = UTF_8; + } try { return decodeNoFallback(cs, buf, startOffset, endOffset); } catch (CharacterCodingException cee) { @@ -290,8 +291,9 @@ public class FileHeader extends DiffEntry { final String[] r = new String[tmp.length]; for (int i = 0; i < tmp.length; i++) { Charset cs = csGuess != null ? csGuess[i] : null; - if (cs == null) - cs = Constants.CHARSET; + if (cs == null) { + cs = UTF_8; + } r[i] = RawParseUtils.decode(cs, tmp[i].toByteArray()); } return r; @@ -429,7 +431,7 @@ public class FileHeader extends DiffEntry { oldPath = QuotedString.GIT_PATH.dequote(buf, bol, sp - 1); oldPath = p1(oldPath); } else { - oldPath = decode(Constants.CHARSET, buf, aStart, sp - 1); + oldPath = decode(UTF_8, buf, aStart, sp - 1); } newPath = oldPath; return eol; @@ -572,7 +574,7 @@ public class FileHeader extends DiffEntry { tab--; if (ptr == tab) tab = end; - r = decode(Constants.CHARSET, buf, ptr, tab - 1); + r = decode(UTF_8, buf, ptr, tab - 1); } if (r.equals(DEV_NULL)) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/FormatError.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/FormatError.java index 1dd24d732e..10ea778343 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/FormatError.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/FormatError.java @@ -43,9 +43,10 @@ package org.eclipse.jgit.patch; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.util.Locale; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.util.RawParseUtils; /** @@ -120,7 +121,7 @@ public class FormatError { */ public String getLineText() { final int eol = RawParseUtils.nextLF(buf, offset); - return RawParseUtils.decode(Constants.CHARSET, buf, offset, eol); + return RawParseUtils.decode(UTF_8, buf, offset, eol); } /** {@inheritDoc} */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java index f27f356c6b..ee18fe7c2f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2009, Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2008-2018, Robin Rosenberg <robin.rosenberg@dewire.com> * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> * and other copyright owners as documented in the project's IP log. * @@ -53,6 +53,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -75,13 +76,25 @@ import org.eclipse.jgit.revwalk.RevWalk; */ public class PlotWalk extends RevWalk { + private Map<AnyObjectId, Set<Ref>> additionalRefMap; + private Map<AnyObjectId, Set<Ref>> reverseRefMap; + private Repository repository; + /** {@inheritDoc} */ @Override public void dispose() { super.dispose(); - reverseRefMap.clear(); + if (reverseRefMap != null) { + reverseRefMap.clear(); + reverseRefMap = null; + } + if (additionalRefMap != null) { + additionalRefMap.clear(); + additionalRefMap = null; + } + repository = null; } /** @@ -93,7 +106,8 @@ public class PlotWalk extends RevWalk { public PlotWalk(Repository repo) { super(repo); super.sort(RevSort.TOPO, true); - reverseRefMap = repo.getAllRefsByPeeledObjectId(); + additionalRefMap = new HashMap<>(); + repository = repo; } /** @@ -105,14 +119,14 @@ public class PlotWalk extends RevWalk { */ public void addAdditionalRefs(Iterable<Ref> refs) throws IOException { for (Ref ref : refs) { - Set<Ref> set = reverseRefMap.get(ref.getObjectId()); + Set<Ref> set = additionalRefMap.get(ref.getObjectId()); if (set == null) set = Collections.singleton(ref); else { set = new HashSet<>(set); set.add(ref); } - reverseRefMap.put(ref.getObjectId(), set); + additionalRefMap.put(ref.getObjectId(), set); } } @@ -141,11 +155,29 @@ public class PlotWalk extends RevWalk { } private Ref[] getRefs(AnyObjectId commitId) { + if (reverseRefMap == null) { + reverseRefMap = repository.getAllRefsByPeeledObjectId(); + for (Map.Entry<AnyObjectId, Set<Ref>> entry : additionalRefMap + .entrySet()) { + Set<Ref> set = reverseRefMap.get(entry.getKey()); + Set<Ref> additional = entry.getValue(); + if (set != null) { + if (additional.size() == 1) { + // It's an unmodifiable singleton set... + additional = new HashSet<>(additional); + } + additional.addAll(set); + } + reverseRefMap.put(entry.getKey(), additional); + } + additionalRefMap.clear(); + additionalRefMap = null; + } Collection<Ref> list = reverseRefMap.get(commitId); - if (list == null) + if (list == null) { return PlotCommit.NO_REFS; - else { - Ref[] tags = list.toArray(new Ref[list.size()]); + } else { + Ref[] tags = list.toArray(new Ref[0]); Arrays.sort(tags, new PlotRefComparator()); return tags; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java index b67f934f6e..86ecd8eaee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java @@ -44,13 +44,14 @@ package org.eclipse.jgit.revwalk; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -168,10 +169,10 @@ public class RevCommit extends RevObject { } } - void parseCanonical(RevWalk walk, byte[] raw) - throws IOException { - if (!walk.shallowCommitsInitialized) - walk.initializeShallowCommits(); + void parseCanonical(RevWalk walk, byte[] raw) throws IOException { + if (!walk.shallowCommitsInitialized) { + walk.initializeShallowCommits(this); + } final MutableObjectId idBuffer = walk.idBuffer; idBuffer.fromString(raw, 5); @@ -182,13 +183,14 @@ public class RevCommit extends RevObject { RevCommit[] pList = new RevCommit[1]; int nParents = 0; for (;;) { - if (raw[ptr] != 'p') + if (raw[ptr] != 'p') { break; + } idBuffer.fromString(raw, ptr + 7); final RevCommit p = walk.lookupCommit(idBuffer); - if (nParents == 0) + if (nParents == 0) { pList[nParents++] = p; - else if (nParents == 1) { + } else if (nParents == 1) { pList = new RevCommit[] { pList[0], p }; nParents = 2; } else { @@ -218,8 +220,9 @@ public class RevCommit extends RevObject { commitTime = RawParseUtils.parseBase10(raw, ptr, null); } - if (walk.isRetainBody()) + if (walk.isRetainBody()) { buffer = raw; + } flags |= PARSED; } @@ -388,6 +391,34 @@ public class RevCommit extends RevObject { } /** + * Parse the gpg signature from the raw buffer. + * <p> + * This method parses and returns the raw content of the gpgsig lines. This + * method is fairly expensive and produces a new byte[] instance on each + * invocation. Callers should invoke this method only if they are certain + * they will need, and should cache the return value for as long as + * necessary to use all information from it. + * <p> + * RevFilter implementations should try to use + * {@link org.eclipse.jgit.util.RawParseUtils} to scan the + * {@link #getRawBuffer()} instead, as this will allow faster evaluation of + * commits. + * + * @return contents of the gpg signature; null if the commit was not signed. + * @since 5.1 + */ + public final @Nullable byte[] getRawGpgSignature() { + final byte[] raw = buffer; + final byte[] header = {'g', 'p', 'g', 's', 'i', 'g'}; + final int start = RawParseUtils.headerStart(header, raw, 0); + if (start < 0) { + return null; + } + final int end = RawParseUtils.headerEnd(raw, start); + return Arrays.copyOfRange(raw, start, end); + } + + /** * Parse the author identity from the raw buffer. * <p> * This method parses and returns the content of the author line, after @@ -539,7 +570,7 @@ public class RevCommit extends RevObject { try { return getEncoding(); } catch (IllegalCharsetNameException | UnsupportedCharsetException e) { - return CHARSET; + return UTF_8; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java index 0050bac0ae..32e7adfb0c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java @@ -45,7 +45,7 @@ package org.eclipse.jgit.revwalk; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.nio.charset.Charset; @@ -169,7 +169,7 @@ public class RevTag extends RevObject { int p = pos.value += 4; // "tag " final int nameEnd = RawParseUtils.nextLF(rawTag, p) - 1; - tagName = RawParseUtils.decode(CHARSET, rawTag, p, nameEnd); + tagName = RawParseUtils.decode(UTF_8, rawTag, p, nameEnd); if (walk.isRetainBody()) buffer = rawTag; @@ -257,7 +257,7 @@ public class RevTag extends RevObject { try { return RawParseUtils.parseEncoding(buffer); } catch (IllegalCharsetNameException | UnsupportedCharsetException e) { - return CHARSET; + return UTF_8; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index a986cfd8ff..4d555d2178 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -1460,17 +1460,44 @@ public class RevWalk implements Iterable<RevCommit>, AutoCloseable { lookupCommit(id).parents = RevCommit.NO_PARENTS; } - void initializeShallowCommits() throws IOException { - if (shallowCommitsInitialized) + /** + * Reads the "shallow" file and applies it by setting the parents of shallow + * commits to an empty array. + * <p> + * There is a sequencing problem if the first commit being parsed is a + * shallow commit, since {@link RevCommit#parseCanonical(RevWalk, byte[])} + * calls this method before its callers add the new commit to the + * {@link RevWalk#objects} map. That means a call from this method to + * {@link #lookupCommit(AnyObjectId)} fails to find that commit and creates + * a new one, which is promptly discarded. + * <p> + * To avoid that, {@link RevCommit#parseCanonical(RevWalk, byte[])} passes + * its commit to this method, so that this method can apply the shallow + * state to it directly and avoid creating the duplicate commit object. + * + * @param rc + * the initial commit being parsed + * @throws IOException + * if the shallow commits file can't be read + */ + void initializeShallowCommits(RevCommit rc) throws IOException { + if (shallowCommitsInitialized) { throw new IllegalStateException( JGitText.get().shallowCommitsAlreadyInitialized); + } shallowCommitsInitialized = true; - if (reader == null) + if (reader == null) { return; + } - for (ObjectId id : reader.getShallowCommits()) - lookupCommit(id).parents = RevCommit.NO_PARENTS; + for (ObjectId id : reader.getShallowCommits()) { + if (id.equals(rc.getId())) { + rc.parents = RevCommit.NO_PARENTS; + } else { + lookupCommit(id).parents = RevCommit.NO_PARENTS; + } + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java index 4b272ba7a4..93b3baa61f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java @@ -49,7 +49,7 @@ package org.eclipse.jgit.storage.file; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayOutputStream; import java.io.File; @@ -166,7 +166,7 @@ public class FileBasedConfig extends StoredConfig { } else { final String decoded; if (isUtf8(in)) { - decoded = RawParseUtils.decode(CHARSET, + decoded = RawParseUtils.decode(UTF_8, in, 3, in.length); utf8Bom = true; } else { @@ -224,7 +224,7 @@ public class FileBasedConfig extends StoredConfig { bos.write(0xEF); bos.write(0xBB); bos.write(0xBF); - bos.write(text.getBytes(CHARSET)); + bos.write(text.getBytes(UTF_8)); out = bos.toByteArray(); } else { out = Constants.encode(text); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java index a0fc57ca1f..f6ec4b90eb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayOutputStream; import java.io.File; @@ -635,7 +635,7 @@ public class AmazonS3 { try { final Mac m = Mac.getInstance(HMAC); m.init(privateKey); - sec = Base64.encodeBytes(m.doFinal(s.toString().getBytes(CHARSET))); + sec = Base64.encodeBytes(m.doFinal(s.toString().getBytes(UTF_8))); } catch (NoSuchAlgorithmException e) { throw new IOException(MessageFormat.format(JGitText.get().noHMACsupport, HMAC, e.getMessage())); } catch (InvalidKeyException e) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java index 38eae1cd48..fcf78ac7b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackConnection.java @@ -57,6 +57,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Set; +import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.NoRemoteRepositoryException; import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.errors.RemoteRepositoryException; @@ -222,6 +223,10 @@ abstract class BasePackConnection extends BaseConnection { } } + // Expecting to get a line in the form "sha1 refname" + if (line.length() < 41 || line.charAt(40) != ' ') { + throw invalidRefAdvertisementLine(line); + } String name = line.substring(41, line.length()); if (avail.isEmpty() && name.equals("capabilities^{}")) { //$NON-NLS-1$ // special line from git-receive-pack to show @@ -229,7 +234,12 @@ abstract class BasePackConnection extends BaseConnection { continue; } - final ObjectId id = ObjectId.fromString(line.substring(0, 40)); + final ObjectId id; + try { + id = ObjectId.fromString(line.substring(0, 40)); + } catch (InvalidObjectIdException e) { + throw invalidRefAdvertisementLine(line); + } if (name.equals(".have")) { //$NON-NLS-1$ additionalHaves.add(id); } else if (name.endsWith("^{}")) { //$NON-NLS-1$ @@ -318,6 +328,10 @@ abstract class BasePackConnection extends BaseConnection { return new PackProtocolException(uri, MessageFormat.format(JGitText.get().duplicateAdvertisementsOf, name)); } + private PackProtocolException invalidRefAdvertisementLine(String line) { + return new PackProtocolException(uri, MessageFormat.format(JGitText.get().invalidRefAdvertisementLine, line)); + } + /** {@inheritDoc} */ @Override public void close() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index 0dfcd8716e..ed7465c82a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -52,7 +52,6 @@ import java.text.MessageFormat; import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.HashSet; import java.util.Set; import org.eclipse.jgit.errors.PackProtocolException; @@ -235,12 +234,12 @@ public abstract class BasePackFetchConnection extends BasePackConnection private boolean noProgress; - private Set<AnyObjectId> minimalNegotiationSet; - private String lockMessage; private PackLock packLock; + private int maxHaves; + /** RPC state, if {@link BasePackConnection#statelessRPC} is true. */ private TemporaryBuffer.Heap state; @@ -261,12 +260,12 @@ public abstract class BasePackFetchConnection extends BasePackConnection if (local != null) { final FetchConfig cfg = getFetchConfig(); allowOfsDelta = cfg.allowOfsDelta; - if (cfg.minimalNegotiation) { - minimalNegotiationSet = new HashSet<>(); - } + maxHaves = cfg.maxHaves; } else { allowOfsDelta = true; + maxHaves = Integer.MAX_VALUE; } + includeTags = transport.getTagOpt() != TagOpt.NO_TAGS; thinPack = transport.isFetchThin(); filterBlobLimit = transport.getFilterBlobLimit(); @@ -294,17 +293,16 @@ public abstract class BasePackFetchConnection extends BasePackConnection static class FetchConfig { final boolean allowOfsDelta; - final boolean minimalNegotiation; + final int maxHaves; FetchConfig(Config c) { allowOfsDelta = c.getBoolean("repack", "usedeltabaseoffset", true); //$NON-NLS-1$ //$NON-NLS-2$ - minimalNegotiation = c.getBoolean("fetch", "useminimalnegotiation", //$NON-NLS-1$ //$NON-NLS-2$ - false); + maxHaves = c.getInt("fetch", "maxhaves", Integer.MAX_VALUE); //$NON-NLS-1$ //$NON-NLS-2$ } - FetchConfig(boolean allowOfsDelta, boolean minimalNegotiation) { + FetchConfig(boolean allowOfsDelta, int maxHaves) { this.allowOfsDelta = allowOfsDelta; - this.minimalNegotiation = minimalNegotiation; + this.maxHaves = maxHaves; } } @@ -518,15 +516,6 @@ public abstract class BasePackFetchConnection extends BasePackConnection } line.append('\n'); p.writeString(line.toString()); - if (minimalNegotiationSet != null) { - Ref current = local.exactRef(r.getName()); - if (current != null) { - ObjectId o = current.getObjectId(); - if (o != null && !o.equals(ObjectId.zeroId())) { - minimalNegotiationSet.add(o); - } - } - } } if (first) { return false; @@ -610,9 +599,6 @@ public abstract class BasePackFetchConnection extends BasePackConnection pckOut.writeString("have " + o.name() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ havesSent++; havesSinceLastContinue++; - if (minimalNegotiationSet != null) { - minimalNegotiationSet.remove(o); - } if ((31 & havesSent) != 0) { // We group the have lines into blocks of 32, each marked @@ -646,16 +632,6 @@ public abstract class BasePackFetchConnection extends BasePackConnection // pack on the remote side. Keep doing that. // resultsPending--; - if (minimalNegotiationSet != null - && minimalNegotiationSet.isEmpty()) { - // Minimal negotiation was requested and we sent out our - // current reference values for our wants, so terminate - // negotiation early. - if (statelessRPC) { - state.writeTo(out, null); - } - break SEND_HAVES; - } break READ_RESULT; case ACK: @@ -686,14 +662,6 @@ public abstract class BasePackFetchConnection extends BasePackConnection if (anr == AckNackResult.ACK_READY) { receivedReady = true; } - if (minimalNegotiationSet != null && minimalNegotiationSet.isEmpty()) { - // Minimal negotiation was requested and we sent out our current reference - // values for our wants, so terminate negotiation early. - if (statelessRPC) { - state.writeTo(out, null); - } - break SEND_HAVES; - } break; } @@ -709,7 +677,8 @@ public abstract class BasePackFetchConnection extends BasePackConnection state.writeTo(out, null); } - if (receivedContinue && havesSinceLastContinue > MAX_HAVES) { + if (receivedContinue && havesSinceLastContinue > MAX_HAVES + || havesSent >= maxHaves) { // Our history must be really different from the remote's. // We just sent a whole slew of have lines, and it did not // recognize any of them. Avoid sending our entire history diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleFetchConnection.java index 449f529447..4b20f6c8b0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleFetchConnection.java @@ -47,6 +47,8 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -66,7 +68,6 @@ import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackLock; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; @@ -173,7 +174,7 @@ class BundleFetchConnection extends BaseFetchConnection { IO.skipFully(bin, 1); done = true; } - line.append(RawParseUtils.decode(Constants.CHARSET, hdrbuf, 0, lf)); + line.append(RawParseUtils.decode(UTF_8, hdrbuf, 0, lf)); } return line.toString(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleWriter.java index f2a261bbe6..56aaede80d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleWriter.java @@ -43,6 +43,8 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -231,7 +233,7 @@ public class BundleWriter { packWriter.setTagTargets(tagTargets); packWriter.preparePack(monitor, inc, exc); - final Writer w = new OutputStreamWriter(os, Constants.CHARSET); + final Writer w = new OutputStreamWriter(os, UTF_8); w.write(TransportBundle.V2_BUNDLE_SIGNATURE); w.write('\n'); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/CapabilitiesV2Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/CapabilitiesV2Request.java new file mode 100644 index 0000000000..b133ab579a --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/CapabilitiesV2Request.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2018, Google LLC. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.transport; + +/** + * Capabilities protocol v2 request. + * + * <p> + * This is used as an input to {@link ProtocolV2Hook}. + * + * @since 5.1 + */ +public final class CapabilitiesV2Request { + private CapabilitiesV2Request() { + } + + /** @return A builder of {@link CapabilitiesV2Request}. */ + public static Builder builder() { + return new Builder(); + } + + /** A builder for {@link CapabilitiesV2Request}. */ + public static final class Builder { + private Builder() { + } + + /** @return CapabilitiesV2Request */ + public CapabilitiesV2Request build() { + return new CapabilitiesV2Request(); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/CredentialsProvider.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/CredentialsProvider.java index 8115946204..f7a295d63d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/CredentialsProvider.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/CredentialsProvider.java @@ -155,7 +155,7 @@ public abstract class CredentialsProvider { */ public boolean get(URIish uri, List<CredentialItem> items) throws UnsupportedCredentialItem { - return get(uri, items.toArray(new CredentialItem[items.size()])); + return get(uri, items.toArray(new CredentialItem[0])); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java new file mode 100644 index 0000000000..853d96905c --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchV2Request.java @@ -0,0 +1,356 @@ +/* + * Copyright (C) 2018, Google LLC. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.transport; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.lib.ObjectId; + +/** + * fetch protocol v2 request. + * + * <p> + * This is used as an input to {@link ProtocolV2Hook}. + * + * @since 5.1 + */ +public final class FetchV2Request { + private final List<ObjectId> peerHas; + + private final TreeMap<String, ObjectId> wantedRefs; + + private final Set<ObjectId> wantsIds; + + private final Set<ObjectId> clientShallowCommits; + + private final int deepenSince; + + private final List<String> deepenNotRefs; + + private final int depth; + + private final long filterBlobLimit; + + private final Set<String> options; + + private final boolean doneReceived; + + private FetchV2Request(List<ObjectId> peerHas, + TreeMap<String, ObjectId> wantedRefs, Set<ObjectId> wantsIds, + Set<ObjectId> clientShallowCommits, int deepenSince, + List<String> deepenNotRefs, int depth, long filterBlobLimit, + boolean doneReceived, Set<String> options) { + this.peerHas = peerHas; + this.wantedRefs = wantedRefs; + this.wantsIds = wantsIds; + this.clientShallowCommits = clientShallowCommits; + this.deepenSince = deepenSince; + this.deepenNotRefs = deepenNotRefs; + this.depth = depth; + this.filterBlobLimit = filterBlobLimit; + this.doneReceived = doneReceived; + this.options = options; + } + + /** + * @return object ids in the "have" lines of the request + */ + @NonNull + List<ObjectId> getPeerHas() { + return this.peerHas; + } + + /** + * @return list of references in the "want-ref" lines of the request + */ + @NonNull + Map<String, ObjectId> getWantedRefs() { + return this.wantedRefs; + } + + /** + * @return object ids in the "want" (and "want-ref") lines of the request + */ + @NonNull + Set<ObjectId> getWantsIds() { + return wantsIds; + } + + /** + * Shallow commits the client already has. + * + * These are sent by the client in "shallow" request lines. + * + * @return set of commits the client has declared as shallow. + */ + @NonNull + Set<ObjectId> getClientShallowCommits() { + return clientShallowCommits; + } + + /** + * The value in a "deepen-since" line in the request, indicating the + * timestamp where to stop fetching/cloning. + * + * @return timestamp in seconds since the epoch, where to stop the shallow + * fetch/clone. Defaults to 0 if not set in the request. + */ + int getDeepenSince() { + return deepenSince; + } + + /** + * @return the refs in "deepen-not" lines in the request. + */ + @NonNull + List<String> getDeepenNotRefs() { + return deepenNotRefs; + } + + /** + * @return the depth set in a "deepen" line. 0 by default. + */ + int getDepth() { + return depth; + } + + /** + * @return the blob limit set in a "filter" line (-1 if not set) + */ + long getFilterBlobLimit() { + return filterBlobLimit; + } + + /** + * @return true if the request had a "done" line + */ + boolean wasDoneReceived() { + return doneReceived; + } + + /** + * Options that tune the expected response from the server, like + * "thin-pack", "no-progress" or "ofs-delta" + * + * These are options listed and well-defined in the git protocol + * specification + * + * @return options found in the request lines + */ + @NonNull + Set<String> getOptions() { + return options; + } + + /** @return A builder of {@link FetchV2Request}. */ + static Builder builder() { + return new Builder(); + } + + + /** A builder for {@link FetchV2Request}. */ + static final class Builder { + List<ObjectId> peerHas = new ArrayList<>(); + + TreeMap<String, ObjectId> wantedRefs = new TreeMap<>(); + + Set<ObjectId> wantsIds = new HashSet<>(); + + Set<ObjectId> clientShallowCommits = new HashSet<>(); + + List<String> deepenNotRefs = new ArrayList<>(); + + Set<String> options = new HashSet<>(); + + int depth; + + int deepenSince; + + long filterBlobLimit = -1; + + boolean doneReceived; + + private Builder() { + } + + /** + * @param objectId + * from a "have" line in a fetch request + * @return the builder + */ + Builder addPeerHas(ObjectId objectId) { + peerHas.add(objectId); + return this; + } + + /** + * From a "want-ref" line in a fetch request + * + * @param refName + * reference name + * @param oid + * object id + * @return the builder + */ + Builder addWantedRef(String refName, ObjectId oid) { + wantedRefs.put(refName, oid); + return this; + } + + /** + * @param option + * fetch request lines acting as options + * @return the builder + */ + Builder addOption(String option) { + options.add(option); + return this; + } + + /** + * @param objectId + * from a "want" line in a fetch request + * @return the builder + */ + Builder addWantsIds(ObjectId objectId) { + wantsIds.add(objectId); + return this; + } + + /** + * @param shallowOid + * from a "shallow" line in the fetch request + * @return the builder + */ + Builder addClientShallowCommit(ObjectId shallowOid) { + this.clientShallowCommits.add(shallowOid); + return this; + } + + /** + * @param d + * from a "deepen" line in the fetch request + * @return the builder + */ + Builder setDepth(int d) { + this.depth = d; + return this; + } + + /** + * @return depth set in the request (via a "deepen" line). Defaulting to + * 0 if not set. + */ + int getDepth() { + return this.depth; + } + + /** + * @return if there has been any "deepen not" line in the request + */ + boolean hasDeepenNotRefs() { + return !deepenNotRefs.isEmpty(); + } + + /** + * @param deepenNotRef reference in a "deepen not" line + * @return the builder + */ + Builder addDeepenNotRef(String deepenNotRef) { + this.deepenNotRefs.add(deepenNotRef); + return this; + } + + /** + * @param value + * Unix timestamp received in a "deepen since" line + * @return the builder + */ + Builder setDeepenSince(int value) { + this.deepenSince = value; + return this; + } + + /** + * @return shallow since value, sent before in a "deepen since" line. 0 + * by default. + */ + int getDeepenSince() { + return this.deepenSince; + } + + /** + * @param filterBlobLimit + * set in a "filter" line + * @return the builder + */ + Builder setFilterBlobLimit(long filterBlobLimit) { + this.filterBlobLimit = filterBlobLimit; + return this; + } + + /** + * Mark that the "done" line has been received. + * + * @return the builder + */ + Builder setDoneReceived() { + this.doneReceived = true; + return this; + } + /** + * @return Initialized fetch request + */ + FetchV2Request build() { + return new FetchV2Request(peerHas, wantedRefs, wantsIds, + clientShallowCommits, deepenSince, deepenNotRefs, + depth, filterBlobLimit, doneReceived, options); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java index 10cd775304..760ac6c1d7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/GitProtocolConstants.java @@ -50,7 +50,7 @@ package org.eclipse.jgit.transport; * * @since 3.2 */ -public class GitProtocolConstants { +public final class GitProtocolConstants { /** * Include tags if we are also including the referenced objects. * @@ -167,6 +167,13 @@ public class GitProtocolConstants { public static final String OPTION_FILTER = "filter"; //$NON-NLS-1$ /** + * The client specified a want-ref expression. + * + * @since 5.1 + */ + public static final String OPTION_WANT_REF = "want-ref"; //$NON-NLS-1$ + + /** * The client supports atomic pushes. If this option is used, the server * will update all refs within one atomic transaction. * @@ -231,6 +238,13 @@ public class GitProtocolConstants { public static final String CAPABILITY_PUSH_OPTIONS = "push-options"; //$NON-NLS-1$ /** + * The server supports the client specifying ref names. + * + * @since 5.1 + */ + public static final String CAPABILITY_REF_IN_WANT = "ref-in-want"; //$NON-NLS-1$ + + /** * The server supports listing refs using protocol v2. * * @since 5.0 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java index 1415334c63..6c26b7026a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.transport; import static java.nio.charset.StandardCharsets.ISO_8859_1; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.security.InvalidKeyException; @@ -102,7 +102,7 @@ public class HMACSHA1NonceGenerator implements NonceGenerator { } String input = path + ":" + String.valueOf(timestamp); //$NON-NLS-1$ - byte[] rawHmac = mac.doFinal(input.getBytes(CHARSET)); + byte[] rawHmac = mac.doFinal(input.getBytes(UTF_8)); return Long.toString(timestamp) + "-" + toHex(rawHmac); //$NON-NLS-1$ } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpAuthMethod.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpAuthMethod.java index fb03190b02..69745eb310 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpAuthMethod.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpAuthMethod.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.util.HttpSupport.HDR_AUTHORIZATION; import static org.eclipse.jgit.util.HttpSupport.HDR_WWW_AUTHENTICATE; @@ -315,7 +315,7 @@ abstract class HttpAuthMethod { @Override void configureRequest(HttpConnection conn) throws IOException { String ident = user + ":" + pass; //$NON-NLS-1$ - String enc = Base64.encodeBytes(ident.getBytes(CHARSET)); + String enc = Base64.encodeBytes(ident.getBytes(UTF_8)); conn.setRequestProperty(HDR_AUTHORIZATION, type.getSchemeName() + " " + enc); //$NON-NLS-1$ } @@ -430,15 +430,15 @@ abstract class HttpAuthMethod { private static String H(String data) { MessageDigest md = newMD5(); - md.update(data.getBytes(CHARSET)); + md.update(data.getBytes(UTF_8)); return LHEX(md.digest()); } private static String KD(String secret, String data) { MessageDigest md = newMD5(); - md.update(secret.getBytes(CHARSET)); + md.update(secret.getBytes(UTF_8)); md.update((byte) ':'); - md.update(data.getBytes(CHARSET)); + md.update(data.getBytes(UTF_8)); return LHEX(md.digest()); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschConfigSessionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschConfigSessionFactory.java index eab3b3c0cc..4e712a5567 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschConfigSessionFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschConfigSessionFactory.java @@ -1,4 +1,5 @@ /* + * Copyright (C) 2018, Sasa Zivkov <sasa.zivkov@sap.com> * Copyright (C) 2016, Mark Ingram <markdingram@gmail.com> * Copyright (C) 2009, Constantine Plotnikov <constantine.plotnikov@gmail.com> * Copyright (C) 2008-2009, Google Inc. @@ -49,6 +50,10 @@ package org.eclipse.jgit.transport; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static org.eclipse.jgit.transport.OpenSshConfig.SSH_PORT; + import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -59,9 +64,11 @@ import java.net.ConnectException; import java.net.UnknownHostException; import java.text.MessageFormat; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; @@ -71,6 +78,8 @@ import org.slf4j.LoggerFactory; import com.jcraft.jsch.ConfigRepository; import com.jcraft.jsch.ConfigRepository.Config; +import com.jcraft.jsch.HostKey; +import com.jcraft.jsch.HostKeyRepository; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; @@ -224,6 +233,9 @@ public abstract class JschConfigSessionFactory extends SshSessionFactory { credentialsProvider)); } safeConfig(session, hc.getConfig()); + if (hc.getConfig().getValue("HostKeyAlgorithms") == null) { //$NON-NLS-1$ + setPreferredKeyTypesOrder(session); + } configure(hc, session); return session; } @@ -239,6 +251,36 @@ public abstract class JschConfigSessionFactory extends SshSessionFactory { "CheckSignatures"); //$NON-NLS-1$ } + private static void setPreferredKeyTypesOrder(Session session) { + HostKeyRepository hkr = session.getHostKeyRepository(); + List<String> known = Stream.of(hkr.getHostKey(hostName(session), null)) + .map(HostKey::getType) + .collect(toList()); + + if (!known.isEmpty()) { + String serverHostKey = "server_host_key"; //$NON-NLS-1$ + String current = session.getConfig(serverHostKey); + if (current == null) { + session.setConfig(serverHostKey, String.join(",", known)); //$NON-NLS-1$ + return; + } + + String knownFirst = Stream.concat( + known.stream(), + Stream.of(current.split(",")) //$NON-NLS-1$ + .filter(s -> !known.contains(s))) + .collect(joining(",")); //$NON-NLS-1$ + session.setConfig(serverHostKey, knownFirst); + } + } + + private static String hostName(Session s) { + if (s.getPort() == SSH_PORT) { + return s.getHost(); + } + return String.format("[%s]:%d", s.getHost(), s.getPort()); //$NON-NLS-1$ + } + private void copyConfigValueToSession(Session session, Config cfg, String from, String to) { String value = cfg.getValue(from); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/LsRefsV2Request.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/LsRefsV2Request.java new file mode 100644 index 0000000000..3aff584a00 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/LsRefsV2Request.java @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2018, Google LLC. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.transport; + +import java.util.Collections; +import java.util.List; + +/** + * ls-refs protocol v2 request. + * + * <p> + * This is used as an input to {@link ProtocolV2Hook}. + * + * @since 5.1 + */ +public final class LsRefsV2Request { + private final List<String> refPrefixes; + + private final boolean symrefs; + + private final boolean peel; + + private LsRefsV2Request(List<String> refPrefixes, boolean symrefs, + boolean peel) { + this.refPrefixes = refPrefixes; + this.symrefs = symrefs; + this.peel = peel; + } + + /** @return ref prefixes that the client requested. */ + public List<String> getRefPrefixes() { + return refPrefixes; + } + + /** @return true if the client requests symbolic references. */ + public boolean getSymrefs() { + return symrefs; + } + + /** @return true if the client requests tags to be peeled. */ + public boolean getPeel() { + return peel; + } + + /** @return A builder of {@link LsRefsV2Request}. */ + public static Builder builder() { + return new Builder(); + } + + /** A builder for {@link LsRefsV2Request}. */ + public static final class Builder { + private List<String> refPrefixes = Collections.emptyList(); + + private boolean symrefs; + + private boolean peel; + + private Builder() { + } + + /** + * @param value + * @return the Builder + */ + public Builder setRefPrefixes(List<String> value) { + refPrefixes = value; + return this; + } + + /** + * @param value + * @return the Builder + */ + public Builder setSymrefs(boolean value) { + symrefs = value; + return this; + } + + /** + * @param value + * @return the Builder + */ + public Builder setPeel(boolean value) { + peel = value; + return this; + } + + /** @return LsRefsV2Request */ + public LsRefsV2Request build() { + return new LsRefsV2Request( + Collections.unmodifiableList(refPrefixes), symrefs, peel); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java index f5ccdc8f13..480055c2e4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/OpenSshConfig.java @@ -524,7 +524,7 @@ public class OpenSshConfig implements ConfigRepository { if (values == null || values.isEmpty()) { return new String[0]; } - return values.toArray(new String[values.size()]); + return values.toArray(new String[0]); } public void setValue(String key, String value) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java index cc556f8af6..c6e19d5762 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java @@ -45,13 +45,14 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; @@ -186,7 +187,7 @@ public class PacketLineIn { if (raw[len - 1] == '\n') len--; - String s = RawParseUtils.decode(Constants.CHARSET, raw, 0, len); + String s = RawParseUtils.decode(UTF_8, raw, 0, len); log.debug("git< " + s); //$NON-NLS-1$ return s; } @@ -218,7 +219,7 @@ public class PacketLineIn { IO.readFully(in, raw, 0, len); - String s = RawParseUtils.decode(Constants.CHARSET, raw, 0, len); + String s = RawParseUtils.decode(UTF_8, raw, 0, len); log.debug("git< " + s); //$NON-NLS-1$ return s; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java index a26d1d77d6..e9400919a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineOut.java @@ -45,6 +45,8 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.io.OutputStream; @@ -141,7 +143,7 @@ public class PacketLineOut { out.write(lenbuffer, 0, 4); out.write(buf, pos, len); if (log.isDebugEnabled()) { - String s = RawParseUtils.decode(Constants.CHARSET, buf, pos, len); + String s = RawParseUtils.decode(UTF_8, buf, pos, len); log.debug("git> " + s); //$NON-NLS-1$ } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProgressSpinner.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProgressSpinner.java index 2364434b0f..41af8078c8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProgressSpinner.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProgressSpinner.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.IOException; import java.io.OutputStream; @@ -141,7 +141,7 @@ public class ProgressSpinner { private void write(String s) { if (write) { try { - out.write(s.getBytes(CHARSET)); + out.write(s.getBytes(UTF_8)); out.flush(); } catch (IOException e) { write = false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Hook.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Hook.java new file mode 100644 index 0000000000..d67b90b6b9 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Hook.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2018, Google LLC. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.transport; + +/** + * Hook to allow callers to be notified on Git protocol v2 requests. + * + * @see UploadPack#setProtocolV2Hook(ProtocolV2Hook) + * @since 5.1 + */ +public interface ProtocolV2Hook { + /** + * The default hook implementation that does nothing. + */ + static ProtocolV2Hook DEFAULT = new ProtocolV2Hook() { + // No override. + }; + + /** + * @param req + * the capabilities request + * @throws ServiceMayNotContinueException + * abort; the message will be sent to the user + * @since 5.1 + */ + default void onCapabilities(CapabilitiesV2Request req) + throws ServiceMayNotContinueException { + // Do nothing by default. + } + + /** + * @param req + * the ls-refs request + * @throws ServiceMayNotContinueException + * abort; the message will be sent to the user + * @since 5.1 + */ + default void onLsRefs(LsRefsV2Request req) + throws ServiceMayNotContinueException { + // Do nothing by default. + } + + /** + * @param req the fetch request + * @throws ServiceMayNotContinueException abort; the message will be sent to the user + */ + default void onFetch(FetchV2Request req) + throws ServiceMayNotContinueException { + // Do nothing by default + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java new file mode 100644 index 0000000000..eae2c6edb9 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ProtocolV2Parser.java @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2018, Google LLC. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.transport; + +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_DEEPEN_RELATIVE; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_FILTER; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_INCLUDE_TAG; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_NO_PROGRESS; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_OFS_DELTA; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND_64K; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_THIN_PACK; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_WANT_REF; + +import java.io.IOException; +import java.text.MessageFormat; + +import org.eclipse.jgit.errors.PackProtocolException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; + +/** + * Parse the incoming git protocol lines from the wire and translate them into a + * Request object. + * + * It requires a transferConfig object to know what the server supports (e.g. + * ref-in-want and/or filters). + */ +final class ProtocolV2Parser { + + private final TransferConfig transferConfig; + + ProtocolV2Parser(TransferConfig transferConfig) { + this.transferConfig = transferConfig; + } + + /** + * Parse the incoming fetch request arguments from the wire. The caller must + * be sure that what is comings is a fetch request before coming here. + * + * This operation requires the reference database to validate incoming + * references. + * + * @param pckIn + * incoming lines + * @param refdb + * reference database (to validate that received references exist + * and point to valid objects) + * @return A FetchV2Request populated with information received from the + * wire. + * @throws PackProtocolException + * incompatible options, wrong type of arguments or other issues + * where the request breaks the protocol. + * @throws IOException + * an IO error prevented reading the incoming message or + * accessing the ref database. + */ + FetchV2Request parseFetchRequest(PacketLineIn pckIn, RefDatabase refdb) + throws PackProtocolException, IOException { + FetchV2Request.Builder reqBuilder = FetchV2Request.builder(); + + // Packs are always sent multiplexed and using full 64K + // lengths. + reqBuilder.addOption(OPTION_SIDE_BAND_64K); + + String line; + + // Currently, we do not support any capabilities, so the next + // line is DELIM. + if ((line = pckIn.readString()) != PacketLineIn.DELIM) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().unexpectedPacketLine, line)); + } + + boolean filterReceived = false; + while ((line = pckIn.readString()) != PacketLineIn.END) { + if (line.startsWith("want ")) { //$NON-NLS-1$ + reqBuilder.addWantsIds(ObjectId.fromString(line.substring(5))); + } else if (transferConfig.isAllowRefInWant() + && line.startsWith(OPTION_WANT_REF + " ")) { //$NON-NLS-1$ + String refName = line.substring(OPTION_WANT_REF.length() + 1); + // TODO(ifrade): This validation should be done after the + // protocol parsing. It is not a protocol problem asking for an + // unexisting ref and we wouldn't need the ref database here + Ref ref = refdb.exactRef(refName); + if (ref == null) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().invalidRefName, refName)); + } + ObjectId oid = ref.getObjectId(); + if (oid == null) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().invalidRefName, refName)); + } + reqBuilder.addWantedRef(refName, oid); + reqBuilder.addWantsIds(oid); + } else if (line.startsWith("have ")) { //$NON-NLS-1$ + reqBuilder.addPeerHas(ObjectId.fromString(line.substring(5))); + } else if (line.equals("done")) { //$NON-NLS-1$ + reqBuilder.setDoneReceived(); + } else if (line.equals(OPTION_THIN_PACK)) { + reqBuilder.addOption(OPTION_THIN_PACK); + } else if (line.equals(OPTION_NO_PROGRESS)) { + reqBuilder.addOption(OPTION_NO_PROGRESS); + } else if (line.equals(OPTION_INCLUDE_TAG)) { + reqBuilder.addOption(OPTION_INCLUDE_TAG); + } else if (line.equals(OPTION_OFS_DELTA)) { + reqBuilder.addOption(OPTION_OFS_DELTA); + } else if (line.startsWith("shallow ")) { //$NON-NLS-1$ + reqBuilder.addClientShallowCommit( + ObjectId.fromString(line.substring(8))); + } else if (line.startsWith("deepen ")) { //$NON-NLS-1$ + int parsedDepth = Integer.parseInt(line.substring(7)); + if (parsedDepth <= 0) { + throw new PackProtocolException( + MessageFormat.format(JGitText.get().invalidDepth, + Integer.valueOf(parsedDepth))); + } + if (reqBuilder.getDeepenSince() != 0) { + throw new PackProtocolException( + JGitText.get().deepenSinceWithDeepen); + } + if (reqBuilder.hasDeepenNotRefs()) { + throw new PackProtocolException( + JGitText.get().deepenNotWithDeepen); + } + reqBuilder.setDepth(parsedDepth); + } else if (line.startsWith("deepen-not ")) { //$NON-NLS-1$ + reqBuilder.addDeepenNotRef(line.substring(11)); + if (reqBuilder.getDepth() != 0) { + throw new PackProtocolException( + JGitText.get().deepenNotWithDeepen); + } + } else if (line.equals(OPTION_DEEPEN_RELATIVE)) { + reqBuilder.addOption(OPTION_DEEPEN_RELATIVE); + } else if (line.startsWith("deepen-since ")) { //$NON-NLS-1$ + int ts = Integer.parseInt(line.substring(13)); + if (ts <= 0) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().invalidTimestamp, line)); + } + if (reqBuilder.getDepth() != 0) { + throw new PackProtocolException( + JGitText.get().deepenSinceWithDeepen); + } + reqBuilder.setDeepenSince(ts); + } else if (transferConfig.isAllowFilter() + && line.startsWith(OPTION_FILTER + ' ')) { + if (filterReceived) { + throw new PackProtocolException( + JGitText.get().tooManyFilters); + } + filterReceived = true; + reqBuilder.setFilterBlobLimit(filterLine( + line.substring(OPTION_FILTER.length() + 1))); + } else { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().unexpectedPacketLine, line)); + } + } + + return reqBuilder.build(); + } + + /** + * Process the content of "filter" line from the protocol. It has a shape + * like "blob:none" or "blob:limit=N", with limit a positive number. + * + * @param blobLine + * the content of the "filter" line in the protocol + * @return N, the limit, defaulting to 0 if "none" + * @throws PackProtocolException + * invalid filter because due to unrecognized format or + * negative/non-numeric filter. + */ + static long filterLine(String blobLine) throws PackProtocolException { + long blobLimit = -1; + + if (blobLine.equals("blob:none")) { //$NON-NLS-1$ + blobLimit = 0; + } else if (blobLine.startsWith("blob:limit=")) { //$NON-NLS-1$ + try { + blobLimit = Long + .parseLong(blobLine.substring("blob:limit=".length())); //$NON-NLS-1$ + } catch (NumberFormatException e) { + throw new PackProtocolException(MessageFormat + .format(JGitText.get().invalidFilter, blobLine)); + } + } + /* + * We must have (1) either "blob:none" or "blob:limit=" set (because we + * only support blob size limits for now), and (2) if the latter, then + * it must be nonnegative. Throw if (1) or (2) is not met. + */ + if (blobLimit < 0) { + throw new PackProtocolException( + MessageFormat.format(JGitText.get().invalidFilter, blobLine)); + } + + return blobLimit; + } + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java index 178c80d22e..f9fddbe889 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.util.RawParseUtils.lastIndexOfTrim; import java.text.SimpleDateFormat; @@ -95,7 +95,7 @@ public class PushCertificateIdent { */ public static PushCertificateIdent parse(String str) { MutableInteger p = new MutableInteger(); - byte[] raw = str.getBytes(CHARSET); + byte[] raw = str.getBytes(UTF_8); int tzBegin = raw.length - 1; tzBegin = lastIndexOfTrim(raw, ' ', tzBegin); if (tzBegin < 0 || raw[tzBegin] != ' ') { @@ -129,7 +129,7 @@ public class PushCertificateIdent { idEnd = raw.length; } } - String id = new String(raw, 0, idEnd, CHARSET); + String id = new String(raw, 0, idEnd, UTF_8); return new PushCertificateIdent(str, id, when * 1000L, tz); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java index 7f5a3408e3..aeca63500a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; import static org.eclipse.jgit.lib.FileMode.TYPE_FILE; @@ -292,7 +292,8 @@ public class PushCertificateStore implements AutoCloseable { ObjectLoader loader = tw.getObjectReader().open(tw.getObjectId(0), OBJ_BLOB); try (InputStream in = loader.openStream(); - Reader r = new BufferedReader(new InputStreamReader(in, CHARSET))) { + Reader r = new BufferedReader( + new InputStreamReader(in, UTF_8))) { return PushCertificateParser.fromReader(r); } } @@ -473,7 +474,7 @@ public class PushCertificateStore implements AutoCloseable { DirCacheEditor editor = dc.editor(); String certText = pc.cert.toText() + pc.cert.getSignature(); - final ObjectId certId = inserter.insert(OBJ_BLOB, certText.getBytes(CHARSET)); + final ObjectId certId = inserter.insert(OBJ_BLOB, certText.getBytes(UTF_8)); boolean any = false; for (ReceiveCommand cmd : pc.cert.getCommands()) { if (byRef != null && !commandsEqual(cmd, byRef.get(cmd.getRefName()))) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java index dc1871b729..4662435ea7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SYMREF; @@ -74,7 +74,7 @@ import org.eclipse.jgit.lib.Repository; public abstract class RefAdvertiser { /** Advertiser which frames lines in a {@link PacketLineOut} format. */ public static class PacketLineOutRefAdvertiser extends RefAdvertiser { - private final CharsetEncoder utf8 = CHARSET.newEncoder(); + private final CharsetEncoder utf8 = UTF_8.newEncoder(); private final PacketLineOut pckOut; private byte[] binArr = new byte[256]; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SideBandInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SideBandInputStream.java index 3100cb444a..90600cbb98 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SideBandInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SideBandInputStream.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.transport.SideBandOutputStream.HDR_SIZE; import java.io.IOException; @@ -57,7 +58,6 @@ import java.util.regex.Pattern; import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; @@ -257,6 +257,6 @@ public class SideBandInputStream extends InputStream { private String readString(int len) throws IOException { final byte[] raw = new byte[len]; IO.readFully(rawIn, raw, 0, len); - return RawParseUtils.decode(Constants.CHARSET, raw, 0, len); + return RawParseUtils.decode(UTF_8, raw, 0, len); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java index 4ae1ccb420..59740c4dc8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransferConfig.java @@ -126,18 +126,38 @@ public class TransferConfig { private final boolean allowInvalidPersonIdent; private final boolean safeForWindows; private final boolean safeForMacOS; + private final boolean allowRefInWant; private final boolean allowTipSha1InWant; private final boolean allowReachableSha1InWant; private final boolean allowFilter; final @Nullable ProtocolVersion protocolVersion; final String[] hideRefs; - TransferConfig(Repository db) { + /** + * Create a configuration honoring the repository's settings. + * + * @param db + * the repository to read settings from. The repository is not + * retained by the new configuration, instead its settings are + * copied during the constructor. + * @since 5.1.4 + */ + public TransferConfig(Repository db) { this(db.getConfig()); } + /** + * Create a configuration honoring settings in a + * {@link org.eclipse.jgit.lib.Config}. + * + * @param rc + * the source to read settings from. The source is not retained + * by the new configuration, instead its settings are copied + * during the constructor. + * @since 5.1.4 + */ @SuppressWarnings("nls") - TransferConfig(Config rc) { + public TransferConfig(Config rc) { boolean fsck = rc.getBoolean("transfer", "fsckobjects", false); fetchFsck = rc.getBoolean("fetch", "fsckobjects", fsck); receiveFsck = rc.getBoolean("receive", "fsckobjects", fsck); @@ -180,6 +200,7 @@ public class TransferConfig { ignore.add(ObjectChecker.ErrorType.ZERO_PADDED_FILEMODE); } + allowRefInWant = rc.getBoolean("uploadpack", "allowrefinwant", false); allowTipSha1InWant = rc.getBoolean( "uploadpack", "allowtipsha1inwant", false); allowReachableSha1InWant = rc.getBoolean( @@ -262,6 +283,14 @@ public class TransferConfig { } /** + * @return true if clients are allowed to specify a "want-ref" line + * @since 5.1 + */ + public boolean isAllowRefInWant() { + return allowRefInWant; + } + + /** * Get {@link org.eclipse.jgit.transport.RefFilter} respecting configured * hidden refs. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index d342ef46df..621c2ea56c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -46,7 +46,7 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.BufferedReader; import java.io.IOException; @@ -144,7 +144,7 @@ public abstract class Transport implements AutoCloseable { private static void scan(ClassLoader ldr, URL url) { try (BufferedReader br = new BufferedReader( - new InputStreamReader(url.openStream(), CHARSET))) { + new InputStreamReader(url.openStream(), UTF_8))) { String line; while ((line = br.readLine()) != null) { line = line.trim(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index c08f400307..8b41ab0466 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.HEAD; import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP; @@ -426,7 +427,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } private BufferedReader toBufferedReader(InputStream in) { - return new BufferedReader(new InputStreamReader(in, Constants.CHARSET)); + return new BufferedReader(new InputStreamReader(in, UTF_8)); } /** {@inheritDoc} */ @@ -483,6 +484,18 @@ public class TransportHttp extends HttpTransport implements WalkTransport, this.headers = headers; } + private NoRemoteRepositoryException createNotFoundException(URIish u, + URL url, String msg) { + String text; + if (msg != null && !msg.isEmpty()) { + text = MessageFormat.format(JGitText.get().uriNotFoundWithMessage, + url, msg); + } else { + text = MessageFormat.format(JGitText.get().uriNotFound, url); + } + return new NoRemoteRepositoryException(u, text); + } + private HttpConnection connect(String service) throws TransportException, NotSupportedException { URL u = getServiceURL(service); @@ -511,8 +524,8 @@ public class TransportHttp extends HttpTransport implements WalkTransport, return conn; case HttpConnection.HTTP_NOT_FOUND: - throw new NoRemoteRepositoryException(uri, - MessageFormat.format(JGitText.get().uriNotFound, u)); + throw createNotFoundException(uri, u, + conn.getResponseMessage()); case HttpConnection.HTTP_UNAUTHORIZED: authMethod = HttpAuthMethod.scanResponse(conn, ignoreTypes); @@ -940,7 +953,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, // Line oriented readable content is likely to compress well. // Request gzip encoding. InputStream is = open(path, AcceptEncoding.GZIP).in; - return new BufferedReader(new InputStreamReader(is, Constants.CHARSET)); + return new BufferedReader(new InputStreamReader(is, UTF_8)); } @Override @@ -1180,9 +1193,8 @@ public class TransportHttp extends HttpTransport implements WalkTransport, return; case HttpConnection.HTTP_NOT_FOUND: - throw new NoRemoteRepositoryException(uri, - MessageFormat.format(JGitText.get().uriNotFound, - conn.getURL())); + throw createNotFoundException(uri, conn.getURL(), + conn.getResponseMessage()); case HttpConnection.HTTP_FORBIDDEN: throw new TransportException(uri, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 1cdf8f7f1b..c753bcdc7a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.transport; import static java.util.function.Function.identity; import static java.util.stream.Collectors.toMap; import static org.eclipse.jgit.lib.Constants.R_TAGS; +import static org.eclipse.jgit.transport.GitProtocolConstants.CAPABILITY_REF_IN_WANT; import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_FETCH; import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_LS_REFS; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT; @@ -268,6 +269,9 @@ public class UploadPack { */ private Map<String, Ref> refs; + /** Hook used while processing Git protocol v2 requests. */ + private ProtocolV2Hook protocolV2Hook = ProtocolV2Hook.DEFAULT; + /** Hook used while advertising the refs to the client. */ private AdvertiseRefsHook advertiseRefsHook = AdvertiseRefsHook.DEFAULT; @@ -297,7 +301,7 @@ public class UploadPack { private final Set<RevObject> commonBase = new HashSet<>(); /** Shallow commits the client already has. */ - private final Set<ObjectId> clientShallowCommits = new HashSet<>(); + private Set<ObjectId> clientShallowCommits = new HashSet<>(); /** Desired depth from the client on a shallow request. */ private int depth; @@ -310,10 +314,10 @@ public class UploadPack { /** * (Possibly short) ref names, ancestors of which the client has asked us - * not to send using --shallow-exclude. Cannot be non-null if depth is + * not to send using --shallow-exclude. Cannot be non-empty if depth is * nonzero. */ - private @Nullable List<String> shallowExcludeRefs; + private List<String> deepenNotRefs = new ArrayList<>(); /** Commit time of the oldest common commit, in seconds. */ private int oldestTime; @@ -583,6 +587,16 @@ public class UploadPack { } /** + * Set the protocol V2 hook. + * + * @param hook + * @since 5.1 + */ + public void setProtocolV2Hook(ProtocolV2Hook hook) { + this.protocolV2Hook = hook; + } + + /** * Set the filter used while advertising the refs to the client. * <p> * Only refs allowed by this filter will be sent to the client. The filter @@ -865,7 +879,7 @@ public class UploadPack { multiAck = MultiAck.OFF; if (!clientShallowCommits.isEmpty()) - verifyClientShallow(); + verifyClientShallow(clientShallowCommits); if (depth != 0) processShallow(null, unshallowCommits, true); if (!clientShallowCommits.isEmpty()) @@ -925,25 +939,19 @@ public class UploadPack { } private void lsRefsV2() throws IOException { - PacketLineOutRefAdvertiser adv = new PacketLineOutRefAdvertiser(pckOut); - String line; - ArrayList<String> refPrefixes = new ArrayList<>(); - boolean needToFindSymrefs = false; - - adv.setUseProtocolV2(true); - - line = pckIn.readString(); - + LsRefsV2Request.Builder builder = LsRefsV2Request.builder(); + List<String> prefixes = new ArrayList<>(); + String line = pckIn.readString(); // Currently, we do not support any capabilities, so the next // line is DELIM if there are arguments or END if not. if (line == PacketLineIn.DELIM) { while ((line = pckIn.readString()) != PacketLineIn.END) { if (line.equals("peel")) { //$NON-NLS-1$ - adv.setDerefTags(true); + builder.setPeel(true); } else if (line.equals("symrefs")) { //$NON-NLS-1$ - needToFindSymrefs = true; + builder.setSymrefs(true); } else if (line.startsWith("ref-prefix ")) { //$NON-NLS-1$ - refPrefixes.add(line.substring("ref-prefix ".length())); //$NON-NLS-1$ + prefixes.add(line.substring("ref-prefix ".length())); //$NON-NLS-1$ } else { throw new PackProtocolException(MessageFormat .format(JGitText.get().unexpectedPacketLine, line)); @@ -953,11 +961,18 @@ public class UploadPack { throw new PackProtocolException(MessageFormat .format(JGitText.get().unexpectedPacketLine, line)); } - rawOut.stopBuffering(); + LsRefsV2Request req = builder.setRefPrefixes(prefixes).build(); - Map<String, Ref> refsToSend = getFilteredRefs(refPrefixes); + protocolV2Hook.onLsRefs(req); - if (needToFindSymrefs) { + rawOut.stopBuffering(); + PacketLineOutRefAdvertiser adv = new PacketLineOutRefAdvertiser(pckOut); + adv.setUseProtocolV2(true); + if (req.getPeel()) { + adv.setDerefTags(true); + } + Map<String, Ref> refsToSend = getFilteredRefs(req.getRefPrefixes()); + if (req.getSymrefs()) { findSymrefs(adv, refsToSend); } @@ -966,12 +981,6 @@ public class UploadPack { } private void fetchV2() throws IOException { - options = new HashSet<>(); - - // Packs are always sent multiplexed and using full 64K - // lengths. - options.add(OPTION_SIDE_BAND_64K); - // Depending on the requestValidator, #processHaveLines may // require that advertised be set. Set it only in the required // circumstances (to avoid a full ref lookup in the case that @@ -984,113 +993,50 @@ public class UploadPack { advertised = refIdSet(getAdvertisedOrDefaultRefs().values()); } - String line; - List<ObjectId> peerHas = new ArrayList<>(); - boolean doneReceived = false; + ProtocolV2Parser parser = new ProtocolV2Parser(transferConfig); + FetchV2Request req = parser.parseFetchRequest(pckIn, + db.getRefDatabase()); + rawOut.stopBuffering(); - // Currently, we do not support any capabilities, so the next - // line is DELIM. - if ((line = pckIn.readString()) != PacketLineIn.DELIM) { - throw new PackProtocolException(MessageFormat - .format(JGitText.get().unexpectedPacketLine, line)); - } + protocolV2Hook.onFetch(req); - boolean includeTag = false; - boolean filterReceived = false; - while ((line = pckIn.readString()) != PacketLineIn.END) { - if (line.startsWith("want ")) { //$NON-NLS-1$ - wantIds.add(ObjectId.fromString(line.substring(5))); - } else if (line.startsWith("have ")) { //$NON-NLS-1$ - peerHas.add(ObjectId.fromString(line.substring(5))); - } else if (line.equals("done")) { //$NON-NLS-1$ - doneReceived = true; - } else if (line.equals(OPTION_THIN_PACK)) { - options.add(OPTION_THIN_PACK); - } else if (line.equals(OPTION_NO_PROGRESS)) { - options.add(OPTION_NO_PROGRESS); - } else if (line.equals(OPTION_INCLUDE_TAG)) { - options.add(OPTION_INCLUDE_TAG); - includeTag = true; - } else if (line.equals(OPTION_OFS_DELTA)) { - options.add(OPTION_OFS_DELTA); - } else if (line.startsWith("shallow ")) { //$NON-NLS-1$ - clientShallowCommits.add(ObjectId.fromString(line.substring(8))); - } else if (line.startsWith("deepen ")) { //$NON-NLS-1$ - depth = Integer.parseInt(line.substring(7)); - if (depth <= 0) { - throw new PackProtocolException( - MessageFormat.format(JGitText.get().invalidDepth, - Integer.valueOf(depth))); - } - if (shallowSince != 0) { - throw new PackProtocolException( - JGitText.get().deepenSinceWithDeepen); - } - if (shallowExcludeRefs != null) { - throw new PackProtocolException( - JGitText.get().deepenNotWithDeepen); - } - } else if (line.startsWith("deepen-not ")) { //$NON-NLS-1$ - List<String> exclude = shallowExcludeRefs; - if (exclude == null) { - exclude = shallowExcludeRefs = new ArrayList<>(); - } - exclude.add(line.substring(11)); - if (depth != 0) { - throw new PackProtocolException( - JGitText.get().deepenNotWithDeepen); - } - } else if (line.equals(OPTION_DEEPEN_RELATIVE)) { - options.add(OPTION_DEEPEN_RELATIVE); - } else if (line.startsWith("deepen-since ")) { //$NON-NLS-1$ - shallowSince = Integer.parseInt(line.substring(13)); - if (shallowSince <= 0) { - throw new PackProtocolException( - MessageFormat.format( - JGitText.get().invalidTimestamp, line)); - } - if (depth != 0) { - throw new PackProtocolException( - JGitText.get().deepenSinceWithDeepen); - } - } else if (transferConfig.isAllowFilter() - && line.startsWith(OPTION_FILTER + ' ')) { - if (filterReceived) { - throw new PackProtocolException(JGitText.get().tooManyFilters); - } - filterReceived = true; - parseFilter(line.substring(OPTION_FILTER.length() + 1)); - } else { - throw new PackProtocolException(MessageFormat - .format(JGitText.get().unexpectedPacketLine, line)); - } - } - rawOut.stopBuffering(); + // TODO(ifrade): Refactor to pass around the Request object, instead of + // copying data back to class fields + options = req.getOptions(); + wantIds.addAll(req.getWantsIds()); + clientShallowCommits = req.getClientShallowCommits(); + depth = req.getDepth(); + shallowSince = req.getDeepenSince(); + filterBlobLimit = req.getFilterBlobLimit(); + deepenNotRefs = req.getDeepenNotRefs(); boolean sectionSent = false; @Nullable List<ObjectId> shallowCommits = null; List<ObjectId> unshallowCommits = new ArrayList<>(); - if (!clientShallowCommits.isEmpty()) { - verifyClientShallow(); + if (!req.getClientShallowCommits().isEmpty()) { + verifyClientShallow(req.getClientShallowCommits()); } - if (depth != 0 || shallowSince != 0 || shallowExcludeRefs != null) { - shallowCommits = new ArrayList<ObjectId>(); + if (req.getDepth() != 0 || req.getDeepenSince() != 0 + || !req.getDeepenNotRefs().isEmpty()) { + shallowCommits = new ArrayList<>(); processShallow(shallowCommits, unshallowCommits, false); } - if (!clientShallowCommits.isEmpty()) - walk.assumeShallow(clientShallowCommits); + if (!req.getClientShallowCommits().isEmpty()) + walk.assumeShallow(req.getClientShallowCommits()); - if (doneReceived) { - processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE)); + if (req.wasDoneReceived()) { + processHaveLines(req.getPeerHas(), ObjectId.zeroId(), + new PacketLineOut(NullOutputStream.INSTANCE)); } else { pckOut.writeString("acknowledgments\n"); //$NON-NLS-1$ - for (ObjectId id : peerHas) { + for (ObjectId id : req.getPeerHas()) { if (walk.getObjectReader().has(id)) { pckOut.writeString("ACK " + id.getName() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$ } } - processHaveLines(peerHas, ObjectId.zeroId(), new PacketLineOut(NullOutputStream.INSTANCE)); + processHaveLines(req.getPeerHas(), ObjectId.zeroId(), + new PacketLineOut(NullOutputStream.INSTANCE)); if (okToGiveUp()) { pckOut.writeString("ready\n"); //$NON-NLS-1$ } else if (commonBase.isEmpty()) { @@ -1099,7 +1045,7 @@ public class UploadPack { sectionSent = true; } - if (doneReceived || okToGiveUp()) { + if (req.wasDoneReceived() || okToGiveUp()) { if (shallowCommits != null) { if (sectionSent) pckOut.writeDelim(); @@ -1113,16 +1059,33 @@ public class UploadPack { sectionSent = true; } + if (!req.getWantedRefs().isEmpty()) { + if (sectionSent) { + pckOut.writeDelim(); + } + pckOut.writeString("wanted-refs\n"); //$NON-NLS-1$ + for (Map.Entry<String, ObjectId> entry : req.getWantedRefs() + .entrySet()) { + pckOut.writeString(entry.getValue().getName() + ' ' + + entry.getKey() + '\n'); + } + sectionSent = true; + } + if (sectionSent) pckOut.writeDelim(); pckOut.writeString("packfile\n"); //$NON-NLS-1$ sendPack(new PackStatistics.Accumulator(), - includeTag + req.getOptions().contains(OPTION_INCLUDE_TAG) ? db.getRefDatabase().getRefsByPrefix(R_TAGS) : null, unshallowCommits); + // sendPack invokes pckOut.end() for us, so we do not + // need to invoke it here. + } else { + // Invoke pckOut.end() by ourselves. + pckOut.end(); } - pckOut.end(); } /* @@ -1159,9 +1122,13 @@ public class UploadPack { ArrayList<String> caps = new ArrayList<>(); caps.add("version 2"); //$NON-NLS-1$ caps.add(COMMAND_LS_REFS); + boolean advertiseRefInWant = transferConfig.isAllowRefInWant() && + db.getConfig().getBoolean("uploadpack", null, //$NON-NLS-1$ + "advertiserefinwant", true); //$NON-NLS-1$ caps.add( COMMAND_FETCH + '=' + (transferConfig.isAllowFilter() ? OPTION_FILTER + ' ' : "") + //$NON-NLS-1$ + (advertiseRefInWant ? CAPABILITY_REF_IN_WANT + ' ' : "") + //$NON-NLS-1$ OPTION_SHALLOW); return caps; } @@ -1172,6 +1139,8 @@ public class UploadPack { // is sent only if this is a bidirectional pipe. (If // not, the client is expected to call // sendAdvertisedRefs() on its own.) + protocolV2Hook + .onCapabilities(CapabilitiesV2Request.builder().build()); for (String s : getV2CapabilityAdvertisement()) { pckOut.writeString(s + "\n"); //$NON-NLS-1$ } @@ -1219,7 +1188,7 @@ public class UploadPack { boolean writeToPckOut) throws IOException { if (options.contains(OPTION_DEEPEN_RELATIVE) || shallowSince != 0 || - shallowExcludeRefs != null) { + !deepenNotRefs.isEmpty()) { // TODO(jonathantanmy): Implement deepen-relative, deepen-since, // and deepen-not. throw new UnsupportedOperationException(); @@ -1270,9 +1239,14 @@ public class UploadPack { } } - private void verifyClientShallow() + /* + * Verify all shallow lines refer to commits + * + * It can mutate the input set (removing missing object ids from it) + */ + private void verifyClientShallow(Set<ObjectId> shallowCommits) throws IOException, PackProtocolException { - AsyncRevObjectQueue q = walk.parseAny(clientShallowCommits, true); + AsyncRevObjectQueue q = walk.parseAny(shallowCommits, true); try { for (;;) { try { @@ -1290,7 +1264,7 @@ public class UploadPack { } catch (MissingObjectException notCommit) { // shallow objects not known at the server are ignored // by git-core upload-pack, match that behavior. - clientShallowCommits.remove(notCommit.getObjectId()); + shallowCommits.remove(notCommit.getObjectId()); continue; } } @@ -1336,6 +1310,8 @@ public class UploadPack { if (useProtocolV2()) { // The equivalent in v2 is only the capabilities // advertisement. + protocolV2Hook + .onCapabilities(CapabilitiesV2Request.builder().build()); for (String s : getV2CapabilityAdvertisement()) { adv.writeOne(s); } @@ -1411,33 +1387,6 @@ public class UploadPack { return msgOut; } - private void parseFilter(String arg) throws PackProtocolException { - if (arg.equals("blob:none")) { //$NON-NLS-1$ - filterBlobLimit = 0; - } else if (arg.startsWith("blob:limit=")) { //$NON-NLS-1$ - try { - filterBlobLimit = Long.parseLong( - arg.substring("blob:limit=".length())); //$NON-NLS-1$ - } catch (NumberFormatException e) { - throw new PackProtocolException( - MessageFormat.format(JGitText.get().invalidFilter, - arg)); - } - } - /* - * We must have (1) either "blob:none" or - * "blob:limit=" set (because we only support - * blob size limits for now), and (2) if the - * latter, then it must be nonnegative. Throw - * if (1) or (2) is not met. - */ - if (filterBlobLimit < 0) { - throw new PackProtocolException( - MessageFormat.format(JGitText.get().invalidFilter, - arg)); - } - } - private void recvWants() throws IOException { boolean isFirst = true; boolean filterReceived = false; @@ -1478,7 +1427,7 @@ public class UploadPack { } filterReceived = true; - parseFilter(arg); + filterBlobLimit = ProtocolV2Parser.filterLine(arg); continue; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java index aa71c9445a..6d4df4fbad 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java @@ -43,6 +43,8 @@ package org.eclipse.jgit.transport; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; @@ -363,7 +365,7 @@ abstract class WalkRemoteObjectDatabase { */ BufferedReader openReader(String path) throws IOException { final InputStream is = open(path).in; - return new BufferedReader(new InputStreamReader(is, Constants.CHARSET)); + return new BufferedReader(new InputStreamReader(is, UTF_8)); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java index 470ed02841..335abe1b5b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java @@ -45,6 +45,8 @@ package org.eclipse.jgit.treewalk; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.nio.ByteBuffer; import java.nio.CharBuffer; @@ -182,7 +184,7 @@ public abstract class AbstractTreeIterator { if (prefix != null && prefix.length() > 0) { final ByteBuffer b; - b = Constants.CHARSET.encode(CharBuffer.wrap(prefix)); + b = UTF_8.encode(CharBuffer.wrap(prefix)); pathLen = b.limit(); path = new byte[Math.max(DEFAULT_PATH_SIZE, pathLen + 1)]; b.get(path, 0, pathLen); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java index d500aae681..69303d6ee3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java @@ -44,10 +44,13 @@ package org.eclipse.jgit.treewalk; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.JGitInternalException; @@ -1045,7 +1048,7 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { final AbstractTreeIterator t = currentHead; final int off = t.pathOffset; final int end = t.pathLen; - return RawParseUtils.decode(Constants.CHARSET, t.path, off, end); + return RawParseUtils.decode(UTF_8, t.path, off, end); } /** @@ -1378,11 +1381,11 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { } static String pathOf(AbstractTreeIterator t) { - return RawParseUtils.decode(Constants.CHARSET, t.path, 0, t.pathLen); + return RawParseUtils.decode(UTF_8, t.path, 0, t.pathLen); } static String pathOf(byte[] buf, int pos, int end) { - return RawParseUtils.decode(Constants.CHARSET, buf, pos, end); + return RawParseUtils.decode(UTF_8, buf, pos, end); } /** @@ -1436,7 +1439,8 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { return null; } return filterCommand.replaceAll("%f", //$NON-NLS-1$ - QuotedString.BOURNE.quote((getPathString()))); + Matcher.quoteReplacement( + QuotedString.BOURNE.quote((getPathString())))); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 179fd46791..7c6bfb9d63 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -46,6 +46,8 @@ package org.eclipse.jgit.treewalk; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -1413,7 +1415,7 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { IteratorState(WorkingTreeOptions options) { this.options = options; - this.nameEncoder = Constants.CHARSET.newEncoder(); + this.nameEncoder = UTF_8.newEncoder(); } void initializeReadBuffer() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Base64.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Base64.java index 442f0793fb..69f8547452 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/Base64.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Base64.java @@ -6,7 +6,7 @@ package org.eclipse.jgit.util; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.text.MessageFormat; import java.util.Arrays; @@ -54,7 +54,7 @@ public class Base64 { + "abcdefghijklmnopqrstuvwxyz" // //$NON-NLS-1$ + "0123456789" // //$NON-NLS-1$ + "+/" // //$NON-NLS-1$ - ).getBytes(CHARSET); + ).getBytes(UTF_8); DEC = new byte[128]; Arrays.fill(DEC, INVALID_DEC); @@ -177,7 +177,7 @@ public class Base64 { e += 4; } - return new String(outBuff, 0, e, CHARSET); + return new String(outBuff, 0, e, UTF_8); } /** @@ -293,7 +293,7 @@ public class Base64 { * @return the decoded data */ public static byte[] decode(String s) { - byte[] bytes = s.getBytes(CHARSET); + byte[] bytes = s.getBytes(UTF_8); return decode(bytes, 0, bytes.length); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 8948e6099a..e5c2922cfa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -640,12 +640,15 @@ public abstract class FS { JGitText.get().commandClosedStderrButDidntExit, desc, PROCESS_EXIT_TIMEOUT), -1); fail.set(true); + return false; } } catch (InterruptedException e) { - LOG.error(MessageFormat.format( - JGitText.get().threadInterruptedWhileRunning, desc), e); + setError(originalError, MessageFormat.format( + JGitText.get().threadInterruptedWhileRunning, desc), -1); + fail.set(true); + return false; } - return false; + return true; } private void setError(IOException e, String message, int exitCode) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java index 9f99e28557..98797dc64f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32.java @@ -169,7 +169,7 @@ public class FS_Win32 extends FS { if (result.isEmpty()) { return NO_ENTRIES; } - return result.toArray(new Entry[result.size()]); + return result.toArray(new Entry[0]); } /** {@inheritDoc} */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java index 3393fbfd1d..e5c8d9d398 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java @@ -43,7 +43,7 @@ package org.eclipse.jgit.util; -import static org.eclipse.jgit.lib.Constants.CHARSET; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.PrintStream; @@ -126,7 +126,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { try { w = readPipe(dir, // new String[] { cygpath, "--windows", "--absolute", pn }, // //$NON-NLS-1$ //$NON-NLS-2$ - CHARSET.name()); + UTF_8.name()); } catch (CommandFailedException e) { LOG.warn(e.getMessage()); return null; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index 2f547c15b0..ecfd31647d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -45,6 +45,8 @@ package org.eclipse.jgit.util; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.File; import java.io.IOException; import java.nio.file.AtomicMoveNotSupportedException; @@ -712,7 +714,7 @@ public class FileUtils { Path nioPath = toPath(file); if (Files.isSymbolicLink(nioPath)) return Files.readSymbolicLink(nioPath).toString() - .getBytes(Constants.CHARSET).length; + .getBytes(UTF_8).length; return Files.size(nioPath); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/QuotedString.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/QuotedString.java index 4d58d0614a..a55cad3705 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/QuotedString.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/QuotedString.java @@ -43,6 +43,8 @@ package org.eclipse.jgit.util; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.util.Arrays; import org.eclipse.jgit.lib.Constants; @@ -181,7 +183,7 @@ public abstract class QuotedString { continue; } } - return RawParseUtils.decode(Constants.CHARSET, r, 0, rPtr); + return RawParseUtils.decode(UTF_8, r, 0, rPtr); } } @@ -294,7 +296,7 @@ public abstract class QuotedString { public String dequote(byte[] in, int inPtr, int inEnd) { if (2 <= inEnd - inPtr && in[inPtr] == '"' && in[inEnd - 1] == '"') return dq(in, inPtr + 1, inEnd - 1); - return RawParseUtils.decode(Constants.CHARSET, in, inPtr, inEnd); + return RawParseUtils.decode(UTF_8, in, inPtr, inEnd); } private static String dq(byte[] in, int inPtr, int inEnd) { @@ -370,7 +372,7 @@ public abstract class QuotedString { } } - return RawParseUtils.decode(Constants.CHARSET, r, 0, rPtr); + return RawParseUtils.decode(UTF_8, r, 0, rPtr); } private GitPathStyle() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java index c22c8de5d1..28f406a49e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java @@ -549,6 +549,62 @@ public final class RawParseUtils { } /** + * Locate the end of the header. Note that headers may be + * more than one line long. + * @param b + * buffer to scan. + * @param ptr + * position within buffer to start looking for the end-of-header. + * @return new position just after the header. This is either + * b.length, or the index of the header's terminating newline. + * @since 5.1 + */ + public static final int headerEnd(final byte[] b, int ptr) { + final int sz = b.length; + while (ptr < sz) { + final byte c = b[ptr++]; + if (c == '\n' && (ptr == sz || b[ptr] != ' ')) { + return ptr - 1; + } + } + return ptr - 1; + } + + /** + * Find the start of the contents of a given header. + * + * @param b + * buffer to scan. + * @param headerName + * header to search for + * @param ptr + * position within buffer to start looking for header at. + * @return new position at the start of the header's contents, -1 for + * not found + * @since 5.1 + */ + public static final int headerStart(byte[] headerName, byte[] b, int ptr) { + // Start by advancing to just past a LF or buffer start + if (ptr != 0) { + ptr = nextLF(b, ptr - 1); + } + while (ptr < b.length - (headerName.length + 1)) { + boolean found = true; + for (int i = 0; i < headerName.length; i++) { + if (headerName[i] != b[ptr++]) { + found = false; + break; + } + } + if (found && b[ptr++] == ' ') { + return ptr; + } + ptr = nextLF(b, ptr); + } + return -1; + } + + /** * Locate the first position before a given character. * * @param b diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/MessageWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/MessageWriter.java index 94e5ef3d74..12d809b513 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/MessageWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/MessageWriter.java @@ -43,13 +43,14 @@ package org.eclipse.jgit.util.io; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; -import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.util.RawParseUtils; /** @@ -80,7 +81,7 @@ public class MessageWriter extends Writer { */ public MessageWriter() { buf = new ByteArrayOutputStream(); - enc = new OutputStreamWriter(getRawStream(), Constants.CHARSET); + enc = new OutputStreamWriter(getRawStream(), UTF_8); } /** {@inheritDoc} */ |