diff options
Diffstat (limited to 'org.eclipse.jgit')
124 files changed, 2833 insertions, 608 deletions
diff --git a/org.eclipse.jgit/.classpath b/org.eclipse.jgit/.classpath index 04a2be7bdb..cfcf24a51e 100644 --- a/org.eclipse.jgit/.classpath +++ b/org.eclipse.jgit/.classpath @@ -2,7 +2,7 @@ <classpath> <classpathentry kind="src" path="src"/> <classpathentry kind="src" path="resources"/> - <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/> + <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/> <classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/> <classpathentry kind="output" path="bin"/> </classpath> diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index eb31d56ed1..4a2663b34e 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -3,8 +3,17 @@ <resource path="META-INF/MANIFEST.MF"> <filter id="924844039"> <message_arguments> - <message_argument value="4.5.7"/> - <message_argument value="4.5.0"/> + <message_argument value="4.6.2"/> + <message_argument value="4.6.0"/> + </message_arguments> + </filter> + </resource> + <resource path="src/org/eclipse/jgit/errors/NoPackSignatureException.java" type="org.eclipse.jgit.errors.NoPackSignatureException"> + <filter id="1108344834"> + <message_arguments> + <message_argument value="4.5"/> + <message_argument value="4.6"/> + <message_argument value="org.eclipse.jgit.errors.NoPackSignatureException"/> </message_arguments> </filter> </resource> @@ -22,6 +31,24 @@ </message_arguments> </filter> </resource> + <resource path="src/org/eclipse/jgit/errors/UnsupportedPackIndexVersionException.java" type="org.eclipse.jgit.errors.UnsupportedPackIndexVersionException"> + <filter id="1108344834"> + <message_arguments> + <message_argument value="4.5"/> + <message_argument value="4.6"/> + <message_argument value="org.eclipse.jgit.errors.UnsupportedPackIndexVersionException"/> + </message_arguments> + </filter> + </resource> + <resource path="src/org/eclipse/jgit/errors/UnsupportedPackVersionException.java" type="org.eclipse.jgit.errors.UnsupportedPackVersionException"> + <filter id="1108344834"> + <message_arguments> + <message_argument value="4.5"/> + <message_argument value="4.6"/> + <message_argument value="org.eclipse.jgit.errors.UnsupportedPackVersionException"/> + </message_arguments> + </filter> + </resource> <resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants"> <filter id="336658481"> <message_arguments> @@ -29,8 +56,29 @@ <message_argument value="CONFIG_KEY_SUPPORTSATOMICFILECREATION"/> </message_arguments> </filter> + <filter id="1141899266"> + <message_arguments> + <message_argument value="4.5"/> + <message_argument value="4.6"/> + <message_argument value="CONFIG_KEY_SUPPORTSATOMICFILECREATION"/> + </message_arguments> + </filter> </resource> <resource path="src/org/eclipse/jgit/util/FS.java" type="org.eclipse.jgit.util.FS"> + <filter id="1141899266"> + <message_arguments> + <message_argument value="4.5"/> + <message_argument value="4.6"/> + <message_argument value="createNewFile(File)"/> + </message_arguments> + </filter> + <filter id="1141899266"> + <message_arguments> + <message_argument value="4.5"/> + <message_argument value="4.6"/> + <message_argument value="supportsAtomicCreateNewFile()"/> + </message_arguments> + </filter> <filter id="1142947843"> <message_arguments> <message_argument value="4.5.6"/> diff --git a/org.eclipse.jgit/.settings/org.eclipse.jdt.core.prefs b/org.eclipse.jgit/.settings/org.eclipse.jdt.core.prefs index bfaf736d6e..4f1759fb3f 100644 --- a/org.eclipse.jgit/.settings/org.eclipse.jdt.core.prefs +++ b/org.eclipse.jgit/.settings/org.eclipse.jdt.core.prefs @@ -7,9 +7,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jgit.annotations.N org.eclipse.jdt.core.compiler.annotation.nullanalysis=enabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -112,7 +112,7 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=error org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=error -org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.compiler.source=1.8 org.eclipse.jdt.core.formatter.align_type_members_on_columns=false org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16 org.eclipse.jdt.core.formatter.alignment_for_arguments_in_annotation=0 diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index 79673a73fd..701efcff12 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -2,12 +2,12 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: %plugin_name Bundle-SymbolicName: org.eclipse.jgit -Bundle-Version: 4.5.8.qualifier +Bundle-Version: 4.6.2.qualifier Bundle-Localization: plugin Bundle-Vendor: %provider_name Bundle-ActivationPolicy: lazy -Export-Package: org.eclipse.jgit.annotations;version="4.5.8", - org.eclipse.jgit.api;version="4.5.8"; +Export-Package: org.eclipse.jgit.annotations;version="4.6.2", + org.eclipse.jgit.api;version="4.6.2"; uses:="org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.diff, @@ -21,60 +21,57 @@ Export-Package: org.eclipse.jgit.annotations;version="4.5.8", org.eclipse.jgit.submodule, org.eclipse.jgit.transport, org.eclipse.jgit.merge", - org.eclipse.jgit.api.errors;version="4.5.8";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.errors", - org.eclipse.jgit.attributes;version="4.5.8", - org.eclipse.jgit.blame;version="4.5.8"; + org.eclipse.jgit.api.errors;version="4.6.2";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.errors", + org.eclipse.jgit.attributes;version="4.6.2", + org.eclipse.jgit.blame;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.diff", - org.eclipse.jgit.diff;version="4.5.8"; + org.eclipse.jgit.diff;version="4.6.2"; uses:="org.eclipse.jgit.patch, org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.util", - org.eclipse.jgit.dircache;version="4.5.8"; + org.eclipse.jgit.dircache;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.util, org.eclipse.jgit.events, org.eclipse.jgit.attributes", - org.eclipse.jgit.errors;version="4.5.8"; + org.eclipse.jgit.errors;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.internal.storage.pack, org.eclipse.jgit.transport, org.eclipse.jgit.dircache", - org.eclipse.jgit.events;version="4.5.8";uses:="org.eclipse.jgit.lib", - org.eclipse.jgit.fnmatch;version="4.5.8", - org.eclipse.jgit.gitrepo;version="4.5.8"; + org.eclipse.jgit.events;version="4.6.2";uses:="org.eclipse.jgit.lib", + org.eclipse.jgit.fnmatch;version="4.6.2", + org.eclipse.jgit.gitrepo;version="4.6.2"; uses:="org.eclipse.jgit.api, org.eclipse.jgit.lib, org.eclipse.jgit.revwalk, org.xml.sax.helpers, org.xml.sax", - org.eclipse.jgit.gitrepo.internal;version="4.5.8";x-internal:=true, - org.eclipse.jgit.hooks;version="4.5.8";uses:="org.eclipse.jgit.lib", - org.eclipse.jgit.ignore;version="4.5.8", - org.eclipse.jgit.ignore.internal;version="4.5.8";x-friends:="org.eclipse.jgit.test", - org.eclipse.jgit.internal;version="4.5.8";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test", - org.eclipse.jgit.internal.ketch;version="4.5.8";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.dfs;version="4.5.8"; - x-friends:="org.eclipse.jgit.test, - org.eclipse.jgit.http.server, - org.eclipse.jgit.http.test", - org.eclipse.jgit.internal.storage.file;version="4.5.8"; + org.eclipse.jgit.gitrepo.internal;version="4.6.2";x-internal:=true, + org.eclipse.jgit.hooks;version="4.6.2";uses:="org.eclipse.jgit.lib", + org.eclipse.jgit.ignore;version="4.6.2", + org.eclipse.jgit.ignore.internal;version="4.6.2";x-friends:="org.eclipse.jgit.test", + org.eclipse.jgit.internal;version="4.6.2";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.test", + org.eclipse.jgit.internal.ketch;version="4.6.2";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", + org.eclipse.jgit.internal.storage.dfs;version="4.6.2";x-friends:="org.eclipse.jgit.test,org.eclipse.jgit.http.server,org.eclipse.jgit.http.test", + org.eclipse.jgit.internal.storage.file;version="4.6.2"; x-friends:="org.eclipse.jgit.test, org.eclipse.jgit.junit, org.eclipse.jgit.junit.http, org.eclipse.jgit.http.server, - org.eclipse.jgit.lfs.server, + org.eclipse.jgit.lfs, org.eclipse.jgit.pgm, org.eclipse.jgit.pgm.test", - org.eclipse.jgit.internal.storage.pack;version="4.5.8";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.reftree;version="4.5.8";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", - org.eclipse.jgit.lib;version="4.5.8"; + org.eclipse.jgit.internal.storage.pack;version="4.6.2";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", + org.eclipse.jgit.internal.storage.reftree;version="4.6.2";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", + org.eclipse.jgit.lib;version="4.6.2"; uses:="org.eclipse.jgit.revwalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.util, @@ -84,32 +81,32 @@ Export-Package: org.eclipse.jgit.annotations;version="4.5.8", org.eclipse.jgit.treewalk, org.eclipse.jgit.transport, org.eclipse.jgit.submodule", - org.eclipse.jgit.merge;version="4.5.8"; + org.eclipse.jgit.merge;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.revwalk, org.eclipse.jgit.diff, org.eclipse.jgit.dircache, org.eclipse.jgit.api", - org.eclipse.jgit.nls;version="4.5.8", - org.eclipse.jgit.notes;version="4.5.8"; + org.eclipse.jgit.nls;version="4.6.2", + org.eclipse.jgit.notes;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.revwalk, org.eclipse.jgit.merge", - org.eclipse.jgit.patch;version="4.5.8";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.diff", - org.eclipse.jgit.revplot;version="4.5.8";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.revwalk", - org.eclipse.jgit.revwalk;version="4.5.8"; + org.eclipse.jgit.patch;version="4.6.2";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.diff", + org.eclipse.jgit.revplot;version="4.6.2";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.revwalk", + org.eclipse.jgit.revwalk;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.treewalk, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.diff, org.eclipse.jgit.revwalk.filter", - org.eclipse.jgit.revwalk.filter;version="4.5.8";uses:="org.eclipse.jgit.revwalk,org.eclipse.jgit.lib,org.eclipse.jgit.util", - org.eclipse.jgit.storage.file;version="4.5.8";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.util", - org.eclipse.jgit.storage.pack;version="4.5.8";uses:="org.eclipse.jgit.lib", - org.eclipse.jgit.submodule;version="4.5.8";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.treewalk.filter,org.eclipse.jgit.treewalk", - org.eclipse.jgit.transport;version="4.5.8"; + org.eclipse.jgit.revwalk.filter;version="4.6.2";uses:="org.eclipse.jgit.revwalk,org.eclipse.jgit.lib,org.eclipse.jgit.util", + org.eclipse.jgit.storage.file;version="4.6.2";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.util", + org.eclipse.jgit.storage.pack;version="4.6.2";uses:="org.eclipse.jgit.lib", + org.eclipse.jgit.submodule;version="4.6.2";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.treewalk.filter,org.eclipse.jgit.treewalk", + org.eclipse.jgit.transport;version="4.6.2"; uses:="org.eclipse.jgit.transport.resolver, org.eclipse.jgit.revwalk, org.eclipse.jgit.internal.storage.pack, @@ -121,24 +118,25 @@ Export-Package: org.eclipse.jgit.annotations;version="4.5.8", org.eclipse.jgit.transport.http, org.eclipse.jgit.errors, org.eclipse.jgit.storage.pack", - org.eclipse.jgit.transport.http;version="4.5.8";uses:="javax.net.ssl", - org.eclipse.jgit.transport.resolver;version="4.5.8";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.transport", - org.eclipse.jgit.treewalk;version="4.5.8"; + org.eclipse.jgit.transport.http;version="4.6.2";uses:="javax.net.ssl", + org.eclipse.jgit.transport.resolver;version="4.6.2";uses:="org.eclipse.jgit.lib,org.eclipse.jgit.transport", + org.eclipse.jgit.treewalk;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.revwalk, org.eclipse.jgit.attributes, org.eclipse.jgit.treewalk.filter, org.eclipse.jgit.util, org.eclipse.jgit.dircache", - org.eclipse.jgit.treewalk.filter;version="4.5.8";uses:="org.eclipse.jgit.treewalk", - org.eclipse.jgit.util;version="4.5.8"; + org.eclipse.jgit.treewalk.filter;version="4.6.2";uses:="org.eclipse.jgit.treewalk", + org.eclipse.jgit.util;version="4.6.2"; uses:="org.eclipse.jgit.lib, org.eclipse.jgit.transport.http, org.eclipse.jgit.storage.file, org.ietf.jgss", - org.eclipse.jgit.util.io;version="4.5.8" -Bundle-RequiredExecutionEnvironment: JavaSE-1.7 -Import-Package: com.googlecode.javaewah;version="[0.7.9,0.8.0)", + org.eclipse.jgit.util.io;version="4.6.2", + org.eclipse.jgit.util.time;version="4.6.2" +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", com.jcraft.jsch;version="[0.1.37,0.2.0)", javax.crypto, javax.net.ssl, diff --git a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF index 7ab2c18cc5..c863b047aa 100644 --- a/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/SOURCE-MANIFEST.MF @@ -3,5 +3,5 @@ Bundle-ManifestVersion: 2 Bundle-Name: org.eclipse.jgit - Sources Bundle-SymbolicName: org.eclipse.jgit.source Bundle-Vendor: Eclipse.org - JGit -Bundle-Version: 4.5.8.qualifier -Eclipse-SourceBundle: org.eclipse.jgit;version="4.5.8.qualifier";roots="." +Bundle-Version: 4.6.2.qualifier +Eclipse-SourceBundle: org.eclipse.jgit;version="4.6.2.qualifier";roots="." diff --git a/org.eclipse.jgit/pom.xml b/org.eclipse.jgit/pom.xml index 2f8a24d5ca..4e31e0b8d7 100644 --- a/org.eclipse.jgit/pom.xml +++ b/org.eclipse.jgit/pom.xml @@ -53,7 +53,7 @@ <parent> <groupId>org.eclipse.jgit</groupId> <artifactId>org.eclipse.jgit-parent</artifactId> - <version>4.5.8-SNAPSHOT</version> + <version>4.6.2-SNAPSHOT</version> </parent> <artifactId>org.eclipse.jgit</artifactId> diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index c949c0aa9b..cde45cfc35 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -39,6 +39,7 @@ bitmapsMustBePrepared=Bitmaps must be prepared before they may be written. blameNotCommittedYet=Not Committed Yet blobNotFound=Blob not found: {0} blobNotFoundForPath=Blob not found: {0} for path: {1} +blockSizeNotPowerOf2=blockSize must be a power of 2 branchNameInvalid=Branch name {0} is not allowed buildingBitmaps=Building bitmaps cachedPacksPreventsIndexCreation=Using cached packs prevents index creation @@ -115,6 +116,7 @@ checkoutConflictWithFiles=Checkout conflict with files: {0} checkoutUnexpectedResult=Checkout returned unexpected result {0} classCastNotA=Not a {0} cloneNonEmptyDirectory=Destination path "{0}" already exists and is not an empty directory +closed=closed collisionOn=Collision on {0} commandRejectedByHook=Rejected by "{0}" hook.\n{1} commandWasCalledInTheWrongState=Command {0} was called in the wrong state @@ -280,6 +282,7 @@ expectedLessThanGot=expected less than ''{0}'', got ''{1}'' expectedPktLineWithService=expected pkt-line with ''# service=-'', got ''{0}'' expectedReceivedContentType=expected Content-Type {0}; received Content-Type {1} expectedReportForRefNotReceived={0}: expected report for ref {1} not received +failedToDetermineFilterDefinition=An exception occured while determining filter definitions failedUpdatingRefs=failed updating refs failureDueToOneOfTheFollowing=Failure due to one of the following: failureUpdatingFETCH_HEAD=Failure updating FETCH_HEAD: {0} @@ -367,6 +370,7 @@ invalidTagOption=Invalid tag option: {0} invalidTimeout=Invalid timeout: {0} invalidTimeUnitValue2=Invalid time unit value: {0}.{1}={2} invalidTimeUnitValue3=Invalid time unit value: {0}.{1}.{2}={3} +invalidTreeZeroLengthName=Cannot append a tree entry with zero-length name invalidURL=Invalid URL {0} invalidWildcards=Invalid wildcards {0} invalidRefSpec=Invalid refspec {0} @@ -464,6 +468,7 @@ packDoesNotMatchIndex=Pack {0} does not match index packedRefsHandleIsStale=packed-refs handle is stale, {0}. retry packetSizeMustBeAtLeast=packet size {0} must be >= {1} packetSizeMustBeAtMost=packet size {0} must be <= {1} +packedRefsCorruptionDetected=packed-refs corruption detected: {0} packfileCorruptionDetected=Packfile corruption detected: {0} packFileInvalid=Pack file invalid: {0} packfileIsTruncated=Packfile {0} is truncated. @@ -602,6 +607,7 @@ tagAlreadyExists=tag ''{0}'' already exists tagNameInvalid=tag name {0} is invalid tagOnRepoWithoutHEADCurrentlyNotSupported=Tag on repository without HEAD currently not supported theFactoryMustNotBeNull=The factory must not be null +timeIsUncertain=Time is uncertain timerAlreadyTerminated=Timer already terminated tooManyIncludeRecursions=Too many recursions; circular includes in config file(s)? topologicalSortRequired=Topological sort required. @@ -671,6 +677,7 @@ unsupportedMark=Mark not supported unsupportedOperationNotAddAtEnd=Not add-at-end: {0} unsupportedPackIndexVersion=Unsupported pack index version {0} unsupportedPackVersion=Unsupported pack version {0}. +unsupportedRepositoryDescription=Repository description not supported updatingHeadFailed=Updating HEAD failed updatingReferences=Updating references updatingRefFailed=Updating the ref {0} to {1} failed. ReturnCode from RefUpdate.update() was {2} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index 1f37833a41..16ec1463c9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -224,6 +224,11 @@ public class AddCommand extends GitCommand<DirCache> { entry.setLength(f.getEntryLength()); entry.setLastModified(f.getEntryLastModified()); long len = f.getEntryContentLength(); + // We read and filter the content multiple times. + // f.getEntryContentLength() reads and filters the input and + // inserter.insert(...) does it again. That's because an + // ObjectInserter needs to know the length before it starts + // inserting. TODO: Fix this by using Buffers. try (InputStream in = f.openEntryStream()) { ObjectId id = inserter.insert(OBJ_BLOB, len, in); entry.setObjectId(id); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java index d803efd649..2a2e07ddda 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java @@ -61,9 +61,9 @@ import org.eclipse.jgit.diff.RawTextComparator; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.io.AutoLFInputStream; @@ -248,11 +248,12 @@ public class BlameCommand extends GitCommand<BlameResult> { rawText = new RawText(inTree); break; case TRUE: - AutoLFInputStream in = new AutoLFInputStream( - new FileInputStream(inTree), true); - // Canonicalization should lead to same or shorter length - // (CRLF to LF), so the file size on disk is an upper size bound - rawText = new RawText(toByteArray(in, (int) inTree.length())); + try (AutoLFInputStream in = new AutoLFInputStream( + new FileInputStream(inTree), true)) { + // Canonicalization should lead to same or shorter length + // (CRLF to LF), so the file size on disk is an upper size bound + rawText = new RawText(toByteArray(in, (int) inTree.length())); + } break; default: throw new IllegalArgumentException( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java index 65508eff40..c17ae5c00d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java @@ -318,7 +318,9 @@ public class CheckoutCommand extends GitCommand<Ref> { if (!dco.getToBeDeleted().isEmpty()) { status = new CheckoutResult(Status.NONDELETED, - dco.getToBeDeleted()); + dco.getToBeDeleted(), + new ArrayList<String>(dco.getUpdated().keySet()), + dco.getRemoved()); } else status = new CheckoutResult(new ArrayList<String>(dco .getUpdated().keySet()), dco.getRemoved()); @@ -365,6 +367,26 @@ public class CheckoutCommand extends GitCommand<Ref> { } /** + * Add multiple slash-separated paths to the list of paths to check out. To + * check out all paths, use {@link #setAllPaths(boolean)}. + * <p> + * If this option is set, neither the {@link #setCreateBranch(boolean)} nor + * {@link #setName(String)} option is considered. In other words, these + * options are exclusive. + * + * @param p + * paths to update in the working tree and index (with + * <code>/</code> as separator) + * @return {@code this} + * @since 4.6 + */ + public CheckoutCommand addPaths(List<String> p) { + checkCallable(); + this.paths.addAll(p); + return this; + } + + /** * Set whether to checkout all paths. * <p> * This options should be used when you want to do a path checkout on the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutResult.java index 6a1bfb8f0e..92a67f46af 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutResult.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutResult.java @@ -113,6 +113,28 @@ public class CheckoutResult { * {@link Status#CONFLICTS} or {@link Status#NONDELETED}. */ CheckoutResult(Status status, List<String> fileList) { + this(status, fileList, null, null); + } + + /** + * Create a new fail result. If status is {@link Status#CONFLICTS}, + * <code>fileList</code> is a list of conflicting files, if status is + * {@link Status#NONDELETED}, <code>fileList</code> is a list of not deleted + * files. All other values ignore <code>fileList</code>. To create a result + * for {@link Status#OK}, see {@link #CheckoutResult(List, List)}. + * + * @param status + * the failure status + * @param fileList + * the list of files to store, status has to be either + * {@link Status#CONFLICTS} or {@link Status#NONDELETED}. + * @param modified + * the modified files + * @param removed + * the removed files. + */ + CheckoutResult(Status status, List<String> fileList, List<String> modified, + List<String> removed) { myStatus = status; if (status == Status.CONFLICTS) this.conflictList = fileList; @@ -123,8 +145,8 @@ public class CheckoutResult { else this.undeletedList = new ArrayList<String>(0); - this.modifiedList = new ArrayList<String>(0); - this.removedList = new ArrayList<String>(0); + this.modifiedList = modified; + this.removedList = removed; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java index e82a69798f..276bf96ea2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CherryPickCommand.java @@ -330,4 +330,15 @@ public class CherryPickCommand extends GitCommand<CherryPickResult> { String headName = Repository.shortenRefName(targetRefName); return headName; } + + @SuppressWarnings("nls") + @Override + public String toString() { + return "CherryPickCommand [repo=" + repo + ",\ncommits=" + commits + + ",\nmainlineParentNumber=" + mainlineParentNumber + + ", noCommit=" + noCommit + ", ourCommitName=" + ourCommitName + + ", reflogPrefix=" + reflogPrefix + ", strategy=" + strategy + + "]"; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java index 38b10971f4..ced1863719 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java @@ -381,6 +381,7 @@ public class MergeCommand extends GitCommand<MergeResult> { .call().getId(); } mergeStatus = MergeStatus.MERGED; + getRepository().autoGC(monitor); } if (commit && squash) { msg = JGitText.get().squashCommitNotUpdatingHEAD; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 2d6a76b390..d10cc3d715 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -400,8 +400,8 @@ public class RebaseCommand extends GitCommand<RebaseResult> { boolean conflicts = false; if (rebaseState.getFile(AUTOSTASH).exists()) { String stash = rebaseState.readFile(AUTOSTASH); - try { - Git.wrap(repo).stashApply().setStashRef(stash) + try (Git git = Git.wrap(repo)) { + git.stashApply().setStashRef(stash) .ignoreRepositoryState(true).setStrategy(strategy) .call(); } catch (StashApplyFailureException e) { @@ -463,8 +463,10 @@ public class RebaseCommand extends GitCommand<RebaseResult> { String oldMessage = commitToPick.getFullMessage(); String newMessage = interactiveHandler .modifyCommitMessage(oldMessage); - newHead = new Git(repo).commit().setMessage(newMessage) - .setAmend(true).setNoVerify(true).call(); + try (Git git = new Git(repo)) { + newHead = git.commit().setMessage(newMessage).setAmend(true) + .setNoVerify(true).call(); + } return null; case EDIT: rebaseState.createFile(AMEND, commitToPick.name()); @@ -693,6 +695,7 @@ public class RebaseCommand extends GitCommand<RebaseResult> { String headName = rebaseState.readFile(HEAD_NAME); updateHead(headName, finalHead, upstreamCommit); boolean stashConflicts = autoStashApply(); + getRepository().autoGC(monitor); FileUtils.delete(rebaseState.getDir(), FileUtils.RECURSIVE); if (stashConflicts) return RebaseResult.STASH_APPLY_CONFLICTS_RESULT; @@ -752,12 +755,12 @@ public class RebaseCommand extends GitCommand<RebaseResult> { GitAPIException, CheckoutConflictException { Ref ref = repo.exactRef(Constants.ORIG_HEAD); ObjectId orig_head = ref == null ? null : ref.getObjectId(); - try { - // we have already commited the cherry-picked commit. + try (Git git = Git.wrap(repo)) { + // we have already committed the cherry-picked commit. // what we need is to have changes introduced by this // commit to be on the index // resetting is a workaround - Git.wrap(repo).reset().setMode(ResetType.SOFT) + git.reset().setMode(ResetType.SOFT) .setRef("HEAD~1").call(); //$NON-NLS-1$ } finally { // set ORIG_HEAD back to where we started because soft diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java index 3ceff843a5..106988d4d4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ResetCommand.java @@ -432,4 +432,12 @@ public class ResetCommand extends GitCommand<Ref> { repo.writeMergeCommitMsg(null); } + @SuppressWarnings("nls") + @Override + public String toString() { + return "ResetCommand [repo=" + repo + ", ref=" + ref + ", mode=" + mode + + ", isReflogDisabled=" + isReflogDisabled + ", filepaths=" + + filepaths + "]"; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java index fbb24c1577..b0f772e0aa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java @@ -133,7 +133,9 @@ public class SubmoduleAddCommand extends */ protected boolean submoduleExists() throws IOException { TreeFilter filter = PathFilter.create(path); - return SubmoduleWalk.forIndex(repo).setFilter(filter).next(); + try (SubmoduleWalk w = SubmoduleWalk.forIndex(repo)) { + return w.setFilter(filter).next(); + } } /** @@ -178,7 +180,11 @@ public class SubmoduleAddCommand extends clone.setURI(resolvedUri); if (monitor != null) clone.setProgressMonitor(monitor); - Repository subRepo = clone.call().getRepository(); + Repository subRepo = null; + try (Git git = clone.call()) { + subRepo = git.getRepository(); + subRepo.incrementOpen(); + } // Save submodule URL to parent repository's config StoredConfig config = repo.getConfig(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleInitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleInitCommand.java index 0ed02acc82..1dbe3681bf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleInitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleInitCommand.java @@ -94,8 +94,7 @@ public class SubmoduleInitCommand extends GitCommand<Collection<String>> { public Collection<String> call() throws GitAPIException { checkCallable(); - try { - SubmoduleWalk generator = SubmoduleWalk.forIndex(repo); + try (SubmoduleWalk generator = SubmoduleWalk.forIndex(repo)) { if (!paths.isEmpty()) generator.setFilter(PathFilterGroup.createFromStrings(paths)); StoredConfig config = repo.getConfig(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleStatusCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleStatusCommand.java index 6e89f9873e..a1ea790d0d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleStatusCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleStatusCommand.java @@ -54,9 +54,9 @@ import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.submodule.SubmoduleStatus; import org.eclipse.jgit.submodule.SubmoduleStatusType; +import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.treewalk.filter.PathFilterGroup; /** @@ -94,8 +94,7 @@ public class SubmoduleStatusCommand extends public Map<String, SubmoduleStatus> call() throws GitAPIException { checkCallable(); - try { - SubmoduleWalk generator = SubmoduleWalk.forIndex(repo); + try (SubmoduleWalk generator = SubmoduleWalk.forIndex(repo)) { if (!paths.isEmpty()) generator.setFilter(PathFilterGroup.createFromStrings(paths)); Map<String, SubmoduleStatus> statuses = new HashMap<String, SubmoduleStatus>(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleSyncCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleSyncCommand.java index 024f0bebde..088eedc2dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleSyncCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleSyncCommand.java @@ -111,8 +111,7 @@ public class SubmoduleSyncCommand extends GitCommand<Map<String, String>> { public Map<String, String> call() throws GitAPIException { checkCallable(); - try { - SubmoduleWalk generator = SubmoduleWalk.forIndex(repo); + try (SubmoduleWalk generator = SubmoduleWalk.forIndex(repo)) { if (!paths.isEmpty()) generator.setFilter(PathFilterGroup.createFromStrings(paths)); Map<String, String> synced = new HashMap<String, String>(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/StashApplyFailureException.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/StashApplyFailureException.java index 1d54f77db8..222c1db2bb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/StashApplyFailureException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/errors/StashApplyFailureException.java @@ -1,7 +1,5 @@ package org.eclipse.jgit.api.errors; -import org.eclipse.jgit.api.errors.GitAPIException; - /** * Thrown from StashApplyCommand when stash apply fails */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesHandler.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesHandler.java index 19e4afdf92..3bf4179e7d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesHandler.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/AttributesHandler.java @@ -54,8 +54,8 @@ import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.treewalk.AbstractTreeIterator; import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.treewalk.WorkingTreeIterator; import org.eclipse.jgit.treewalk.TreeWalk.OperationType; +import org.eclipse.jgit.treewalk.WorkingTreeIterator; /** * The attributes handler knows how to retrieve, parse and merge attributes from diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommand.java new file mode 100644 index 0000000000..10be58880c --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommand.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.attributes; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * An abstraction for JGit's builtin implementations for hooks and filters. + * Instead of spawning an external processes to start a filter/hook and to pump + * data from/to stdin/stdout these builtin commmands may be used. They are + * constructed by {@link FilterCommandFactory}. + * + * @since 4.6 + */ +public abstract class FilterCommand { + /** + * The {@link InputStream} this command should read from + */ + protected InputStream in; + + /** + * The {@link OutputStream} this command should write to + */ + protected OutputStream out; + + /** + * @param in + * The {@link InputStream} this command should read from + * @param out + * The {@link OutputStream} this command should write to + */ + public FilterCommand(InputStream in, OutputStream out) { + this.in = in; + this.out = out; + } + + /** + * Execute the command. The command is supposed to read data from + * {@link #in} and to write the result to {@link #out}. It returns the + * number of bytes it read from {@link #in}. It should be called in a loop + * until it returns -1 signaling that the {@link InputStream} is completely + * processed. + * + * @return the number of bytes read from the {@link InputStream} or -1. -1 + * means that the {@link InputStream} is completely processed. + * @throws IOException + * when {@link IOException} occured while reading from + * {@link #in} or writing to {@link #out} + * + */ + public abstract int run() throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandFactory.java new file mode 100644 index 0000000000..6b973da35f --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandFactory.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.attributes; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.eclipse.jgit.lib.Repository; + +/** + * The factory responsible for creating instances of {@link FilterCommand}. + * + * @since 4.6 + */ +public interface FilterCommandFactory { + /** + * Create a new {@link FilterCommand}. + * + * @param db + * the repository this command should work on + * @param in + * the {@link InputStream} this command should read from + * @param out + * the {@link OutputStream} this command should write to + * @return the created {@link FilterCommand} + * @throws IOException + * thrown when the command constructor throws an IOException + */ + public FilterCommand create(Repository db, InputStream in, + OutputStream out) throws IOException; + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java new file mode 100644 index 0000000000..3fbaedb051 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/FilterCommandRegistry.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2016, Matthias Sohn <matthias.sohn@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.attributes; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.eclipse.jgit.lib.Repository; + +/** + * Registry for built-in filters + * + * @since 4.6 + */ +public class FilterCommandRegistry { + private static ConcurrentHashMap<String, FilterCommandFactory> filterCommandRegistry = new ConcurrentHashMap<>(); + + /** + * Registers a {@link FilterCommandFactory} responsible for creating + * {@link FilterCommand}s for a certain command name. If the factory f1 is + * registered for the name "jgit://builtin/x" then a call to + * <code>getCommand("jgit://builtin/x", ...)</code> will call + * <code>f1(...)</code> to create a new instance of {@link FilterCommand} + * + * @param filterCommandName + * the command name for which this factory is registered + * @param factory + * the factory responsible for creating {@link FilterCommand}s + * for the specified name + * @return the previous factory associated with <tt>commandName</tt>, or + * <tt>null</tt> if there was no mapping for <tt>commandName</tt> + */ + public static FilterCommandFactory register(String filterCommandName, + FilterCommandFactory factory) { + return filterCommandRegistry.put(filterCommandName, factory); + } + + /** + * Unregisters the {@link FilterCommandFactory} registered for the given + * command name + * + * @param filterCommandName + * the FilterCommandFactory's filter command name + * @return the previous factory associated with <tt>filterCommandName</tt>, + * or <tt>null</tt> if there was no mapping for <tt>commandName</tt> + */ + public static FilterCommandFactory unregister(String filterCommandName) { + return filterCommandRegistry.remove(filterCommandName); + } + + /** + * Checks whether any {@link FilterCommandFactory} is registered for a given + * command name + * + * @param filterCommandName + * the name for which the registry should be checked + * @return <code>true</code> if any factory was registered for the name + */ + public static boolean isRegistered(String filterCommandName) { + return filterCommandRegistry.containsKey(filterCommandName); + } + + /** + * @return Set of commandNames for which a {@link FilterCommandFactory} is + * registered + */ + public static Set<String> getRegisteredFilterCommands() { + return filterCommandRegistry.keySet(); + } + + /** + * Creates a new {@link FilterCommand} for the given name. A factory must be + * registered for the name in advance. + * + * @param filterCommandName + * The name for which a new {@link FilterCommand} should be + * created + * @param db + * the repository this command should work on + * @param in + * the {@link InputStream} this {@link FilterCommand} should read + * from + * @param out + * the {@link OutputStream} this {@link FilterCommand} should + * write to + * @return the command if a command could be created or <code>null</code> if + * there was no factory registered for that name + * @throws IOException + */ + public static FilterCommand createFilterCommand(String filterCommandName, + Repository db, InputStream in, OutputStream out) + throws IOException { + FilterCommandFactory cf = filterCommandRegistry.get(filterCommandName); + return (cf == null) ? null : cf.create(db, in, out); + } + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java index b9101c028c..b0cf8be076 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.dircache; import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.EOFException; import java.io.File; import java.io.FileInputStream; @@ -86,7 +87,6 @@ import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.MutableInteger; import org.eclipse.jgit.util.NB; import org.eclipse.jgit.util.TemporaryBuffer; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; /** * Support for the Git dircache (aka index file). @@ -634,9 +634,9 @@ public class DirCache { public void write() throws IOException { final LockFile tmp = myLock; requireLocked(tmp); - try { - writeTo(liveFile.getParentFile(), - new SafeBufferedOutputStream(tmp.getOutputStream())); + try (OutputStream o = tmp.getOutputStream(); + OutputStream bo = new BufferedOutputStream(o)) { + writeTo(liveFile.getParentFile(), bo); } catch (IOException err) { tmp.unlock(); throw err; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 8af7e46a07..c3184437b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -54,6 +54,8 @@ import java.util.List; import java.util.Map; import org.eclipse.jgit.api.errors.FilterFailedException; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.errors.CheckoutConflictException; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -86,11 +88,15 @@ import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.SystemReader; import org.eclipse.jgit.util.io.EolStreamTypeUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class handles checking out one or two trees merging with the index. */ public class DirCacheCheckout { + private static Logger LOG = LoggerFactory.getLogger(DirCacheCheckout.class); + private static final int MAX_EXCEPTION_TEXT_SIZE = 10 * 1024; /** @@ -1303,45 +1309,19 @@ public class DirCacheCheckout { } else { nonNullEolStreamType = EolStreamType.DIRECT; } - OutputStream channel = EolStreamTypeUtil.wrapOutputStream( - new FileOutputStream(tmpFile), nonNullEolStreamType); - if (checkoutMetadata.smudgeFilterCommand != null) { - ProcessBuilder filterProcessBuilder = fs.runInShell( - checkoutMetadata.smudgeFilterCommand, new String[0]); - filterProcessBuilder.directory(repo.getWorkTree()); - filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, - repo.getDirectory().getAbsolutePath()); - ExecutionResult result; - int rc; - try { - // TODO: wire correctly with AUTOCRLF - result = fs.execute(filterProcessBuilder, ol.openStream()); - rc = result.getRc(); - if (rc == 0) { - result.getStdout().writeTo(channel, - NullProgressMonitor.INSTANCE); + try (OutputStream channel = EolStreamTypeUtil.wrapOutputStream( + new FileOutputStream(tmpFile), nonNullEolStreamType)) { + if (checkoutMetadata.smudgeFilterCommand != null) { + if (FilterCommandRegistry + .isRegistered(checkoutMetadata.smudgeFilterCommand)) { + runBuiltinFilterCommand(repo, checkoutMetadata, ol, + channel); + } else { + runExternalFilterCommand(repo, entry, checkoutMetadata, ol, + fs, channel); } - } catch (IOException | InterruptedException e) { - throw new IOException(new FilterFailedException(e, - checkoutMetadata.smudgeFilterCommand, - entry.getPathString())); - - } finally { - channel.close(); - } - if (rc != 0) { - throw new IOException(new FilterFailedException(rc, - checkoutMetadata.smudgeFilterCommand, - entry.getPathString(), - result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE), - RawParseUtils.decode(result.getStderr() - .toByteArray(MAX_EXCEPTION_TEXT_SIZE)))); - } - } else { - try { + } else { ol.copyTo(channel); - } finally { - channel.close(); } } // The entry needs to correspond to the on-disk filesize. If the content @@ -1382,6 +1362,63 @@ public class DirCacheCheckout { entry.setLastModified(fs.lastModified(f)); } + // Run an external filter command + private static void runExternalFilterCommand(Repository repo, + DirCacheEntry entry, + CheckoutMetadata checkoutMetadata, ObjectLoader ol, FS fs, + OutputStream channel) throws IOException { + ProcessBuilder filterProcessBuilder = fs.runInShell( + checkoutMetadata.smudgeFilterCommand, new String[0]); + filterProcessBuilder.directory(repo.getWorkTree()); + filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, + repo.getDirectory().getAbsolutePath()); + ExecutionResult result; + int rc; + try { + // TODO: wire correctly with AUTOCRLF + result = fs.execute(filterProcessBuilder, ol.openStream()); + rc = result.getRc(); + if (rc == 0) { + result.getStdout().writeTo(channel, + NullProgressMonitor.INSTANCE); + } + } catch (IOException | InterruptedException e) { + throw new IOException(new FilterFailedException(e, + checkoutMetadata.smudgeFilterCommand, + entry.getPathString())); + } + if (rc != 0) { + throw new IOException(new FilterFailedException(rc, + checkoutMetadata.smudgeFilterCommand, + entry.getPathString(), + result.getStdout().toByteArray(MAX_EXCEPTION_TEXT_SIZE), + RawParseUtils.decode(result.getStderr() + .toByteArray(MAX_EXCEPTION_TEXT_SIZE)))); + } + } + + // Run a builtin filter command + private static void runBuiltinFilterCommand(Repository repo, + CheckoutMetadata checkoutMetadata, ObjectLoader ol, + OutputStream channel) throws MissingObjectException, IOException { + FilterCommand command = null; + try { + command = FilterCommandRegistry.createFilterCommand( + checkoutMetadata.smudgeFilterCommand, repo, ol.openStream(), + channel); + } catch (IOException e) { + LOG.error(JGitText.get().failedToDetermineFilterDefinition, e); + // In case an IOException occurred during creating of the command + // then proceed as if there would not have been a builtin filter. + ol.copyTo(channel); + } + if (command != null) { + while (command.run() != -1) { + // loop as long as command.run() tells there is work to do + } + } + } + @SuppressWarnings("deprecation") private static void checkValidPath(CanonicalTreeParser t) throws InvalidPathException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index 2370ae14c7..8a35d35fea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -215,10 +215,13 @@ public class ManifestParser extends DefaultHandler { attributes.getValue("dest"))); //$NON-NLS-1$ } else if ("include".equals(qName)) { //$NON-NLS-1$ String name = attributes.getValue("name"); //$NON-NLS-1$ - InputStream is = null; if (includedReader != null) { - try { - is = includedReader.readIncludeFile(name); + try (InputStream is = includedReader.readIncludeFile(name)) { + if (is == null) { + throw new SAXException( + RepoText.get().errorIncludeNotImplemented); + } + read(is); } catch (Exception e) { throw new SAXException(MessageFormat.format( RepoText.get().errorIncludeFile, name), e); @@ -226,22 +229,13 @@ public class ManifestParser extends DefaultHandler { } else if (filename != null) { int index = filename.lastIndexOf('/'); String path = filename.substring(0, index + 1) + name; - try { - is = new FileInputStream(path); + try (InputStream is = new FileInputStream(path)) { + read(is); } catch (IOException e) { throw new SAXException(MessageFormat.format( RepoText.get().errorIncludeFile, path), e); } } - if (is == null) { - throw new SAXException( - RepoText.get().errorIncludeNotImplemented); - } - try { - read(is); - } catch (IOException e) { - throw new SAXException(e); - } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 9b7f094d0f..86dbabca0b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -180,17 +180,11 @@ public class RepoCommand extends GitCommand<RevCommit> { public byte[] readFile(String uri, String ref, String path) throws GitAPIException, IOException { File dir = FileUtils.createTempDir("jgit_", ".git", null); //$NON-NLS-1$ //$NON-NLS-2$ - Repository repo = Git - .cloneRepository() - .setBare(true) - .setDirectory(dir) - .setURI(uri) - .call() - .getRepository(); - try { + try (Git git = Git.cloneRepository().setBare(true).setDirectory(dir) + .setURI(uri).call(); + Repository repo = git.getRepository()) { return readFileFromRepo(repo, ref, path); } finally { - repo.close(); FileUtils.delete(dir, FileUtils.RECURSIVE); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java index eb081adb19..ef67d49419 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/ignore/FastIgnoreRule.java @@ -42,10 +42,11 @@ */ package org.eclipse.jgit.ignore; +import static org.eclipse.jgit.ignore.internal.IMatcher.NO_MATCH; +import static org.eclipse.jgit.ignore.internal.Strings.isDirectoryPattern; import static org.eclipse.jgit.ignore.internal.Strings.stripTrailing; import static org.eclipse.jgit.ignore.internal.Strings.stripTrailingWhitespace; -import static org.eclipse.jgit.ignore.internal.Strings.isDirectoryPattern; -import static org.eclipse.jgit.ignore.internal.IMatcher.NO_MATCH; + import org.eclipse.jgit.errors.InvalidPatternException; import org.eclipse.jgit.ignore.internal.IMatcher; import org.eclipse.jgit.ignore.internal.PathMatcher; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index d32e873c19..df2cc5080e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -98,6 +98,7 @@ public class JGitText extends TranslationBundle { /***/ public String blameNotCommittedYet; /***/ public String blobNotFound; /***/ public String blobNotFoundForPath; + /***/ public String blockSizeNotPowerOf2; /***/ public String branchNameInvalid; /***/ public String buildingBitmaps; /***/ public String cachedPacksPreventsIndexCreation; @@ -174,6 +175,7 @@ public class JGitText extends TranslationBundle { /***/ public String checkoutUnexpectedResult; /***/ public String classCastNotA; /***/ public String cloneNonEmptyDirectory; + /***/ public String closed; /***/ public String collisionOn; /***/ public String commandRejectedByHook; /***/ public String commandWasCalledInTheWrongState; @@ -339,6 +341,7 @@ public class JGitText extends TranslationBundle { /***/ public String expectedPktLineWithService; /***/ public String expectedReceivedContentType; /***/ public String expectedReportForRefNotReceived; + /***/ public String failedToDetermineFilterDefinition; /***/ public String failedUpdatingRefs; /***/ public String failureDueToOneOfTheFollowing; /***/ public String failureUpdatingFETCH_HEAD; @@ -425,6 +428,7 @@ public class JGitText extends TranslationBundle { /***/ public String invalidTimeout; /***/ public String invalidTimeUnitValue2; /***/ public String invalidTimeUnitValue3; + /***/ public String invalidTreeZeroLengthName; /***/ public String invalidURL; /***/ public String invalidWildcards; /***/ public String invalidRefSpec; @@ -523,6 +527,7 @@ public class JGitText extends TranslationBundle { /***/ public String packedRefsHandleIsStale; /***/ public String packetSizeMustBeAtLeast; /***/ public String packetSizeMustBeAtMost; + /***/ public String packedRefsCorruptionDetected; /***/ public String packfileCorruptionDetected; /***/ public String packFileInvalid; /***/ public String packfileIsTruncated; @@ -662,6 +667,7 @@ public class JGitText extends TranslationBundle { /***/ public String tagOnRepoWithoutHEADCurrentlyNotSupported; /***/ public String transactionAborted; /***/ public String theFactoryMustNotBeNull; + /***/ public String timeIsUncertain; /***/ public String timerAlreadyTerminated; /***/ public String tooManyIncludeRecursions; /***/ public String topologicalSortRequired; @@ -730,6 +736,7 @@ public class JGitText extends TranslationBundle { /***/ public String unsupportedOperationNotAddAtEnd; /***/ public String unsupportedPackIndexVersion; /***/ public String unsupportedPackVersion; + /***/ public String unsupportedRepositoryDescription; /***/ public String updatingHeadFailed; /***/ public String updatingReferences; /***/ public String updatingRefFailed; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java index 014eab2b45..1221ddd8d2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java @@ -43,10 +43,12 @@ package org.eclipse.jgit.internal.ketch; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.eclipse.jgit.internal.ketch.KetchConstants.TERM; import java.io.IOException; import java.util.List; +import java.util.concurrent.TimeoutException; import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.ObjectId; @@ -55,6 +57,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.TreeFormatter; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.util.time.ProposedTimestamp; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,9 +78,11 @@ class ElectionRound extends Round { void start() throws IOException { ObjectId id; try (Repository git = leader.openRepository(); + ProposedTimestamp ts = getSystem().getClock().propose(); ObjectInserter inserter = git.newObjectInserter()) { - id = bumpTerm(git, inserter); + id = bumpTerm(git, ts, inserter); inserter.flush(); + blockUntil(ts); } runAsync(id); } @@ -91,12 +96,17 @@ class ElectionRound extends Round { return term; } - private ObjectId bumpTerm(Repository git, ObjectInserter inserter) - throws IOException { + private ObjectId bumpTerm(Repository git, ProposedTimestamp ts, + ObjectInserter inserter) throws IOException { CommitBuilder b = new CommitBuilder(); if (!ObjectId.zeroId().equals(acceptedOldIndex)) { try (RevWalk rw = new RevWalk(git)) { RevCommit c = rw.parseCommit(acceptedOldIndex); + if (getSystem().requireMonotonicLeaderElections()) { + if (ts.read(SECONDS) < c.getCommitTime()) { + throw new TimeIsUncertainException(); + } + } b.setTreeId(c.getTree()); b.setParentId(acceptedOldIndex); term = parseTerm(c.getFooterLines(TERM)) + 1; @@ -116,7 +126,7 @@ class ElectionRound extends Round { msg.append(' ').append(tag); } - b.setAuthor(leader.getSystem().newCommitter()); + b.setAuthor(leader.getSystem().newCommitter(ts)); b.setCommitter(b.getAuthor()); b.setMessage(msg.toString()); @@ -138,4 +148,12 @@ class ElectionRound extends Round { } return Long.parseLong(s, 10); } + + private void blockUntil(ProposedTimestamp ts) throws IOException { + try { + ts.blockUntil(getSystem().getMaxWaitForMonotonicClock()); + } catch (InterruptedException | TimeoutException e) { + throw new TimeIsUncertainException(e); + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java index 71e872e3fa..33f526e52c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java @@ -53,6 +53,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_NAME; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; import java.net.URISyntaxException; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -67,6 +68,9 @@ import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.RemoteConfig; import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.util.time.MonotonicClock; +import org.eclipse.jgit.util.time.MonotonicSystemClock; +import org.eclipse.jgit.util.time.ProposedTimestamp; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,6 +92,7 @@ public class KetchSystem { } private final ScheduledExecutorService executor; + private final MonotonicClock clock; private final String txnNamespace; private final String txnAccepted; private final String txnCommitted; @@ -95,7 +100,7 @@ public class KetchSystem { /** Create a default system with a thread pool of 1 thread per CPU. */ public KetchSystem() { - this(defaultExecutor(), DEFAULT_TXN_NAMESPACE); + this(defaultExecutor(), new MonotonicSystemClock(), DEFAULT_TXN_NAMESPACE); } /** @@ -103,13 +108,17 @@ public class KetchSystem { * * @param executor * thread pool to run background operations. + * @param clock + * clock to create timestamps. * @param txnNamespace * reference namespace for the RefTree graph and associated * transaction state. Must begin with {@code "refs/"} and end * with {@code '/'}, for example {@code "refs/txn/"}. */ - public KetchSystem(ScheduledExecutorService executor, String txnNamespace) { + public KetchSystem(ScheduledExecutorService executor, MonotonicClock clock, + String txnNamespace) { this.executor = executor; + this.clock = clock; this.txnNamespace = txnNamespace; this.txnAccepted = txnNamespace + ACCEPTED; this.txnCommitted = txnNamespace + COMMITTED; @@ -121,6 +130,28 @@ public class KetchSystem { return executor; } + /** @return clock to obtain timestamps from. */ + public MonotonicClock getClock() { + return clock; + } + + /** + * @return how long the leader will wait for the {@link #getClock()}'s + * {@code ProposedTimestamp} used in commits proposed to the RefTree + * graph ({@link #getTxnAccepted()}). Defaults to 5 seconds. + */ + public Duration getMaxWaitForMonotonicClock() { + return Duration.ofSeconds(5); + } + + /** + * @return true if elections should require monotonically increasing commit + * timestamps. This requires a very good {@link MonotonicClock}. + */ + public boolean requireMonotonicLeaderElections() { + return false; + } + /** * Get the namespace used for the RefTree graph and transaction management. * @@ -145,27 +176,32 @@ public class KetchSystem { return txnStage; } - /** @return identity line for the committer header of a RefTreeGraph. */ - public PersonIdent newCommitter() { + /** + * @param time + * timestamp for the committer. + * @return identity line for the committer header of a RefTreeGraph. + */ + public PersonIdent newCommitter(ProposedTimestamp time) { String name = "ketch"; //$NON-NLS-1$ String email = "ketch@system"; //$NON-NLS-1$ - return new PersonIdent(name, email); + return new PersonIdent(name, email, time); } /** * Construct a random tag to identify a candidate during leader election. * <p> * Multiple processes trying to elect themselves leaders at exactly the same - * time (rounded to seconds) using the same {@link #newCommitter()} identity - * strings, for the same term, may generate the same ObjectId for the - * election commit and falsely assume they have both won. + * time (rounded to seconds) using the same + * {@link #newCommitter(ProposedTimestamp)} identity strings, for the same + * term, may generate the same ObjectId for the election commit and falsely + * assume they have both won. * <p> * Candidates add this tag to their election ballot commit to disambiguate * the election. The tag only needs to be unique for a given triplet of - * {@link #newCommitter()}, system time (rounded to seconds), and term. If - * every replica in the system uses a unique {@code newCommitter} (such as - * including the host name after the {@code "@"} in the email address) the - * tag could be the empty string. + * {@link #newCommitter(ProposedTimestamp)}, system time (rounded to + * seconds), and term. If every replica in the system uses a unique + * {@code newCommitter} (such as including the host name after the + * {@code "@"} in the email address) the tag could be the empty string. * <p> * The default implementation generates a few bytes of random data. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java index e297bca45e..907eecbef4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java @@ -64,6 +64,8 @@ import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.time.MonotonicClock; +import org.eclipse.jgit.util.time.ProposedTimestamp; /** Ketch replica running on the same system as the {@link KetchLeader}. */ public class LocalReplica extends KetchReplica { @@ -119,9 +121,11 @@ public class LocalReplica extends KetchReplica { getSystem().getExecutor().execute(new Runnable() { @Override public void run() { - try (Repository git = getLeader().openRepository()) { + MonotonicClock clk = getSystem().getClock(); + try (Repository git = getLeader().openRepository(); + ProposedTimestamp ts = clk.propose()) { try { - update(git, req); + update(git, req, ts); req.done(git); } catch (Throwable err) { req.setException(git, err); @@ -139,8 +143,8 @@ public class LocalReplica extends KetchReplica { throw new IOException(KetchText.get().cannotFetchFromLocalReplica); } - private void update(Repository git, ReplicaPushRequest req) - throws IOException { + private void update(Repository git, ReplicaPushRequest req, + ProposedTimestamp ts) throws IOException { RefDatabase refdb = git.getRefDatabase(); CommitMethod method = getCommitMethod(); @@ -156,7 +160,8 @@ public class LocalReplica extends KetchReplica { } BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.setRefLogIdent(getSystem().newCommitter()); + batch.addProposedTimestamp(ts); + batch.setRefLogIdent(getSystem().newCommitter(ts)); batch.setRefLogMessage("ketch", false); //$NON-NLS-1$ batch.setAllowNonFastForwards(true); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java index 0876eb5dbd..12d3f4c9c8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java @@ -67,6 +67,7 @@ import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.PushCertificate; import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.time.ProposedTimestamp; /** * A proposal to be applied in a Ketch system. @@ -123,6 +124,8 @@ public class Proposal { private PersonIdent author; private String message; private PushCertificate pushCert; + + private List<ProposedTimestamp> timestamps; private final List<Runnable> listeners = new CopyOnWriteArrayList<>(); private final AtomicReference<State> state = new AtomicReference<>(NEW); @@ -223,6 +226,31 @@ public class Proposal { } /** + * @return timestamps that Ketch must block for. These may have been used as + * commit times inside the objects involved in the proposal. + */ + public List<ProposedTimestamp> getProposedTimestamps() { + if (timestamps != null) { + return timestamps; + } + return Collections.emptyList(); + } + + /** + * Request the proposal to wait for the affected timestamps to resolve. + * + * @param ts + * @return {@code this}. + */ + public Proposal addProposedTimestamp(ProposedTimestamp ts) { + if (timestamps == null) { + timestamps = new ArrayList<>(4); + } + timestamps.add(ts); + return this; + } + + /** * Add a callback to be invoked when the proposal is done. * <p> * A proposal is done when it has entered either {@link State#EXECUTED} or diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java index d34477ab26..ddd7059fc2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java @@ -46,12 +46,16 @@ package org.eclipse.jgit.internal.ketch; import static org.eclipse.jgit.internal.ketch.Proposal.State.RUNNING; import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.storage.reftree.Command; @@ -65,6 +69,7 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.time.ProposedTimestamp; /** A {@link Round} that aggregates and sends user {@link Proposal}s. */ class ProposalRound extends Round { @@ -123,8 +128,10 @@ class ProposalRound extends Round { } try { ObjectId id; - try (Repository git = leader.openRepository()) { - id = insertProposals(git); + try (Repository git = leader.openRepository(); + ProposedTimestamp ts = getSystem().getClock().propose()) { + id = insertProposals(git, ts); + blockUntil(ts); } runAsync(id); } catch (NoOp e) { @@ -143,16 +150,16 @@ class ProposalRound extends Round { } } - private ObjectId insertProposals(Repository git) + private ObjectId insertProposals(Repository git, ProposedTimestamp ts) throws IOException, NoOp { ObjectId id; try (ObjectInserter inserter = git.newObjectInserter()) { // TODO(sop) Process signed push certificates. if (queuedTree != null) { - id = insertSingleProposal(git, inserter); + id = insertSingleProposal(git, ts, inserter); } else { - id = insertMultiProposal(git, inserter); + id = insertMultiProposal(git, ts, inserter); } stageCommands = makeStageList(git, inserter); @@ -161,7 +168,7 @@ class ProposalRound extends Round { return id; } - private ObjectId insertSingleProposal(Repository git, + private ObjectId insertSingleProposal(Repository git, ProposedTimestamp ts, ObjectInserter inserter) throws IOException, NoOp { // Fast path: tree is passed in with all proposals applied. ObjectId treeId = queuedTree.writeTree(inserter); @@ -183,13 +190,13 @@ class ProposalRound extends Round { if (!ObjectId.zeroId().equals(acceptedOldIndex)) { b.setParentId(acceptedOldIndex); } - b.setCommitter(leader.getSystem().newCommitter()); + b.setCommitter(leader.getSystem().newCommitter(ts)); b.setAuthor(p.getAuthor() != null ? p.getAuthor() : b.getCommitter()); b.setMessage(message(p)); return inserter.insert(b); } - private ObjectId insertMultiProposal(Repository git, + private ObjectId insertMultiProposal(Repository git, ProposedTimestamp ts, ObjectInserter inserter) throws IOException, NoOp { // The tree was not passed in, or there are multiple proposals // each needing their own commit. Reset the tree and replay each @@ -208,7 +215,7 @@ class ProposalRound extends Round { } } - PersonIdent committer = leader.getSystem().newCommitter(); + PersonIdent committer = leader.getSystem().newCommitter(ts); for (Proposal p : todo) { if (!tree.apply(p.getCommands())) { // This should not occur, previously during queuing the @@ -292,6 +299,20 @@ class ProposalRound extends Round { return b.makeStageList(newObjs, git, inserter); } + private void blockUntil(ProposedTimestamp ts) + throws TimeIsUncertainException { + List<ProposedTimestamp> times = todo.stream() + .flatMap(p -> p.getProposedTimestamps().stream()) + .collect(Collectors.toCollection(ArrayList::new)); + times.add(ts); + + try { + Duration maxWait = getSystem().getMaxWaitForMonotonicClock(); + ProposedTimestamp.blockUntil(times, maxWait); + } catch (InterruptedException | TimeoutException e) { + throw new TimeIsUncertainException(e); + } + } private static class NoOp extends Exception { private static final long serialVersionUID = 1L; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java index 6f4a178673..396fbdd722 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java @@ -44,13 +44,13 @@ package org.eclipse.jgit.internal.ketch; import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; +import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NODELETE; import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; import java.io.IOException; import java.util.ArrayList; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java index 1335b85cca..dd8e568c7f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java @@ -75,6 +75,10 @@ abstract class Round { this.acceptedOldIndex = head; } + KetchSystem getSystem() { + return leader.getSystem(); + } + /** * Creates a commit for {@code refs/txn/accepted} and calls * {@link #runAsync(AnyObjectId)} to begin execution of the round across diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java new file mode 100644 index 0000000000..7223f553ca --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.ketch; + +import java.io.IOException; + +import org.eclipse.jgit.internal.JGitText; + +class TimeIsUncertainException extends IOException { + private static final long serialVersionUID = 1L; + + TimeIsUncertainException() { + super(JGitText.get().timeIsUncertain); + } + + TimeIsUncertainException(Exception e) { + super(JGitText.get().timeIsUncertain, e); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java index ecd4b23c25..f7decf1324 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java @@ -145,6 +145,8 @@ public final class DfsBlockCache { * <p> * If a pack file has a native size, a whole multiple of the native size * will be used until it matches this size. + * <p> + * The value for blockSize must be a power of 2. */ private final int blockSize; @@ -175,13 +177,14 @@ public final class DfsBlockCache { /** Number of bytes currently loaded in the cache. */ private volatile long liveBytes; + @SuppressWarnings("unchecked") private DfsBlockCache(final DfsBlockCacheConfig cfg) { tableSize = tableSize(cfg); if (tableSize < 1) throw new IllegalArgumentException(JGitText.get().tSizeMustBeGreaterOrEqual1); table = new AtomicReferenceArray<HashEntry>(tableSize); - loadLocks = new ReentrantLock[32]; + loadLocks = new ReentrantLock[cfg.getConcurrencyLevel()]; for (int i = 0; i < loadLocks.length; i++) loadLocks[i] = new ReentrantLock(true /* fair */); @@ -260,20 +263,22 @@ public final class DfsBlockCache { // TODO This table grows without bound. It needs to clean up // entries that aren't in cache anymore, and aren't being used // by a live DfsObjDatabase reference. - synchronized (packCache) { - DfsPackFile pack = packCache.get(dsc); - if (pack != null && pack.invalid()) { - packCache.remove(dsc); - pack = null; - } - if (pack == null) { - if (key == null) - key = new DfsPackKey(); - pack = new DfsPackFile(this, dsc, key); - packCache.put(dsc, pack); - } + + DfsPackFile pack = packCache.get(dsc); + if (pack != null && !pack.invalid()) { return pack; } + + // 'pack' either didn't exist or was invalid. Compute a new + // entry atomically (guaranteed by ConcurrentHashMap). + return packCache.compute(dsc, (k, v) -> { + if (v != null && !v.invalid()) { // valid value added by + return v; // another thread + } else { + return new DfsPackFile( + this, dsc, key != null ? key : new DfsPackKey()); + } + }); } private int hash(int packHash, long off) { @@ -416,6 +421,7 @@ public final class DfsBlockCache { clockLock.unlock(); } + @SuppressWarnings("unchecked") private void addToClock(Ref ref, int credit) { clockLock.lock(); try { @@ -500,9 +506,7 @@ public final class DfsBlockCache { } void remove(DfsPackFile pack) { - synchronized (packCache) { - packCache.remove(pack.getPackDescription()); - } + packCache.remove(pack.getPackDescription()); } private int slot(DfsPackKey pack, long position) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index a7d13defdc..089bfa471d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -47,6 +47,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_LIMIT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_SIZE; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO; import java.text.MessageFormat; @@ -65,12 +66,14 @@ public class DfsBlockCacheConfig { private long blockLimit; private int blockSize; private double streamRatio; + private int concurrencyLevel; /** Create a default configuration. */ public DfsBlockCacheConfig() { setBlockLimit(32 * MB); setBlockSize(64 * KB); setStreamRatio(0.30); + setConcurrencyLevel(32); } /** @@ -103,10 +106,38 @@ public class DfsBlockCacheConfig { /** * @param newSize * size in bytes of a single window read in from the pack file. + * The value must be a power of 2. * @return {@code this} */ public DfsBlockCacheConfig setBlockSize(final int newSize) { - blockSize = Math.max(512, newSize); + int size = Math.max(512, newSize); + if ((size & (size - 1)) != 0) { + throw new IllegalArgumentException( + JGitText.get().blockSizeNotPowerOf2); + } + blockSize = size; + return this; + } + + /** + * @return the estimated number of threads concurrently accessing the cache. + * <b>Default is 32.</b> + * @since 4.6 + */ + public int getConcurrencyLevel() { + return concurrencyLevel; + } + + /** + * @param newConcurrencyLevel + * the estimated number of threads concurrently accessing the + * cache. + * @return {@code this} + * @since 4.6 + */ + public DfsBlockCacheConfig setConcurrencyLevel( + final int newConcurrencyLevel) { + concurrencyLevel = newConcurrencyLevel; return this; } @@ -154,6 +185,12 @@ public class DfsBlockCacheConfig { CONFIG_KEY_BLOCK_SIZE, getBlockSize())); + setConcurrencyLevel(rc.getInt( + CONFIG_CORE_SECTION, + CONFIG_DFS_SECTION, + CONFIG_KEY_CONCURRENCY_LEVEL, + getConcurrencyLevel())); + String v = rc.getString( CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index a5e920a75e..c179e77786 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -312,8 +312,7 @@ public class DfsInserter extends ObjectInserter { } DfsOutputStream os = db.writeFile(pack, INDEX); - try { - CountingOutputStream cnt = new CountingOutputStream(os); + try (CountingOutputStream cnt = new CountingOutputStream(os)) { if (buf != null) buf.writeTo(cnt, null); else @@ -321,7 +320,9 @@ public class DfsInserter extends ObjectInserter { pack.addFileExt(INDEX); pack.setFileSize(INDEX, cnt.getCount()); } finally { - os.close(); + if (buf != null) { + buf.close(); + } } return packIndex; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 2f61dea0d5..8c9329503f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -113,6 +113,7 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs { DfsReader(DfsObjDatabase db) { this.db = db; + this.streamFileThreshold = db.getReaderOptions().getStreamFileThreshold(); } DfsReaderOptions getOptions() { @@ -125,10 +126,6 @@ public final class DfsReader extends ObjectReader implements ObjectReuseAsIs { return baseCache; } - int getStreamFileThreshold() { - return getOptions().getStreamFileThreshold(); - } - @Override public ObjectReader newReader() { return new DfsReader(db); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefUpdate.java index d872f973c4..1f26fe35ff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefUpdate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsRefUpdate.java @@ -47,9 +47,9 @@ import java.io.IOException; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Ref.Storage; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.lib.Ref.Storage; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevTag; import org.eclipse.jgit.revwalk.RevWalk; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java index 6f390a4b3b..fd213977a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/InMemoryRepository.java @@ -16,6 +16,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.ObjectId; @@ -54,6 +55,7 @@ public class InMemoryRepository extends DfsRepository { private final DfsObjDatabase objdb; private final RefDatabase refdb; + private String gitwebDescription; private boolean performsAtomicTransactions = true; /** @@ -94,6 +96,17 @@ public class InMemoryRepository extends DfsRepository { performsAtomicTransactions = atomic; } + @Override + @Nullable + public String getGitwebDescription() { + return gitwebDescription; + } + + @Override + public void setGitwebDescription(@Nullable String d) { + gitwebDescription = d; + } + private class MemObjDatabase extends DfsObjDatabase { private List<DfsPackDescription> packs = new ArrayList<DfsPackDescription>(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java index 3c101e621a..30e973ecf4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackBitmapIndex.java @@ -43,11 +43,11 @@ package org.eclipse.jgit.internal.storage.file; -import com.googlecode.javaewah.EWAHCompressedBitmap; - import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import com.googlecode.javaewah.EWAHCompressedBitmap; + /** * Base implementation of the PackBitmapIndex. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitSet.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitSet.java index ddb5ff0cac..bafae87cd8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitSet.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitSet.java @@ -96,7 +96,7 @@ final class BitSet { } if (lastNonEmptyWord != 0) - compressed.add(lastNonEmptyWord); + compressed.addWord(lastNonEmptyWord); if (runningEmptyWords > 0) { compressed.addStreamOfEmptyWords(false, runningEmptyWords); @@ -107,7 +107,7 @@ final class BitSet { } int bitsThatMatter = 64 - Long.numberOfLeadingZeros(lastNonEmptyWord); if (bitsThatMatter > 0) - compressed.add(lastNonEmptyWord, bitsThatMatter); + compressed.addWord(lastNonEmptyWord, bitsThatMatter); return compressed; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java index b27bcc4258..b18a06f9c9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BitmapIndexImpl.java @@ -47,9 +47,6 @@ import java.text.MessageFormat; import java.util.Iterator; import java.util.NoSuchElementException; -import com.googlecode.javaewah.EWAHCompressedBitmap; -import com.googlecode.javaewah.IntIterator; - import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BitmapIndex; @@ -59,6 +56,9 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; import org.eclipse.jgit.util.BlockList; +import com.googlecode.javaewah.EWAHCompressedBitmap; +import com.googlecode.javaewah.IntIterator; + /** A compressed bitmap representation of the entire object graph. */ public class BitmapIndexImpl implements BitmapIndex { private static final int EXTRA_BITS = 10 * 1024; @@ -504,10 +504,10 @@ public class BitmapIndexImpl implements BitmapIndex { static final EWAHCompressedBitmap ones(int sizeInBits) { EWAHCompressedBitmap mask = new EWAHCompressedBitmap(); mask.addStreamOfEmptyWords( - true, sizeInBits / EWAHCompressedBitmap.wordinbits); - int remaining = sizeInBits % EWAHCompressedBitmap.wordinbits; + true, sizeInBits / EWAHCompressedBitmap.WORD_IN_BITS); + int remaining = sizeInBits % EWAHCompressedBitmap.WORD_IN_BITS; if (remaining > 0) - mask.add((1L << remaining) - 1, remaining); + mask.addWord((1L << remaining) - 1, remaining); return mask; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index 53fd37e534..0388acbbaf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -53,9 +53,13 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.text.MessageFormat; +import java.text.ParseException; import java.util.HashSet; +import java.util.Objects; import java.util.Set; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesNodeProvider; import org.eclipse.jgit.errors.ConfigInvalidException; @@ -63,15 +67,16 @@ import org.eclipse.jgit.events.ConfigChangedEvent; import org.eclipse.jgit.events.ConfigChangedListener; import org.eclipse.jgit.events.IndexChangedEvent; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepository; +import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; import org.eclipse.jgit.lib.BaseRepositoryBuilder; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig.HideDotFiles; import org.eclipse.jgit.lib.CoreConfig.SymLinks; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; @@ -79,8 +84,11 @@ import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.IO; +import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.SystemReader; @@ -110,16 +118,13 @@ import org.eclipse.jgit.util.SystemReader; * */ public class FileRepository extends Repository { - private final FileBasedConfig systemConfig; + private static final String UNNAMED = "Unnamed repository; edit this file to name it for gitweb."; //$NON-NLS-1$ + private final FileBasedConfig systemConfig; private final FileBasedConfig userConfig; - private final FileBasedConfig repoConfig; - private final RefDatabase refs; - private final ObjectDirectory objectDatabase; - private FileSnapshot snapshot; /** @@ -177,10 +182,12 @@ public class FileRepository extends Repository { getFS()); else systemConfig = new FileBasedConfig(null, FS.DETECTED) { + @Override public void load() { // empty, do not load } + @Override public boolean isOutdated() { // regular class would bomb here return false; @@ -197,6 +204,7 @@ public class FileRepository extends Repository { loadRepoConfig(); repoConfig.addChangeListener(new ConfigChangedListener() { + @Override public void onConfigChanged(ConfigChangedEvent event) { fireEvent(event); } @@ -279,6 +287,7 @@ public class FileRepository extends Repository { * @throws IOException * in case of IO problem */ + @Override public void create(boolean bare) throws IOException { final FileBasedConfig cfg = getConfig(); if (cfg.getFile().exists()) { @@ -376,21 +385,20 @@ public class FileRepository extends Repository { return objectDatabase.getDirectory(); } - /** - * @return the object database which stores this repository's data. - */ + /** @return the object database storing this repository's data. */ + @Override public ObjectDirectory getObjectDatabase() { return objectDatabase; } /** @return the reference database which stores the reference namespace. */ + @Override public RefDatabase getRefDatabase() { return refs; } - /** - * @return the configuration of this repository - */ + /** @return the configuration of this repository. */ + @Override public FileBasedConfig getConfig() { if (systemConfig.isOutdated()) { try { @@ -416,6 +424,59 @@ public class FileRepository extends Repository { return repoConfig; } + @Override + @Nullable + public String getGitwebDescription() throws IOException { + String d; + try { + d = RawParseUtils.decode(IO.readFully(descriptionFile())); + } catch (FileNotFoundException err) { + return null; + } + if (d != null) { + d = d.trim(); + if (d.isEmpty() || UNNAMED.equals(d)) { + return null; + } + } + return d; + } + + @Override + public void setGitwebDescription(@Nullable String description) + throws IOException { + String old = getGitwebDescription(); + if (Objects.equals(old, description)) { + return; + } + + File path = descriptionFile(); + LockFile lock = new LockFile(path); + if (!lock.lock()) { + throw new IOException(MessageFormat.format(JGitText.get().lockError, + path.getAbsolutePath())); + } + try { + String d = description; + if (d != null) { + d = d.trim(); + if (!d.isEmpty()) { + d += '\n'; + } + } else { + d = ""; //$NON-NLS-1$ + } + lock.write(Constants.encode(d)); + lock.commit(); + } finally { + lock.unlock(); + } + } + + private File descriptionFile() { + return new File(getDirectory(), "description"); //$NON-NLS-1$ + } + /** * Objects known to exist but not expressed by {@link #getAllRefs()}. * <p> @@ -426,6 +487,7 @@ public class FileRepository extends Repository { * * @return unmodifiable collection of other known objects. */ + @Override public Set<ObjectId> getAdditionalHaves() { HashSet<ObjectId> r = new HashSet<ObjectId>(); for (AlternateHandle d : objectDatabase.myAlternates()) { @@ -464,9 +526,7 @@ public class FileRepository extends Repository { detectIndexChanges(); } - /** - * Detect index changes. - */ + /** Detect index changes. */ private void detectIndexChanges() { if (isBare()) return; @@ -490,6 +550,7 @@ public class FileRepository extends Repository { * named ref does not exist. * @throws IOException the ref could not be accessed. */ + @Override public ReflogReader getReflogReader(String refName) throws IOException { Ref ref = findRef(refName); if (ref != null) @@ -527,6 +588,7 @@ public class FileRepository extends Repository { globalAttributesNode = new GlobalAttributesNode(repo); } + @Override public AttributesNode getInfoAttributesNode() throws IOException { if (infoAttributesNode instanceof InfoAttributesNode) infoAttributesNode = ((InfoAttributesNode) infoAttributesNode) @@ -534,6 +596,7 @@ public class FileRepository extends Repository { return infoAttributesNode; } + @Override public AttributesNode getGlobalAttributesNode() throws IOException { if (globalAttributesNode instanceof GlobalAttributesNode) globalAttributesNode = ((GlobalAttributesNode) globalAttributesNode) @@ -555,4 +618,16 @@ public class FileRepository extends Repository { } + @Override + public void autoGC(ProgressMonitor monitor) { + GC gc = new GC(this); + gc.setPackConfig(new PackConfig(this)); + gc.setProgressMonitor(monitor); + gc.setAuto(true); + try { + gc.gc(); + } catch (ParseException | IOException e) { + throw new JGitInternalException(JGitText.get().gcFailed, e); + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index fbd1dbf696..be49f780a0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -293,12 +293,6 @@ public class FileSnapshot { return false; } - // Our lastRead flag may be old, refresh and retry - lastRead = System.currentTimeMillis(); - if (notRacyClean(lastRead)) { - return false; - } - // We last read this path too close to its last observed // modification time. We may have missed a modification. // Scan again, to ensure we still see the same state. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index a3e9430b13..b608416868 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -52,6 +52,10 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.text.ParseException; @@ -62,12 +66,16 @@ import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.dircache.DirCacheIterator; @@ -100,6 +108,8 @@ import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.GitDateParser; import org.eclipse.jgit.util.SystemReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A garbage collector for git {@link FileRepository}. Instances of this class @@ -109,10 +119,27 @@ import org.eclipse.jgit.util.SystemReader; * adapted to FileRepositories. */ public class GC { + private final static Logger LOG = LoggerFactory + .getLogger(GC.class); + private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$ private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$ + private static final Pattern PATTERN_LOOSE_OBJECT = Pattern + .compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$ + + private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$ + + private static final String BITMAP_EXT = "." //$NON-NLS-1$ + + PackExt.BITMAP_INDEX.getExtension(); + + private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$ + + private static final int DEFAULT_AUTOPACKLIMIT = 50; + + private static final int DEFAULT_AUTOLIMIT = 6700; + private final FileRepository repo; private ProgressMonitor pm; @@ -143,6 +170,11 @@ public class GC { private long lastRepackTime; /** + * Whether gc should do automatic housekeeping + */ + private boolean automatic; + + /** * Creates a new garbage collector with default values. An expirationTime of * two weeks and <code>null</code> as progress monitor will be used. * @@ -163,6 +195,10 @@ public class GC { * <li>prune all loose objects which are now reachable by packs</li> * </ul> * + * If {@link #setAuto(boolean)} was set to {@code true} {@code gc} will + * first check whether any housekeeping is required; if not, it exits + * without performing any work. + * * @return the collection of {@link PackFile}'s which are newly created * @throws IOException * @throws ParseException @@ -170,6 +206,9 @@ public class GC { * parsed */ public Collection<PackFile> gc() throws IOException, ParseException { + if (automatic && !needGc()) { + return Collections.emptyList(); + } pm.start(6 /* tasks */); packRefs(); // TODO: implement reflog_expire(pm, repo); @@ -325,45 +364,48 @@ public class GC { Set<ObjectId> indexObjects = null; File objects = repo.getObjectsDirectory(); String[] fanout = objects.list(); - if (fanout != null && fanout.length > 0) { - pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects, - fanout.length); - try { - for (String d : fanout) { - pm.update(1); - if (d.length() != 2) + if (fanout == null || fanout.length == 0) { + return; + } + pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects, + fanout.length); + try { + for (String d : fanout) { + pm.update(1); + if (d.length() != 2) + continue; + File[] entries = new File(objects, d).listFiles(); + if (entries == null) + continue; + for (File f : entries) { + String fName = f.getName(); + if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2) continue; - File[] entries = new File(objects, d).listFiles(); - if (entries == null) + if (repo.getFS().lastModified(f) >= expireDate) continue; - for (File f : entries) { - String fName = f.getName(); - if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2) - continue; - if (repo.getFS().lastModified(f) >= expireDate) + try { + ObjectId id = ObjectId.fromString(d + fName); + if (objectsToKeep.contains(id)) continue; - try { - ObjectId id = ObjectId.fromString(d + fName); - if (objectsToKeep.contains(id)) - continue; - if (indexObjects == null) - indexObjects = listNonHEADIndexObjects(); - if (indexObjects.contains(id)) - continue; - deletionCandidates.put(id, f); - } catch (IllegalArgumentException notAnObject) { - // ignoring the file that does not represent loose - // object + if (indexObjects == null) + indexObjects = listNonHEADIndexObjects(); + if (indexObjects.contains(id)) continue; - } + deletionCandidates.put(id, f); + } catch (IllegalArgumentException notAnObject) { + // ignoring the file that does not represent loose + // object + continue; } } - } finally { - pm.endTask(); } + } finally { + pm.endTask(); } - if (deletionCandidates.isEmpty()) + + if (deletionCandidates.isEmpty()) { return; + } // From the set of current refs remove all those which have been handled // during last repack(). Only those refs will survive which have been @@ -433,12 +475,19 @@ public class GC { // loose objects. Make a last check, though, to avoid deleting objects // that could have been referenced while the candidates list was being // built (by an incoming push, for example). + Set<File> touchedFanout = new HashSet<>(); for (File f : deletionCandidates.values()) { if (f.lastModified() < expireDate) { f.delete(); + touchedFanout.add(f.getParentFile()); } } + for (File f : touchedFanout) { + FileUtils.delete(f, + FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS); + } + repo.getObjectDatabase().close(); } @@ -625,6 +674,7 @@ public class GC { throw new IOException(e); } prunePacked(); + deleteOrphans(); lastPackedRefs = refsBefore; lastRepackTime = time; @@ -632,6 +682,48 @@ public class GC { } /** + * Deletes orphans + * <p> + * A file is considered an orphan if it is either a "bitmap" or an index + * file, and its corresponding pack file is missing in the list. + * </p> + */ + private void deleteOrphans() { + Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(), + "pack"); //$NON-NLS-1$ + List<String> fileNames = null; + try (Stream<Path> files = Files.list(packDir)) { + fileNames = files.map(path -> path.getFileName().toString()) + .filter(name -> { + return (name.endsWith(PACK_EXT) + || name.endsWith(BITMAP_EXT) + || name.endsWith(INDEX_EXT)); + }).sorted(Collections.reverseOrder()) + .collect(Collectors.toList()); + } catch (IOException e1) { + // ignore + } + if (fileNames == null) { + return; + } + + String base = null; + for (String n : fileNames) { + if (n.endsWith(PACK_EXT)) { + base = n.substring(0, n.lastIndexOf('.')); + } else { + if (base == null || !n.startsWith(base)) { + try { + Files.delete(new File(packDir.toFile(), n).toPath()); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + } + } + } + + /** * @param ref * the ref which log should be inspected * @param minTime only reflog entries not older then this time are processed @@ -1081,4 +1173,114 @@ public class GC { this.packExpire = packExpire; packExpireAgeMillis = -1; } + + /** + * Set the {@code gc --auto} option. + * + * With this option, gc checks whether any housekeeping is required; if not, + * it exits without performing any work. Some JGit commands run + * {@code gc --auto} after performing operations that could create many + * loose objects. + * <p/> + * Housekeeping is required if there are too many loose objects or too many + * packs in the repository. If the number of loose objects exceeds the value + * of the gc.auto option JGit GC consolidates all existing packs into a + * single pack (equivalent to {@code -A} option), whereas git-core would + * combine all loose objects into a single pack using {@code repack -d -l}. + * Setting the value of {@code gc.auto} to 0 disables automatic packing of + * loose objects. + * <p/> + * If the number of packs exceeds the value of {@code gc.autoPackLimit}, + * then existing packs (except those marked with a .keep file) are + * consolidated into a single pack by using the {@code -A} option of repack. + * Setting {@code gc.autoPackLimit} to 0 disables automatic consolidation of + * packs. + * <p/> + * Like git the following jgit commands run auto gc: + * <ul> + * <li>fetch</li> + * <li>merge</li> + * <li>rebase</li> + * <li>receive-pack</li> + * </ul> + * The auto gc for receive-pack can be suppressed by setting the config + * option {@code receive.autogc = false} + * + * @param auto + * defines whether gc should do automatic housekeeping + * @since 4.5 + */ + public void setAuto(boolean auto) { + this.automatic = auto; + } + + private boolean needGc() { + if (tooManyPacks()) { + addRepackAllOption(); + } else if (!tooManyLooseObjects()) { + return false; + } + // TODO run pre-auto-gc hook, if it fails return false + return true; + } + + private void addRepackAllOption() { + // TODO: if JGit GC is enhanced to support repack's option -l this + // method needs to be implemented + } + + /** + * @return {@code true} if number of packs > gc.autopacklimit (default 50) + */ + boolean tooManyPacks() { + int autopacklimit = repo.getConfig().getInt( + ConfigConstants.CONFIG_GC_SECTION, + ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, + DEFAULT_AUTOPACKLIMIT); + if (autopacklimit <= 0) { + return false; + } + // JGit always creates two packfiles, one for the objects reachable from + // branches, and another one for the rest + return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1); + } + + /** + * Quickly estimate number of loose objects, SHA1 is distributed evenly so + * counting objects in one directory (bucket 17) is sufficient + * + * @return {@code true} if number of loose objects > gc.auto (default 6700) + */ + boolean tooManyLooseObjects() { + int auto = repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION, + ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT); + if (auto <= 0) { + return false; + } + int n = 0; + int threshold = (auto + 255) / 256; + Path dir = repo.getObjectsDirectory().toPath().resolve("17"); //$NON-NLS-1$ + if (!Files.exists(dir)) { + return false; + } + try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, + new DirectoryStream.Filter<Path>() { + + public boolean accept(Path file) throws IOException { + return Files.isRegularFile(file) && PATTERN_LOOSE_OBJECT + .matcher(file.getFileName().toString()) + .matches(); + } + })) { + for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); + iter.next()) { + if (++n > threshold) { + return true; + } + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + return false; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java index e743cb4aff..7fb8e6d644 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndex.java @@ -49,13 +49,13 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; -import com.googlecode.javaewah.EWAHCompressedBitmap; - import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.ObjectId; +import com.googlecode.javaewah.EWAHCompressedBitmap; + /** * Logical representation of the bitmap data stored in the pack index. * {@link ObjectId}s are encoded as a single integer in the range [0, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java index 4ff09a148f..956e8de0a7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexBuilder.java @@ -50,8 +50,6 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; -import com.googlecode.javaewah.EWAHCompressedBitmap; - import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; @@ -63,6 +61,8 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; import org.eclipse.jgit.util.BlockList; +import com.googlecode.javaewah.EWAHCompressedBitmap; + /** * Helper for constructing {@link PackBitmapIndex}es. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java index 7cd68b625e..2c462a78b7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexRemapper.java @@ -47,15 +47,15 @@ import java.util.Collections; import java.util.Iterator; import java.util.NoSuchElementException; -import com.googlecode.javaewah.EWAHCompressedBitmap; -import com.googlecode.javaewah.IntIterator; - import org.eclipse.jgit.internal.storage.file.BasePackBitmapIndex.StoredBitmap; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.BitmapIndex; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; +import com.googlecode.javaewah.EWAHCompressedBitmap; +import com.googlecode.javaewah.IntIterator; + /** * A PackBitmapIndex that remaps the bitmaps in the previous index to the * positions in the new pack index. Note, unlike typical PackBitmapIndex diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java index a7ab00db2d..9d2c70b4c6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexV1.java @@ -49,8 +49,6 @@ import java.io.InputStream; import java.text.MessageFormat; import java.util.Arrays; -import com.googlecode.javaewah.EWAHCompressedBitmap; - import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -59,6 +57,8 @@ import org.eclipse.jgit.lib.ObjectIdOwnerMap; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.NB; +import com.googlecode.javaewah.EWAHCompressedBitmap; + /** * Support for the pack bitmap index v1 format. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexWriterV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexWriterV1.java index 8325e2ebd1..f8f02e95e3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexWriterV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackBitmapIndexWriterV1.java @@ -50,12 +50,11 @@ import java.io.OutputStream; import java.security.DigestOutputStream; import java.text.MessageFormat; -import com.googlecode.javaewah.EWAHCompressedBitmap; - import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder.StoredEntry; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; + +import com.googlecode.javaewah.EWAHCompressedBitmap; /** * Creates the version 1 pack bitmap index files. @@ -74,7 +73,7 @@ public class PackBitmapIndexWriterV1 { */ public PackBitmapIndexWriterV1(final OutputStream dst) { out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst - : new SafeBufferedOutputStream(dst), + : new BufferedOutputStream(dst), Constants.newMessageDigest()); dataOutput = new SimpleDataOutput(out); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java index 6dfe74bf83..51539112e7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java @@ -56,7 +56,6 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.NB; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; /** * Creates a table of contents to support random access by {@link PackFile}. @@ -183,7 +182,7 @@ public abstract class PackIndexWriter { */ protected PackIndexWriter(final OutputStream dst) { out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst - : new SafeBufferedOutputStream(dst), + : new BufferedOutputStream(dst), Constants.newMessageDigest()); tmp = new byte[4 + Constants.OBJECT_ID_LENGTH]; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index cd98539102..a5d380df53 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -850,6 +850,11 @@ public class RefDirectory extends RefDatabase { } int sp = p.indexOf(' '); + if (sp < 0) { + throw new IOException(MessageFormat.format( + JGitText.get().packedRefsCorruptionDetected, + packedRefsFile.getAbsolutePath())); + } ObjectId id = ObjectId.fromString(p.substring(0, sp)); String name = copy(p, sp + 1, p.length()); ObjectIdRef cur; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index a2c0561ae1..a742d1747e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -94,12 +94,14 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { WindowCursor(FileObjectDatabase db) { this.db = db; this.createdFromInserter = null; + this.streamFileThreshold = WindowCache.getStreamFileThreshold(); } WindowCursor(FileObjectDatabase db, @Nullable ObjectDirectoryInserter createdFromInserter) { this.db = db; this.createdFromInserter = createdFromInserter; + this.streamFileThreshold = WindowCache.getStreamFileThreshold(); } DeltaBaseCache getDeltaBaseCache() { @@ -337,10 +339,6 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { } } - int getStreamFileThreshold() { - return WindowCache.getStreamFileThreshold(); - } - @Override @Nullable public ObjectInserter getCreatedFromInserter() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackOutputStream.java index be1e3d4715..59166e6206 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackOutputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackOutputStream.java @@ -183,7 +183,7 @@ public final class PackOutputStream extends OutputStream { public final void writeHeader(ObjectToPack otp, long rawLength) throws IOException { ObjectToPack b = otp.getDeltaBase(); - if (b != null && (b.isWritten() & ofsDelta)) { + if (b != null && (b.isWritten() & ofsDelta)) { // Non-short-circuit logic is intentional int n = objectHeader(rawLength, OBJ_OFS_DELTA, headerBuffer); n = ofsDelta(count - b.getOffset(), headerBuffer, n); write(headerBuffer, 0, n); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 691867aba8..8b4d2e6d35 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -1597,14 +1597,15 @@ public class PackWriter implements AutoCloseable { } } - TemporaryBuffer.Heap delta = delta(otp); - out.writeHeader(otp, delta.length()); + try (TemporaryBuffer.Heap delta = delta(otp)) { + out.writeHeader(otp, delta.length()); - Deflater deflater = deflater(); - deflater.reset(); - DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater); - delta.writeTo(dst, null); - dst.finish(); + Deflater deflater = deflater(); + deflater.reset(); + DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater); + delta.writeTo(dst, null); + dst.finish(); + } typeStats.cntDeltas++; typeStats.deltaBytes += out.length() - otp.getOffset(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapWalker.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapWalker.java index d9ac9ef16b..2ec4d568c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapWalker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapWalker.java @@ -49,11 +49,11 @@ import java.util.Set; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.lib.BitmapIndex; +import org.eclipse.jgit.lib.BitmapIndex.Bitmap; +import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.BitmapIndex.Bitmap; -import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevCommit; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java index d383abf316..2ef0f20d8d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java @@ -43,7 +43,6 @@ package org.eclipse.jgit.internal.storage.reftree; -import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.eclipse.jgit.lib.Constants.R_REFS; import static org.eclipse.jgit.lib.Constants.encode; @@ -52,6 +51,7 @@ import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; import static org.eclipse.jgit.lib.FileMode.TYPE_TREE; import static org.eclipse.jgit.lib.Ref.Storage.NEW; import static org.eclipse.jgit.lib.Ref.Storage.PACKED; +import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; import java.io.IOException; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchRefUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchRefUpdate.java index 8550ec3a3f..653c9f66b6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchRefUpdate.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchRefUpdate.java @@ -49,18 +49,21 @@ import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_RE import java.io.IOException; import java.text.MessageFormat; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.concurrent.TimeoutException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.PushCertificate; import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.time.ProposedTimestamp; /** * Batch of reference updates to be applied to a repository. @@ -69,6 +72,17 @@ import org.eclipse.jgit.transport.ReceiveCommand; * server is making changes to more than one reference at a time. */ public class BatchRefUpdate { + /** + * Maximum delay the calling thread will tolerate while waiting for a + * {@code MonotonicClock} to resolve associated {@link ProposedTimestamp}s. + * <p> + * A default of 5 seconds was chosen by guessing. A common assumption is + * clock skew between machines on the same LAN using an NTP server also on + * the same LAN should be under 5 seconds. 5 seconds is also not that long + * for a large `git push` operation to complete. + */ + private static final Duration MAX_WAIT = Duration.ofSeconds(5); + private final RefDatabase refdb; /** Commands to apply during this batch. */ @@ -95,6 +109,9 @@ public class BatchRefUpdate { /** Push options associated with this update. */ private List<String> pushOptions; + /** Associated timestamps that should be blocked on before update. */ + private List<ProposedTimestamp> timestamps; + /** * Initialize a new batch update. * @@ -314,6 +331,32 @@ public class BatchRefUpdate { } /** + * @return list of timestamps the batch must wait for. + * @since 4.6 + */ + public List<ProposedTimestamp> getProposedTimestamps() { + if (timestamps != null) { + return Collections.unmodifiableList(timestamps); + } + return Collections.emptyList(); + } + + /** + * Request the batch to wait for the affected timestamps to resolve. + * + * @param ts + * @return {@code this}. + * @since 4.6 + */ + public BatchRefUpdate addProposedTimestamp(ProposedTimestamp ts) { + if (timestamps == null) { + timestamps = new ArrayList<>(4); + } + timestamps.add(ts); + return this; + } + + /** * Execute this batch update. * <p> * The default implementation of this method performs a sequential reference @@ -348,6 +391,9 @@ public class BatchRefUpdate { } return; } + if (!blockUntilTimestamps(MAX_WAIT)) { + return; + } if (options != null) { pushOptions = options; @@ -433,6 +479,33 @@ public class BatchRefUpdate { } /** + * Wait for timestamps to be in the past, aborting commands on timeout. + * + * @param maxWait + * maximum amount of time to wait for timestamps to resolve. + * @return true if timestamps were successfully waited for; false if + * commands were aborted. + * @since 4.6 + */ + protected boolean blockUntilTimestamps(Duration maxWait) { + if (timestamps == null) { + return true; + } + try { + ProposedTimestamp.blockUntil(timestamps, maxWait); + return true; + } catch (TimeoutException | InterruptedException e) { + String msg = JGitText.get().timeIsUncertain; + for (ReceiveCommand c : commands) { + if (c.getResult() == NOT_ATTEMPTED) { + c.setResult(REJECTED_OTHER_REASON, msg); + } + } + return false; + } + } + + /** * Execute this batch update without option strings. * * @param walk diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index e3f8ba5b5b..87a95b938a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -65,6 +65,12 @@ public class ConfigConstants { /** The "dfs" section */ public static final String CONFIG_DFS_SECTION = "dfs"; + /** + * The "receive" section + * @since 4.6 + */ + public static final String CONFIG_RECEIVE_SECTION = "receive"; + /** The "user" section */ public static final String CONFIG_USER_SECTION = "user"; @@ -101,6 +107,12 @@ public class ConfigConstants { */ public static final String CONFIG_PULL_SECTION = "pull"; + /** + * The "filter" section + * @since 4.6 + */ + public static final String CONFIG_FILTER_SECTION = "filter"; + /** The "algorithm" key */ public static final String CONFIG_KEY_ALGORITHM = "algorithm"; @@ -108,6 +120,24 @@ public class ConfigConstants { public static final String CONFIG_KEY_AUTOCRLF = "autocrlf"; /** + * The "auto" key + * @since 4.6 + */ + public static final String CONFIG_KEY_AUTO = "auto"; + + /** + * The "autogc" key + * @since 4.6 + */ + public static final String CONFIG_KEY_AUTOGC = "autogc"; + + /** + * The "autopacklimit" key + * @since 4.6 + */ + public static final String CONFIG_KEY_AUTOPACKLIMIT = "autopacklimit"; + + /** * The "eol" key * * @since 4.3 @@ -145,6 +175,13 @@ public class ConfigConstants { /** The "blockSize" key */ public static final String CONFIG_KEY_BLOCK_SIZE = "blockSize"; + /** + * The "concurrencyLevel" key + * + * @since 4.6 + */ + public static final String CONFIG_KEY_CONCURRENCY_LEVEL = "concurrencyLevel"; + /** The "deltaBaseCacheLimit" key */ public static final String CONFIG_KEY_DELTA_BASE_CACHE_LIMIT = "deltaBaseCacheLimit"; @@ -337,4 +374,11 @@ public class ConfigConstants { * @since 4.0 */ public static final String CONFIG_KEY_STREAM_RATIO = "streamRatio"; + + /** + * Flag in the filter section whether to use JGit's implementations of + * filters and hooks + * @since 4.6 + */ + public static final String CONFIG_KEY_USEJGITBUILTIN = "useJGitBuiltin"; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index d30edaf41b..ff80672f80 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -391,6 +391,13 @@ public final class Constants { */ public static final String ATTR_FILTER_TYPE_SMUDGE = "smudge"; + /** + * Builtin filter commands start with this prefix + * + * @since 4.6 + */ + public static final String BUILTIN_FILTER_PREFIX = "jgit://builtin/"; + /** Name of the ignore file */ public static final String DOT_GIT_IGNORE = ".gitignore"; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java index 9e474f86a8..af6a4fb919 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/IndexDiff.java @@ -65,7 +65,6 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StopWalkException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.submodule.SubmoduleWalk.IgnoreSubmoduleMode; @@ -73,8 +72,8 @@ import org.eclipse.jgit.treewalk.AbstractTreeIterator; import org.eclipse.jgit.treewalk.EmptyTreeIterator; import org.eclipse.jgit.treewalk.FileTreeIterator; import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.treewalk.WorkingTreeIterator; import org.eclipse.jgit.treewalk.TreeWalk.OperationType; +import org.eclipse.jgit.treewalk.WorkingTreeIterator; import org.eclipse.jgit.treewalk.filter.AndTreeFilter; import org.eclipse.jgit.treewalk.filter.IndexDiffFilter; import org.eclipse.jgit.treewalk.filter.SkipWorkTreeFilter; @@ -248,7 +247,7 @@ public class IndexDiff { private final Repository repository; - private final RevTree tree; + private final AnyObjectId tree; private TreeFilter filter = null; @@ -311,10 +310,13 @@ public class IndexDiff { public IndexDiff(Repository repository, ObjectId objectId, WorkingTreeIterator workingTreeIterator) throws IOException { this.repository = repository; - if (objectId != null) - tree = new RevWalk(repository).parseTree(objectId); - else + if (objectId != null) { + try (RevWalk rw = new RevWalk(repository)) { + tree = rw.parseTree(objectId); + } + } else { tree = null; + } this.initialWorkingTreeIterator = workingTreeIterator; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java index 0b5efd77d4..feecbd81c0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java @@ -709,11 +709,12 @@ public class ObjectChecker { return ptr; } - @SuppressWarnings("resource") @Nullable private ObjectId idFor(int objType, byte[] raw) { if (skipList != null) { - return new ObjectInserter.Formatter().idFor(objType, raw); + try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { + return fmt.idFor(objType, raw); + } } return null; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java index 4edb38c5df..2a2d67d25a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java @@ -44,15 +44,15 @@ package org.eclipse.jgit.lib; -import org.eclipse.jgit.errors.InvalidObjectIdException; -import org.eclipse.jgit.util.NB; -import org.eclipse.jgit.util.RawParseUtils; - import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import org.eclipse.jgit.errors.InvalidObjectIdException; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.RawParseUtils; + /** * A SHA-1 abstraction. */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectIdOwnerMap.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectIdOwnerMap.java index 442261cbd5..95cb976372 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectIdOwnerMap.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectIdOwnerMap.java @@ -143,6 +143,7 @@ public class ObjectIdOwnerMap<V extends ObjectIdOwnerMap.Entry> * object to find. * @return true if the mapping exists for this object; false otherwise. */ + @Override public boolean contains(final AnyObjectId toFind) { return get(toFind) != null; } @@ -219,20 +220,20 @@ public class ObjectIdOwnerMap<V extends ObjectIdOwnerMap.Entry> return size == 0; } + @Override public Iterator<V> iterator() { return new Iterator<V>() { private int found; - private int dirIdx; - private int tblIdx; - private V next; + @Override public boolean hasNext() { return found < size; } + @Override public V next() { if (next != null) return found(next); @@ -261,6 +262,7 @@ public class ObjectIdOwnerMap<V extends ObjectIdOwnerMap.Entry> return v; } + @Override public void remove() { throw new UnsupportedOperationException(); } @@ -341,7 +343,7 @@ public class ObjectIdOwnerMap<V extends ObjectIdOwnerMap.Entry> /** Type of entry stored in the {@link ObjectIdOwnerMap}. */ public static abstract class Entry extends ObjectId { - Entry next; + transient Entry next; /** * Initialize this entry with a specific ObjectId. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java index b23145d798..372da98939 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectReader.java @@ -66,6 +66,13 @@ public abstract class ObjectReader implements AutoCloseable { public static final int OBJ_ANY = -1; /** + * The threshold at which a file will be streamed rather than loaded + * entirely into memory. + * @since 4.6 + */ + protected int streamFileThreshold; + + /** * Construct a new reader from the same data. * <p> * Applications can use this method to build a new reader from the same data @@ -445,6 +452,29 @@ public abstract class ObjectReader implements AutoCloseable { public abstract void close(); /** + * Sets the threshold at which a file will be streamed rather than loaded + * entirely into memory + * + * @param threshold + * the new threshold + * @since 4.6 + */ + public void setStreamFileThreshold(int threshold) { + streamFileThreshold = threshold; + } + + /** + * Returns the threshold at which a file will be streamed rather than loaded + * entirely into memory + * + * @return the threshold in bytes + * @since 4.6 + */ + public int getStreamFileThreshold() { + return streamFileThreshold; + } + + /** * Wraps a delegate ObjectReader. * * @since 4.4 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index e08a98529d..627ccaa206 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -53,6 +53,7 @@ import java.util.TimeZone; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.util.SystemReader; +import org.eclipse.jgit.util.time.ProposedTimestamp; /** * A combination of a person identity and time in Git. @@ -189,6 +190,19 @@ public class PersonIdent implements Serializable { } /** + * Construct a new {@link PersonIdent} with current time. + * + * @param aName + * @param aEmailAddress + * @param when + * @since 4.6 + */ + public PersonIdent(String aName, String aEmailAddress, + ProposedTimestamp when) { + this(aName, aEmailAddress, when.millis()); + } + + /** * Copy a PersonIdent, but alter the clone's time stamp * * @param pi diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java index 4ebe5fedf3..75a3592213 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java @@ -43,6 +43,7 @@ package org.eclipse.jgit.lib; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -53,7 +54,6 @@ import java.util.List; import org.eclipse.jgit.lib.RebaseTodoLine.Action; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; /** * Offers methods to read and write files formatted like the git-rebase-todo @@ -216,9 +216,8 @@ public class RebaseTodoFile { */ public void writeRebaseTodoFile(String path, List<RebaseTodoLine> steps, boolean append) throws IOException { - OutputStream fw = new SafeBufferedOutputStream(new FileOutputStream( - new File(repo.getDirectory(), path), append)); - try { + try (OutputStream fw = new BufferedOutputStream(new FileOutputStream( + new File(repo.getDirectory(), path), append))) { StringBuilder sb = new StringBuilder(); for (RebaseTodoLine step : steps) { sb.setLength(0); @@ -234,8 +233,6 @@ public class RebaseTodoFile { sb.append('\n'); fw.write(Constants.encode(sb.toString())); } - } finally { - fw.close(); } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ReflogEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ReflogEntry.java index e2102e54f0..0504646ee7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ReflogEntry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ReflogEntry.java @@ -42,9 +42,6 @@ */ package org.eclipse.jgit.lib; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.PersonIdent; - /** * Parsed reflog entry * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index aba5242b14..c5b2ef8e5b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -52,6 +52,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.net.URISyntaxException; import java.text.MessageFormat; import java.util.Collection; @@ -80,6 +81,7 @@ import org.eclipse.jgit.events.IndexChangedListener; import org.eclipse.jgit.events.ListenerList; import org.eclipse.jgit.events.RepositoryEvent; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.GC; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; @@ -93,7 +95,6 @@ import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.RawParseUtils; import org.eclipse.jgit.util.SystemReader; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,8 +107,7 @@ import org.slf4j.LoggerFactory; * This class is thread-safe. */ public abstract class Repository implements AutoCloseable { - private static Logger LOG = LoggerFactory.getLogger(Repository.class); - + private static final Logger LOG = LoggerFactory.getLogger(Repository.class); private static final ListenerList globalListeners = new ListenerList(); /** @return the global listener list observing all events in this JVM. */ @@ -245,7 +245,6 @@ public abstract class Repository implements AutoCloseable { @NonNull public abstract AttributesNodeProvider createAttributesNodeProvider(); - /** * @return the used file system abstraction, or or {@code null} if * repository isn't local. @@ -653,7 +652,10 @@ public abstract class Repository implements AutoCloseable { // detached name = Constants.HEAD; if (!Repository.isValidRefName("x/" + name)) //$NON-NLS-1$ - throw new RevisionSyntaxException(revstr); + throw new RevisionSyntaxException(MessageFormat + .format(JGitText.get().invalidRefName, + name), + revstr); Ref ref = getRef(name); name = null; if (ref == null) @@ -703,7 +705,10 @@ public abstract class Repository implements AutoCloseable { if (name.equals("")) //$NON-NLS-1$ name = Constants.HEAD; if (!Repository.isValidRefName("x/" + name)) //$NON-NLS-1$ - throw new RevisionSyntaxException(revstr); + throw new RevisionSyntaxException(MessageFormat + .format(JGitText.get().invalidRefName, + name), + revstr); Ref ref = getRef(name); name = null; if (ref == null) @@ -752,7 +757,9 @@ public abstract class Repository implements AutoCloseable { return null; name = revstr.substring(done); if (!Repository.isValidRefName("x/" + name)) //$NON-NLS-1$ - throw new RevisionSyntaxException(revstr); + throw new RevisionSyntaxException( + MessageFormat.format(JGitText.get().invalidRefName, name), + revstr); if (getRef(name) != null) return name; return resolveSimple(name); @@ -869,6 +876,7 @@ public abstract class Repository implements AutoCloseable { } /** Decrement the use count, and maybe close resources. */ + @Override public void close() { int newCount = useCnt.decrementAndGet(); if (newCount == 0) { @@ -902,8 +910,9 @@ public abstract class Repository implements AutoCloseable { getRefDatabase().close(); } - @NonNull @SuppressWarnings("nls") + @Override + @NonNull public String toString() { String desc; File directory = getDirectory(); @@ -1175,7 +1184,7 @@ public abstract class Repository implements AutoCloseable { // we want DirCache to inform us so that we can inform registered // listeners about index changes IndexChangedListener l = new IndexChangedListener() { - + @Override public void onIndexChanged(IndexChangedEvent event) { notifyIndexChanged(); } @@ -1183,15 +1192,6 @@ public abstract class Repository implements AutoCloseable { return DirCache.lock(this, l); } - static byte[] gitInternalSlash(byte[] bytes) { - if (File.separatorChar == '/') - return bytes; - for (int i=0; i<bytes.length; ++i) - if (bytes[i] == File.separatorChar) - bytes[i] = '/'; - return bytes; - } - /** * @return an important state */ @@ -1445,6 +1445,33 @@ public abstract class Repository implements AutoCloseable { } /** + * Read the {@code GIT_DIR/description} file for gitweb. + * + * @return description text; null if no description has been configured. + * @throws IOException + * description cannot be accessed. + * @since 4.6 + */ + @Nullable + public String getGitwebDescription() throws IOException { + return null; + } + + /** + * Set the {@code GIT_DIR/description} file for gitweb. + * + * @param description + * new description; null to clear the description. + * @throws IOException + * description cannot be persisted. + * @since 4.6 + */ + public void setGitwebDescription(@Nullable String description) + throws IOException { + throw new IOException(JGitText.get().unsupportedRepositoryDescription); + } + + /** * @param refName * @return a {@link ReflogReader} for the supplied refname, or {@code null} * if the named ref does not exist. @@ -1781,15 +1808,12 @@ public abstract class Repository implements AutoCloseable { throws FileNotFoundException, IOException { File headsFile = new File(getDirectory(), filename); if (heads != null) { - BufferedOutputStream bos = new SafeBufferedOutputStream( - new FileOutputStream(headsFile)); - try { + try (OutputStream bos = new BufferedOutputStream( + new FileOutputStream(headsFile))) { for (ObjectId id : heads) { id.copyTo(bos); bos.write('\n'); } - } finally { - bos.close(); } } else { FileUtils.delete(headsFile, FileUtils.SKIP_MISSING); @@ -1845,4 +1869,22 @@ public abstract class Repository implements AutoCloseable { return getConfig() .getSubsections(ConfigConstants.CONFIG_REMOTE_SECTION); } + + /** + * Check whether any housekeeping is required; if yes, run garbage + * collection; if not, exit without performing any work. Some JGit commands + * run autoGC after performing operations that could create many loose + * objects. + * <p/> + * Currently this option is supported for repositories of type + * {@code FileRepository} only. See {@link GC#setAuto(boolean)} for + * configuration details. + * + * @param monitor + * to report progress + * @since 4.6 + */ + public void autoGC(ProgressMonitor monitor) { + // default does nothing + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java index 7a8d246df2..2f1a9e1cda 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCache.java @@ -45,12 +45,8 @@ package org.eclipse.jgit.lib; import java.io.File; import java.io.IOException; -import java.lang.ref.Reference; -import java.lang.ref.SoftReference; import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -150,7 +146,7 @@ public class RepositoryCache { public static void close(@NonNull final Repository db) { if (db.getDirectory() != null) { FileKey key = FileKey.exact(db.getDirectory(), db.getFS()); - cache.unregisterAndCloseRepository(key, db); + cache.unregisterAndCloseRepository(key); } } @@ -202,8 +198,7 @@ public class RepositoryCache { return false; } FileKey key = new FileKey(gitDir, repo.getFS()); - Reference<Repository> repoRef = cache.cacheMap.get(key); - return repoRef != null && repoRef.get() == repo; + return cache.cacheMap.get(key) == repo; } /** Unregister all repositories from the cache. */ @@ -219,7 +214,7 @@ public class RepositoryCache { cache.configureEviction(repositoryCacheConfig); } - private final ConcurrentHashMap<Key, Reference<Repository>> cacheMap; + private final ConcurrentHashMap<Key, Repository> cacheMap; private final Lock[] openLocks; @@ -228,7 +223,7 @@ public class RepositoryCache { private volatile long expireAfter; private RepositoryCache() { - cacheMap = new ConcurrentHashMap<Key, Reference<Repository>>(); + cacheMap = new ConcurrentHashMap<>(); openLocks = new Lock[4]; for (int i = 0; i < openLocks.length; i++) { openLocks[i] = new Lock(); @@ -261,19 +256,15 @@ public class RepositoryCache { } } - @SuppressWarnings("resource") private Repository openRepository(final Key location, final boolean mustExist) throws IOException { - Reference<Repository> ref = cacheMap.get(location); - Repository db = ref != null ? ref.get() : null; + Repository db = cacheMap.get(location); if (db == null) { synchronized (lockFor(location)) { - ref = cacheMap.get(location); - db = ref != null ? ref.get() : null; + db = cacheMap.get(location); if (db == null) { db = location.open(mustExist); - ref = new SoftReference<Repository>(db); - cacheMap.put(location, ref); + cacheMap.put(location, db); } else { db.incrementOpen(); } @@ -285,16 +276,13 @@ public class RepositoryCache { } private void registerRepository(final Key location, final Repository db) { - SoftReference<Repository> newRef = new SoftReference<Repository>(db); - Reference<Repository> oldRef = cacheMap.put(location, newRef); - Repository oldDb = oldRef != null ? oldRef.get() : null; + Repository oldDb = cacheMap.put(location, db); if (oldDb != null) oldDb.close(); } private Repository unregisterRepository(final Key location) { - Reference<Repository> oldRef = cacheMap.remove(location); - return oldRef != null ? oldRef.get() : null; + return cacheMap.remove(location); } private boolean isExpired(Repository db) { @@ -302,8 +290,7 @@ public class RepositoryCache { && (System.currentTimeMillis() - db.closedAt.get() > expireAfter); } - private void unregisterAndCloseRepository(final Key location, - Repository db) { + private void unregisterAndCloseRepository(final Key location) { synchronized (lockFor(location)) { Repository oldDb = unregisterRepository(location); if (oldDb != null) { @@ -317,8 +304,7 @@ public class RepositoryCache { } private void clearAllExpired() { - for (Reference<Repository> ref : cacheMap.values()) { - Repository db = ref.get(); + for (Repository db : cacheMap.values()) { if (isExpired(db)) { RepositoryCache.close(db); } @@ -326,9 +312,8 @@ public class RepositoryCache { } private void clearAll() { - for (Iterator<Map.Entry<Key, Reference<Repository>>> i = cacheMap - .entrySet().iterator(); i.hasNext();) { - unregisterAndCloseRepository(i.next().getKey(), null); + for (Key k : cacheMap.keySet()) { + unregisterAndCloseRepository(k); } } @@ -442,6 +427,7 @@ public class RepositoryCache { return path; } + @Override public Repository open(final boolean mustExist) throws IOException { if (mustExist && !isGitRepository(path, fs)) throw new RepositoryNotFoundException(path); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCacheConfig.java index 428dea3e67..28cdaae443 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RepositoryCacheConfig.java @@ -53,8 +53,8 @@ public class RepositoryCacheConfig { /** * Set cleanupDelayMillis to this value in order to switch off time-based - * cache eviction. The JVM can still expire cache entries when heap memory - * runs low. + * cache eviction. Expired cache entries will only be evicted when + * RepositoryCache.clearExpired or RepositoryCache.clear are called. */ public static final long NO_CLEANUP = 0; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TreeFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TreeFormatter.java index 065b8f46b6..777ce94aa0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TreeFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TreeFormatter.java @@ -53,6 +53,7 @@ import static org.eclipse.jgit.lib.FileMode.TREE; import java.io.IOException; import org.eclipse.jgit.errors.CorruptObjectException; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevTree; @@ -193,6 +194,34 @@ public class TreeFormatter { */ public void append(byte[] nameBuf, int namePos, int nameLen, FileMode mode, AnyObjectId id) { + append(nameBuf, namePos, nameLen, mode, id, false); + } + + /** + * Append any entry to the tree. + * + * @param nameBuf + * buffer holding the name of the entry. The name should be UTF-8 + * encoded, but file name encoding is not a well defined concept + * in Git. + * @param namePos + * first position within {@code nameBuf} of the name data. + * @param nameLen + * number of bytes from {@code nameBuf} to use as the name. + * @param mode + * mode describing the treatment of {@code id}. + * @param id + * the ObjectId to store in this entry. + * @param allowEmptyName + * allow an empty filename (creating a corrupt tree) + * @since 4.6 + */ + public void append(byte[] nameBuf, int namePos, int nameLen, FileMode mode, + AnyObjectId id, boolean allowEmptyName) { + if (nameLen == 0 && !allowEmptyName) { + throw new IllegalArgumentException( + JGitText.get().invalidTreeZeroLengthName); + } if (fmtBuf(nameBuf, namePos, nameLen, mode)) { id.copyRawTo(buf, ptr); ptr += OBJECT_ID_LENGTH; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeConfig.java index 9125ddfd68..83b143b90d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeConfig.java @@ -46,8 +46,8 @@ import java.io.IOException; import org.eclipse.jgit.api.MergeCommand.FastForwardMode; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Config.SectionParser; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Repository; /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java index 383c1f8fef..40ea77e8ec 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java @@ -44,9 +44,9 @@ package org.eclipse.jgit.patch; import static org.eclipse.jgit.lib.Constants.encodeASCII; -import static org.eclipse.jgit.patch.FileHeader.isHunkHdr; import static org.eclipse.jgit.patch.FileHeader.NEW_NAME; import static org.eclipse.jgit.patch.FileHeader.OLD_NAME; +import static org.eclipse.jgit.patch.FileHeader.isHunkHdr; import static org.eclipse.jgit.util.RawParseUtils.match; import static org.eclipse.jgit.util.RawParseUtils.nextLF; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotCommit.java index 2e8aab885a..98bcd1accf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotCommit.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotCommit.java @@ -44,8 +44,8 @@ package org.eclipse.jgit.revplot; import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; /** * A commit reference to a commit in the DAG. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/MergeBaseGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/MergeBaseGenerator.java index f1d7dc8361..3609d46e30 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/MergeBaseGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/MergeBaseGenerator.java @@ -45,6 +45,7 @@ package org.eclipse.jgit.revwalk; import java.io.IOException; import java.text.MessageFormat; +import java.util.LinkedList; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.MissingObjectException; @@ -85,12 +86,15 @@ class MergeBaseGenerator extends Generator { private int recarryMask; + private int mergeBaseAncestor = -1; + private LinkedList<RevCommit> ret = new LinkedList<RevCommit>(); + MergeBaseGenerator(final RevWalk w) { walker = w; pending = new DateRevQueue(); } - void init(final AbstractRevQueue p) { + void init(final AbstractRevQueue p) throws IOException { try { for (;;) { final RevCommit c = p.next(); @@ -98,17 +102,25 @@ class MergeBaseGenerator extends Generator { break; add(c); } - } finally { - // Always free the flags immediately. This ensures the flags - // will be available for reuse when the walk resets. - // - walker.freeFlag(branchMask); - // Setup the condition used by carryOntoOne to detect a late // merge base and produce it on the next round. // recarryTest = branchMask | POPPED; recarryMask = branchMask | POPPED | MERGE_BASE; + mergeBaseAncestor = walker.allocFlag(); + + for (;;) { + RevCommit c = _next(); + if (c == null) { + break; + } + ret.add(c); + } + } finally { + // Always free the flags immediately. This ensures the flags + // will be available for reuse when the walk resets. + // + walker.freeFlag(branchMask | mergeBaseAncestor); } } @@ -131,8 +143,7 @@ class MergeBaseGenerator extends Generator { return 0; } - @Override - RevCommit next() throws MissingObjectException, + private RevCommit _next() throws MissingObjectException, IncorrectObjectTypeException, IOException { for (;;) { final RevCommit c = pending.next(); @@ -156,7 +167,7 @@ class MergeBaseGenerator extends Generator { // also flagged as being popped, so that they do not // generate to the caller. // - carry |= MERGE_BASE; + carry |= MERGE_BASE | mergeBaseAncestor; } carryOntoHistory(c, carry); @@ -179,6 +190,18 @@ class MergeBaseGenerator extends Generator { } } + @Override + RevCommit next() throws MissingObjectException, + IncorrectObjectTypeException, IOException { + while (!ret.isEmpty()) { + RevCommit commit = ret.remove(); + if ((commit.flags & mergeBaseAncestor) == 0) { + return commit; + } + } + return null; + } + private void carryOntoHistory(RevCommit c, final int carry) { for (;;) { final RevCommit[] pList = c.parents; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java index 9a38846442..2d02ad5a9b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/FileBasedConfig.java @@ -55,8 +55,8 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.text.MessageFormat; -import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.FileSnapshot; import org.eclipse.jgit.internal.storage.file.LockFile; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java index 4069a64535..1aebaddacd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java @@ -529,21 +529,29 @@ public class AmazonS3 { Integer.valueOf(HttpSupport.response(c)), c.getResponseMessage())); final InputStream errorStream = c.getErrorStream(); - if (errorStream == null) + if (errorStream == null) { return err; + } - final ByteArrayOutputStream b = new ByteArrayOutputStream(); - byte[] buf = new byte[2048]; - for (;;) { - final int n = errorStream.read(buf); - if (n < 0) - break; - if (n > 0) - b.write(buf, 0, n); + try { + final ByteArrayOutputStream b = new ByteArrayOutputStream(); + byte[] buf = new byte[2048]; + for (;;) { + final int n = errorStream.read(buf); + if (n < 0) { + break; + } + if (n > 0) { + b.write(buf, 0, n); + } + } + buf = b.toByteArray(); + if (buf.length > 0) { + err.initCause(new IOException("\n" + new String(buf))); //$NON-NLS-1$ + } + } finally { + errorStream.close(); } - buf = b.toByteArray(); - if (buf.length > 0) - err.initCause(new IOException("\n" + new String(buf))); //$NON-NLS-1$ return err; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index 754cf361a9..0dd907f97e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -71,7 +71,6 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevCommitList; import org.eclipse.jgit.revwalk.RevFlag; @@ -80,6 +79,7 @@ import org.eclipse.jgit.revwalk.RevSort; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter; import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck; import org.eclipse.jgit.transport.PacketLineIn.AckNackResult; import org.eclipse.jgit.util.TemporaryBuffer; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BaseReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BaseReceivePack.java index 0724eac701..4d0803a339 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BaseReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BaseReceivePack.java @@ -69,6 +69,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.PackProtocolException; @@ -268,6 +269,7 @@ public abstract class BaseReceivePack { private PushCertificateParser pushCertificateParser; private SignedPushConfig signedPushConfig; private PushCertificate pushCert; + private ReceivedPackStatistics stats; /** * Get the push certificate used to verify the pusher's identity. @@ -1115,6 +1117,18 @@ public abstract class BaseReceivePack { } /** + * Returns the statistics on the received pack if available. This should be + * called after {@link #receivePack} is called. + * + * @return ReceivedPackStatistics + * @since 4.6 + */ + @Nullable + public ReceivedPackStatistics getReceivedPackStatistics() { + return stats; + } + + /** * Receive a list of commands from the input. * * @throws IOException @@ -1307,6 +1321,7 @@ public abstract class BaseReceivePack { parser.setMaxObjectSizeLimit(maxObjectSizeLimit); packLock = parser.parse(receiving, resolving); packSize = Long.valueOf(parser.getPackSize()); + stats = parser.getReceivedPackStatistics(); ins.flush(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonClient.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonClient.java index 22f343899d..23e3379f91 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonClient.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonClient.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.transport; import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -52,7 +53,6 @@ import java.net.Socket; import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException; import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; /** Active network client of {@link Daemon}. */ public class DaemonClient { @@ -95,7 +95,7 @@ public class DaemonClient { void execute(final Socket sock) throws IOException, ServiceNotEnabledException, ServiceNotAuthorizedException { rawIn = new BufferedInputStream(sock.getInputStream()); - rawOut = new SafeBufferedOutputStream(sock.getOutputStream()); + rawOut = new BufferedOutputStream(sock.getOutputStream()); if (0 < daemon.getTimeout()) sock.setSoTimeout(daemon.getTimeout() * 1000); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonService.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonService.java index 4e7e12399c..ec6f24273d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonService.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/DaemonService.java @@ -47,8 +47,8 @@ package org.eclipse.jgit.transport; import java.io.IOException; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.Config.SectionParser; +import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException; import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java index 622680a27f..319ae1edc7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HMACSHA1NonceGenerator.java @@ -52,7 +52,6 @@ import javax.crypto.spec.SecretKeySpec; import org.eclipse.jgit.internal.storage.dfs.DfsRepository; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.NonceGenerator; import org.eclipse.jgit.transport.PushCertificate.NonceStatus; /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/InsecureCipherFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/InsecureCipherFactory.java new file mode 100644 index 0000000000..73384a1162 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/InsecureCipherFactory.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.transport; + +import java.security.NoSuchAlgorithmException; + +import javax.crypto.Cipher; +import javax.crypto.NoSuchPaddingException; + +/** + * <b>DO NOT USE</b> Factory to create any cipher. + * <p> + * This is a hack for {@link WalkEncryption} to create any cipher configured by + * the end-user. Using this class allows JGit to violate ErrorProne's security + * recommendations (<a + * href="http://errorprone.info/bugpattern/InsecureCryptoUsage" + * >InsecureCryptoUsage</a>), which is not secure. + */ +class InsecureCipherFactory { + static Cipher create(String algo) + throws NoSuchAlgorithmException, NoSuchPaddingException { + return Cipher.getInstance(algo); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschSession.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschSession.java index 1dfe5d9797..fa27bfce5f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschSession.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/JschSession.java @@ -48,15 +48,14 @@ package org.eclipse.jgit.transport; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.util.io.StreamCopyThread; +import org.eclipse.jgit.util.io.IsolatedOutputStream; import com.jcraft.jsch.Channel; import com.jcraft.jsch.ChannelExec; @@ -178,33 +177,12 @@ public class JschSession implements RemoteSession { // that we spawn a background thread to shuttle data through a pipe, // as we can issue an interrupted write out of that. Its slower, so // we only use this route if there is a timeout. - final OutputStream out = channel.getOutputStream(); + OutputStream out = channel.getOutputStream(); if (timeout <= 0) { outputStream = out; } else { - final PipedInputStream pipeIn = new PipedInputStream(); - final StreamCopyThread copier = new StreamCopyThread(pipeIn, - out); - final PipedOutputStream pipeOut = new PipedOutputStream(pipeIn) { - @Override - public void flush() throws IOException { - super.flush(); - copier.flush(); - } - - @Override - public void close() throws IOException { - super.close(); - try { - copier.join(timeout * 1000); - } catch (InterruptedException e) { - // Just wake early, the thread will terminate - // anyway. - } - } - }; - copier.start(); - outputStream = pipeOut; + IsolatedOutputStream i = new IsolatedOutputStream(out); + outputStream = new BufferedOutputStream(i, 16 * 1024); } errStream = channel.getErrStream(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java index b96fe885e1..4bbe3f8813 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java @@ -186,6 +186,9 @@ public abstract class PackParser { /** Git object size limit */ private long maxObjectSizeLimit; + private final ReceivedPackStatistics.Builder stats = + new ReceivedPackStatistics.Builder(); + /** * Initialize a pack parser. * @@ -455,8 +458,8 @@ public abstract class PackParser { } /** - * Get the size of the parsed pack. - * + * Get the size of the newly created pack. + * <p> * This will also include the pack index size if an index was created. This * method should only be called after pack parsing is finished. * @@ -469,6 +472,18 @@ public abstract class PackParser { } /** + * Returns the statistics of the parsed pack. + * <p> + * This should only be called after pack parsing is finished. + * + * @return {@link ReceivedPackStatistics} + * @since 4.6 + */ + public ReceivedPackStatistics getReceivedPackStatistics() { + return stats.build(); + } + + /** * Parse the pack stream. * * @param progress @@ -626,6 +641,7 @@ public abstract class PackParser { private void resolveDeltas(DeltaVisit visit, final int type, ObjectTypeAndSize info, ProgressMonitor progress) throws IOException { + stats.addDeltaObject(type); do { progress.update(1); info = openDatabase(visit.delta, info); @@ -919,6 +935,7 @@ public abstract class PackParser { // Cleanup all resources associated with our input parsing. private void endInput() { + stats.setNumBytesRead(streamPosition()); in = null; } @@ -947,12 +964,14 @@ public abstract class PackParser { case Constants.OBJ_TREE: case Constants.OBJ_BLOB: case Constants.OBJ_TAG: + stats.addWholeObject(typeCode); onBeginWholeObject(streamPosition, typeCode, sz); onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr); whole(streamPosition, typeCode, sz); break; case Constants.OBJ_OFS_DELTA: { + stats.addOffsetDelta(); c = readFrom(Source.INPUT); hdrBuf[hdrPtr++] = (byte) c; long ofs = c & 127; @@ -975,6 +994,7 @@ public abstract class PackParser { } case Constants.OBJ_REF_DELTA: { + stats.addRefDelta(); c = fill(Source.INPUT, 20); final ObjectId base = ObjectId.fromRaw(buf, c); System.arraycopy(buf, c, hdrBuf, hdrPtr, 20); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java index 871a6f752b..4c8acd2cc2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateIdent.java @@ -44,7 +44,6 @@ package org.eclipse.jgit.transport; import static java.nio.charset.StandardCharsets.UTF_8; - import static org.eclipse.jgit.util.RawParseUtils.lastIndexOfTrim; import java.text.SimpleDateFormat; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index cc20d50a7f..393e25a2a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java @@ -56,7 +56,9 @@ import java.util.List; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.UnpackException; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.ReceiveCommand.Result; import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser; @@ -307,9 +309,19 @@ public class ReceivePack extends BaseReceivePack { throw new UnpackException(unpackError); } postReceive.onPostReceive(this, filterCommands(Result.OK)); + autoGc(); } } + private void autoGc() { + Repository repo = getRepository(); + if (!repo.getConfig().getBoolean(ConfigConstants.CONFIG_RECEIVE_SECTION, + ConfigConstants.CONFIG_KEY_AUTOGC, true)) { + return; + } + repo.autoGC(NullProgressMonitor.INSTANCE); + } + @Override protected String getLockMessageProcessName() { return "jgit receive-pack"; //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivedPackStatistics.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivedPackStatistics.java new file mode 100644 index 0000000000..052d5506f8 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivedPackStatistics.java @@ -0,0 +1,231 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.transport; + +import org.eclipse.jgit.lib.Constants; + +/** + * Statistics about {@link PackParser}. + * + * @since 4.6 + */ +public class ReceivedPackStatistics { + private long numBytesRead; + + private long numWholeCommit; + private long numWholeTree; + private long numWholeBlob; + private long numWholeTag; + private long numOfsDelta; + private long numRefDelta; + + private long numDeltaCommit; + private long numDeltaTree; + private long numDeltaBlob; + private long numDeltaTag; + + /** @return number of bytes read from the input stream */ + public long getNumBytesRead() { + return numBytesRead; + } + + /** @return number of whole commit objects in the pack */ + public long getNumWholeCommit() { + return numWholeCommit; + } + + /** @return number of whole tree objects in the pack */ + public long getNumWholeTree() { + return numWholeTree; + } + + /** @return number of whole blob objects in the pack */ + public long getNumWholeBlob() { + return numWholeBlob; + } + + /** @return number of whole tag objects in the pack */ + public long getNumWholeTag() { + return numWholeTag; + } + + /** @return number of offset delta objects in the pack */ + public long getNumOfsDelta() { + return numOfsDelta; + } + + /** @return number of ref delta objects in the pack */ + public long getNumRefDelta() { + return numRefDelta; + } + + /** @return number of delta commit objects in the pack */ + public long getNumDeltaCommit() { + return numDeltaCommit; + } + + /** @return number of delta tree objects in the pack */ + public long getNumDeltaTree() { + return numDeltaTree; + } + + /** @return number of delta blob objects in the pack */ + public long getNumDeltaBlob() { + return numDeltaBlob; + } + + /** @return number of delta tag objects in the pack */ + public long getNumDeltaTag() { + return numDeltaTag; + } + + /** A builder for {@link ReceivedPackStatistics}. */ + public static class Builder { + private long numBytesRead; + + private long numWholeCommit; + private long numWholeTree; + private long numWholeBlob; + private long numWholeTag; + private long numOfsDelta; + private long numRefDelta; + + private long numDeltaCommit; + private long numDeltaTree; + private long numDeltaBlob; + private long numDeltaTag; + + /** + * @param numBytesRead number of bytes read from the input stream + * @return this + */ + public Builder setNumBytesRead(long numBytesRead) { + this.numBytesRead = numBytesRead; + return this; + } + + /** + * Increment a whole object count. + * + * @param type OBJ_COMMIT, OBJ_TREE, OBJ_BLOB, or OBJ_TAG + * @return this + */ + public Builder addWholeObject(int type) { + switch (type) { + case Constants.OBJ_COMMIT: + numWholeCommit++; + break; + case Constants.OBJ_TREE: + numWholeTree++; + break; + case Constants.OBJ_BLOB: + numWholeBlob++; + break; + case Constants.OBJ_TAG: + numWholeTag++; + break; + default: + throw new IllegalArgumentException( + type + " cannot be a whole object"); //$NON-NLS-1$ + } + return this; + } + + /** @return this */ + public Builder addOffsetDelta() { + numOfsDelta++; + return this; + } + + /** @return this */ + public Builder addRefDelta() { + numRefDelta++; + return this; + } + + /** + * Increment a delta object count. + * + * @param type OBJ_COMMIT, OBJ_TREE, OBJ_BLOB, or OBJ_TAG + * @return this + */ + public Builder addDeltaObject(int type) { + switch (type) { + case Constants.OBJ_COMMIT: + numDeltaCommit++; + break; + case Constants.OBJ_TREE: + numDeltaTree++; + break; + case Constants.OBJ_BLOB: + numDeltaBlob++; + break; + case Constants.OBJ_TAG: + numDeltaTag++; + break; + default: + throw new IllegalArgumentException( + "delta should be a delta to a whole object. " + //$NON-NLS-1$ + type + " cannot be a whole object"); //$NON-NLS-1$ + } + return this; + } + + ReceivedPackStatistics build() { + ReceivedPackStatistics s = new ReceivedPackStatistics(); + s.numBytesRead = numBytesRead; + s.numWholeCommit = numWholeCommit; + s.numWholeTree = numWholeTree; + s.numWholeBlob = numWholeBlob; + s.numWholeTag = numWholeTag; + s.numOfsDelta = numOfsDelta; + s.numRefDelta = numRefDelta; + s.numDeltaCommit = numDeltaCommit; + s.numDeltaTree = numDeltaTree; + s.numDeltaBlob = numDeltaBlob; + s.numDeltaTag = numDeltaTag; + return s; + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ServiceMayNotContinueException.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ServiceMayNotContinueException.java index ce5ccaa65d..81c5da3c78 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ServiceMayNotContinueException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ServiceMayNotContinueException.java @@ -44,6 +44,7 @@ package org.eclipse.jgit.transport; import java.io.IOException; + import javax.servlet.http.HttpServletResponse; import org.eclipse.jgit.internal.JGitText; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index bc4843a8af..df860695df 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -1199,6 +1199,9 @@ public abstract class Transport implements AutoCloseable { final FetchResult result = new FetchResult(); new FetchProcess(this, toFetch).execute(monitor, result); + + local.autoGC(monitor); + return result; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportBundleFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportBundleFile.java index bbc0d0aa73..9b0834133b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportBundleFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportBundleFile.java @@ -94,7 +94,7 @@ class TransportBundleFile extends Transport implements TransportBundle { public Transport open(URIish uri, Repository local, String remoteName) throws NotSupportedException, TransportException { if ("bundle".equals(uri.getScheme())) { //$NON-NLS-1$ - File path = local.getFS().resolve(new File("."), uri.getPath()); //$NON-NLS-1$ + File path = FS.DETECTED.resolve(new File("."), uri.getPath()); //$NON-NLS-1$ return new TransportBundleFile(local, uri, path); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java index a7f42fd873..c6e4c50801 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitAnon.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.transport; import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -62,7 +63,6 @@ import org.eclipse.jgit.errors.NotSupportedException; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; /** * Transport through a git-daemon waiting for anonymous TCP connections. @@ -182,7 +182,7 @@ class TransportGitAnon extends TcpTransport implements PackTransport { OutputStream sOut = sock.getOutputStream(); sIn = new BufferedInputStream(sIn); - sOut = new SafeBufferedOutputStream(sOut); + sOut = new BufferedOutputStream(sOut); init(sIn, sOut); service("git-upload-pack", pckOut); //$NON-NLS-1$ @@ -221,7 +221,7 @@ class TransportGitAnon extends TcpTransport implements PackTransport { OutputStream sOut = sock.getOutputStream(); sIn = new BufferedInputStream(sIn); - sOut = new SafeBufferedOutputStream(sOut); + sOut = new BufferedOutputStream(sOut); init(sIn, sOut); service("git-receive-pack", pckOut); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java index 52f0f04562..da98e8c9ea 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportGitSsh.java @@ -64,11 +64,11 @@ import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.QuotedString; import org.eclipse.jgit.util.SystemReader; import org.eclipse.jgit.util.io.MessageWriter; import org.eclipse.jgit.util.io.StreamCopyThread; -import org.eclipse.jgit.util.FS; /** * Transport through an SSH tunnel. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index 1166080f2a..96a6fe1d01 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -45,7 +45,9 @@ package org.eclipse.jgit.transport; +import static org.eclipse.jgit.lib.Constants.HEAD; import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; +import static org.eclipse.jgit.util.HttpSupport.ENCODING_X_GZIP; import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT; import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_ENCODING; @@ -128,6 +130,25 @@ public class TransportHttp extends HttpTransport implements WalkTransport, private static final String SVC_RECEIVE_PACK = "git-receive-pack"; //$NON-NLS-1$ + /** + * Accept-Encoding header in the HTTP request + * (https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html). + * + * @since 4.6 + */ + public enum AcceptEncoding { + /** + * Do not specify an Accept-Encoding header. In most servers this + * results in the content being transmitted as-is. + */ + UNSPECIFIED, + + /** + * Accept gzip content encoding. + */ + GZIP + } + static final TransportProtocol PROTO_HTTP = new TransportProtocol() { private final String[] schemeNames = { "http", "https" }; //$NON-NLS-1$ //$NON-NLS-2$ @@ -324,12 +345,15 @@ public class TransportHttp extends HttpTransport implements WalkTransport, br.close(); } - if (!refs.containsKey(Constants.HEAD)) { + if (!refs.containsKey(HEAD)) { // If HEAD was not published in the info/refs file (it usually // is not there) download HEAD by itself as a loose file and do // the resolution by hand. // - HttpConnection conn = httpOpen(new URL(baseUrl, Constants.HEAD)); + HttpConnection conn = httpOpen( + METHOD_GET, + new URL(baseUrl, HEAD), + AcceptEncoding.GZIP); int status = HttpSupport.response(conn); switch (status) { case HttpConnection.HTTP_OK: { @@ -341,11 +365,11 @@ public class TransportHttp extends HttpTransport implements WalkTransport, Ref r = refs.get(target); if (r == null) r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null); - r = new SymbolicRef(Constants.HEAD, r); + r = new SymbolicRef(HEAD, r); refs.put(r.getName(), r); } else if (line != null && ObjectId.isId(line)) { Ref r = new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, - Constants.HEAD, ObjectId.fromString(line)); + HEAD, ObjectId.fromString(line)); refs.put(r.getName(), r); } } finally { @@ -455,7 +479,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, Collection<Type> ignoreTypes = null; for (;;) { try { - final HttpConnection conn = httpOpen(u); + final HttpConnection conn = httpOpen(METHOD_GET, u, AcceptEncoding.GZIP); if (useSmartHttp) { String exp = "application/x-" + service + "-advertisement"; //$NON-NLS-1$ //$NON-NLS-2$ conn.setRequestProperty(HDR_ACCEPT, exp + ", */*"); //$NON-NLS-1$ @@ -529,21 +553,37 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } } - final HttpConnection httpOpen(URL u) throws IOException { - return httpOpen(METHOD_GET, u); + /** + * Open an HTTP connection, setting the accept-encoding request header to gzip. + * + * @param method HTTP request method + * @param u url of the HTTP connection + * @return the HTTP connection + * @throws IOException + * @since 3.3 + * @deprecated use {@link #httpOpen(String, URL, AcceptEncoding)} instead. + */ + @Deprecated + protected HttpConnection httpOpen(String method, URL u) throws IOException { + return httpOpen(method, u, AcceptEncoding.GZIP); } /** * Open an HTTP connection. * - * @param method - * @param u - * @return the connection + * @param method HTTP request method + * @param u url of the HTTP connection + * @param acceptEncoding accept-encoding header option + * @return the HTTP connection * @throws IOException - * @since 3.3 + * @since 4.6 */ - protected HttpConnection httpOpen(String method, URL u) - throws IOException { + protected HttpConnection httpOpen(String method, URL u, + AcceptEncoding acceptEncoding) throws IOException { + if (method == null || u == null || acceptEncoding == null) { + throw new NullPointerException(); + } + final Proxy proxy = HttpSupport.proxyFor(proxySelector, u); HttpConnection conn = connectionFactory.create(u, proxy); @@ -553,7 +593,9 @@ public class TransportHttp extends HttpTransport implements WalkTransport, conn.setRequestMethod(method); conn.setUseCaches(false); - conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP); + if (acceptEncoding == AcceptEncoding.GZIP) { + conn.setRequestProperty(HDR_ACCEPT_ENCODING, ENCODING_GZIP); + } conn.setRequestProperty(HDR_PRAGMA, "no-cache"); //$NON-NLS-1$ if (UserAgent.get() != null) { conn.setRequestProperty(HDR_USER_AGENT, UserAgent.get()); @@ -575,7 +617,7 @@ public class TransportHttp extends HttpTransport implements WalkTransport, final InputStream openInputStream(HttpConnection conn) throws IOException { InputStream input = conn.getInputStream(); - if (ENCODING_GZIP.equals(conn.getHeaderField(HDR_CONTENT_ENCODING))) + if (isGzipContent(conn)) input = new GZIPInputStream(input); return input; } @@ -591,6 +633,11 @@ public class TransportHttp extends HttpTransport implements WalkTransport, return expType.equals(actType); } + private boolean isGzipContent(final HttpConnection c) { + return ENCODING_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING)) + || ENCODING_X_GZIP.equals(c.getHeaderField(HDR_CONTENT_ENCODING)); + } + private void readSmartHeaders(final InputStream in, final String service) throws IOException { // A smart reply will have a '#' after the first 4 bytes, but @@ -655,6 +702,14 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } @Override + BufferedReader openReader(String path) throws IOException { + // Line oriented readable content is likely to compress well. + // Request gzip encoding. + InputStream is = open(path, AcceptEncoding.GZIP).in; + return new BufferedReader(new InputStreamReader(is, Constants.CHARSET)); + } + + @Override Collection<String> getPackNames() throws IOException { final Collection<String> packs = new ArrayList<String>(); try { @@ -679,14 +734,25 @@ public class TransportHttp extends HttpTransport implements WalkTransport, @Override FileStream open(final String path) throws IOException { + return open(path, AcceptEncoding.UNSPECIFIED); + } + + FileStream open(String path, AcceptEncoding acceptEncoding) + throws IOException { final URL base = httpObjectsUrl; final URL u = new URL(base, path); - final HttpConnection c = httpOpen(u); + final HttpConnection c = httpOpen(METHOD_GET, u, acceptEncoding); switch (HttpSupport.response(c)) { case HttpConnection.HTTP_OK: final InputStream in = openInputStream(c); - final int len = c.getContentLength(); - return new FileStream(in, len); + // If content is being gzipped and then transferred, the content + // length in the header is the zipped content length, not the + // actual content length. + if (!isGzipContent(c)) { + final int len = c.getContentLength(); + return new FileStream(in, len); + } + return new FileStream(in); case HttpConnection.HTTP_NOT_FOUND: throw new FileNotFoundException(u.toString()); default: @@ -832,7 +898,10 @@ public class TransportHttp extends HttpTransport implements WalkTransport, } void openStream() throws IOException { - conn = httpOpen(METHOD_POST, new URL(baseUrl, serviceName)); + conn = httpOpen( + METHOD_POST, + new URL(baseUrl, serviceName), + AcceptEncoding.GZIP); conn.setInstanceFollowRedirects(false); conn.setDoOutput(true); conn.setRequestProperty(HDR_CONTENT_TYPE, requestType); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java index 292ba7ed8a..1528c71426 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportLocal.java @@ -48,6 +48,7 @@ package org.eclipse.jgit.transport; import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -67,7 +68,6 @@ import org.eclipse.jgit.transport.resolver.ReceivePackFactory; import org.eclipse.jgit.transport.resolver.UploadPackFactory; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.io.MessageWriter; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; import org.eclipse.jgit.util.io.StreamCopyThread; /** @@ -258,7 +258,7 @@ class TransportLocal extends Transport implements PackTransport { OutputStream upOut = uploadPack.getOutputStream(); upIn = new BufferedInputStream(upIn); - upOut = new SafeBufferedOutputStream(upOut); + upOut = new BufferedOutputStream(upOut); init(upIn, upOut); readAdvertisedRefs(); @@ -311,7 +311,7 @@ class TransportLocal extends Transport implements PackTransport { OutputStream rpOut = receivePack.getOutputStream(); rpIn = new BufferedInputStream(rpIn); - rpOut = new SafeBufferedOutputStream(rpOut); + rpOut = new BufferedOutputStream(rpOut); init(rpIn, rpOut); readAdvertisedRefs(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 0b1848c311..b1cc5ded54 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -45,8 +45,8 @@ package org.eclipse.jgit.transport; import static org.eclipse.jgit.lib.RefDatabase.ALL; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT; -import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_INCLUDE_TAG; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK_DETAILED; @@ -312,6 +312,7 @@ public class UploadPack { private PackStatistics statistics; + @SuppressWarnings("deprecation") private UploadPackLogger logger = UploadPackLogger.NULL; /** @@ -1424,6 +1425,7 @@ public class UploadPack { } } + @SuppressWarnings("deprecation") private void sendPack(final boolean sideband) throws IOException { ProgressMonitor pm = NullProgressMonitor.INSTANCE; OutputStream packOut = rawOut; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java index 85ebecc450..0588634d2a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPackLogger.java @@ -56,7 +56,7 @@ import org.eclipse.jgit.internal.storage.pack.PackWriter; * @deprecated use {@link PostUploadHook} instead */ @Deprecated -public interface UploadPackLogger { +public interface UploadPackLogger { // TODO remove in JGit 5.0 /** A simple no-op logger. */ public static final UploadPackLogger NULL = new UploadPackLogger() { public void onPackStatistics(PackWriter.Statistics stats) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java index fe03bdc867..4c3fdd84f2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkEncryption.java @@ -143,35 +143,6 @@ abstract class WalkEncryption { } } - // PBEParameterSpec factory for Java (version <= 7). - // Does not support AlgorithmParameterSpec. - static PBEParameterSpec java7PBEParameterSpec(byte[] salt, - int iterationCount) { - return new PBEParameterSpec(salt, iterationCount); - } - - // PBEParameterSpec factory for Java (version >= 8). - // Adds support for AlgorithmParameterSpec. - static PBEParameterSpec java8PBEParameterSpec(byte[] salt, - int iterationCount, AlgorithmParameterSpec paramSpec) { - try { - @SuppressWarnings("boxing") - PBEParameterSpec instance = PBEParameterSpec.class - .getConstructor(byte[].class, int.class, - AlgorithmParameterSpec.class) - .newInstance(salt, iterationCount, paramSpec); - return instance; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - // Current runtime version. - // https://docs.oracle.com/javase/7/docs/technotes/guides/versioning/spec/versioning2.html - static double javaVersion() { - return Double.parseDouble(System.getProperty("java.specification.version")); //$NON-NLS-1$ - } - /** * JetS3t compatibility reference: <a href= * "https://bitbucket.org/jmurty/jets3t/src/156c00eb160598c2e9937fd6873f00d3190e28ca/src/org/jets3t/service/security/EncryptionUtil.java"> @@ -204,7 +175,7 @@ abstract class WalkEncryption { // Size 16, see com.sun.crypto.provider.AESConstants.AES_BLOCK_SIZE static final byte[] ZERO_AES_IV = new byte[16]; - private static final String cryptoVer = VERSION; + private static final String CRYPTO_VER = VERSION; private final String cryptoAlg; @@ -217,7 +188,7 @@ abstract class WalkEncryption { cryptoAlg = algo; // Verify if cipher is present. - Cipher cipher = Cipher.getInstance(cryptoAlg); + Cipher cipher = InsecureCipherFactory.create(cryptoAlg); // Standard names are not case-sensitive. // http://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html @@ -233,9 +204,7 @@ abstract class WalkEncryption { boolean useIV = cryptoName.contains("AES"); //$NON-NLS-1$ // PBEParameterSpec algorithm parameters are supported from Java 8. - boolean isJava8 = javaVersion() >= 1.8; - - if (useIV && isJava8) { + if (useIV) { // Support IV where possible: // * since JCE provider uses random IV for PBE/AES // * and there is no place to store dynamic IV in JetS3t V2 @@ -245,34 +214,33 @@ abstract class WalkEncryption { // https://bitbucket.org/jmurty/jets3t/raw/156c00eb160598c2e9937fd6873f00d3190e28ca/src/org/jets3t/service/security/EncryptionUtil.java // http://cr.openjdk.java.net/~mullan/webrevs/ascarpin/webrev.00/raw_files/new/src/share/classes/com/sun/crypto/provider/PBES2Core.java IvParameterSpec paramIV = new IvParameterSpec(ZERO_AES_IV); - paramSpec = java8PBEParameterSpec(SALT, ITERATIONS, paramIV); + paramSpec = new PBEParameterSpec(SALT, ITERATIONS, paramIV); } else { // Strict legacy JetS3t V2 compatibility, with no IV support. - paramSpec = java7PBEParameterSpec(SALT, ITERATIONS); + paramSpec = new PBEParameterSpec(SALT, ITERATIONS); } // Verify if cipher + key are allowed by policy. cipher.init(Cipher.ENCRYPT_MODE, secretKey, paramSpec); cipher.doFinal(); - } @Override void request(final HttpURLConnection u, final String prefix) { - u.setRequestProperty(prefix + JETS3T_CRYPTO_VER, cryptoVer); + u.setRequestProperty(prefix + JETS3T_CRYPTO_VER, CRYPTO_VER); u.setRequestProperty(prefix + JETS3T_CRYPTO_ALG, cryptoAlg); } @Override void validate(final HttpURLConnection u, final String prefix) throws IOException { - validateImpl(u, prefix, cryptoVer, cryptoAlg); + validateImpl(u, prefix, CRYPTO_VER, cryptoAlg); } @Override OutputStream encrypt(final OutputStream os) throws IOException { try { - final Cipher cipher = Cipher.getInstance(cryptoAlg); + final Cipher cipher = InsecureCipherFactory.create(cryptoAlg); cipher.init(Cipher.ENCRYPT_MODE, secretKey, paramSpec); return new CipherOutputStream(os, cipher); } catch (GeneralSecurityException e) { @@ -283,7 +251,7 @@ abstract class WalkEncryption { @Override InputStream decrypt(final InputStream in) throws IOException { try { - final Cipher cipher = Cipher.getInstance(cryptoAlg); + final Cipher cipher = InsecureCipherFactory.create(cryptoAlg); cipher.init(Cipher.DECRYPT_MODE, secretKey, paramSpec); return new CipherInputStream(in, cipher); } catch (GeneralSecurityException e) { @@ -374,7 +342,7 @@ abstract class WalkEncryption { String keySalt = props.getProperty(profile + X_KEY_SALT, DEFAULT_KEY_SALT); // Verify if cipher is present. - Cipher cipher = Cipher.getInstance(cipherAlgo); + Cipher cipher = InsecureCipherFactory.create(cipherAlgo); // Verify if key factory is present. SecretKeyFactory factory = SecretKeyFactory.getInstance(keyAlgo); @@ -432,7 +400,7 @@ abstract class WalkEncryption { @Override OutputStream encrypt(OutputStream output) throws IOException { try { - Cipher cipher = Cipher.getInstance(cipherAlgo); + Cipher cipher = InsecureCipherFactory.create(cipherAlgo); cipher.init(Cipher.ENCRYPT_MODE, secretKey); AlgorithmParameters params = cipher.getParameters(); if (params == null) { @@ -489,7 +457,7 @@ abstract class WalkEncryption { JGitText.get().unsupportedEncryptionVersion, vers)); } try { - decryptCipher = Cipher.getInstance(cipherAlgo); + decryptCipher = InsecureCipherFactory.create(cipherAlgo); if (cont.isEmpty()) { decryptCipher.init(Cipher.DECRYPT_MODE, secretKey); } else { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java index 17edfdc4fb..13d4a24b02 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java @@ -195,7 +195,7 @@ class WalkFetchConnection extends BaseFetchConnection { local = wt.local; objCheck = wt.getObjectChecker(); inserter = local.newObjectInserter(); - reader = local.newObjectReader(); + reader = inserter.newReader(); remotes = new ArrayList<WalkRemoteObjectDatabase>(); remotes.add(w); @@ -240,6 +240,12 @@ class WalkFetchConnection extends BaseFetchConnection { downloadObject(monitor, id); process(id); } + + try { + inserter.flush(); + } catch (IOException e) { + throw new TransportException(e.getMessage(), e); + } } public Collection<PackLock> getPackLocks() { @@ -652,7 +658,6 @@ class WalkFetchConnection extends BaseFetchConnection { Constants.typeString(type), Integer.valueOf(compressed.length))); } - inserter.flush(); } private Collection<WalkRemoteObjectDatabase> expandOneAlternate( @@ -876,14 +881,17 @@ class WalkFetchConnection extends BaseFetchConnection { void downloadPack(final ProgressMonitor monitor) throws IOException { String name = "pack/" + packName; //$NON-NLS-1$ WalkRemoteObjectDatabase.FileStream s = connection.open(name); - PackParser parser = inserter.newPackParser(s.in); - parser.setAllowThin(false); - parser.setObjectChecker(objCheck); - parser.setLockMessage(lockMessage); - PackLock lock = parser.parse(monitor); - if (lock != null) - packLocks.add(lock); - inserter.flush(); + try { + PackParser parser = inserter.newPackParser(s.in); + parser.setAllowThin(false); + parser.setObjectChecker(objCheck); + parser.setLockMessage(lockMessage); + PackLock lock = parser.parse(monitor); + if (lock != null) + packLocks.add(lock); + } finally { + s.in.close(); + } } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java index 4eaf3f9d8a..7b449c752a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java @@ -45,6 +45,7 @@ package org.eclipse.jgit.transport; import static org.eclipse.jgit.transport.WalkRemoteObjectDatabase.ROOT_DIR; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; @@ -69,7 +70,6 @@ import org.eclipse.jgit.lib.Ref.Storage; import org.eclipse.jgit.lib.RefWriter; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.RemoteRefUpdate.Status; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; /** * Generic push support for dumb transport protocols. @@ -262,21 +262,15 @@ class WalkPushConnection extends BaseConnection implements PushConnection { // Write the pack file, then the index, as readers look the // other direction (index, then pack file). // - final String wt = "Put " + base.substring(0, 12); //$NON-NLS-1$ - OutputStream os = dest.writeFile(pathPack, monitor, wt + "..pack"); //$NON-NLS-1$ - try { - os = new SafeBufferedOutputStream(os); + String wt = "Put " + base.substring(0, 12); //$NON-NLS-1$ + try (OutputStream os = new BufferedOutputStream( + dest.writeFile(pathPack, monitor, wt + "..pack"))) { //$NON-NLS-1$ writer.writePack(monitor, monitor, os); - } finally { - os.close(); } - os = dest.writeFile(pathIdx, monitor, wt + "..idx"); //$NON-NLS-1$ - try { - os = new SafeBufferedOutputStream(os); + try (OutputStream os = new BufferedOutputStream( + dest.writeFile(pathIdx, monitor, wt + "..idx"))) { //$NON-NLS-1$ writer.writeIndex(os); - } finally { - os.close(); } // Record the pack at the start of the pack info list. This diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java index e47913ea39..24f30ed206 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkRemoteObjectDatabase.java @@ -133,6 +133,9 @@ abstract class WalkRemoteObjectDatabase { * Callers such as {@link WalkFetchConnection} are prepared to handle this * by validating the content received, and assuming content that fails to * match its hash is an incorrectly phrased FileNotFoundException. + * <p> + * This method is recommended for already compressed files like loose objects + * and pack files. For text files, see {@link #openReader(String)}. * * @param path * location of the file to read, relative to this objects @@ -346,8 +349,8 @@ abstract class WalkRemoteObjectDatabase { /** * Open a buffered reader around a file. * <p> - * This is shorthand for calling {@link #open(String)} and then wrapping it - * in a reader suitable for line oriented files like the alternates list. + * This method is suitable for for reading line-oriented resources like + * <code>info/packs</code>, <code>info/refs</code>, and the alternates list. * * @return a stream to read from the file. Never null. * @param path diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java index 07fc829db4..6d4c342284 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/AbstractTreeIterator.java @@ -49,8 +49,8 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.CharBuffer; -import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesHandler; +import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.dircache.DirCacheCheckout; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java index db81e1af9b..afa2ed9cb8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java @@ -426,4 +426,9 @@ public class FileTreeIterator extends WorkingTreeIterator { protected byte[] idSubmodule(final Entry e) { return idSubmodule(getDirectory(), e); } + + @Override + protected String readSymlinkTarget(Entry entry) throws IOException { + return fs.readSymLink(getEntryFile()); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java index 911b7ffa1a..5dfebe9ca7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java @@ -53,10 +53,11 @@ import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.attributes.Attributes; +import org.eclipse.jgit.attributes.AttributesHandler; import org.eclipse.jgit.attributes.AttributesNodeProvider; import org.eclipse.jgit.attributes.AttributesProvider; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.dircache.DirCacheBuildIterator; -import org.eclipse.jgit.attributes.AttributesHandler; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -64,6 +65,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.StopWalkException; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; @@ -313,6 +315,8 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { private Config config; + private Set<String> filterCommands; + /** * Create a new tree walker for a given repository. * @@ -357,6 +361,8 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { if (repo != null) { config = repo.getConfig(); attributesNodeProvider = repo.createAttributesNodeProvider(); + filterCommands = FilterCommandRegistry + .getRegisteredFilterCommands(); } else { config = null; attributesNodeProvider = null; @@ -1367,10 +1373,22 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { String filterCommand = filterCommandsByNameDotType.get(key); if (filterCommand != null) return filterCommand; - filterCommand = config.getString(Constants.ATTR_FILTER, + filterCommand = config.getString(ConfigConstants.CONFIG_FILTER_SECTION, filterDriverName, filterCommandType); - if (filterCommand != null) + boolean useBuiltin = config.getBoolean( + ConfigConstants.CONFIG_FILTER_SECTION, + filterDriverName, ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, false); + if (useBuiltin) { + String builtinFilterCommand = Constants.BUILTIN_FILTER_PREFIX + + filterDriverName + '/' + filterCommandType; + if (filterCommands != null + && filterCommands.contains(builtinFilterCommand)) { + filterCommand = builtinFilterCommand; + } + } + if (filterCommand != null) { filterCommandsByNameDotType.put(key, filterCommand); + } return filterCommand; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 9a3fa8060b..52477cb573 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -65,6 +65,8 @@ import java.util.Comparator; import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesRule; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.diff.RawText; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -93,6 +95,8 @@ import org.eclipse.jgit.util.Holder; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.Paths; import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.eclipse.jgit.util.TemporaryBuffer.LocalFile; import org.eclipse.jgit.util.io.AutoLFInputStream; import org.eclipse.jgit.util.io.EolStreamTypeUtil; @@ -263,7 +267,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { // If there is a matching DirCacheIterator, we can reuse // its idBuffer, but only if we appear to be clean against // the cached index information for the path. - // DirCacheIterator i = state.walk.getTree(state.dirCacheTree, DirCacheIterator.class); if (i != null) { @@ -393,15 +396,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { if (len <= MAXIMUM_FILE_SIZE_TO_READ_FULLY) { ByteBuffer rawbuf = IO.readWholeStream(is, (int) len); - byte[] raw = rawbuf.array(); - int n = rawbuf.limit(); - if (!isBinary(raw, n)) { - rawbuf = filterClean(raw, n, opType); - raw = rawbuf.array(); - n = rawbuf.limit(); - } - canonLen = n; - return new ByteArrayInputStream(raw, 0, n); + rawbuf = filterClean(rawbuf.array(), rawbuf.limit(), opType); + canonLen = rawbuf.limit(); + return new ByteArrayInputStream(rawbuf.array(), 0, (int) canonLen); } if (getCleanFilterCommand() == null && isBinary(e)) { @@ -429,10 +426,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { } } - private static boolean isBinary(byte[] content, int sz) { - return RawText.isBinary(content, sz); - } - private static boolean isBinary(Entry entry) throws IOException { InputStream in = entry.openInputStream(); try { @@ -461,6 +454,16 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { in = handleAutoCRLF(in, opType); String filterCommand = getCleanFilterCommand(); if (filterCommand != null) { + if (FilterCommandRegistry.isRegistered(filterCommand)) { + LocalFile buffer = new TemporaryBuffer.LocalFile(null); + FilterCommand command = FilterCommandRegistry + .createFilterCommand(filterCommand, repository, in, + buffer); + while (command.run() != -1) { + // loop as long as command.run() tells there is work to do + } + return buffer.openInputStream(); + } FS fs = repository.getFS(); ProcessBuilder filterProcessBuilder = fs.runInShell(filterCommand, new String[0]); @@ -1005,10 +1008,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return false; } else { - if (mode == FileMode.SYMLINK.getBits()) - return !new File(readContentAsNormalizedString(current())) - .equals(new File((readContentAsNormalizedString(entry, - reader)))); + if (mode == FileMode.SYMLINK.getBits()) { + return !new File(readSymlinkTarget(current())).equals( + new File(readContentAsNormalizedString(entry, reader))); + } // Content differs: that's a real change, perhaps if (reader == null) // deprecated use, do no further checks return true; @@ -1054,12 +1057,30 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return FS.detect().normalize(RawParseUtils.decode(cachedBytes)); } - private static String readContentAsNormalizedString(Entry entry) throws IOException { + /** + * Reads the target of a symlink as a string. This default implementation + * fully reads the entry's input stream and converts it to a normalized + * string. Subclasses may override to provide more specialized + * implementations. + * + * @param entry + * to read + * @return the entry's content as a normalized string + * @throws IOException + * if the entry cannot be read or does not denote a symlink + * @since 4.6 + */ + protected String readSymlinkTarget(Entry entry) throws IOException { + if (!entry.getMode().equals(FileMode.SYMLINK)) { + throw new java.nio.file.NotLinkException(entry.getName()); + } long length = entry.getLength(); byte[] content = new byte[(int) length]; - InputStream is = entry.openInputStream(); - IO.readFully(is, content, 0, (int) length); - return FS.detect().normalize(RawParseUtils.decode(content)); + try (InputStream is = entry.openInputStream()) { + int bytesRead = IO.readFully(is, content, 0); + return FS.detect() + .normalize(RawParseUtils.decode(content, 0, bytesRead)); + } } private static long computeLength(InputStream in) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java index dea07c1973..112ce8fb9f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java @@ -44,8 +44,8 @@ package org.eclipse.jgit.treewalk; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Config.SectionParser; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.CoreConfig.AutoCRLF; import org.eclipse.jgit.lib.CoreConfig.CheckStat; import org.eclipse.jgit.lib.CoreConfig.EOL; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java index e14096e598..f1f6053e3b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java @@ -45,8 +45,8 @@ package org.eclipse.jgit.util; import java.util.regex.Pattern; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 530b1d4bc2..4a2a2d6f3d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -165,7 +165,7 @@ public abstract class FS { /** The auto-detected implementation selected for this operating system and JRE. */ public static final FS DETECTED = detect(); - private static FSFactory factory; + private volatile static FSFactory factory; /** * Auto-detect the appropriate file system abstraction. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java index 4ecaf9c8ee..f63c437e4e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java @@ -99,6 +99,7 @@ public class FS_POSIX extends FS { } } + @SuppressWarnings("boxing") private void determineAtomicFileCreationSupport() { // @TODO: enhance SystemReader to support this without copying code Boolean ret = getAtomicFileCreationSupportOption( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java index 202645b9d9..251381ab33 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/HttpSupport.java @@ -144,6 +144,12 @@ public class HttpSupport { /** The {@code gzip} encoding value for {@link #HDR_ACCEPT_ENCODING}. */ public static final String ENCODING_GZIP = "gzip"; //$NON-NLS-1$ + /** + * The {@code x-gzip} encoding value for {@link #HDR_ACCEPT_ENCODING}. + * @since 4.6 + */ + public static final String ENCODING_X_GZIP = "x-gzip"; //$NON-NLS-1$ + /** The standard {@code text/plain} MIME type. */ public static final String TEXT_PLAIN = "text/plain"; //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 9860ef070f..b36fc2391a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -56,10 +56,12 @@ import java.text.SimpleDateFormat; import java.util.Locale; import java.util.TimeZone; -import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectChecker; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.util.time.MonotonicClock; +import org.eclipse.jgit.util.time.MonotonicSystemClock; /** * Interface to read values from the system. @@ -231,6 +233,14 @@ public abstract class SystemReader { public abstract long getCurrentTime(); /** + * @return clock instance preferred by this system. + * @since 4.6 + */ + public MonotonicClock getClock() { + return new MonotonicSystemClock(); + } + + /** * @param when TODO * @return the local time zone */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java index 3cd5929c7f..57bcfbd5e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java @@ -56,7 +56,6 @@ import java.util.ArrayList; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.util.io.SafeBufferedOutputStream; /** * A fully buffered output stream. @@ -360,7 +359,7 @@ public abstract class TemporaryBuffer extends OutputStream { overflow.write(b.buffer, 0, b.count); blocks = null; - overflow = new SafeBufferedOutputStream(overflow, Block.SZ); + overflow = new BufferedOutputStream(overflow, Block.SZ); overflow.write(last.buffer, 0, last.count); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/IsolatedOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/IsolatedOutputStream.java new file mode 100644 index 0000000000..3598a7ba87 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/IsolatedOutputStream.java @@ -0,0 +1,243 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.util.io; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.eclipse.jgit.internal.JGitText; + +/** + * OutputStream isolated from interrupts. + * <p> + * Wraps an OutputStream to prevent interrupts during writes from being made + * visible to that stream instance. This works around buggy or difficult + * OutputStream implementations like JSch that cannot gracefully handle an + * interrupt during write. + * <p> + * Every write (or flush) requires a context switch to another thread. Callers + * should wrap this stream with {@code BufferedOutputStream} using a suitable + * buffer size to amortize the cost of context switches. + * + * @since 4.6 + */ +public class IsolatedOutputStream extends OutputStream { + private final OutputStream dst; + private final ExecutorService copier; + private Future<Void> pending; + + /** + * Wraps an OutputStream. + * + * @param out + * stream to send all writes to. + */ + public IsolatedOutputStream(OutputStream out) { + dst = out; + copier = new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<Runnable>(1), new NamedThreadFactory()); + } + + @Override + public void write(int ch) throws IOException { + write(new byte[] { (byte) ch }, 0, 1); + } + + @Override + public void write(final byte[] buf, final int pos, final int cnt) + throws IOException { + checkClosed(); + execute(new Callable<Void>() { + @Override + public Void call() throws IOException { + dst.write(buf, pos, cnt); + return null; + } + }); + } + + @Override + public void flush() throws IOException { + checkClosed(); + execute(new Callable<Void>() { + @Override + public Void call() throws IOException { + dst.flush(); + return null; + } + }); + } + + @Override + public void close() throws IOException { + if (!copier.isShutdown()) { + try { + if (pending == null || tryCleanClose()) { + cleanClose(); + } else { + dirtyClose(); + } + } finally { + copier.shutdown(); + } + } + } + + private boolean tryCleanClose() { + /* + * If the caller stopped waiting for a prior write or flush, they could + * be trying to close a stream that is still in-use. Check if the prior + * operation ended in a predictable way. + */ + try { + pending.get(0, TimeUnit.MILLISECONDS); + pending = null; + return true; + } catch (TimeoutException | InterruptedException e) { + return false; + } catch (ExecutionException e) { + pending = null; + return true; + } + } + + private void cleanClose() throws IOException { + execute(new Callable<Void>() { + @Override + public Void call() throws IOException { + dst.close(); + return null; + } + }); + } + + private void dirtyClose() throws IOException { + /* + * Interrupt any still pending write or flush operation. This may cause + * massive failures inside of the stream, but its going to be closed as + * the next step. + */ + pending.cancel(true); + + Future<Void> close; + try { + close = copier.submit(new Callable<Void>() { + @Override + public Void call() throws IOException { + dst.close(); + return null; + } + }); + } catch (RejectedExecutionException e) { + throw new IOException(e); + } + try { + close.get(200, TimeUnit.MILLISECONDS); + } catch (InterruptedException | TimeoutException e) { + close.cancel(true); + throw new IOException(e); + } catch (ExecutionException e) { + throw new IOException(e.getCause()); + } + } + + private void checkClosed() throws IOException { + if (copier.isShutdown()) { + throw new IOException(JGitText.get().closed); + } + } + + private void execute(Callable<Void> task) throws IOException { + if (pending != null) { + // Check (and rethrow) any prior failed operation. + checkedGet(pending); + } + try { + pending = copier.submit(task); + } catch (RejectedExecutionException e) { + throw new IOException(e); + } + checkedGet(pending); + pending = null; + } + + private static void checkedGet(Future<Void> future) throws IOException { + try { + future.get(); + } catch (InterruptedException e) { + throw interrupted(e); + } catch (ExecutionException e) { + throw new IOException(e.getCause()); + } + } + + private static InterruptedIOException interrupted(InterruptedException c) { + InterruptedIOException e = new InterruptedIOException(); + e.initCause(c); + return e; + } + + private static class NamedThreadFactory implements ThreadFactory { + private static final AtomicInteger cnt = new AtomicInteger(); + + @Override + public Thread newThread(Runnable r) { + int n = cnt.incrementAndGet(); + String name = IsolatedOutputStream.class.getSimpleName() + '-' + n; + return new Thread(r, name); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/SafeBufferedOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/SafeBufferedOutputStream.java index 68f250da09..84c339889b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/SafeBufferedOutputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/SafeBufferedOutputStream.java @@ -43,20 +43,13 @@ package org.eclipse.jgit.util.io; import java.io.BufferedOutputStream; -import java.io.IOException; import java.io.OutputStream; /** - * A BufferedOutputStream that throws an error if the final flush fails on - * close. - * <p> - * Java's BufferedOutputStream swallows errors that occur when the output stream - * tries to write the final bytes to the output during close. This may result in - * corrupted files without notice. - * </p> + * @deprecated use BufferedOutputStream in Java 8 and later. */ +@Deprecated public class SafeBufferedOutputStream extends BufferedOutputStream { - /** * @see BufferedOutputStream#BufferedOutputStream(OutputStream) * @param out @@ -76,13 +69,4 @@ public class SafeBufferedOutputStream extends BufferedOutputStream { public SafeBufferedOutputStream(OutputStream out, int size) { super(out, size); } - - @Override - public void close() throws IOException { - try { - flush(); - } finally { - super.close(); - } - } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/StreamCopyThread.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/StreamCopyThread.java index 8d39a22ac2..7aba0a583d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/StreamCopyThread.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/StreamCopyThread.java @@ -47,7 +47,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.io.OutputStream; -import java.util.concurrent.atomic.AtomicInteger; /** Thread to copy from an input stream to an output stream. */ public class StreamCopyThread extends Thread { @@ -59,7 +58,8 @@ public class StreamCopyThread extends Thread { private volatile boolean done; - private final AtomicInteger flushCount = new AtomicInteger(0); + /** Lock held by flush to avoid interrupting a write. */ + private final Object writeLock; /** * Create a thread to copy data from an input stream to an output stream. @@ -75,6 +75,7 @@ public class StreamCopyThread extends Thread { setName(Thread.currentThread().getName() + "-StreamCopy"); //$NON-NLS-1$ src = i; dst = o; + writeLock = new Object(); } /** @@ -84,9 +85,11 @@ public class StreamCopyThread extends Thread { * happen at some future point in time, when the thread wakes up to process * the request. */ + @Deprecated public void flush() { - flushCount.incrementAndGet(); - interrupt(); + synchronized (writeLock) { + interrupt(); + } } /** @@ -113,25 +116,23 @@ public class StreamCopyThread extends Thread { public void run() { try { final byte[] buf = new byte[BUFFER_SIZE]; - int flushCountBeforeRead = 0; boolean readInterrupted = false; for (;;) { try { if (readInterrupted) { - dst.flush(); - readInterrupted = false; - if (!flushCount.compareAndSet(flushCountBeforeRead, 0)) { - // There was a flush() call since last blocked read. - // Set interrupt status, so next blocked read will throw - // an InterruptedIOException and we will flush again. - interrupt(); + synchronized (writeLock) { + boolean interruptedAgain = Thread.interrupted(); + dst.flush(); + if (interruptedAgain) { + interrupt(); + } } + readInterrupted = false; } if (done) break; - flushCountBeforeRead = flushCount.get(); final int n; try { n = src.read(buf); @@ -142,20 +143,12 @@ public class StreamCopyThread extends Thread { if (n < 0) break; - boolean writeInterrupted = false; - for (;;) { - try { - dst.write(buf, 0, n); - } catch (InterruptedIOException wakey) { - writeInterrupted = true; - continue; - } - - // set interrupt status, which will be checked - // when we block in src.read - if (writeInterrupted || flushCount.get() > 0) + synchronized (writeLock) { + boolean writeInterrupted = Thread.interrupted(); + dst.write(buf, 0, n); + if (writeInterrupted) { interrupt(); - break; + } } } catch (IOException e) { break; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/time/MonotonicClock.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/MonotonicClock.java new file mode 100644 index 0000000000..794d851903 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/MonotonicClock.java @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.util.time; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +/** + * A provider of time. + * <p> + * Clocks should provide wall clock time, obtained from a reasonable clock + * source, such as the local system clock. + * <p> + * MonotonicClocks provide the following behavior, with the assertion always + * being true if {@link ProposedTimestamp#blockUntil(Duration)} is used: + * + * <pre> + * MonotonicClock clk = ...; + * long r1; + * try (ProposedTimestamp t1 = clk.propose()) { + * r1 = t1.millis(); + * t1.blockUntil(...); + * } + * + * try (ProposedTimestamp t2 = clk.propose()) { + * assert t2.millis() > r1; + * } + * </pre> + * + * @since 4.6 + */ +public interface MonotonicClock { + /** + * Obtain a timestamp close to "now". + * <p> + * Proposed times are close to "now", but may not yet be certainly in the + * past. This allows the calling thread to interleave other useful work + * while waiting for the clock instance to create an assurance it will never + * in the future propose a time earlier than the returned time. + * <p> + * A hypothetical implementation could read the local system clock (managed + * by NTP) and return that proposal, concurrently sending network messages + * to closely collaborating peers in the same cluster to also ensure their + * system clocks are ahead of this time. In such an implementation the + * {@link ProposedTimestamp#blockUntil(Duration)} method would wait for + * replies from the peers indicating their own system clocks have moved past + * the proposed time. + * + * @return "now". The value can be immediately accessed by + * {@link ProposedTimestamp#read(TimeUnit)} and friends, but the + * caller must use {@link ProposedTimestamp#blockUntil(Duration)} to + * ensure ordering holds. + */ + ProposedTimestamp propose(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/time/MonotonicSystemClock.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/MonotonicSystemClock.java new file mode 100644 index 0000000000..a9f483063c --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/MonotonicSystemClock.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.util.time; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A {@link MonotonicClock} based on {@code System.currentTimeMillis}. + * + * @since 4.6 + */ +public class MonotonicSystemClock implements MonotonicClock { + private static final AtomicLong before = new AtomicLong(); + + private static long nowMicros() { + long now = MILLISECONDS.toMicros(System.currentTimeMillis()); + for (;;) { + long o = before.get(); + long n = Math.max(o + 1, now); + if (before.compareAndSet(o, n)) { + return n; + } + } + } + + @Override + public ProposedTimestamp propose() { + final long u = nowMicros(); + return new ProposedTimestamp() { + @Override + public long read(TimeUnit unit) { + return unit.convert(u, MICROSECONDS); + } + + @Override + public void blockUntil(Duration maxWait) { + // Assume system clock never goes backwards. + } + }; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java new file mode 100644 index 0000000000..c09ab32b68 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.util.time; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.Iterator; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * A timestamp generated by {@link MonotonicClock#propose()}. + * <p> + * ProposedTimestamp implements AutoCloseable so that implementations can + * release resources associated with obtaining certainty about time elapsing. + * For example the constructing MonotonicClock may start network IO with peers + * when creating the ProposedTimestamp, and {@link #close()} can ensure those + * network resources are released in a timely fashion. + * + * @since 4.6 + */ +public abstract class ProposedTimestamp implements AutoCloseable { + /** + * Wait for several timestamps. + * + * @param times + * timestamps to wait on. + * @param maxWait + * how long to wait for the timestamps. + * @throws InterruptedException + * current thread was interrupted before the waiting process + * completed normally. + * @throws TimeoutException + * the timeout was reached without the proposed timestamp become + * certainly in the past. + */ + public static void blockUntil(Iterable<ProposedTimestamp> times, + Duration maxWait) throws TimeoutException, InterruptedException { + Iterator<ProposedTimestamp> itr = times.iterator(); + if (!itr.hasNext()) { + return; + } + + long now = System.currentTimeMillis(); + long deadline = now + maxWait.toMillis(); + for (;;) { + long w = deadline - now; + if (w < 0) { + throw new TimeoutException(); + } + itr.next().blockUntil(Duration.ofMillis(w)); + if (itr.hasNext()) { + now = System.currentTimeMillis(); + } else { + break; + } + } + } + + /** + * Read the timestamp as {@code unit} since the epoch. + * <p> + * The timestamp value for a specific {@code ProposedTimestamp} object never + * changes, and can be read before {@link #blockUntil(Duration)}. + * + * @param unit + * what unit to return the timestamp in. The timestamp will be + * rounded if the unit is bigger than the clock's granularity. + * @return {@code unit} since the epoch. + */ + public abstract long read(TimeUnit unit); + + /** + * Wait for this proposed timestamp to be certainly in the recent past. + * <p> + * This method forces the caller to wait up to {@code timeout} for + * {@code this} to pass sufficiently into the past such that the creating + * {@link MonotonicClock} instance will not create an earlier timestamp. + * + * @param maxWait + * how long the implementation may block the caller. + * @throws InterruptedException + * current thread was interrupted before the waiting process + * completed normally. + * @throws TimeoutException + * the timeout was reached without the proposed timestamp + * becoming certainly in the past. + */ + public abstract void blockUntil(Duration maxWait) + throws InterruptedException, TimeoutException; + + /** @return milliseconds since epoch; {@code read(MILLISECONDS}). */ + public long millis() { + return read(MILLISECONDS); + } + + /** @return microseconds since epoch; {@code read(MICROSECONDS}). */ + public long micros() { + return read(MICROSECONDS); + } + + /** @return time since epoch, with up to microsecond resolution. */ + public Instant instant() { + long usec = micros(); + long secs = usec / 1000000L; + long nanos = (usec % 1000000L) * 1000L; + return Instant.ofEpochSecond(secs, nanos); + } + + /** @return time since epoch, with up to microsecond resolution. */ + public Timestamp timestamp() { + return Timestamp.from(instant()); + } + + /** @return time since epoch, with up to millisecond resolution. */ + public Date date() { + return new Date(millis()); + } + + /** Release resources allocated by this timestamp. */ + @Override + public void close() { + // Do nothing by default. + } + + @Override + public String toString() { + return instant().toString(); + } +} |