* stable-4.7: Run auto GC in the background Change-Id: I5e25765f65d833f13cbe99696ef33055d7f5c4cf Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>tags/v4.8.0.201706111038-r
@@ -62,6 +62,7 @@ import java.util.TreeSet; | |||
import org.eclipse.jgit.dircache.DirCache; | |||
import org.eclipse.jgit.dircache.DirCacheEntry; | |||
import org.eclipse.jgit.internal.storage.file.FileRepository; | |||
import org.eclipse.jgit.lib.ConfigConstants; | |||
import org.eclipse.jgit.lib.Constants; | |||
import org.eclipse.jgit.lib.ObjectId; | |||
import org.eclipse.jgit.lib.PersonIdent; | |||
@@ -121,6 +122,12 @@ public abstract class LocalDiskRepositoryTestCase { | |||
mockSystemReader = new MockSystemReader(); | |||
mockSystemReader.userGitConfig = new FileBasedConfig(new File(tmp, | |||
"usergitconfig"), FS.DETECTED); | |||
// We have to set autoDetach to false for tests, because tests expect to be able | |||
// to clean up by recursively removing the repository, and background GC might be | |||
// in the middle of writing or deleting files, which would disrupt this. | |||
mockSystemReader.userGitConfig.setBoolean(ConfigConstants.CONFIG_GC_SECTION, | |||
null, ConfigConstants.CONFIG_KEY_AUTODETACH, false); | |||
mockSystemReader.userGitConfig.save(); | |||
ceilTestDirectories(getCeilings()); | |||
SystemReader.setInstance(mockSystemReader); | |||
@@ -74,6 +74,7 @@ public class ConfigTest extends CLIRepositoryTestCase { | |||
String[] output = execute("git config --list"); | |||
List<String> expect = new ArrayList<>(); | |||
expect.add("gc.autoDetach=false"); | |||
expect.add("core.filemode=" + !isWindows); | |||
expect.add("core.logallrefupdates=true"); | |||
if (isMac) |
@@ -1,5 +1,21 @@ | |||
<?xml version="1.0" encoding="UTF-8" standalone="no"?> | |||
<component id="org.eclipse.jgit" version="2"> | |||
<resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants"> | |||
<filter id="1141899266"> | |||
<message_arguments> | |||
<message_argument value="4.7"/> | |||
<message_argument value="4.8"/> | |||
<message_argument value="CONFIG_KEY_AUTODETACH"/> | |||
</message_arguments> | |||
</filter> | |||
<filter id="1141899266"> | |||
<message_arguments> | |||
<message_argument value="4.7"/> | |||
<message_argument value="4.8"/> | |||
<message_argument value="CONFIG_KEY_LOGEXPIRY"/> | |||
</message_arguments> | |||
</filter> | |||
</resource> | |||
<resource path="src/org/eclipse/jgit/merge/MergeStrategy.java" type="org.eclipse.jgit.merge.MergeStrategy"> | |||
<filter comment="OSGi semantic versioning allows breaking implementors of an API in a minor version" id="336695337"> | |||
<message_arguments> |
@@ -299,6 +299,8 @@ flagNotFromThis={0} not from this. | |||
flagsAlreadyCreated={0} flags already created. | |||
funnyRefname=funny refname | |||
gcFailed=Garbage collection failed. | |||
gcLogExists=A previous GC run reported an error: ''{0}''. Automatic gc will fail until ''{1}'' is removed. | |||
gcTooManyUnpruned=Too many loose, unpruneable objects after garbage collection. Consider adjusting gc.auto or gc.pruneExpire. | |||
gitmodulesNotFound=.gitmodules not found in tree. | |||
headRequiredToStash=HEAD required to stash local changes | |||
hoursAgo={0} hours ago |
@@ -358,6 +358,8 @@ public class JGitText extends TranslationBundle { | |||
/***/ public String flagsAlreadyCreated; | |||
/***/ public String funnyRefname; | |||
/***/ public String gcFailed; | |||
/***/ public String gcLogExists; | |||
/***/ public String gcTooManyUnpruned; | |||
/***/ public String gitmodulesNotFound; | |||
/***/ public String headRequiredToStash; | |||
/***/ public String hoursAgo; |
@@ -637,12 +637,18 @@ public class FileRepository extends Repository { | |||
} | |||
private boolean shouldAutoDetach() { | |||
return getConfig().getBoolean(ConfigConstants.CONFIG_GC_SECTION, | |||
ConfigConstants.CONFIG_KEY_AUTODETACH, true); | |||
} | |||
@Override | |||
public void autoGC(ProgressMonitor monitor) { | |||
GC gc = new GC(this); | |||
gc.setPackConfig(new PackConfig(this)); | |||
gc.setProgressMonitor(monitor); | |||
gc.setAuto(true); | |||
gc.setBackground(shouldAutoDetach()); | |||
try { | |||
gc.gc(); | |||
} catch (ParseException | IOException e) { |
@@ -50,6 +50,8 @@ import java.io.File; | |||
import java.io.FileOutputStream; | |||
import java.io.IOException; | |||
import java.io.OutputStream; | |||
import java.io.PrintWriter; | |||
import java.io.StringWriter; | |||
import java.nio.channels.Channels; | |||
import java.nio.channels.FileChannel; | |||
import java.nio.file.DirectoryStream; | |||
@@ -73,11 +75,17 @@ import java.util.Map; | |||
import java.util.Objects; | |||
import java.util.Set; | |||
import java.util.TreeMap; | |||
import java.util.concurrent.Callable; | |||
import java.util.concurrent.ExecutionException; | |||
import java.util.concurrent.ExecutorService; | |||
import java.util.concurrent.Executors; | |||
import java.util.concurrent.Future; | |||
import java.util.regex.Pattern; | |||
import java.util.stream.Collectors; | |||
import java.util.stream.Stream; | |||
import org.eclipse.jgit.annotations.NonNull; | |||
import org.eclipse.jgit.api.errors.JGitInternalException; | |||
import org.eclipse.jgit.dircache.DirCacheIterator; | |||
import org.eclipse.jgit.errors.CancelledException; | |||
import org.eclipse.jgit.errors.CorruptObjectException; | |||
@@ -143,6 +151,8 @@ public class GC { | |||
private static final int DEFAULT_AUTOLIMIT = 6700; | |||
private static ExecutorService executor = Executors.newFixedThreadPool(1); | |||
private final FileRepository repo; | |||
private ProgressMonitor pm; | |||
@@ -177,6 +187,11 @@ public class GC { | |||
*/ | |||
private boolean automatic; | |||
/** | |||
* Whether to run gc in a background thread | |||
*/ | |||
private boolean background; | |||
/** | |||
* Creates a new garbage collector with default values. An expirationTime of | |||
* two weeks and <code>null</code> as progress monitor will be used. | |||
@@ -202,13 +217,73 @@ public class GC { | |||
* first check whether any housekeeping is required; if not, it exits | |||
* without performing any work. | |||
* | |||
* If {@link #setBackground(boolean)} was set to {@code true} | |||
* {@code collectGarbage} will start the gc in the background, and then | |||
* return immediately. In this case, errors will not be reported except in | |||
* gc.log. | |||
* | |||
* @return the collection of {@link PackFile}'s which are newly created | |||
* @throws IOException | |||
* @throws ParseException | |||
* If the configuration parameter "gc.pruneexpire" couldn't be | |||
* parsed | |||
*/ | |||
// TODO(ms): in 5.0 change signature and return Future<Collection<PackFile>> | |||
public Collection<PackFile> gc() throws IOException, ParseException { | |||
final GcLog gcLog = background ? new GcLog(repo) : null; | |||
if (gcLog != null && !gcLog.lock(background)) { | |||
// there is already a background gc running | |||
return Collections.emptyList(); | |||
} | |||
Callable<Collection<PackFile>> gcTask = () -> { | |||
try { | |||
Collection<PackFile> newPacks = doGc(); | |||
if (automatic && tooManyLooseObjects() && gcLog != null) { | |||
String message = JGitText.get().gcTooManyUnpruned; | |||
gcLog.write(message); | |||
gcLog.commit(); | |||
} | |||
return newPacks; | |||
} catch (IOException | ParseException e) { | |||
if (background) { | |||
if (gcLog == null) { | |||
// Lacking a log, there's no way to report this. | |||
return Collections.emptyList(); | |||
} | |||
try { | |||
gcLog.write(e.getMessage()); | |||
StringWriter sw = new StringWriter(); | |||
e.printStackTrace(new PrintWriter(sw)); | |||
gcLog.write(sw.toString()); | |||
gcLog.commit(); | |||
} catch (IOException e2) { | |||
e2.addSuppressed(e); | |||
LOG.error(e2.getMessage(), e2); | |||
} | |||
} else { | |||
throw new JGitInternalException(e.getMessage(), e); | |||
} | |||
} finally { | |||
if (gcLog != null) { | |||
gcLog.unlock(); | |||
} | |||
} | |||
return Collections.emptyList(); | |||
}; | |||
Future<Collection<PackFile>> result = executor.submit(gcTask); | |||
if (background) { | |||
// TODO(ms): in 5.0 change signature and return the Future | |||
return Collections.emptyList(); | |||
} | |||
try { | |||
return result.get(); | |||
} catch (InterruptedException | ExecutionException e) { | |||
throw new IOException(e); | |||
} | |||
} | |||
private Collection<PackFile> doGc() throws IOException, ParseException { | |||
if (automatic && !needGc()) { | |||
return Collections.emptyList(); | |||
} | |||
@@ -1365,6 +1440,14 @@ public class GC { | |||
this.automatic = auto; | |||
} | |||
/** | |||
* @param background | |||
* whether to run the gc in a background thread. | |||
*/ | |||
void setBackground(boolean background) { | |||
this.background = background; | |||
} | |||
private boolean needGc() { | |||
if (tooManyPacks()) { | |||
addRepackAllOption(); | |||
@@ -1403,8 +1486,7 @@ public class GC { | |||
* @return {@code true} if number of loose objects > gc.auto (default 6700) | |||
*/ | |||
boolean tooManyLooseObjects() { | |||
int auto = repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION, | |||
ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT); | |||
int auto = getLooseObjectLimit(); | |||
if (auto <= 0) { | |||
return false; | |||
} | |||
@@ -1436,4 +1518,9 @@ public class GC { | |||
} | |||
return false; | |||
} | |||
private int getLooseObjectLimit() { | |||
return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION, | |||
ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT); | |||
} | |||
} |
@@ -0,0 +1,191 @@ | |||
/* | |||
* Copyright (C) 2017 Two Sigma Open Source | |||
* and other copyright owners as documented in the project's IP log. | |||
* | |||
* This program and the accompanying materials are made available | |||
* under the terms of the Eclipse Distribution License v1.0 which | |||
* accompanies this distribution, is reproduced below, and is | |||
* available at http://www.eclipse.org/org/documents/edl-v10.php | |||
* | |||
* All rights reserved. | |||
* | |||
* Redistribution and use in source and binary forms, with or | |||
* without modification, are permitted provided that the following | |||
* conditions are met: | |||
* | |||
* - Redistributions of source code must retain the above copyright | |||
* notice, this list of conditions and the following disclaimer. | |||
* | |||
* - Redistributions in binary form must reproduce the above | |||
* copyright notice, this list of conditions and the following | |||
* disclaimer in the documentation and/or other materials provided | |||
* with the distribution. | |||
* | |||
* - Neither the name of the Eclipse Foundation, Inc. nor the | |||
* names of its contributors may be used to endorse or promote | |||
* products derived from this software without specific prior | |||
* written permission. | |||
* | |||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND | |||
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, | |||
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | |||
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR | |||
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | |||
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; | |||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, | |||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | |||
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF | |||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
*/ | |||
package org.eclipse.jgit.internal.storage.file; | |||
import org.eclipse.jgit.api.errors.JGitInternalException; | |||
import org.eclipse.jgit.internal.JGitText; | |||
import org.eclipse.jgit.lib.ConfigConstants; | |||
import org.eclipse.jgit.util.GitDateParser; | |||
import org.eclipse.jgit.util.SystemReader; | |||
import static java.nio.charset.StandardCharsets.UTF_8; | |||
import java.io.BufferedReader; | |||
import java.io.File; | |||
import java.io.IOException; | |||
import java.nio.file.Files; | |||
import java.nio.file.NoSuchFileException; | |||
import java.nio.file.attribute.FileTime; | |||
import java.text.MessageFormat; | |||
import java.text.ParseException; | |||
import java.time.Instant; | |||
/** | |||
* This class manages the gc.log file for a {@link FileRepository}. | |||
*/ | |||
class GcLog { | |||
private final FileRepository repo; | |||
private final File logFile; | |||
private final LockFile lock; | |||
private Instant gcLogExpire; | |||
private static final String LOG_EXPIRY_DEFAULT = "1.day.ago"; //$NON-NLS-1$ | |||
private boolean nonEmpty = false; | |||
/** | |||
* Construct a GcLog object for a {@link FileRepository} | |||
* | |||
* @param repo | |||
* the repository | |||
*/ | |||
GcLog(FileRepository repo) { | |||
this.repo = repo; | |||
logFile = new File(repo.getDirectory(), "gc.log"); //$NON-NLS-1$ | |||
lock = new LockFile(logFile); | |||
} | |||
private Instant getLogExpiry() throws ParseException { | |||
if (gcLogExpire == null) { | |||
String logExpiryStr = repo.getConfig().getString( | |||
ConfigConstants.CONFIG_GC_SECTION, null, | |||
ConfigConstants.CONFIG_KEY_LOGEXPIRY); | |||
if (logExpiryStr == null) { | |||
logExpiryStr = LOG_EXPIRY_DEFAULT; | |||
} | |||
gcLogExpire = GitDateParser.parse(logExpiryStr, null, | |||
SystemReader.getInstance().getLocale()).toInstant(); | |||
} | |||
return gcLogExpire; | |||
} | |||
private boolean autoGcBlockedByOldLockFile(boolean background) { | |||
try { | |||
FileTime lastModified = Files.getLastModifiedTime(logFile.toPath()); | |||
if (lastModified.toInstant().compareTo(getLogExpiry()) > 0) { | |||
// There is an existing log file, which is too recent to ignore | |||
if (!background) { | |||
try (BufferedReader reader = Files | |||
.newBufferedReader(logFile.toPath())) { | |||
char[] buf = new char[1000]; | |||
int len = reader.read(buf, 0, 1000); | |||
String oldError = new String(buf, 0, len); | |||
throw new JGitInternalException(MessageFormat.format( | |||
JGitText.get().gcLogExists, oldError, logFile)); | |||
} | |||
} | |||
return true; | |||
} | |||
} catch (NoSuchFileException e) { | |||
// No existing log file, OK. | |||
} catch (IOException | ParseException e) { | |||
throw new JGitInternalException(e.getMessage(), e); | |||
} | |||
return false; | |||
} | |||
/** | |||
* Lock the GC log file for updates | |||
* | |||
* @param background | |||
* If true, and if gc.log already exists, unlock and return false | |||
* @return {@code true} if we hold the lock | |||
*/ | |||
boolean lock(boolean background) { | |||
try { | |||
if (!lock.lock()) { | |||
return false; | |||
} | |||
} catch (IOException e) { | |||
throw new JGitInternalException(e.getMessage(), e); | |||
} | |||
if (autoGcBlockedByOldLockFile(background)) { | |||
lock.unlock(); | |||
return false; | |||
} | |||
return true; | |||
} | |||
/** | |||
* Unlock (roll back) the GC log lock | |||
*/ | |||
void unlock() { | |||
lock.unlock(); | |||
} | |||
/** | |||
* Commit changes to the gc log, if there have been any writes. Otherwise, | |||
* just unlock and delete the existing file (if any) | |||
* | |||
* @return true if committing (or unlocking/deleting) succeeds. | |||
*/ | |||
boolean commit() { | |||
if (nonEmpty) { | |||
return lock.commit(); | |||
} else { | |||
logFile.delete(); | |||
lock.unlock(); | |||
return true; | |||
} | |||
} | |||
/** | |||
* Write to the pending gc log. Content will be committed upon a call to | |||
* commit() | |||
* | |||
* @param content | |||
* The content to write | |||
* @throws IOException | |||
*/ | |||
void write(String content) throws IOException { | |||
if (content.length() > 0) { | |||
nonEmpty = true; | |||
} | |||
lock.write(content.getBytes(UTF_8)); | |||
} | |||
} |
@@ -290,6 +290,20 @@ public class ConfigConstants { | |||
*/ | |||
public static final String CONFIG_KEY_PRUNEPACKEXPIRE = "prunepackexpire"; | |||
/** | |||
* The "logexpiry" key | |||
* | |||
* @since 4.7 | |||
*/ | |||
public static final String CONFIG_KEY_LOGEXPIRY = "logExpiry"; | |||
/** | |||
* The "autodetach" key | |||
* | |||
* @since 4.7 | |||
*/ | |||
public static final String CONFIG_KEY_AUTODETACH = "autoDetach"; | |||
/** | |||
* The "aggressiveDepth" key | |||
* @since 3.6 |