diff options
8 files changed, 217 insertions, 1 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java index 38b10971f4..ced1863719 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/MergeCommand.java @@ -381,6 +381,7 @@ public class MergeCommand extends GitCommand<MergeResult> { .call().getId(); } mergeStatus = MergeStatus.MERGED; + getRepository().autoGC(monitor); } if (commit && squash) { msg = JGitText.get().squashCommitNotUpdatingHEAD; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 2d6a76b390..c5c0cfb821 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -693,6 +693,7 @@ public class RebaseCommand extends GitCommand<RebaseResult> { String headName = rebaseState.readFile(HEAD_NAME); updateHead(headName, finalHead, upstreamCommit); boolean stashConflicts = autoStashApply(); + getRepository().autoGC(monitor); FileUtils.delete(rebaseState.getDir(), FileUtils.RECURSIVE); if (stashConflicts) return RebaseResult.STASH_APPLY_CONFLICTS_RESULT; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index 53fd37e534..5b93399a76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -53,9 +53,11 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.text.MessageFormat; +import java.text.ParseException; import java.util.HashSet; import java.util.Set; +import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesNodeProvider; import org.eclipse.jgit.errors.ConfigInvalidException; @@ -63,15 +65,16 @@ import org.eclipse.jgit.events.ConfigChangedEvent; import org.eclipse.jgit.events.ConfigChangedListener; import org.eclipse.jgit.events.IndexChangedEvent; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepository; +import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; import org.eclipse.jgit.lib.BaseRepositoryBuilder; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig.HideDotFiles; import org.eclipse.jgit.lib.CoreConfig.SymLinks; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; @@ -79,6 +82,7 @@ import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.StringUtils; @@ -555,4 +559,17 @@ public class FileRepository extends Repository { } + @Override + public void autoGC(ProgressMonitor monitor) { + GC gc = new GC(this); + gc.setPackConfig(new PackConfig(this)); + gc.setProgressMonitor(monitor); + gc.setAuto(true); + try { + gc.gc(); + } catch (ParseException | IOException e) { + throw new JGitInternalException(JGitText.get().gcFailed, e); + } + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 2e8da8fc9b..f55e15f5f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -52,6 +52,9 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.text.MessageFormat; import java.text.ParseException; @@ -62,12 +65,14 @@ import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.TreeMap; +import java.util.regex.Pattern; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.dircache.DirCacheIterator; @@ -100,6 +105,8 @@ import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.GitDateParser; import org.eclipse.jgit.util.SystemReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A garbage collector for git {@link FileRepository}. Instances of this class @@ -109,10 +116,20 @@ import org.eclipse.jgit.util.SystemReader; * adapted to FileRepositories. */ public class GC { + private final static Logger LOG = LoggerFactory + .getLogger(GC.class); + private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$ private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$ + private static final Pattern PATTERN_LOOSE_OBJECT = Pattern + .compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$ + + private static final int DEFAULT_AUTOPACKLIMIT = 50; + + private static final int DEFAULT_AUTOLIMIT = 6700; + private final FileRepository repo; private ProgressMonitor pm; @@ -143,6 +160,11 @@ public class GC { private long lastRepackTime; /** + * Whether gc should do automatic housekeeping + */ + private boolean automatic; + + /** * Creates a new garbage collector with default values. An expirationTime of * two weeks and <code>null</code> as progress monitor will be used. * @@ -163,6 +185,10 @@ public class GC { * <li>prune all loose objects which are now reachable by packs</li> * </ul> * + * If {@link #setAuto(boolean)} was set to {@code true} {@code gc} will + * first check whether any housekeeping is required; if not, it exits + * without performing any work. + * * @return the collection of {@link PackFile}'s which are newly created * @throws IOException * @throws ParseException @@ -170,6 +196,9 @@ public class GC { * parsed */ public Collection<PackFile> gc() throws IOException, ParseException { + if (automatic && !needGc()) { + return Collections.emptyList(); + } pm.start(6 /* tasks */); packRefs(); // TODO: implement reflog_expire(pm, repo); @@ -1076,4 +1105,114 @@ public class GC { this.packExpire = packExpire; packExpireAgeMillis = -1; } + + /** + * Set the {@code gc --auto} option. + * + * With this option, gc checks whether any housekeeping is required; if not, + * it exits without performing any work. Some JGit commands run + * {@code gc --auto} after performing operations that could create many + * loose objects. + * <p/> + * Housekeeping is required if there are too many loose objects or too many + * packs in the repository. If the number of loose objects exceeds the value + * of the gc.auto option JGit GC consolidates all existing packs into a + * single pack (equivalent to {@code -A} option), whereas git-core would + * combine all loose objects into a single pack using {@code repack -d -l}. + * Setting the value of {@code gc.auto} to 0 disables automatic packing of + * loose objects. + * <p/> + * If the number of packs exceeds the value of {@code gc.autoPackLimit}, + * then existing packs (except those marked with a .keep file) are + * consolidated into a single pack by using the {@code -A} option of repack. + * Setting {@code gc.autoPackLimit} to 0 disables automatic consolidation of + * packs. + * <p/> + * Like git the following jgit commands run auto gc: + * <ul> + * <li>fetch</li> + * <li>merge</li> + * <li>rebase</li> + * <li>receive-pack</li> + * </ul> + * The auto gc for receive-pack can be suppressed by setting the config + * option {@code receive.autogc = false} + * + * @param auto + * defines whether gc should do automatic housekeeping + * @since 4.5 + */ + public void setAuto(boolean auto) { + this.automatic = auto; + } + + private boolean needGc() { + if (tooManyPacks()) { + addRepackAllOption(); + } else if (!tooManyLooseObjects()) { + return false; + } + // TODO run pre-auto-gc hook, if it fails return false + return true; + } + + private void addRepackAllOption() { + // TODO: if JGit GC is enhanced to support repack's option -l this + // method needs to be implemented + } + + /** + * @return {@code true} if number of packs > gc.autopacklimit (default 50) + */ + private boolean tooManyPacks() { + int autopacklimit = repo.getConfig().getInt( + ConfigConstants.CONFIG_GC_SECTION, + ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, + DEFAULT_AUTOPACKLIMIT); + if (autopacklimit <= 0) { + return false; + } + // JGit always creates two packfiles, one for the objects reachable from + // branches, and another one for the rest + return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1); + } + + /** + * Quickly estimate number of loose objects, SHA1 is distributed evenly so + * counting objects in one directory (bucket 17) is sufficient + * + * @return {@code true} if number of loose objects > gc.auto (default 6700) + */ + private boolean tooManyLooseObjects() { + int auto = repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION, + ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT); + if (auto <= 0) { + return false; + } + int n = 0; + int threshold = (auto + 255) / 256; + Path dir = repo.getObjectsDirectory().toPath().resolve("17"); //$NON-NLS-1$ + if (!Files.exists(dir)) { + return false; + } + try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, + new DirectoryStream.Filter<Path>() { + + public boolean accept(Path file) throws IOException { + return Files.isRegularFile(file) && PATTERN_LOOSE_OBJECT + .matcher(file.getFileName().toString()) + .matches(); + } + })) { + Iterator<Path> iter = stream.iterator(); + while (iter.hasNext()) { + if (n++ > threshold) { + return true; + } + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + return false; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 9a1f565d71..4d3e118011 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -65,6 +65,12 @@ public class ConfigConstants { /** The "dfs" section */ public static final String CONFIG_DFS_SECTION = "dfs"; + /** + * The "receive" section + * @since 4.6 + */ + public static final String CONFIG_RECEIVE_SECTION = "receive"; + /** The "user" section */ public static final String CONFIG_USER_SECTION = "user"; @@ -108,6 +114,24 @@ public class ConfigConstants { public static final String CONFIG_KEY_AUTOCRLF = "autocrlf"; /** + * The "auto" key + * @since 4.6 + */ + public static final String CONFIG_KEY_AUTO = "auto"; + + /** + * The "autogc" key + * @since 4.6 + */ + public static final String CONFIG_KEY_AUTOGC = "autogc"; + + /** + * The "autopacklimit" key + * @since 4.6 + */ + public static final String CONFIG_KEY_AUTOPACKLIMIT = "autopacklimit"; + + /** * The "eol" key * * @since 4.3 diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index e9ff504a42..1909037eeb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -80,6 +80,7 @@ import org.eclipse.jgit.events.IndexChangedListener; import org.eclipse.jgit.events.ListenerList; import org.eclipse.jgit.events.RepositoryEvent; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.GC; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; @@ -1836,4 +1837,22 @@ public abstract class Repository implements AutoCloseable { return getConfig() .getSubsections(ConfigConstants.CONFIG_REMOTE_SECTION); } + + /** + * Check whether any housekeeping is required; if yes, run garbage + * collection; if not, exit without performing any work. Some JGit commands + * run autoGC after performing operations that could create many loose + * objects. + * <p/> + * Currently this option is supported for repositories of type + * {@code FileRepository} only. See {@link GC#setAuto(boolean)} for + * configuration details. + * + * @param monitor + * to report progress + * @since 4.6 + */ + public void autoGC(ProgressMonitor monitor) { + // default does nothing + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index cc20d50a7f..393e25a2a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java @@ -56,7 +56,9 @@ import java.util.List; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.UnpackException; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.transport.ReceiveCommand.Result; import org.eclipse.jgit.transport.RefAdvertiser.PacketLineOutRefAdvertiser; @@ -307,9 +309,19 @@ public class ReceivePack extends BaseReceivePack { throw new UnpackException(unpackError); } postReceive.onPostReceive(this, filterCommands(Result.OK)); + autoGc(); } } + private void autoGc() { + Repository repo = getRepository(); + if (!repo.getConfig().getBoolean(ConfigConstants.CONFIG_RECEIVE_SECTION, + ConfigConstants.CONFIG_KEY_AUTOGC, true)) { + return; + } + repo.autoGC(NullProgressMonitor.INSTANCE); + } + @Override protected String getLockMessageProcessName() { return "jgit receive-pack"; //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index bc4843a8af..df860695df 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -1199,6 +1199,9 @@ public abstract class Transport implements AutoCloseable { final FetchResult result = new FetchResult(); new FetchProcess(this, toFetch).execute(monitor, result); + + local.autoGC(monitor); + return result; } |