Add config parameter gc.prunePackExpire for packfile expiration

JGit's Garbage Collector is repacking relevant objects into new
packfiles and is afterwards deleting the now obsolete packfiles. But to
prevent problems caused by race conditions JGit was not deleting
packfiles when they are too young. The same mechanism as for loose
objects and the config parameter gc.pruneExpire was used.
But JGit was reusing the parameter gc.pruneExpire also for packfiles
which may cause a lot of filesystem consumption if gc.pruneExpire was
set to the default of 2 weeks. Only two weeks after packfile creation gc
was allowed to delete this packfile.

This change introduces a new config paramter gc.prunePackExpire with a
default of "1.hour". This parameter is used when packfiles are deleted.
Only packfiles older than the specified time can be deleted.

For loose objects the behaviour is not changed and only the old
parameter gc.pruneExpire is relevant.

Change-Id: I6209efb05678b15153bd22479dc13486907a44f8

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java
index f549fb5cdf71869604cb078d253aa711db290bca..41a1a5d3f5a32104ecb20167f56ed6fa7bd87152 100644 (file)
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcBasicPackingTest.java
@@ -213,9 +213,20 @@ public class GcBasicPackingTest extends GcTestCase {
                assertEquals(9, stats.numberOfPackedObjects);
                assertEquals(2, stats.numberOfPackFiles);
 
+               // repack again but now without a grace period for loose objects. Since
+               // we don't have loose objects anymore this shouldn't change anything
+               gc.setExpireAgeMillis(0);
+               gc.gc();
+               stats = gc.getStatistics();
+               assertEquals(0, stats.numberOfLooseObjects);
+               // if objects exist in multiple packFiles then they are counted multiple
+               // times
+               assertEquals(9, stats.numberOfPackedObjects);
+               assertEquals(2, stats.numberOfPackFiles);
+
                // repack again but now without a grace period for packfiles. We should
                // end up with one packfile
-               gc.setExpireAgeMillis(0);
+               gc.setPackExpireAgeMillis(0);
                gc.gc();
                stats = gc.getStatistics();
                assertEquals(0, stats.numberOfLooseObjects);

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java
index 5fda0708673b5607c50b476c571d8f5ba754f09e..49e1f6f3da7dd0ebbede49029d2564f15536823f 100644 (file)
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/pack/GcCommitSelectionTest.java
@@ -95,7 +95,8 @@ public class GcCommitSelectionTest extends GcTestCase {
                        }
                        currentCommits = nextCommitCount;
 
-                       gc.setExpireAgeMillis(0); // immediately delete old packs
+                       gc.setPackExpireAgeMillis(0); // immediately delete old packs
+                       gc.setExpireAgeMillis(0);
                        gc.gc();
                        assertEquals(currentCommits * 3, // commit/tree/object
                                        gc.getStatistics().numberOfPackedObjects);
@@ -158,7 +159,8 @@ public class GcCommitSelectionTest extends GcTestCase {
                        }
                        currentCommits = nextCommitCount;
 
-                       gc.setExpireAgeMillis(0); // immediately delete old packs
+                       gc.setPackExpireAgeMillis(0); // immediately delete old packs
+                       gc.setExpireAgeMillis(0);
                        gc.gc();
                        assertEquals(currentCommits + " commits: ", expectedBitmapCount,
                                        gc.getStatistics().numberOfBitmaps);
@@ -198,7 +200,8 @@ public class GcCommitSelectionTest extends GcTestCase {
                final int commitsForShallowBranches = 100;
 
                // Excessive branch history pruning, one old branch.
-               gc.setExpireAgeMillis(0); // immediately delete old packs
+               gc.setPackExpireAgeMillis(0); // immediately delete old packs
+               gc.setExpireAgeMillis(0);
                gc.gc();
                assertEquals(
                                commitsForSparseBranch + commitsForFullBranch

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
index 49f9335aed0fdaa9260cb62639846e27f1e645e9..9cdb753221be3fe3bfca02937c05e3de3995115d 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
@@ -111,6 +111,8 @@ import org.eclipse.jgit.util.SystemReader;
 public class GC {
        private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$
 
+       private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$
+
        private final FileRepository repo;
 
        private ProgressMonitor pm;
@@ -119,6 +121,10 @@ public class GC {
 
        private Date expire;
 
+       private long packExpireAgeMillis = -1;
+
+       private Date packExpire;
+
        private PackConfig pconfig = null;
 
        /**
@@ -186,7 +192,7 @@ public class GC {
         */
        private void deleteOldPacks(Collection<PackFile> oldPacks,
                        Collection<PackFile> newPacks) throws ParseException {
-               long expireDate = getExpireDate();
+               long packExpireDate = getPackExpireDate();
                oldPackLoop: for (PackFile oldPack : oldPacks) {
                        String oldName = oldPack.getPackName();
                        // check whether an old pack file is also among the list of new
@@ -196,7 +202,7 @@ public class GC {
                                        continue oldPackLoop;
 
                        if (!oldPack.shouldBeKept()
-                                       && oldPack.getPackFile().lastModified() < expireDate) {
+                                       && oldPack.getPackFile().lastModified() < packExpireDate) {
                                oldPack.close();
                                prunePack(oldName);
                        }
@@ -449,6 +455,26 @@ public class GC {
                return expireDate;
        }
 
+       private long getPackExpireDate() throws ParseException {
+               long packExpireDate = Long.MAX_VALUE;
+
+               if (packExpire == null && packExpireAgeMillis == -1) {
+                       String prunePackExpireStr = repo.getConfig().getString(
+                                       ConfigConstants.CONFIG_GC_SECTION, null,
+                                       ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
+                       if (prunePackExpireStr == null)
+                               prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
+                       packExpire = GitDateParser.parse(prunePackExpireStr, null,
+                                       SystemReader.getInstance().getLocale());
+                       packExpireAgeMillis = -1;
+               }
+               if (packExpire != null)
+                       packExpireDate = packExpire.getTime();
+               if (packExpireAgeMillis != -1)
+                       packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
+               return packExpireDate;
+       }
+
        /**
         * Remove all entries from a map which key is the id of an object referenced
         * by the given ObjectWalk
@@ -982,6 +1008,20 @@ public class GC {
                expire = null;
        }
 
+       /**
+        * During gc() or prune() packfiles which are created or modified in the
+        * last <code>packExpireAgeMillis</code> milliseconds will not be deleted.
+        * Only older packfiles may be deleted. If set to 0 then every packfile is a
+        * candidate for deletion.
+        *
+        * @param packExpireAgeMillis
+        *            minimal age of packfiles to be deleted in milliseconds.
+        */
+       public void setPackExpireAgeMillis(long packExpireAgeMillis) {
+               this.packExpireAgeMillis = packExpireAgeMillis;
+               expire = null;
+       }
+
        /**
         * Set the PackConfig used when (re-)writing packfiles. This allows to
         * influence how packs are written and to implement something similar to
@@ -1011,4 +1051,18 @@ public class GC {
                this.expire = expire;
                expireAgeMillis = -1;
        }
+
+       /**
+        * During gc() or prune() packfiles which are created or modified after or
+        * at <code>packExpire</code> will not be deleted. Only older packfiles may
+        * be deleted. If set to null then every packfile is a candidate for
+        * deletion.
+        *
+        * @param packExpire
+        *            instant in time which defines packfile expiration
+        */
+       public void setPackExpire(Date packExpire) {
+               this.packExpire = packExpire;
+               packExpireAgeMillis = -1;
+       }
 }

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
index 054d1930170ce6f813cc45de37315020d670e84e..3233eba00802eb021a951f1c2e024345e687df25 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
@@ -241,6 +241,12 @@ public class ConfigConstants {
        /** The "pruneexpire" key */
        public static final String CONFIG_KEY_PRUNEEXPIRE = "pruneexpire";
 
+       /**
+        * The "prunepackexpire" key
+        * @since 4.3
+        */
+       public static final String CONFIG_KEY_PRUNEPACKEXPIRE = "prunepackexpire";
+
        /**
         * The "aggressiveDepth" key
         * @since 3.6