JGit's Garbage Collector is repacking relevant objects into new packfiles and is afterwards deleting the now obsolete packfiles. But to prevent problems caused by race conditions JGit was not deleting packfiles when they are too young. The same mechanism as for loose objects and the config parameter gc.pruneExpire was used. But JGit was reusing the parameter gc.pruneExpire also for packfiles which may cause a lot of filesystem consumption if gc.pruneExpire was set to the default of 2 weeks. Only two weeks after packfile creation gc was allowed to delete this packfile. This change introduces a new config paramter gc.prunePackExpire with a default of "1.hour". This parameter is used when packfiles are deleted. Only packfiles older than the specified time can be deleted. For loose objects the behaviour is not changed and only the old parameter gc.pruneExpire is relevant. Change-Id: I6209efb05678b15153bd22479dc13486907a44f8tags/v4.3.0.201604071810-r
assertEquals(9, stats.numberOfPackedObjects); | assertEquals(9, stats.numberOfPackedObjects); | ||||
assertEquals(2, stats.numberOfPackFiles); | assertEquals(2, stats.numberOfPackFiles); | ||||
// repack again but now without a grace period for loose objects. Since | |||||
// we don't have loose objects anymore this shouldn't change anything | |||||
gc.setExpireAgeMillis(0); | |||||
gc.gc(); | |||||
stats = gc.getStatistics(); | |||||
assertEquals(0, stats.numberOfLooseObjects); | |||||
// if objects exist in multiple packFiles then they are counted multiple | |||||
// times | |||||
assertEquals(9, stats.numberOfPackedObjects); | |||||
assertEquals(2, stats.numberOfPackFiles); | |||||
// repack again but now without a grace period for packfiles. We should | // repack again but now without a grace period for packfiles. We should | ||||
// end up with one packfile | // end up with one packfile | ||||
gc.setExpireAgeMillis(0); | |||||
gc.setPackExpireAgeMillis(0); | |||||
gc.gc(); | gc.gc(); | ||||
stats = gc.getStatistics(); | stats = gc.getStatistics(); | ||||
assertEquals(0, stats.numberOfLooseObjects); | assertEquals(0, stats.numberOfLooseObjects); |
} | } | ||||
currentCommits = nextCommitCount; | currentCommits = nextCommitCount; | ||||
gc.setExpireAgeMillis(0); // immediately delete old packs | |||||
gc.setPackExpireAgeMillis(0); // immediately delete old packs | |||||
gc.setExpireAgeMillis(0); | |||||
gc.gc(); | gc.gc(); | ||||
assertEquals(currentCommits * 3, // commit/tree/object | assertEquals(currentCommits * 3, // commit/tree/object | ||||
gc.getStatistics().numberOfPackedObjects); | gc.getStatistics().numberOfPackedObjects); | ||||
} | } | ||||
currentCommits = nextCommitCount; | currentCommits = nextCommitCount; | ||||
gc.setExpireAgeMillis(0); // immediately delete old packs | |||||
gc.setPackExpireAgeMillis(0); // immediately delete old packs | |||||
gc.setExpireAgeMillis(0); | |||||
gc.gc(); | gc.gc(); | ||||
assertEquals(currentCommits + " commits: ", expectedBitmapCount, | assertEquals(currentCommits + " commits: ", expectedBitmapCount, | ||||
gc.getStatistics().numberOfBitmaps); | gc.getStatistics().numberOfBitmaps); | ||||
final int commitsForShallowBranches = 100; | final int commitsForShallowBranches = 100; | ||||
// Excessive branch history pruning, one old branch. | // Excessive branch history pruning, one old branch. | ||||
gc.setExpireAgeMillis(0); // immediately delete old packs | |||||
gc.setPackExpireAgeMillis(0); // immediately delete old packs | |||||
gc.setExpireAgeMillis(0); | |||||
gc.gc(); | gc.gc(); | ||||
assertEquals( | assertEquals( | ||||
commitsForSparseBranch + commitsForFullBranch | commitsForSparseBranch + commitsForFullBranch |
public class GC { | public class GC { | ||||
private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$ | private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$ | ||||
private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$ | |||||
private final FileRepository repo; | private final FileRepository repo; | ||||
private ProgressMonitor pm; | private ProgressMonitor pm; | ||||
private Date expire; | private Date expire; | ||||
private long packExpireAgeMillis = -1; | |||||
private Date packExpire; | |||||
private PackConfig pconfig = null; | private PackConfig pconfig = null; | ||||
/** | /** | ||||
*/ | */ | ||||
private void deleteOldPacks(Collection<PackFile> oldPacks, | private void deleteOldPacks(Collection<PackFile> oldPacks, | ||||
Collection<PackFile> newPacks) throws ParseException { | Collection<PackFile> newPacks) throws ParseException { | ||||
long expireDate = getExpireDate(); | |||||
long packExpireDate = getPackExpireDate(); | |||||
oldPackLoop: for (PackFile oldPack : oldPacks) { | oldPackLoop: for (PackFile oldPack : oldPacks) { | ||||
String oldName = oldPack.getPackName(); | String oldName = oldPack.getPackName(); | ||||
// check whether an old pack file is also among the list of new | // check whether an old pack file is also among the list of new | ||||
continue oldPackLoop; | continue oldPackLoop; | ||||
if (!oldPack.shouldBeKept() | if (!oldPack.shouldBeKept() | ||||
&& oldPack.getPackFile().lastModified() < expireDate) { | |||||
&& oldPack.getPackFile().lastModified() < packExpireDate) { | |||||
oldPack.close(); | oldPack.close(); | ||||
prunePack(oldName); | prunePack(oldName); | ||||
} | } | ||||
return expireDate; | return expireDate; | ||||
} | } | ||||
private long getPackExpireDate() throws ParseException { | |||||
long packExpireDate = Long.MAX_VALUE; | |||||
if (packExpire == null && packExpireAgeMillis == -1) { | |||||
String prunePackExpireStr = repo.getConfig().getString( | |||||
ConfigConstants.CONFIG_GC_SECTION, null, | |||||
ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE); | |||||
if (prunePackExpireStr == null) | |||||
prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT; | |||||
packExpire = GitDateParser.parse(prunePackExpireStr, null, | |||||
SystemReader.getInstance().getLocale()); | |||||
packExpireAgeMillis = -1; | |||||
} | |||||
if (packExpire != null) | |||||
packExpireDate = packExpire.getTime(); | |||||
if (packExpireAgeMillis != -1) | |||||
packExpireDate = System.currentTimeMillis() - packExpireAgeMillis; | |||||
return packExpireDate; | |||||
} | |||||
/** | /** | ||||
* Remove all entries from a map which key is the id of an object referenced | * Remove all entries from a map which key is the id of an object referenced | ||||
* by the given ObjectWalk | * by the given ObjectWalk | ||||
expire = null; | expire = null; | ||||
} | } | ||||
/** | |||||
* During gc() or prune() packfiles which are created or modified in the | |||||
* last <code>packExpireAgeMillis</code> milliseconds will not be deleted. | |||||
* Only older packfiles may be deleted. If set to 0 then every packfile is a | |||||
* candidate for deletion. | |||||
* | |||||
* @param packExpireAgeMillis | |||||
* minimal age of packfiles to be deleted in milliseconds. | |||||
*/ | |||||
public void setPackExpireAgeMillis(long packExpireAgeMillis) { | |||||
this.packExpireAgeMillis = packExpireAgeMillis; | |||||
expire = null; | |||||
} | |||||
/** | /** | ||||
* Set the PackConfig used when (re-)writing packfiles. This allows to | * Set the PackConfig used when (re-)writing packfiles. This allows to | ||||
* influence how packs are written and to implement something similar to | * influence how packs are written and to implement something similar to | ||||
this.expire = expire; | this.expire = expire; | ||||
expireAgeMillis = -1; | expireAgeMillis = -1; | ||||
} | } | ||||
/** | |||||
* During gc() or prune() packfiles which are created or modified after or | |||||
* at <code>packExpire</code> will not be deleted. Only older packfiles may | |||||
* be deleted. If set to null then every packfile is a candidate for | |||||
* deletion. | |||||
* | |||||
* @param packExpire | |||||
* instant in time which defines packfile expiration | |||||
*/ | |||||
public void setPackExpire(Date packExpire) { | |||||
this.packExpire = packExpire; | |||||
packExpireAgeMillis = -1; | |||||
} | |||||
} | } |
/** The "pruneexpire" key */ | /** The "pruneexpire" key */ | ||||
public static final String CONFIG_KEY_PRUNEEXPIRE = "pruneexpire"; | public static final String CONFIG_KEY_PRUNEEXPIRE = "pruneexpire"; | ||||
/** | |||||
* The "prunepackexpire" key | |||||
* @since 4.3 | |||||
*/ | |||||
public static final String CONFIG_KEY_PRUNEPACKEXPIRE = "prunepackexpire"; | |||||
/** | /** | ||||
* The "aggressiveDepth" key | * The "aggressiveDepth" key | ||||
* @since 3.6 | * @since 3.6 |