You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

GC.java 50KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616
  1. /*
  2. * Copyright (C) 2012, Christian Halstrick <christian.halstrick@sap.com>
  3. * Copyright (C) 2011, Shawn O. Pearce <spearce@spearce.org>
  4. * and other copyright owners as documented in the project's IP log.
  5. *
  6. * This program and the accompanying materials are made available
  7. * under the terms of the Eclipse Distribution License v1.0 which
  8. * accompanies this distribution, is reproduced below, and is
  9. * available at http://www.eclipse.org/org/documents/edl-v10.php
  10. *
  11. * All rights reserved.
  12. *
  13. * Redistribution and use in source and binary forms, with or
  14. * without modification, are permitted provided that the following
  15. * conditions are met:
  16. *
  17. * - Redistributions of source code must retain the above copyright
  18. * notice, this list of conditions and the following disclaimer.
  19. *
  20. * - Redistributions in binary form must reproduce the above
  21. * copyright notice, this list of conditions and the following
  22. * disclaimer in the documentation and/or other materials provided
  23. * with the distribution.
  24. *
  25. * - Neither the name of the Eclipse Foundation, Inc. nor the
  26. * names of its contributors may be used to endorse or promote
  27. * products derived from this software without specific prior
  28. * written permission.
  29. *
  30. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  31. * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  32. * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  33. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  34. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  35. * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  37. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  38. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  39. * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  40. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  41. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  42. * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  43. */
  44. package org.eclipse.jgit.internal.storage.file;
  45. import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
  46. import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
  47. import java.io.File;
  48. import java.io.FileOutputStream;
  49. import java.io.IOException;
  50. import java.io.OutputStream;
  51. import java.io.PrintWriter;
  52. import java.io.StringWriter;
  53. import java.nio.channels.Channels;
  54. import java.nio.channels.FileChannel;
  55. import java.nio.file.DirectoryNotEmptyException;
  56. import java.nio.file.DirectoryStream;
  57. import java.nio.file.Files;
  58. import java.nio.file.Path;
  59. import java.nio.file.StandardCopyOption;
  60. import java.text.MessageFormat;
  61. import java.text.ParseException;
  62. import java.time.Instant;
  63. import java.time.temporal.ChronoUnit;
  64. import java.util.ArrayList;
  65. import java.util.Collection;
  66. import java.util.Collections;
  67. import java.util.Comparator;
  68. import java.util.Date;
  69. import java.util.HashMap;
  70. import java.util.HashSet;
  71. import java.util.Iterator;
  72. import java.util.LinkedList;
  73. import java.util.List;
  74. import java.util.Map;
  75. import java.util.Objects;
  76. import java.util.Set;
  77. import java.util.TreeMap;
  78. import java.util.concurrent.Callable;
  79. import java.util.concurrent.ExecutorService;
  80. import java.util.regex.Pattern;
  81. import java.util.stream.Collectors;
  82. import java.util.stream.Stream;
  83. import org.eclipse.jgit.annotations.NonNull;
  84. import org.eclipse.jgit.dircache.DirCacheIterator;
  85. import org.eclipse.jgit.errors.CancelledException;
  86. import org.eclipse.jgit.errors.CorruptObjectException;
  87. import org.eclipse.jgit.errors.IncorrectObjectTypeException;
  88. import org.eclipse.jgit.errors.MissingObjectException;
  89. import org.eclipse.jgit.errors.NoWorkTreeException;
  90. import org.eclipse.jgit.internal.JGitText;
  91. import org.eclipse.jgit.internal.storage.pack.PackExt;
  92. import org.eclipse.jgit.internal.storage.pack.PackWriter;
  93. import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
  94. import org.eclipse.jgit.lib.ConfigConstants;
  95. import org.eclipse.jgit.lib.Constants;
  96. import org.eclipse.jgit.lib.FileMode;
  97. import org.eclipse.jgit.lib.NullProgressMonitor;
  98. import org.eclipse.jgit.lib.ObjectId;
  99. import org.eclipse.jgit.lib.ObjectIdSet;
  100. import org.eclipse.jgit.lib.ObjectLoader;
  101. import org.eclipse.jgit.lib.ObjectReader;
  102. import org.eclipse.jgit.lib.ProgressMonitor;
  103. import org.eclipse.jgit.lib.Ref;
  104. import org.eclipse.jgit.lib.Ref.Storage;
  105. import org.eclipse.jgit.lib.RefDatabase;
  106. import org.eclipse.jgit.lib.ReflogEntry;
  107. import org.eclipse.jgit.lib.ReflogReader;
  108. import org.eclipse.jgit.lib.internal.WorkQueue;
  109. import org.eclipse.jgit.revwalk.ObjectWalk;
  110. import org.eclipse.jgit.revwalk.RevObject;
  111. import org.eclipse.jgit.revwalk.RevWalk;
  112. import org.eclipse.jgit.storage.pack.PackConfig;
  113. import org.eclipse.jgit.treewalk.TreeWalk;
  114. import org.eclipse.jgit.treewalk.filter.TreeFilter;
  115. import org.eclipse.jgit.util.FileUtils;
  116. import org.eclipse.jgit.util.GitDateParser;
  117. import org.eclipse.jgit.util.SystemReader;
  118. import org.slf4j.Logger;
  119. import org.slf4j.LoggerFactory;
  120. /**
  121. * A garbage collector for git
  122. * {@link org.eclipse.jgit.internal.storage.file.FileRepository}. Instances of
  123. * this class are not thread-safe. Don't use the same instance from multiple
  124. * threads.
  125. *
  126. * This class started as a copy of DfsGarbageCollector from Shawn O. Pearce
  127. * adapted to FileRepositories.
  128. */
  129. public class GC {
  130. private final static Logger LOG = LoggerFactory
  131. .getLogger(GC.class);
  132. private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$
  133. private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$
  134. private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
  135. .compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$
  136. private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$
  137. private static final String BITMAP_EXT = "." //$NON-NLS-1$
  138. + PackExt.BITMAP_INDEX.getExtension();
  139. private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$
  140. private static final String KEEP_EXT = "." + PackExt.KEEP.getExtension(); //$NON-NLS-1$
  141. private static final int DEFAULT_AUTOPACKLIMIT = 50;
  142. private static final int DEFAULT_AUTOLIMIT = 6700;
  143. private static volatile ExecutorService executor;
  144. /**
  145. * Set the executor for running auto-gc in the background. If no executor is
  146. * set JGit's own WorkQueue will be used.
  147. *
  148. * @param e
  149. * the executor to be used for running auto-gc
  150. * @since 4.8
  151. */
  152. public static void setExecutor(ExecutorService e) {
  153. executor = e;
  154. }
  155. private final FileRepository repo;
  156. private ProgressMonitor pm;
  157. private long expireAgeMillis = -1;
  158. private Date expire;
  159. private long packExpireAgeMillis = -1;
  160. private Date packExpire;
  161. private PackConfig pconfig;
  162. /**
  163. * the refs which existed during the last call to {@link #repack()}. This is
  164. * needed during {@link #prune(Set)} where we can optimize by looking at the
  165. * difference between the current refs and the refs which existed during
  166. * last {@link #repack()}.
  167. */
  168. private Collection<Ref> lastPackedRefs;
  169. /**
  170. * Holds the starting time of the last repack() execution. This is needed in
  171. * prune() to inspect only those reflog entries which have been added since
  172. * last repack().
  173. */
  174. private long lastRepackTime;
  175. /**
  176. * Whether gc should do automatic housekeeping
  177. */
  178. private boolean automatic;
  179. /**
  180. * Whether to run gc in a background thread
  181. */
  182. private boolean background;
  183. /**
  184. * Creates a new garbage collector with default values. An expirationTime of
  185. * two weeks and <code>null</code> as progress monitor will be used.
  186. *
  187. * @param repo
  188. * the repo to work on
  189. */
  190. public GC(FileRepository repo) {
  191. this.repo = repo;
  192. this.pconfig = new PackConfig(repo);
  193. this.pm = NullProgressMonitor.INSTANCE;
  194. }
  195. /**
  196. * Runs a garbage collector on a
  197. * {@link org.eclipse.jgit.internal.storage.file.FileRepository}. It will
  198. * <ul>
  199. * <li>pack loose references into packed-refs</li>
  200. * <li>repack all reachable objects into new pack files and delete the old
  201. * pack files</li>
  202. * <li>prune all loose objects which are now reachable by packs</li>
  203. * </ul>
  204. *
  205. * If {@link #setAuto(boolean)} was set to {@code true} {@code gc} will
  206. * first check whether any housekeeping is required; if not, it exits
  207. * without performing any work.
  208. *
  209. * If {@link #setBackground(boolean)} was set to {@code true}
  210. * {@code collectGarbage} will start the gc in the background, and then
  211. * return immediately. In this case, errors will not be reported except in
  212. * gc.log.
  213. *
  214. * @return the collection of
  215. * {@link org.eclipse.jgit.internal.storage.file.PackFile}'s which
  216. * are newly created
  217. * @throws java.io.IOException
  218. * @throws java.text.ParseException
  219. * If the configuration parameter "gc.pruneexpire" couldn't be
  220. * parsed
  221. */
  222. // TODO(ms): in 5.0 change signature and return Future<Collection<PackFile>>
  223. @SuppressWarnings("FutureReturnValueIgnored")
  224. public Collection<PackFile> gc() throws IOException, ParseException {
  225. if (!background) {
  226. return doGc();
  227. }
  228. final GcLog gcLog = new GcLog(repo);
  229. if (!gcLog.lock()) {
  230. // there is already a background gc running
  231. return Collections.emptyList();
  232. }
  233. Callable<Collection<PackFile>> gcTask = () -> {
  234. try {
  235. Collection<PackFile> newPacks = doGc();
  236. if (automatic && tooManyLooseObjects()) {
  237. String message = JGitText.get().gcTooManyUnpruned;
  238. gcLog.write(message);
  239. gcLog.commit();
  240. }
  241. return newPacks;
  242. } catch (IOException | ParseException e) {
  243. try {
  244. gcLog.write(e.getMessage());
  245. StringWriter sw = new StringWriter();
  246. e.printStackTrace(new PrintWriter(sw));
  247. gcLog.write(sw.toString());
  248. gcLog.commit();
  249. } catch (IOException e2) {
  250. e2.addSuppressed(e);
  251. LOG.error(e2.getMessage(), e2);
  252. }
  253. } finally {
  254. gcLog.unlock();
  255. }
  256. return Collections.emptyList();
  257. };
  258. // TODO(ms): in 5.0 change signature and return the Future
  259. executor().submit(gcTask);
  260. return Collections.emptyList();
  261. }
  262. private ExecutorService executor() {
  263. return (executor != null) ? executor : WorkQueue.getExecutor();
  264. }
  265. private Collection<PackFile> doGc() throws IOException, ParseException {
  266. if (automatic && !needGc()) {
  267. return Collections.emptyList();
  268. }
  269. pm.start(6 /* tasks */);
  270. packRefs();
  271. // TODO: implement reflog_expire(pm, repo);
  272. Collection<PackFile> newPacks = repack();
  273. prune(Collections.emptySet());
  274. // TODO: implement rerere_gc(pm);
  275. return newPacks;
  276. }
  277. /**
  278. * Loosen objects in a pack file which are not also in the newly-created
  279. * pack files.
  280. *
  281. * @param inserter
  282. * @param reader
  283. * @param pack
  284. * @param existing
  285. * @throws IOException
  286. */
  287. private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
  288. throws IOException {
  289. for (PackIndex.MutableEntry entry : pack) {
  290. ObjectId oid = entry.toObjectId();
  291. if (existing.contains(oid)) {
  292. continue;
  293. }
  294. existing.add(oid);
  295. ObjectLoader loader = reader.open(oid);
  296. inserter.insert(loader.getType(),
  297. loader.getSize(),
  298. loader.openStream(),
  299. true /* create this object even though it's a duplicate */);
  300. }
  301. }
  302. /**
  303. * Delete old pack files. What is 'old' is defined by specifying a set of
  304. * old pack files and a set of new pack files. Each pack file contained in
  305. * old pack files but not contained in new pack files will be deleted. If
  306. * preserveOldPacks is set, keep a copy of the pack file in the preserve
  307. * directory. If an expirationDate is set then pack files which are younger
  308. * than the expirationDate will not be deleted nor preserved.
  309. * <p>
  310. * If we're not immediately expiring loose objects, loosen any objects
  311. * in the old pack files which aren't in the new pack files.
  312. *
  313. * @param oldPacks
  314. * @param newPacks
  315. * @throws ParseException
  316. * @throws IOException
  317. */
  318. private void deleteOldPacks(Collection<PackFile> oldPacks,
  319. Collection<PackFile> newPacks) throws ParseException, IOException {
  320. HashSet<ObjectId> ids = new HashSet<>();
  321. for (PackFile pack : newPacks) {
  322. for (PackIndex.MutableEntry entry : pack) {
  323. ids.add(entry.toObjectId());
  324. }
  325. }
  326. ObjectReader reader = repo.newObjectReader();
  327. ObjectDirectory dir = repo.getObjectDatabase();
  328. ObjectDirectoryInserter inserter = dir.newInserter();
  329. boolean shouldLoosen = !"now".equals(getPruneExpireStr()) && //$NON-NLS-1$
  330. getExpireDate() < Long.MAX_VALUE;
  331. prunePreserved();
  332. long packExpireDate = getPackExpireDate();
  333. oldPackLoop: for (PackFile oldPack : oldPacks) {
  334. checkCancelled();
  335. String oldName = oldPack.getPackName();
  336. // check whether an old pack file is also among the list of new
  337. // pack files. Then we must not delete it.
  338. for (PackFile newPack : newPacks)
  339. if (oldName.equals(newPack.getPackName()))
  340. continue oldPackLoop;
  341. if (!oldPack.shouldBeKept()
  342. && repo.getFS()
  343. .lastModifiedInstant(oldPack.getPackFile())
  344. .toEpochMilli() < packExpireDate) {
  345. oldPack.close();
  346. if (shouldLoosen) {
  347. loosen(inserter, reader, oldPack, ids);
  348. }
  349. prunePack(oldName);
  350. }
  351. }
  352. // close the complete object database. That's my only chance to force
  353. // rescanning and to detect that certain pack files are now deleted.
  354. repo.getObjectDatabase().close();
  355. }
  356. /**
  357. * Deletes old pack file, unless 'preserve-oldpacks' is set, in which case it
  358. * moves the pack file to the preserved directory
  359. *
  360. * @param packFile
  361. * @param packName
  362. * @param ext
  363. * @param deleteOptions
  364. * @throws IOException
  365. */
  366. private void removeOldPack(File packFile, String packName, PackExt ext,
  367. int deleteOptions) throws IOException {
  368. if (pconfig.isPreserveOldPacks()) {
  369. File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
  370. FileUtils.mkdir(oldPackDir, true);
  371. String oldPackName = "pack-" + packName + ".old-" + ext.getExtension(); //$NON-NLS-1$ //$NON-NLS-2$
  372. File oldPackFile = new File(oldPackDir, oldPackName);
  373. FileUtils.rename(packFile, oldPackFile);
  374. } else {
  375. FileUtils.delete(packFile, deleteOptions);
  376. }
  377. }
  378. /**
  379. * Delete the preserved directory including all pack files within
  380. */
  381. private void prunePreserved() {
  382. if (pconfig.isPrunePreserved()) {
  383. try {
  384. FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
  385. FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
  386. } catch (IOException e) {
  387. // Deletion of the preserved pack files failed. Silently return.
  388. }
  389. }
  390. }
  391. /**
  392. * Delete files associated with a single pack file. First try to delete the
  393. * ".pack" file because on some platforms the ".pack" file may be locked and
  394. * can't be deleted. In such a case it is better to detect this early and
  395. * give up on deleting files for this packfile. Otherwise we may delete the
  396. * ".index" file and when failing to delete the ".pack" file we are left
  397. * with a ".pack" file without a ".index" file.
  398. *
  399. * @param packName
  400. */
  401. private void prunePack(String packName) {
  402. PackExt[] extensions = PackExt.values();
  403. try {
  404. // Delete the .pack file first and if this fails give up on deleting
  405. // the other files
  406. int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
  407. for (PackExt ext : extensions)
  408. if (PackExt.PACK.equals(ext)) {
  409. File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
  410. removeOldPack(f, packName, ext, deleteOptions);
  411. break;
  412. }
  413. // The .pack file has been deleted. Delete as many as the other
  414. // files as you can.
  415. deleteOptions |= FileUtils.IGNORE_ERRORS;
  416. for (PackExt ext : extensions) {
  417. if (!PackExt.PACK.equals(ext)) {
  418. File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
  419. removeOldPack(f, packName, ext, deleteOptions);
  420. }
  421. }
  422. } catch (IOException e) {
  423. // Deletion of the .pack file failed. Silently return.
  424. }
  425. }
  426. /**
  427. * Like "git prune-packed" this method tries to prune all loose objects
  428. * which can be found in packs. If certain objects can't be pruned (e.g.
  429. * because the filesystem delete operation fails) this is silently ignored.
  430. *
  431. * @throws java.io.IOException
  432. */
  433. public void prunePacked() throws IOException {
  434. ObjectDirectory objdb = repo.getObjectDatabase();
  435. Collection<PackFile> packs = objdb.getPacks();
  436. File objects = repo.getObjectsDirectory();
  437. String[] fanout = objects.list();
  438. if (fanout != null && fanout.length > 0) {
  439. pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
  440. try {
  441. for (String d : fanout) {
  442. checkCancelled();
  443. pm.update(1);
  444. if (d.length() != 2)
  445. continue;
  446. String[] entries = new File(objects, d).list();
  447. if (entries == null)
  448. continue;
  449. for (String e : entries) {
  450. checkCancelled();
  451. if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
  452. continue;
  453. ObjectId id;
  454. try {
  455. id = ObjectId.fromString(d + e);
  456. } catch (IllegalArgumentException notAnObject) {
  457. // ignoring the file that does not represent loose
  458. // object
  459. continue;
  460. }
  461. boolean found = false;
  462. for (PackFile p : packs) {
  463. checkCancelled();
  464. if (p.hasObject(id)) {
  465. found = true;
  466. break;
  467. }
  468. }
  469. if (found)
  470. FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
  471. | FileUtils.SKIP_MISSING
  472. | FileUtils.IGNORE_ERRORS);
  473. }
  474. }
  475. } finally {
  476. pm.endTask();
  477. }
  478. }
  479. }
  480. /**
  481. * Like "git prune" this method tries to prune all loose objects which are
  482. * unreferenced. If certain objects can't be pruned (e.g. because the
  483. * filesystem delete operation fails) this is silently ignored.
  484. *
  485. * @param objectsToKeep
  486. * a set of objects which should explicitly not be pruned
  487. * @throws java.io.IOException
  488. * @throws java.text.ParseException
  489. * If the configuration parameter "gc.pruneexpire" couldn't be
  490. * parsed
  491. */
  492. public void prune(Set<ObjectId> objectsToKeep) throws IOException,
  493. ParseException {
  494. long expireDate = getExpireDate();
  495. // Collect all loose objects which are old enough, not referenced from
  496. // the index and not in objectsToKeep
  497. Map<ObjectId, File> deletionCandidates = new HashMap<>();
  498. Set<ObjectId> indexObjects = null;
  499. File objects = repo.getObjectsDirectory();
  500. String[] fanout = objects.list();
  501. if (fanout == null || fanout.length == 0) {
  502. return;
  503. }
  504. pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
  505. fanout.length);
  506. try {
  507. for (String d : fanout) {
  508. checkCancelled();
  509. pm.update(1);
  510. if (d.length() != 2)
  511. continue;
  512. File dir = new File(objects, d);
  513. File[] entries = dir.listFiles();
  514. if (entries == null || entries.length == 0) {
  515. FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
  516. continue;
  517. }
  518. for (File f : entries) {
  519. checkCancelled();
  520. String fName = f.getName();
  521. if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
  522. continue;
  523. if (repo.getFS().lastModifiedInstant(f)
  524. .toEpochMilli() >= expireDate) {
  525. continue;
  526. }
  527. try {
  528. ObjectId id = ObjectId.fromString(d + fName);
  529. if (objectsToKeep.contains(id))
  530. continue;
  531. if (indexObjects == null)
  532. indexObjects = listNonHEADIndexObjects();
  533. if (indexObjects.contains(id))
  534. continue;
  535. deletionCandidates.put(id, f);
  536. } catch (IllegalArgumentException notAnObject) {
  537. // ignoring the file that does not represent loose
  538. // object
  539. }
  540. }
  541. }
  542. } finally {
  543. pm.endTask();
  544. }
  545. if (deletionCandidates.isEmpty()) {
  546. return;
  547. }
  548. checkCancelled();
  549. // From the set of current refs remove all those which have been handled
  550. // during last repack(). Only those refs will survive which have been
  551. // added or modified since the last repack. Only these can save existing
  552. // loose refs from being pruned.
  553. Collection<Ref> newRefs;
  554. if (lastPackedRefs == null || lastPackedRefs.isEmpty())
  555. newRefs = getAllRefs();
  556. else {
  557. Map<String, Ref> last = new HashMap<>();
  558. for (Ref r : lastPackedRefs) {
  559. last.put(r.getName(), r);
  560. }
  561. newRefs = new ArrayList<>();
  562. for (Ref r : getAllRefs()) {
  563. Ref old = last.get(r.getName());
  564. if (!equals(r, old)) {
  565. newRefs.add(r);
  566. }
  567. }
  568. }
  569. if (!newRefs.isEmpty()) {
  570. // There are new/modified refs! Check which loose objects are now
  571. // referenced by these modified refs (or their reflogentries).
  572. // Remove these loose objects
  573. // from the deletionCandidates. When the last candidate is removed
  574. // leave this method.
  575. ObjectWalk w = new ObjectWalk(repo);
  576. try {
  577. for (Ref cr : newRefs) {
  578. checkCancelled();
  579. w.markStart(w.parseAny(cr.getObjectId()));
  580. }
  581. if (lastPackedRefs != null)
  582. for (Ref lpr : lastPackedRefs) {
  583. w.markUninteresting(w.parseAny(lpr.getObjectId()));
  584. }
  585. removeReferenced(deletionCandidates, w);
  586. } finally {
  587. w.dispose();
  588. }
  589. }
  590. if (deletionCandidates.isEmpty())
  591. return;
  592. // Since we have not left the method yet there are still
  593. // deletionCandidates. Last chance for these objects not to be pruned is
  594. // that they are referenced by reflog entries. Even refs which currently
  595. // point to the same object as during last repack() may have
  596. // additional reflog entries not handled during last repack()
  597. ObjectWalk w = new ObjectWalk(repo);
  598. try {
  599. for (Ref ar : getAllRefs())
  600. for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
  601. checkCancelled();
  602. w.markStart(w.parseAny(id));
  603. }
  604. if (lastPackedRefs != null)
  605. for (Ref lpr : lastPackedRefs) {
  606. checkCancelled();
  607. w.markUninteresting(w.parseAny(lpr.getObjectId()));
  608. }
  609. removeReferenced(deletionCandidates, w);
  610. } finally {
  611. w.dispose();
  612. }
  613. if (deletionCandidates.isEmpty())
  614. return;
  615. checkCancelled();
  616. // delete all candidates which have survived: these are unreferenced
  617. // loose objects. Make a last check, though, to avoid deleting objects
  618. // that could have been referenced while the candidates list was being
  619. // built (by an incoming push, for example).
  620. Set<File> touchedFanout = new HashSet<>();
  621. for (File f : deletionCandidates.values()) {
  622. if (f.lastModified() < expireDate) {
  623. f.delete();
  624. touchedFanout.add(f.getParentFile());
  625. }
  626. }
  627. for (File f : touchedFanout) {
  628. FileUtils.delete(f,
  629. FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
  630. }
  631. repo.getObjectDatabase().close();
  632. }
  633. private long getExpireDate() throws ParseException {
  634. long expireDate = Long.MAX_VALUE;
  635. if (expire == null && expireAgeMillis == -1) {
  636. String pruneExpireStr = getPruneExpireStr();
  637. if (pruneExpireStr == null)
  638. pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
  639. expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
  640. .getInstance().getLocale());
  641. expireAgeMillis = -1;
  642. }
  643. if (expire != null)
  644. expireDate = expire.getTime();
  645. if (expireAgeMillis != -1)
  646. expireDate = System.currentTimeMillis() - expireAgeMillis;
  647. return expireDate;
  648. }
  649. private String getPruneExpireStr() {
  650. return repo.getConfig().getString(
  651. ConfigConstants.CONFIG_GC_SECTION, null,
  652. ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
  653. }
  654. private long getPackExpireDate() throws ParseException {
  655. long packExpireDate = Long.MAX_VALUE;
  656. if (packExpire == null && packExpireAgeMillis == -1) {
  657. String prunePackExpireStr = repo.getConfig().getString(
  658. ConfigConstants.CONFIG_GC_SECTION, null,
  659. ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
  660. if (prunePackExpireStr == null)
  661. prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
  662. packExpire = GitDateParser.parse(prunePackExpireStr, null,
  663. SystemReader.getInstance().getLocale());
  664. packExpireAgeMillis = -1;
  665. }
  666. if (packExpire != null)
  667. packExpireDate = packExpire.getTime();
  668. if (packExpireAgeMillis != -1)
  669. packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
  670. return packExpireDate;
  671. }
  672. /**
  673. * Remove all entries from a map which key is the id of an object referenced
  674. * by the given ObjectWalk
  675. *
  676. * @param id2File
  677. * @param w
  678. * @throws MissingObjectException
  679. * @throws IncorrectObjectTypeException
  680. * @throws IOException
  681. */
  682. private void removeReferenced(Map<ObjectId, File> id2File,
  683. ObjectWalk w) throws MissingObjectException,
  684. IncorrectObjectTypeException, IOException {
  685. RevObject ro = w.next();
  686. while (ro != null) {
  687. checkCancelled();
  688. if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
  689. return;
  690. }
  691. ro = w.next();
  692. }
  693. ro = w.nextObject();
  694. while (ro != null) {
  695. checkCancelled();
  696. if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
  697. return;
  698. }
  699. ro = w.nextObject();
  700. }
  701. }
  702. private static boolean equals(Ref r1, Ref r2) {
  703. if (r1 == null || r2 == null) {
  704. return false;
  705. }
  706. if (r1.isSymbolic()) {
  707. return r2.isSymbolic() && r1.getTarget().getName()
  708. .equals(r2.getTarget().getName());
  709. }
  710. return !r2.isSymbolic()
  711. && Objects.equals(r1.getObjectId(), r2.getObjectId());
  712. }
  713. /**
  714. * Packs all non-symbolic, loose refs into packed-refs.
  715. *
  716. * @throws java.io.IOException
  717. */
  718. public void packRefs() throws IOException {
  719. Collection<Ref> refs = repo.getRefDatabase()
  720. .getRefsByPrefix(Constants.R_REFS);
  721. List<String> refsToBePacked = new ArrayList<>(refs.size());
  722. pm.beginTask(JGitText.get().packRefs, refs.size());
  723. try {
  724. for (Ref ref : refs) {
  725. checkCancelled();
  726. if (!ref.isSymbolic() && ref.getStorage().isLoose())
  727. refsToBePacked.add(ref.getName());
  728. pm.update(1);
  729. }
  730. ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
  731. } finally {
  732. pm.endTask();
  733. }
  734. }
  735. /**
  736. * Packs all objects which reachable from any of the heads into one pack
  737. * file. Additionally all objects which are not reachable from any head but
  738. * which are reachable from any of the other refs (e.g. tags), special refs
  739. * (e.g. FETCH_HEAD) or index are packed into a separate pack file. Objects
  740. * included in pack files which have a .keep file associated are never
  741. * repacked. All old pack files which existed before are deleted.
  742. *
  743. * @return a collection of the newly created pack files
  744. * @throws java.io.IOException
  745. * when during reading of refs, index, packfiles, objects,
  746. * reflog-entries or during writing to the packfiles
  747. * {@link java.io.IOException} occurs
  748. */
  749. public Collection<PackFile> repack() throws IOException {
  750. Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
  751. long time = System.currentTimeMillis();
  752. Collection<Ref> refsBefore = getAllRefs();
  753. Set<ObjectId> allHeadsAndTags = new HashSet<>();
  754. Set<ObjectId> allHeads = new HashSet<>();
  755. Set<ObjectId> allTags = new HashSet<>();
  756. Set<ObjectId> nonHeads = new HashSet<>();
  757. Set<ObjectId> txnHeads = new HashSet<>();
  758. Set<ObjectId> tagTargets = new HashSet<>();
  759. Set<ObjectId> indexObjects = listNonHEADIndexObjects();
  760. RefDatabase refdb = repo.getRefDatabase();
  761. for (Ref ref : refsBefore) {
  762. checkCancelled();
  763. nonHeads.addAll(listRefLogObjects(ref, 0));
  764. if (ref.isSymbolic() || ref.getObjectId() == null) {
  765. continue;
  766. }
  767. if (isHead(ref)) {
  768. allHeads.add(ref.getObjectId());
  769. } else if (isTag(ref)) {
  770. allTags.add(ref.getObjectId());
  771. } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
  772. txnHeads.add(ref.getObjectId());
  773. } else {
  774. nonHeads.add(ref.getObjectId());
  775. }
  776. if (ref.getPeeledObjectId() != null) {
  777. tagTargets.add(ref.getPeeledObjectId());
  778. }
  779. }
  780. List<ObjectIdSet> excluded = new LinkedList<>();
  781. for (PackFile f : repo.getObjectDatabase().getPacks()) {
  782. checkCancelled();
  783. if (f.shouldBeKept())
  784. excluded.add(f.getIndex());
  785. }
  786. // Don't exclude tags that are also branch tips
  787. allTags.removeAll(allHeads);
  788. allHeadsAndTags.addAll(allHeads);
  789. allHeadsAndTags.addAll(allTags);
  790. // Hoist all branch tips and tags earlier in the pack file
  791. tagTargets.addAll(allHeadsAndTags);
  792. nonHeads.addAll(indexObjects);
  793. // Combine the GC_REST objects into the GC pack if requested
  794. if (pconfig.getSinglePack()) {
  795. allHeadsAndTags.addAll(nonHeads);
  796. nonHeads.clear();
  797. }
  798. List<PackFile> ret = new ArrayList<>(2);
  799. PackFile heads = null;
  800. if (!allHeadsAndTags.isEmpty()) {
  801. heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
  802. tagTargets, excluded);
  803. if (heads != null) {
  804. ret.add(heads);
  805. excluded.add(0, heads.getIndex());
  806. }
  807. }
  808. if (!nonHeads.isEmpty()) {
  809. PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
  810. tagTargets, excluded);
  811. if (rest != null)
  812. ret.add(rest);
  813. }
  814. if (!txnHeads.isEmpty()) {
  815. PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
  816. null, excluded);
  817. if (txn != null)
  818. ret.add(txn);
  819. }
  820. try {
  821. deleteOldPacks(toBeDeleted, ret);
  822. } catch (ParseException e) {
  823. // TODO: the exception has to be wrapped into an IOException because
  824. // throwing the ParseException directly would break the API, instead
  825. // we should throw a ConfigInvalidException
  826. throw new IOException(e);
  827. }
  828. prunePacked();
  829. deleteEmptyRefsFolders();
  830. deleteOrphans();
  831. deleteTempPacksIdx();
  832. lastPackedRefs = refsBefore;
  833. lastRepackTime = time;
  834. return ret;
  835. }
  836. private static boolean isHead(Ref ref) {
  837. return ref.getName().startsWith(Constants.R_HEADS);
  838. }
  839. private static boolean isTag(Ref ref) {
  840. return ref.getName().startsWith(Constants.R_TAGS);
  841. }
  842. private void deleteEmptyRefsFolders() throws IOException {
  843. Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
  844. // Avoid deleting a folder that was created after the threshold so that concurrent
  845. // operations trying to create a reference are not impacted
  846. Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
  847. try (Stream<Path> entries = Files.list(refs)) {
  848. Iterator<Path> iterator = entries.iterator();
  849. while (iterator.hasNext()) {
  850. try (Stream<Path> s = Files.list(iterator.next())) {
  851. s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
  852. }
  853. }
  854. }
  855. }
  856. private boolean canBeSafelyDeleted(Path path, Instant threshold) {
  857. try {
  858. return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
  859. }
  860. catch (IOException e) {
  861. LOG.warn(MessageFormat.format(
  862. JGitText.get().cannotAccessLastModifiedForSafeDeletion,
  863. path), e);
  864. return false;
  865. }
  866. }
  867. private void deleteDir(Path dir) {
  868. try (Stream<Path> dirs = Files.walk(dir)) {
  869. dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
  870. .forEach(this::delete);
  871. } catch (IOException e) {
  872. LOG.error(e.getMessage(), e);
  873. }
  874. }
  875. private boolean isDirectory(Path p) {
  876. return p.toFile().isDirectory();
  877. }
  878. private void delete(Path d) {
  879. try {
  880. Files.delete(d);
  881. } catch (DirectoryNotEmptyException e) {
  882. // Don't log
  883. } catch (IOException e) {
  884. LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
  885. e);
  886. }
  887. }
  888. /**
  889. * Deletes orphans
  890. * <p>
  891. * A file is considered an orphan if it is either a "bitmap" or an index
  892. * file, and its corresponding pack file is missing in the list.
  893. * </p>
  894. */
  895. private void deleteOrphans() {
  896. Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
  897. List<String> fileNames = null;
  898. try (Stream<Path> files = Files.list(packDir)) {
  899. fileNames = files.map(path -> path.getFileName().toString())
  900. .filter(name -> (name.endsWith(PACK_EXT)
  901. || name.endsWith(BITMAP_EXT)
  902. || name.endsWith(INDEX_EXT)
  903. || name.endsWith(KEEP_EXT)))
  904. // sort files with same base name in the order:
  905. // .pack, .keep, .index, .bitmap to avoid look ahead
  906. .sorted(Collections.reverseOrder())
  907. .collect(Collectors.toList());
  908. } catch (IOException e) {
  909. LOG.error(e.getMessage(), e);
  910. return;
  911. }
  912. if (fileNames == null) {
  913. return;
  914. }
  915. String base = null;
  916. for (String n : fileNames) {
  917. if (n.endsWith(PACK_EXT) || n.endsWith(KEEP_EXT)) {
  918. base = n.substring(0, n.lastIndexOf('.'));
  919. } else {
  920. if (base == null || !n.startsWith(base)) {
  921. try {
  922. Path delete = packDir.resolve(n);
  923. Files.delete(delete);
  924. LOG.warn(JGitText.get().deletedOrphanInPackDir, delete);
  925. } catch (IOException e) {
  926. LOG.error(e.getMessage(), e);
  927. }
  928. }
  929. }
  930. }
  931. }
  932. private void deleteTempPacksIdx() {
  933. Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
  934. Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
  935. if (!Files.exists(packDir)) {
  936. return;
  937. }
  938. try (DirectoryStream<Path> stream =
  939. Files.newDirectoryStream(packDir, "gc_*_tmp")) { //$NON-NLS-1$
  940. stream.forEach(t -> {
  941. try {
  942. Instant lastModified = Files.getLastModifiedTime(t)
  943. .toInstant();
  944. if (lastModified.isBefore(threshold)) {
  945. Files.deleteIfExists(t);
  946. }
  947. } catch (IOException e) {
  948. LOG.error(e.getMessage(), e);
  949. }
  950. });
  951. } catch (IOException e) {
  952. LOG.error(e.getMessage(), e);
  953. }
  954. }
  955. /**
  956. * @param ref
  957. * the ref which log should be inspected
  958. * @param minTime only reflog entries not older then this time are processed
  959. * @return the {@link ObjectId}s contained in the reflog
  960. * @throws IOException
  961. */
  962. private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
  963. ReflogReader reflogReader = repo.getReflogReader(ref.getName());
  964. if (reflogReader == null) {
  965. return Collections.emptySet();
  966. }
  967. List<ReflogEntry> rlEntries = reflogReader
  968. .getReverseEntries();
  969. if (rlEntries == null || rlEntries.isEmpty())
  970. return Collections.emptySet();
  971. Set<ObjectId> ret = new HashSet<>();
  972. for (ReflogEntry e : rlEntries) {
  973. if (e.getWho().getWhen().getTime() < minTime)
  974. break;
  975. ObjectId newId = e.getNewId();
  976. if (newId != null && !ObjectId.zeroId().equals(newId))
  977. ret.add(newId);
  978. ObjectId oldId = e.getOldId();
  979. if (oldId != null && !ObjectId.zeroId().equals(oldId))
  980. ret.add(oldId);
  981. }
  982. return ret;
  983. }
  984. /**
  985. * Returns a collection of all refs and additional refs.
  986. *
  987. * Additional refs which don't start with "refs/" are not returned because
  988. * they should not save objects from being garbage collected. Examples for
  989. * such references are ORIG_HEAD, MERGE_HEAD, FETCH_HEAD and
  990. * CHERRY_PICK_HEAD.
  991. *
  992. * @return a collection of refs pointing to live objects.
  993. * @throws IOException
  994. */
  995. private Collection<Ref> getAllRefs() throws IOException {
  996. RefDatabase refdb = repo.getRefDatabase();
  997. Collection<Ref> refs = refdb.getRefs();
  998. List<Ref> addl = refdb.getAdditionalRefs();
  999. if (!addl.isEmpty()) {
  1000. List<Ref> all = new ArrayList<>(refs.size() + addl.size());
  1001. all.addAll(refs);
  1002. // add additional refs which start with refs/
  1003. for (Ref r : addl) {
  1004. checkCancelled();
  1005. if (r.getName().startsWith(Constants.R_REFS)) {
  1006. all.add(r);
  1007. }
  1008. }
  1009. return all;
  1010. }
  1011. return refs;
  1012. }
  1013. /**
  1014. * Return a list of those objects in the index which differ from whats in
  1015. * HEAD
  1016. *
  1017. * @return a set of ObjectIds of changed objects in the index
  1018. * @throws IOException
  1019. * @throws CorruptObjectException
  1020. * @throws NoWorkTreeException
  1021. */
  1022. private Set<ObjectId> listNonHEADIndexObjects()
  1023. throws CorruptObjectException, IOException {
  1024. if (repo.isBare()) {
  1025. return Collections.emptySet();
  1026. }
  1027. try (TreeWalk treeWalk = new TreeWalk(repo)) {
  1028. treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
  1029. ObjectId headID = repo.resolve(Constants.HEAD);
  1030. if (headID != null) {
  1031. try (RevWalk revWalk = new RevWalk(repo)) {
  1032. treeWalk.addTree(revWalk.parseTree(headID));
  1033. }
  1034. }
  1035. treeWalk.setFilter(TreeFilter.ANY_DIFF);
  1036. treeWalk.setRecursive(true);
  1037. Set<ObjectId> ret = new HashSet<>();
  1038. while (treeWalk.next()) {
  1039. checkCancelled();
  1040. ObjectId objectId = treeWalk.getObjectId(0);
  1041. switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
  1042. case FileMode.TYPE_MISSING:
  1043. case FileMode.TYPE_GITLINK:
  1044. continue;
  1045. case FileMode.TYPE_TREE:
  1046. case FileMode.TYPE_FILE:
  1047. case FileMode.TYPE_SYMLINK:
  1048. ret.add(objectId);
  1049. continue;
  1050. default:
  1051. throw new IOException(MessageFormat.format(
  1052. JGitText.get().corruptObjectInvalidMode3,
  1053. String.format("%o", //$NON-NLS-1$
  1054. Integer.valueOf(treeWalk.getRawMode(0))),
  1055. (objectId == null) ? "null" : objectId.name(), //$NON-NLS-1$
  1056. treeWalk.getPathString(), //
  1057. repo.getIndexFile()));
  1058. }
  1059. }
  1060. return ret;
  1061. }
  1062. }
  1063. private PackFile writePack(@NonNull Set<? extends ObjectId> want,
  1064. @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
  1065. Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
  1066. throws IOException {
  1067. checkCancelled();
  1068. File tmpPack = null;
  1069. Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
  1070. // INDEX entries must be returned last, so the pack
  1071. // scanner does pick up the new pack until all the
  1072. // PackExt entries have been written.
  1073. if (o1 == o2) {
  1074. return 0;
  1075. }
  1076. if (o1 == PackExt.INDEX) {
  1077. return 1;
  1078. }
  1079. if (o2 == PackExt.INDEX) {
  1080. return -1;
  1081. }
  1082. return Integer.signum(o1.hashCode() - o2.hashCode());
  1083. });
  1084. try (PackWriter pw = new PackWriter(
  1085. pconfig,
  1086. repo.newObjectReader())) {
  1087. // prepare the PackWriter
  1088. pw.setDeltaBaseAsOffset(true);
  1089. pw.setReuseDeltaCommits(false);
  1090. if (tagTargets != null) {
  1091. pw.setTagTargets(tagTargets);
  1092. }
  1093. if (excludeObjects != null)
  1094. for (ObjectIdSet idx : excludeObjects)
  1095. pw.excludeObjects(idx);
  1096. pw.preparePack(pm, want, have, PackWriter.NONE, tags);
  1097. if (pw.getObjectCount() == 0)
  1098. return null;
  1099. checkCancelled();
  1100. // create temporary files
  1101. String id = pw.computeName().getName();
  1102. File packdir = repo.getObjectDatabase().getPackDirectory();
  1103. tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$
  1104. final String tmpBase = tmpPack.getName()
  1105. .substring(0, tmpPack.getName().lastIndexOf('.'));
  1106. File tmpIdx = new File(packdir, tmpBase + ".idx_tmp"); //$NON-NLS-1$
  1107. tmpExts.put(INDEX, tmpIdx);
  1108. if (!tmpIdx.createNewFile())
  1109. throw new IOException(MessageFormat.format(
  1110. JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
  1111. // write the packfile
  1112. try (FileOutputStream fos = new FileOutputStream(tmpPack);
  1113. FileChannel channel = fos.getChannel();
  1114. OutputStream channelStream = Channels
  1115. .newOutputStream(channel)) {
  1116. pw.writePack(pm, pm, channelStream);
  1117. channel.force(true);
  1118. }
  1119. // write the packindex
  1120. try (FileOutputStream fos = new FileOutputStream(tmpIdx);
  1121. FileChannel idxChannel = fos.getChannel();
  1122. OutputStream idxStream = Channels
  1123. .newOutputStream(idxChannel)) {
  1124. pw.writeIndex(idxStream);
  1125. idxChannel.force(true);
  1126. }
  1127. if (pw.prepareBitmapIndex(pm)) {
  1128. File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp"); //$NON-NLS-1$
  1129. tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
  1130. if (!tmpBitmapIdx.createNewFile())
  1131. throw new IOException(MessageFormat.format(
  1132. JGitText.get().cannotCreateIndexfile,
  1133. tmpBitmapIdx.getPath()));
  1134. try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
  1135. FileChannel idxChannel = fos.getChannel();
  1136. OutputStream idxStream = Channels
  1137. .newOutputStream(idxChannel)) {
  1138. pw.writeBitmapIndex(idxStream);
  1139. idxChannel.force(true);
  1140. }
  1141. }
  1142. // rename the temporary files to real files
  1143. File realPack = nameFor(id, ".pack"); //$NON-NLS-1$
  1144. repo.getObjectDatabase().closeAllPackHandles(realPack);
  1145. tmpPack.setReadOnly();
  1146. FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
  1147. for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
  1148. File tmpExt = tmpEntry.getValue();
  1149. tmpExt.setReadOnly();
  1150. File realExt = nameFor(id,
  1151. "." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$
  1152. try {
  1153. FileUtils.rename(tmpExt, realExt,
  1154. StandardCopyOption.ATOMIC_MOVE);
  1155. } catch (IOException e) {
  1156. File newExt = new File(realExt.getParentFile(),
  1157. realExt.getName() + ".new"); //$NON-NLS-1$
  1158. try {
  1159. FileUtils.rename(tmpExt, newExt,
  1160. StandardCopyOption.ATOMIC_MOVE);
  1161. } catch (IOException e2) {
  1162. newExt = tmpExt;
  1163. e = e2;
  1164. }
  1165. throw new IOException(MessageFormat.format(
  1166. JGitText.get().panicCantRenameIndexFile, newExt,
  1167. realExt), e);
  1168. }
  1169. }
  1170. boolean interrupted = false;
  1171. try {
  1172. FileSnapshot snapshot = FileSnapshot.save(realPack);
  1173. if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
  1174. snapshot.waitUntilNotRacy();
  1175. }
  1176. } catch (InterruptedException e) {
  1177. interrupted = true;
  1178. }
  1179. try {
  1180. return repo.getObjectDatabase().openPack(realPack);
  1181. } finally {
  1182. if (interrupted) {
  1183. // Re-set interrupted flag
  1184. Thread.currentThread().interrupt();
  1185. }
  1186. }
  1187. } finally {
  1188. if (tmpPack != null && tmpPack.exists())
  1189. tmpPack.delete();
  1190. for (File tmpExt : tmpExts.values()) {
  1191. if (tmpExt.exists())
  1192. tmpExt.delete();
  1193. }
  1194. }
  1195. }
  1196. private File nameFor(String name, String ext) {
  1197. File packdir = repo.getObjectDatabase().getPackDirectory();
  1198. return new File(packdir, "pack-" + name + ext); //$NON-NLS-1$
  1199. }
  1200. private void checkCancelled() throws CancelledException {
  1201. if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
  1202. throw new CancelledException(JGitText.get().operationCanceled);
  1203. }
  1204. }
  1205. /**
  1206. * A class holding statistical data for a FileRepository regarding how many
  1207. * objects are stored as loose or packed objects
  1208. */
  1209. public static class RepoStatistics {
  1210. /**
  1211. * The number of objects stored in pack files. If the same object is
  1212. * stored in multiple pack files then it is counted as often as it
  1213. * occurs in pack files.
  1214. */
  1215. public long numberOfPackedObjects;
  1216. /**
  1217. * The number of pack files
  1218. */
  1219. public long numberOfPackFiles;
  1220. /**
  1221. * The number of objects stored as loose objects.
  1222. */
  1223. public long numberOfLooseObjects;
  1224. /**
  1225. * The sum of the sizes of all files used to persist loose objects.
  1226. */
  1227. public long sizeOfLooseObjects;
  1228. /**
  1229. * The sum of the sizes of all pack files.
  1230. */
  1231. public long sizeOfPackedObjects;
  1232. /**
  1233. * The number of loose refs.
  1234. */
  1235. public long numberOfLooseRefs;
  1236. /**
  1237. * The number of refs stored in pack files.
  1238. */
  1239. public long numberOfPackedRefs;
  1240. /**
  1241. * The number of bitmaps in the bitmap indices.
  1242. */
  1243. public long numberOfBitmaps;
  1244. @Override
  1245. public String toString() {
  1246. final StringBuilder b = new StringBuilder();
  1247. b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$
  1248. b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$
  1249. b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$
  1250. b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$
  1251. b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$
  1252. b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$
  1253. b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$
  1254. b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$
  1255. return b.toString();
  1256. }
  1257. }
  1258. /**
  1259. * Returns information about objects and pack files for a FileRepository.
  1260. *
  1261. * @return information about objects and pack files for a FileRepository
  1262. * @throws java.io.IOException
  1263. */
  1264. public RepoStatistics getStatistics() throws IOException {
  1265. RepoStatistics ret = new RepoStatistics();
  1266. Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
  1267. for (PackFile f : packs) {
  1268. ret.numberOfPackedObjects += f.getIndex().getObjectCount();
  1269. ret.numberOfPackFiles++;
  1270. ret.sizeOfPackedObjects += f.getPackFile().length();
  1271. if (f.getBitmapIndex() != null)
  1272. ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
  1273. }
  1274. File objDir = repo.getObjectsDirectory();
  1275. String[] fanout = objDir.list();
  1276. if (fanout != null && fanout.length > 0) {
  1277. for (String d : fanout) {
  1278. if (d.length() != 2)
  1279. continue;
  1280. File[] entries = new File(objDir, d).listFiles();
  1281. if (entries == null)
  1282. continue;
  1283. for (File f : entries) {
  1284. if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
  1285. continue;
  1286. ret.numberOfLooseObjects++;
  1287. ret.sizeOfLooseObjects += f.length();
  1288. }
  1289. }
  1290. }
  1291. RefDatabase refDb = repo.getRefDatabase();
  1292. for (Ref r : refDb.getRefs()) {
  1293. Storage storage = r.getStorage();
  1294. if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
  1295. ret.numberOfLooseRefs++;
  1296. if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
  1297. ret.numberOfPackedRefs++;
  1298. }
  1299. return ret;
  1300. }
  1301. /**
  1302. * Set the progress monitor used for garbage collection methods.
  1303. *
  1304. * @param pm a {@link org.eclipse.jgit.lib.ProgressMonitor} object.
  1305. * @return this
  1306. */
  1307. public GC setProgressMonitor(ProgressMonitor pm) {
  1308. this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
  1309. return this;
  1310. }
  1311. /**
  1312. * During gc() or prune() each unreferenced, loose object which has been
  1313. * created or modified in the last <code>expireAgeMillis</code> milliseconds
  1314. * will not be pruned. Only older objects may be pruned. If set to 0 then
  1315. * every object is a candidate for pruning.
  1316. *
  1317. * @param expireAgeMillis
  1318. * minimal age of objects to be pruned in milliseconds.
  1319. */
  1320. public void setExpireAgeMillis(long expireAgeMillis) {
  1321. this.expireAgeMillis = expireAgeMillis;
  1322. expire = null;
  1323. }
  1324. /**
  1325. * During gc() or prune() packfiles which are created or modified in the
  1326. * last <code>packExpireAgeMillis</code> milliseconds will not be deleted.
  1327. * Only older packfiles may be deleted. If set to 0 then every packfile is a
  1328. * candidate for deletion.
  1329. *
  1330. * @param packExpireAgeMillis
  1331. * minimal age of packfiles to be deleted in milliseconds.
  1332. */
  1333. public void setPackExpireAgeMillis(long packExpireAgeMillis) {
  1334. this.packExpireAgeMillis = packExpireAgeMillis;
  1335. expire = null;
  1336. }
  1337. /**
  1338. * Set the PackConfig used when (re-)writing packfiles. This allows to
  1339. * influence how packs are written and to implement something similar to
  1340. * "git gc --aggressive"
  1341. *
  1342. * @param pconfig
  1343. * the {@link org.eclipse.jgit.storage.pack.PackConfig} used when
  1344. * writing packs
  1345. */
  1346. public void setPackConfig(@NonNull PackConfig pconfig) {
  1347. this.pconfig = pconfig;
  1348. }
  1349. /**
  1350. * During gc() or prune() each unreferenced, loose object which has been
  1351. * created or modified after or at <code>expire</code> will not be pruned.
  1352. * Only older objects may be pruned. If set to null then every object is a
  1353. * candidate for pruning.
  1354. *
  1355. * @param expire
  1356. * instant in time which defines object expiration
  1357. * objects with modification time before this instant are expired
  1358. * objects with modification time newer or equal to this instant
  1359. * are not expired
  1360. */
  1361. public void setExpire(Date expire) {
  1362. this.expire = expire;
  1363. expireAgeMillis = -1;
  1364. }
  1365. /**
  1366. * During gc() or prune() packfiles which are created or modified after or
  1367. * at <code>packExpire</code> will not be deleted. Only older packfiles may
  1368. * be deleted. If set to null then every packfile is a candidate for
  1369. * deletion.
  1370. *
  1371. * @param packExpire
  1372. * instant in time which defines packfile expiration
  1373. */
  1374. public void setPackExpire(Date packExpire) {
  1375. this.packExpire = packExpire;
  1376. packExpireAgeMillis = -1;
  1377. }
  1378. /**
  1379. * Set the {@code gc --auto} option.
  1380. *
  1381. * With this option, gc checks whether any housekeeping is required; if not,
  1382. * it exits without performing any work. Some JGit commands run
  1383. * {@code gc --auto} after performing operations that could create many
  1384. * loose objects.
  1385. * <p>
  1386. * Housekeeping is required if there are too many loose objects or too many
  1387. * packs in the repository. If the number of loose objects exceeds the value
  1388. * of the gc.auto option JGit GC consolidates all existing packs into a
  1389. * single pack (equivalent to {@code -A} option), whereas git-core would
  1390. * combine all loose objects into a single pack using {@code repack -d -l}.
  1391. * Setting the value of {@code gc.auto} to 0 disables automatic packing of
  1392. * loose objects.
  1393. * <p>
  1394. * If the number of packs exceeds the value of {@code gc.autoPackLimit},
  1395. * then existing packs (except those marked with a .keep file) are
  1396. * consolidated into a single pack by using the {@code -A} option of repack.
  1397. * Setting {@code gc.autoPackLimit} to 0 disables automatic consolidation of
  1398. * packs.
  1399. * <p>
  1400. * Like git the following jgit commands run auto gc:
  1401. * <ul>
  1402. * <li>fetch</li>
  1403. * <li>merge</li>
  1404. * <li>rebase</li>
  1405. * <li>receive-pack</li>
  1406. * </ul>
  1407. * The auto gc for receive-pack can be suppressed by setting the config
  1408. * option {@code receive.autogc = false}
  1409. *
  1410. * @param auto
  1411. * defines whether gc should do automatic housekeeping
  1412. */
  1413. public void setAuto(boolean auto) {
  1414. this.automatic = auto;
  1415. }
  1416. /**
  1417. * @param background
  1418. * whether to run the gc in a background thread.
  1419. */
  1420. void setBackground(boolean background) {
  1421. this.background = background;
  1422. }
  1423. private boolean needGc() {
  1424. if (tooManyPacks()) {
  1425. addRepackAllOption();
  1426. } else {
  1427. return tooManyLooseObjects();
  1428. }
  1429. // TODO run pre-auto-gc hook, if it fails return false
  1430. return true;
  1431. }
  1432. private void addRepackAllOption() {
  1433. // TODO: if JGit GC is enhanced to support repack's option -l this
  1434. // method needs to be implemented
  1435. }
  1436. /**
  1437. * @return {@code true} if number of packs > gc.autopacklimit (default 50)
  1438. */
  1439. boolean tooManyPacks() {
  1440. int autopacklimit = repo.getConfig().getInt(
  1441. ConfigConstants.CONFIG_GC_SECTION,
  1442. ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
  1443. DEFAULT_AUTOPACKLIMIT);
  1444. if (autopacklimit <= 0) {
  1445. return false;
  1446. }
  1447. // JGit always creates two packfiles, one for the objects reachable from
  1448. // branches, and another one for the rest
  1449. return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
  1450. }
  1451. /**
  1452. * Quickly estimate number of loose objects, SHA1 is distributed evenly so
  1453. * counting objects in one directory (bucket 17) is sufficient
  1454. *
  1455. * @return {@code true} if number of loose objects > gc.auto (default 6700)
  1456. */
  1457. boolean tooManyLooseObjects() {
  1458. int auto = getLooseObjectLimit();
  1459. if (auto <= 0) {
  1460. return false;
  1461. }
  1462. int n = 0;
  1463. int threshold = (auto + 255) / 256;
  1464. Path dir = repo.getObjectsDirectory().toPath().resolve("17"); //$NON-NLS-1$
  1465. if (!dir.toFile().exists()) {
  1466. return false;
  1467. }
  1468. try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
  1469. Path fileName = file.getFileName();
  1470. return file.toFile().isFile() && fileName != null
  1471. && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
  1472. .matches();
  1473. })) {
  1474. for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
  1475. .next()) {
  1476. if (++n > threshold) {
  1477. return true;
  1478. }
  1479. }
  1480. } catch (IOException e) {
  1481. LOG.error(e.getMessage(), e);
  1482. }
  1483. return false;
  1484. }
  1485. private int getLooseObjectLimit() {
  1486. return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
  1487. ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
  1488. }
  1489. }