You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

DfsGarbageCollectorTest.java 23KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691
  1. package org.eclipse.jgit.internal.storage.dfs;
  2. import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
  3. import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST;
  4. import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
  5. import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
  6. import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
  7. import static org.junit.Assert.assertEquals;
  8. import static org.junit.Assert.assertFalse;
  9. import static org.junit.Assert.assertNotNull;
  10. import static org.junit.Assert.assertSame;
  11. import static org.junit.Assert.assertTrue;
  12. import static org.junit.Assert.fail;
  13. import java.io.IOException;
  14. import java.util.Collections;
  15. import java.util.concurrent.TimeUnit;
  16. import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
  17. import org.eclipse.jgit.junit.MockSystemReader;
  18. import org.eclipse.jgit.junit.TestRepository;
  19. import org.eclipse.jgit.lib.AnyObjectId;
  20. import org.eclipse.jgit.lib.Ref;
  21. import org.eclipse.jgit.lib.Repository;
  22. import org.eclipse.jgit.revwalk.RevBlob;
  23. import org.eclipse.jgit.revwalk.RevCommit;
  24. import org.eclipse.jgit.revwalk.RevWalk;
  25. import org.eclipse.jgit.storage.pack.PackConfig;
  26. import org.eclipse.jgit.util.SystemReader;
  27. import org.junit.After;
  28. import org.junit.Before;
  29. import org.junit.Test;
  30. public class DfsGarbageCollectorTest {
  31. private TestRepository<InMemoryRepository> git;
  32. private InMemoryRepository repo;
  33. private DfsObjDatabase odb;
  34. private MockSystemReader mockSystemReader;
  35. @Before
  36. public void setUp() throws IOException {
  37. DfsRepositoryDescription desc = new DfsRepositoryDescription("test");
  38. git = new TestRepository<>(new InMemoryRepository(desc));
  39. repo = git.getRepository();
  40. odb = repo.getObjectDatabase();
  41. mockSystemReader = new MockSystemReader();
  42. SystemReader.setInstance(mockSystemReader);
  43. }
  44. @After
  45. public void tearDown() {
  46. SystemReader.setInstance(null);
  47. }
  48. @Test
  49. public void testCollectionWithNoGarbage() throws Exception {
  50. RevCommit commit0 = commit().message("0").create();
  51. RevCommit commit1 = commit().message("1").parent(commit0).create();
  52. git.update("master", commit1);
  53. assertTrue("commit0 reachable", isReachable(repo, commit0));
  54. assertTrue("commit1 reachable", isReachable(repo, commit1));
  55. // Packs start out as INSERT.
  56. assertEquals(2, odb.getPacks().length);
  57. for (DfsPackFile pack : odb.getPacks()) {
  58. assertEquals(INSERT, pack.getPackDescription().getPackSource());
  59. }
  60. gcNoTtl();
  61. // Single GC pack present with all objects.
  62. assertEquals(1, odb.getPacks().length);
  63. DfsPackFile pack = odb.getPacks()[0];
  64. assertEquals(GC, pack.getPackDescription().getPackSource());
  65. assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
  66. assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
  67. }
  68. @Test
  69. public void testRacyNoReusePrefersSmaller() throws Exception {
  70. StringBuilder msg = new StringBuilder();
  71. for (int i = 0; i < 100; i++) {
  72. msg.append(i).append(": i am a teapot\n");
  73. }
  74. RevBlob a = git.blob(msg.toString());
  75. RevCommit c0 = git.commit()
  76. .add("tea", a)
  77. .message("0")
  78. .create();
  79. msg.append("short and stout\n");
  80. RevBlob b = git.blob(msg.toString());
  81. RevCommit c1 = git.commit().parent(c0).tick(1)
  82. .add("tea", b)
  83. .message("1")
  84. .create();
  85. git.update("master", c1);
  86. PackConfig cfg = new PackConfig();
  87. cfg.setReuseObjects(false);
  88. cfg.setReuseDeltas(false);
  89. cfg.setDeltaCompress(false);
  90. cfg.setThreads(1);
  91. DfsGarbageCollector gc = new DfsGarbageCollector(repo);
  92. gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL
  93. gc.setPackConfig(cfg);
  94. run(gc);
  95. assertEquals(1, odb.getPacks().length);
  96. DfsPackDescription large = odb.getPacks()[0].getPackDescription();
  97. assertSame(PackSource.GC, large.getPackSource());
  98. cfg.setDeltaCompress(true);
  99. gc = new DfsGarbageCollector(repo);
  100. gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL
  101. gc.setPackConfig(cfg);
  102. run(gc);
  103. assertEquals(1, odb.getPacks().length);
  104. DfsPackDescription small = odb.getPacks()[0].getPackDescription();
  105. assertSame(PackSource.GC, small.getPackSource());
  106. assertTrue(
  107. "delta compression pack is smaller",
  108. small.getFileSize(PACK) < large.getFileSize(PACK));
  109. assertTrue(
  110. "large pack is older",
  111. large.getLastModified() < small.getLastModified());
  112. // Forcefully reinsert the older larger GC pack.
  113. odb.commitPack(Collections.singleton(large), null);
  114. odb.clearCache();
  115. assertEquals(2, odb.getPacks().length);
  116. gc = new DfsGarbageCollector(repo);
  117. gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL
  118. run(gc);
  119. assertEquals(1, odb.getPacks().length);
  120. DfsPackDescription rebuilt = odb.getPacks()[0].getPackDescription();
  121. assertEquals(small.getFileSize(PACK), rebuilt.getFileSize(PACK));
  122. }
  123. @Test
  124. public void testCollectionWithGarbage() throws Exception {
  125. RevCommit commit0 = commit().message("0").create();
  126. RevCommit commit1 = commit().message("1").parent(commit0).create();
  127. git.update("master", commit0);
  128. assertTrue("commit0 reachable", isReachable(repo, commit0));
  129. assertFalse("commit1 garbage", isReachable(repo, commit1));
  130. gcNoTtl();
  131. assertEquals(2, odb.getPacks().length);
  132. DfsPackFile gc = null;
  133. DfsPackFile garbage = null;
  134. for (DfsPackFile pack : odb.getPacks()) {
  135. DfsPackDescription d = pack.getPackDescription();
  136. if (d.getPackSource() == GC) {
  137. gc = pack;
  138. } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
  139. garbage = pack;
  140. } else {
  141. fail("unexpected " + d.getPackSource());
  142. }
  143. }
  144. assertNotNull("created GC pack", gc);
  145. assertTrue(isObjectInPack(commit0, gc));
  146. assertNotNull("created UNREACHABLE_GARBAGE pack", garbage);
  147. assertTrue(isObjectInPack(commit1, garbage));
  148. }
  149. @Test
  150. public void testCollectionWithGarbageAndGarbagePacksPurged()
  151. throws Exception {
  152. RevCommit commit0 = commit().message("0").create();
  153. RevCommit commit1 = commit().message("1").parent(commit0).create();
  154. git.update("master", commit0);
  155. gcWithTtl();
  156. // The repository should have a GC pack with commit0 and an
  157. // UNREACHABLE_GARBAGE pack with commit1.
  158. assertEquals(2, odb.getPacks().length);
  159. boolean gcPackFound = false;
  160. boolean garbagePackFound = false;
  161. for (DfsPackFile pack : odb.getPacks()) {
  162. DfsPackDescription d = pack.getPackDescription();
  163. if (d.getPackSource() == GC) {
  164. gcPackFound = true;
  165. assertTrue("has commit0", isObjectInPack(commit0, pack));
  166. assertFalse("no commit1", isObjectInPack(commit1, pack));
  167. } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
  168. garbagePackFound = true;
  169. assertFalse("no commit0", isObjectInPack(commit0, pack));
  170. assertTrue("has commit1", isObjectInPack(commit1, pack));
  171. } else {
  172. fail("unexpected " + d.getPackSource());
  173. }
  174. }
  175. assertTrue("gc pack found", gcPackFound);
  176. assertTrue("garbage pack found", garbagePackFound);
  177. gcWithTtl();
  178. // The gc operation should have removed UNREACHABLE_GARBAGE pack along with commit1.
  179. DfsPackFile[] packs = odb.getPacks();
  180. assertEquals(1, packs.length);
  181. assertEquals(GC, packs[0].getPackDescription().getPackSource());
  182. assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
  183. assertFalse("no commit1", isObjectInPack(commit1, packs[0]));
  184. }
  185. @Test
  186. public void testCollectionWithGarbageAndRereferencingGarbage()
  187. throws Exception {
  188. RevCommit commit0 = commit().message("0").create();
  189. RevCommit commit1 = commit().message("1").parent(commit0).create();
  190. git.update("master", commit0);
  191. gcWithTtl();
  192. // The repository should have a GC pack with commit0 and an
  193. // UNREACHABLE_GARBAGE pack with commit1.
  194. assertEquals(2, odb.getPacks().length);
  195. boolean gcPackFound = false;
  196. boolean garbagePackFound = false;
  197. for (DfsPackFile pack : odb.getPacks()) {
  198. DfsPackDescription d = pack.getPackDescription();
  199. if (d.getPackSource() == GC) {
  200. gcPackFound = true;
  201. assertTrue("has commit0", isObjectInPack(commit0, pack));
  202. assertFalse("no commit1", isObjectInPack(commit1, pack));
  203. } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
  204. garbagePackFound = true;
  205. assertFalse("no commit0", isObjectInPack(commit0, pack));
  206. assertTrue("has commit1", isObjectInPack(commit1, pack));
  207. } else {
  208. fail("unexpected " + d.getPackSource());
  209. }
  210. }
  211. assertTrue("gc pack found", gcPackFound);
  212. assertTrue("garbage pack found", garbagePackFound);
  213. git.update("master", commit1);
  214. gcWithTtl();
  215. // The gc operation should have removed the UNREACHABLE_GARBAGE pack and
  216. // moved commit1 into GC pack.
  217. DfsPackFile[] packs = odb.getPacks();
  218. assertEquals(1, packs.length);
  219. assertEquals(GC, packs[0].getPackDescription().getPackSource());
  220. assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
  221. assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
  222. }
  223. @Test
  224. public void testCollectionWithPureGarbageAndGarbagePacksPurged()
  225. throws Exception {
  226. RevCommit commit0 = commit().message("0").create();
  227. RevCommit commit1 = commit().message("1").parent(commit0).create();
  228. gcWithTtl();
  229. // The repository should have a single UNREACHABLE_GARBAGE pack with commit0
  230. // and commit1.
  231. DfsPackFile[] packs = odb.getPacks();
  232. assertEquals(1, packs.length);
  233. assertEquals(UNREACHABLE_GARBAGE, packs[0].getPackDescription().getPackSource());
  234. assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
  235. assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
  236. gcWithTtl();
  237. // The gc operation should have removed UNREACHABLE_GARBAGE pack along
  238. // with commit0 and commit1.
  239. assertEquals(0, odb.getPacks().length);
  240. }
  241. @Test
  242. public void testCollectionWithPureGarbageAndRereferencingGarbage()
  243. throws Exception {
  244. RevCommit commit0 = commit().message("0").create();
  245. RevCommit commit1 = commit().message("1").parent(commit0).create();
  246. gcWithTtl();
  247. // The repository should have a single UNREACHABLE_GARBAGE pack with commit0
  248. // and commit1.
  249. DfsPackFile[] packs = odb.getPacks();
  250. assertEquals(1, packs.length);
  251. DfsPackDescription pack = packs[0].getPackDescription();
  252. assertEquals(UNREACHABLE_GARBAGE, pack.getPackSource());
  253. assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
  254. assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
  255. git.update("master", commit0);
  256. gcWithTtl();
  257. // The gc operation should have moved commit0 into the GC pack and
  258. // removed UNREACHABLE_GARBAGE along with commit1.
  259. packs = odb.getPacks();
  260. assertEquals(1, packs.length);
  261. pack = packs[0].getPackDescription();
  262. assertEquals(GC, pack.getPackSource());
  263. assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
  264. assertFalse("no commit1", isObjectInPack(commit1, packs[0]));
  265. }
  266. @Test
  267. public void testCollectionWithGarbageCoalescence() throws Exception {
  268. RevCommit commit0 = commit().message("0").create();
  269. RevCommit commit1 = commit().message("1").parent(commit0).create();
  270. git.update("master", commit0);
  271. for (int i = 0; i < 3; i++) {
  272. commit1 = commit().message("g" + i).parent(commit1).create();
  273. // Make sure we don't have more than 1 UNREACHABLE_GARBAGE pack
  274. // because they're coalesced.
  275. gcNoTtl();
  276. assertEquals(1, countPacks(UNREACHABLE_GARBAGE));
  277. }
  278. }
  279. @Test
  280. public void testCollectionWithGarbageNoCoalescence() throws Exception {
  281. RevCommit commit0 = commit().message("0").create();
  282. RevCommit commit1 = commit().message("1").parent(commit0).create();
  283. git.update("master", commit0);
  284. for (int i = 0; i < 3; i++) {
  285. commit1 = commit().message("g" + i).parent(commit1).create();
  286. DfsGarbageCollector gc = new DfsGarbageCollector(repo);
  287. gc.setCoalesceGarbageLimit(0);
  288. gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
  289. run(gc);
  290. assertEquals(1 + i, countPacks(UNREACHABLE_GARBAGE));
  291. }
  292. }
  293. @Test
  294. public void testCollectionWithGarbageCoalescenceWithShortTtl()
  295. throws Exception {
  296. RevCommit commit0 = commit().message("0").create();
  297. RevCommit commit1 = commit().message("1").parent(commit0).create();
  298. git.update("master", commit0);
  299. // Create commits at 1 minute intervals with 1 hour ttl.
  300. for (int i = 0; i < 100; i++) {
  301. mockSystemReader.tick(60);
  302. commit1 = commit().message("g" + i).parent(commit1).create();
  303. DfsGarbageCollector gc = new DfsGarbageCollector(repo);
  304. gc.setGarbageTtl(1, TimeUnit.HOURS);
  305. run(gc);
  306. // Make sure we don't have more than 4 UNREACHABLE_GARBAGE packs
  307. // because all the packs that are created in a 20 minutes interval
  308. // should be coalesced and the packs older than 60 minutes should be
  309. // removed due to ttl.
  310. int count = countPacks(UNREACHABLE_GARBAGE);
  311. assertTrue("Garbage pack count should not exceed 4, but found "
  312. + count, count <= 4);
  313. }
  314. }
  315. @Test
  316. public void testCollectionWithGarbageCoalescenceWithLongTtl()
  317. throws Exception {
  318. RevCommit commit0 = commit().message("0").create();
  319. RevCommit commit1 = commit().message("1").parent(commit0).create();
  320. git.update("master", commit0);
  321. // Create commits at 1 hour intervals with 2 days ttl.
  322. for (int i = 0; i < 100; i++) {
  323. mockSystemReader.tick(3600);
  324. commit1 = commit().message("g" + i).parent(commit1).create();
  325. DfsGarbageCollector gc = new DfsGarbageCollector(repo);
  326. gc.setGarbageTtl(2, TimeUnit.DAYS);
  327. run(gc);
  328. // Make sure we don't have more than 3 UNREACHABLE_GARBAGE packs
  329. // because all the packs that are created in a single day should
  330. // be coalesced and the packs older than 2 days should be
  331. // removed due to ttl.
  332. int count = countPacks(UNREACHABLE_GARBAGE);
  333. assertTrue("Garbage pack count should not exceed 3, but found "
  334. + count, count <= 3);
  335. }
  336. }
  337. @Test
  338. public void testEstimateGcPackSizeInNewRepo() throws Exception {
  339. RevCommit commit0 = commit().message("0").create();
  340. RevCommit commit1 = commit().message("1").parent(commit0).create();
  341. git.update("master", commit1);
  342. // Packs start out as INSERT.
  343. long inputPacksSize = 32;
  344. assertEquals(2, odb.getPacks().length);
  345. for (DfsPackFile pack : odb.getPacks()) {
  346. assertEquals(INSERT, pack.getPackDescription().getPackSource());
  347. inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
  348. }
  349. gcNoTtl();
  350. // INSERT packs are combined into a single GC pack.
  351. assertEquals(1, odb.getPacks().length);
  352. DfsPackFile pack = odb.getPacks()[0];
  353. assertEquals(GC, pack.getPackDescription().getPackSource());
  354. assertEquals(inputPacksSize,
  355. pack.getPackDescription().getEstimatedPackSize());
  356. }
  357. @Test
  358. public void testEstimateGcPackSizeWithAnExistingGcPack() throws Exception {
  359. RevCommit commit0 = commit().message("0").create();
  360. RevCommit commit1 = commit().message("1").parent(commit0).create();
  361. git.update("master", commit1);
  362. gcNoTtl();
  363. RevCommit commit2 = commit().message("2").parent(commit1).create();
  364. git.update("master", commit2);
  365. // There will be one INSERT pack and one GC pack.
  366. assertEquals(2, odb.getPacks().length);
  367. boolean gcPackFound = false;
  368. boolean insertPackFound = false;
  369. long inputPacksSize = 32;
  370. for (DfsPackFile pack : odb.getPacks()) {
  371. DfsPackDescription d = pack.getPackDescription();
  372. if (d.getPackSource() == GC) {
  373. gcPackFound = true;
  374. } else if (d.getPackSource() == INSERT) {
  375. insertPackFound = true;
  376. } else {
  377. fail("unexpected " + d.getPackSource());
  378. }
  379. inputPacksSize += d.getFileSize(PACK) - 32;
  380. }
  381. assertTrue(gcPackFound);
  382. assertTrue(insertPackFound);
  383. gcNoTtl();
  384. // INSERT pack is combined into the GC pack.
  385. DfsPackFile pack = odb.getPacks()[0];
  386. assertEquals(GC, pack.getPackDescription().getPackSource());
  387. assertEquals(inputPacksSize,
  388. pack.getPackDescription().getEstimatedPackSize());
  389. }
  390. @Test
  391. public void testEstimateGcRestPackSizeInNewRepo() throws Exception {
  392. RevCommit commit0 = commit().message("0").create();
  393. RevCommit commit1 = commit().message("1").parent(commit0).create();
  394. git.update("refs/notes/note1", commit1);
  395. // Packs start out as INSERT.
  396. long inputPacksSize = 32;
  397. assertEquals(2, odb.getPacks().length);
  398. for (DfsPackFile pack : odb.getPacks()) {
  399. assertEquals(INSERT, pack.getPackDescription().getPackSource());
  400. inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
  401. }
  402. gcNoTtl();
  403. // INSERT packs are combined into a single GC_REST pack.
  404. assertEquals(1, odb.getPacks().length);
  405. DfsPackFile pack = odb.getPacks()[0];
  406. assertEquals(GC_REST, pack.getPackDescription().getPackSource());
  407. assertEquals(inputPacksSize,
  408. pack.getPackDescription().getEstimatedPackSize());
  409. }
  410. @Test
  411. public void testEstimateGcRestPackSizeWithAnExistingGcPack()
  412. throws Exception {
  413. RevCommit commit0 = commit().message("0").create();
  414. RevCommit commit1 = commit().message("1").parent(commit0).create();
  415. git.update("refs/notes/note1", commit1);
  416. gcNoTtl();
  417. RevCommit commit2 = commit().message("2").parent(commit1).create();
  418. git.update("refs/notes/note2", commit2);
  419. // There will be one INSERT pack and one GC_REST pack.
  420. assertEquals(2, odb.getPacks().length);
  421. boolean gcRestPackFound = false;
  422. boolean insertPackFound = false;
  423. long inputPacksSize = 32;
  424. for (DfsPackFile pack : odb.getPacks()) {
  425. DfsPackDescription d = pack.getPackDescription();
  426. if (d.getPackSource() == GC_REST) {
  427. gcRestPackFound = true;
  428. } else if (d.getPackSource() == INSERT) {
  429. insertPackFound = true;
  430. } else {
  431. fail("unexpected " + d.getPackSource());
  432. }
  433. inputPacksSize += d.getFileSize(PACK) - 32;
  434. }
  435. assertTrue(gcRestPackFound);
  436. assertTrue(insertPackFound);
  437. gcNoTtl();
  438. // INSERT pack is combined into the GC_REST pack.
  439. DfsPackFile pack = odb.getPacks()[0];
  440. assertEquals(GC_REST, pack.getPackDescription().getPackSource());
  441. assertEquals(inputPacksSize,
  442. pack.getPackDescription().getEstimatedPackSize());
  443. }
  444. @Test
  445. public void testEstimateGcPackSizesWithGcAndGcRestPacks() throws Exception {
  446. RevCommit commit0 = commit().message("0").create();
  447. git.update("head", commit0);
  448. RevCommit commit1 = commit().message("1").parent(commit0).create();
  449. git.update("refs/notes/note1", commit1);
  450. gcNoTtl();
  451. RevCommit commit2 = commit().message("2").parent(commit1).create();
  452. git.update("refs/notes/note2", commit2);
  453. // There will be one INSERT, one GC and one GC_REST packs.
  454. assertEquals(3, odb.getPacks().length);
  455. boolean gcPackFound = false;
  456. boolean gcRestPackFound = false;
  457. boolean insertPackFound = false;
  458. long gcPackSize = 0;
  459. long gcRestPackSize = 0;
  460. long insertPackSize = 0;
  461. for (DfsPackFile pack : odb.getPacks()) {
  462. DfsPackDescription d = pack.getPackDescription();
  463. if (d.getPackSource() == GC) {
  464. gcPackFound = true;
  465. gcPackSize = d.getFileSize(PACK);
  466. } else if (d.getPackSource() == GC_REST) {
  467. gcRestPackFound = true;
  468. gcRestPackSize = d.getFileSize(PACK);
  469. } else if (d.getPackSource() == INSERT) {
  470. insertPackFound = true;
  471. insertPackSize = d.getFileSize(PACK);
  472. } else {
  473. fail("unexpected " + d.getPackSource());
  474. }
  475. }
  476. assertTrue(gcPackFound);
  477. assertTrue(gcRestPackFound);
  478. assertTrue(insertPackFound);
  479. gcNoTtl();
  480. // In this test INSERT pack would be combined into the GC_REST pack.
  481. // But, as there is no good heuristic to know whether the new packs will
  482. // be combined into a GC pack or GC_REST packs, the new pick size is
  483. // considered while estimating both the GC and GC_REST packs.
  484. assertEquals(2, odb.getPacks().length);
  485. gcPackFound = false;
  486. gcRestPackFound = false;
  487. for (DfsPackFile pack : odb.getPacks()) {
  488. DfsPackDescription d = pack.getPackDescription();
  489. if (d.getPackSource() == GC) {
  490. gcPackFound = true;
  491. assertEquals(gcPackSize + insertPackSize - 32,
  492. pack.getPackDescription().getEstimatedPackSize());
  493. } else if (d.getPackSource() == GC_REST) {
  494. gcRestPackFound = true;
  495. assertEquals(gcRestPackSize + insertPackSize - 32,
  496. pack.getPackDescription().getEstimatedPackSize());
  497. } else {
  498. fail("unexpected " + d.getPackSource());
  499. }
  500. }
  501. assertTrue(gcPackFound);
  502. assertTrue(gcRestPackFound);
  503. }
  504. @Test
  505. public void testEstimateUnreachableGarbagePackSize() throws Exception {
  506. RevCommit commit0 = commit().message("0").create();
  507. RevCommit commit1 = commit().message("1").parent(commit0).create();
  508. git.update("master", commit0);
  509. assertTrue("commit0 reachable", isReachable(repo, commit0));
  510. assertFalse("commit1 garbage", isReachable(repo, commit1));
  511. // Packs start out as INSERT.
  512. long packSize0 = 0;
  513. long packSize1 = 0;
  514. assertEquals(2, odb.getPacks().length);
  515. for (DfsPackFile pack : odb.getPacks()) {
  516. DfsPackDescription d = pack.getPackDescription();
  517. assertEquals(INSERT, d.getPackSource());
  518. if (isObjectInPack(commit0, pack)) {
  519. packSize0 = d.getFileSize(PACK);
  520. } else if (isObjectInPack(commit1, pack)) {
  521. packSize1 = d.getFileSize(PACK);
  522. } else {
  523. fail("expected object not found in the pack");
  524. }
  525. }
  526. gcNoTtl();
  527. assertEquals(2, odb.getPacks().length);
  528. for (DfsPackFile pack : odb.getPacks()) {
  529. DfsPackDescription d = pack.getPackDescription();
  530. if (d.getPackSource() == GC) {
  531. // Even though just commit0 will end up in GC pack, because
  532. // there is no good way to know that up front, both the pack
  533. // sizes are considered while computing the estimated size of GC
  534. // pack.
  535. assertEquals(packSize0 + packSize1 - 32,
  536. d.getEstimatedPackSize());
  537. } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
  538. // commit1 is moved to UNREACHABLE_GARBAGE pack.
  539. assertEquals(packSize1, d.getEstimatedPackSize());
  540. } else {
  541. fail("unexpected " + d.getPackSource());
  542. }
  543. }
  544. }
  545. private TestRepository<InMemoryRepository>.CommitBuilder commit() {
  546. return git.commit();
  547. }
  548. private void gcNoTtl() throws IOException {
  549. DfsGarbageCollector gc = new DfsGarbageCollector(repo);
  550. gc.setGarbageTtl(0, TimeUnit.MILLISECONDS); // disable TTL
  551. run(gc);
  552. }
  553. private void gcWithTtl() throws IOException {
  554. // Move the clock forward by 1 minute and use the same as ttl.
  555. mockSystemReader.tick(60);
  556. DfsGarbageCollector gc = new DfsGarbageCollector(repo);
  557. gc.setGarbageTtl(1, TimeUnit.MINUTES);
  558. run(gc);
  559. }
  560. private void run(DfsGarbageCollector gc) throws IOException {
  561. // adjust the current time that will be used by the gc operation.
  562. mockSystemReader.tick(1);
  563. assertTrue("gc repacked", gc.pack(null));
  564. odb.clearCache();
  565. }
  566. private static boolean isReachable(Repository repo, AnyObjectId id)
  567. throws IOException {
  568. try (RevWalk rw = new RevWalk(repo)) {
  569. for (Ref ref : repo.getAllRefs().values()) {
  570. rw.markStart(rw.parseCommit(ref.getObjectId()));
  571. }
  572. for (RevCommit next; (next = rw.next()) != null;) {
  573. if (AnyObjectId.equals(next, id)) {
  574. return true;
  575. }
  576. }
  577. }
  578. return false;
  579. }
  580. private boolean isObjectInPack(AnyObjectId id, DfsPackFile pack)
  581. throws IOException {
  582. try (DfsReader reader = odb.newReader()) {
  583. return pack.hasObject(reader, id);
  584. }
  585. }
  586. private int countPacks(PackSource source) throws IOException {
  587. int cnt = 0;
  588. for (DfsPackFile pack : odb.getPacks()) {
  589. if (pack.getPackDescription().getPackSource() == source) {
  590. cnt++;
  591. }
  592. }
  593. return cnt;
  594. }
  595. }