GC: Delete stale temporary packs and indexes

When a GC operation is interrupted, temporary packs and indexes can be
left on the pack folder. In big, busy repositories this can lead to
significant amounts of wasted disk space if this interruption is done
with a certain frequency.

Remove stale temporary packs and indexes at the end of the GC process so
they do not accumulate. To avoid interfering with a possible concurrent
JGit GC process in the same repository, only delete temporary files that
are older than one day.

Change-Id: If9b6c1e57fac8a6a0ecc0a703089634caba4caae
Signed-off-by: Hector Caballero <hector.caballero@ericsson.com>

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTemporaryFilesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTemporaryFilesTest.java
new file mode 100644 (file)
index 0000000..59d544e
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcTemporaryFilesTest.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2017 Ericsson
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.time.Instant;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class GcTemporaryFilesTest extends GcTestCase {
+       private static final String TEMP_IDX = "gc_1234567890.idx_tmp";
+
+       private static final String TEMP_PACK = "gc_1234567890.pack_tmp";
+
+       private File packDir;
+
+       @Override
+       @Before
+       public void setUp() throws Exception {
+               super.setUp();
+               packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
+                               "pack").toFile(); //$NON-NLS-1$
+       }
+
+       @Test
+       public void oldTempPacksAndIdxAreDeleted() throws Exception {
+               File tempIndex = new File(packDir, TEMP_IDX);
+               File tempPack = new File(packDir, TEMP_PACK);
+               if (!packDir.exists() || !packDir.isDirectory()) {
+                       assertTrue(packDir.mkdirs());
+               }
+               assertTrue(tempPack.createNewFile());
+               assertTrue(tempIndex.createNewFile());
+               assertTrue(tempIndex.exists());
+               assertTrue(tempPack.exists());
+               long _24HoursBefore = Instant.now().toEpochMilli()
+                               - 24 * 60 * 62 * 1000;
+               tempIndex.setLastModified(_24HoursBefore);
+               tempPack.setLastModified(_24HoursBefore);
+               gc.gc();
+               assertFalse(tempIndex.exists());
+               assertFalse(tempPack.exists());
+       }
+
+       @Test
+       public void recentTempPacksAndIdxAreNotDeleted() throws Exception {
+               File tempIndex = new File(packDir, TEMP_IDX);
+               File tempPack = new File(packDir, TEMP_PACK);
+               if (!packDir.exists() || !packDir.isDirectory()) {
+                       assertTrue(packDir.mkdirs());
+               }
+               assertTrue(tempPack.createNewFile());
+               assertTrue(tempIndex.createNewFile());
+               assertTrue(tempIndex.exists());
+               assertTrue(tempPack.exists());
+               gc.gc();
+               assertTrue(tempIndex.exists());
+               assertTrue(tempPack.exists());
+       }
+}

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
index f992a334084f994e1a9f38d88852321ad37e30af..928e52d6b4c83c1a87616379258848e434c74ef1 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
@@ -61,6 +61,8 @@ import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.text.MessageFormat;
 import java.text.ParseException;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -903,6 +905,7 @@ public class GC {
                }
                prunePacked();
                deleteOrphans();
+               deleteTempPacksIdx();
 
                lastPackedRefs = refsBefore;
                lastRepackTime = time;
@@ -959,6 +962,28 @@ public class GC {
                }
        }
 
+       private void deleteTempPacksIdx() {
+               Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
+                               "pack"); //$NON-NLS-1$
+               Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
+               try {
+                       Files.newDirectoryStream(packDir, "gc_*_tmp") //$NON-NLS-1$
+                                       .forEach(t -> {
+                                               try {
+                                                       Instant lastModified = Files.getLastModifiedTime(t)
+                                                                       .toInstant();
+                                                       if (lastModified.isBefore(threshold)) {
+                                                               Files.deleteIfExists(t);
+                                                       }
+                                               } catch (IOException e) {
+                                                       LOG.error(e.getMessage(), e);
+                                               }
+                                       });
+               } catch (IOException e) {
+                       LOG.error(e.getMessage(), e);
+               }
+       }
+
        /**
         * @param ref
         *            the ref which log should be inspected