GC: Remove empty references folders

After packaging references, the folders containing these references are
not deleted. In a busy repository, this causes operations to slow down
as traversing the references tree becomes longer.

Delete empty reference folders after the loose references have been
packed.
To avoid deleting a folder that was just created by another concurrent
operation, only delete folders that were not modified in the last 30
seconds.

Signed-off-by: Hector Oswaldo Caballero <hector.caballero@ericsson.com>
Change-Id: Ie79447d6121271cf5e25171be377ea396c7028e0

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java
new file mode 100644 (file)
index 0000000..76e1534
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright (C) 2018 Ericsson
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class GcDeleteEmptyRefsFoldersTest extends GcTestCase {
+       private static final String REF_FOLDER_01 = "01";
+       private static final String REF_FOLDER_02 = "02";
+
+       private Path refsDir;
+       private Path heads;
+
+       @Override
+       @Before
+       public void setUp() throws Exception {
+               super.setUp();
+               refsDir = Paths.get(repo.getDirectory().getAbsolutePath())
+                               .resolve("refs");
+               heads = refsDir.resolve("heads");
+       }
+
+       @Test
+       public void emptyRefFoldersAreDeleted() throws Exception {
+               FileTime fileTime = FileTime.from(Instant.now().minusSeconds(31));
+               Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01));
+               Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02));
+               Files.setLastModifiedTime(refDir01, fileTime);
+               Files.setLastModifiedTime(refDir02, fileTime);
+               assertTrue(refDir01.toFile().exists());
+               assertTrue(refDir02.toFile().exists());
+               gc.gc();
+
+               assertFalse(refDir01.toFile().exists());
+               assertFalse(refDir02.toFile().exists());
+       }
+
+       @Test
+       public void emptyRefFoldersAreKeptIfTheyAreTooRecent()
+                       throws Exception {
+               Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01));
+               Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02));
+               assertTrue(refDir01.toFile().exists());
+               assertTrue(refDir02.toFile().exists());
+               gc.gc();
+
+               assertTrue(refDir01.toFile().exists());
+               assertTrue(refDir02.toFile().exists());
+       }
+
+       @Test
+       public void nonEmptyRefsFoldersAreKept() throws Exception {
+               Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01));
+               Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02));
+               Path ref01 = Files.createFile(refDir01.resolve("ref01"));
+               Path ref02 = Files.createFile(refDir01.resolve("ref02"));
+               assertTrue(refDir01.toFile().exists());
+               assertTrue(refDir02.toFile().exists());
+               assertTrue(ref01.toFile().exists());
+               assertTrue(ref02.toFile().exists());
+               gc.gc();
+               assertTrue(refDir01.toFile().exists());
+               assertTrue(refDir02.toFile().exists());
+               assertTrue(ref01.toFile().exists());
+               assertTrue(ref02.toFile().exists());
+       }
+}

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
index 5669d49ae0afc61fa91922b86bdb3e326d34452f..56a742d5052da9f93d2539a8e3555a46896db66a 100644 (file)
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
@@ -65,6 +65,7 @@ import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -902,6 +903,7 @@ public class GC {
                        throw new IOException(e);
                }
                prunePacked();
+               deleteEmptyRefsFolders();
                deleteOrphans();
                deleteTempPacksIdx();
 
@@ -918,6 +920,49 @@ public class GC {
                return ref.getName().startsWith(Constants.R_TAGS);
        }
 
+       private void deleteEmptyRefsFolders() throws IOException {
+               Path refs = repo.getDirectory().toPath().resolve("refs"); //$NON-NLS-1$
+               try (Stream<Path> entries = Files.list(refs)) {
+                       Iterator<Path> iterator = entries.iterator();
+                       while (iterator.hasNext()) {
+                               try (Stream<Path> s = Files.list(iterator.next())) {
+                                       s.forEach(this::deleteDir);
+                               }
+                       }
+               }
+       }
+
+       private void deleteDir(Path dir) {
+               try (Stream<Path> dirs = Files.walk(dir)) {
+                       dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
+                                       .forEach(this::delete);
+               } catch (IOException e) {
+                       LOG.error(e.getMessage(), e);
+               }
+       }
+
+       private boolean isDirectory(Path p) {
+               return p.toFile().isDirectory();
+       }
+
+       private boolean delete(Path d) {
+               try {
+                       // Avoid deleting a folder that was just created so that concurrent
+                       // operations trying to create a reference are not impacted
+                       Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
+                       Instant lastModified = Files.getLastModifiedTime(d).toInstant();
+                       if (lastModified.isBefore(threshold)) {
+                               // If the folder is not empty, the delete operation will fail
+                               // silently. This is a cheaper alternative to filtering the
+                               // stream in the calling method.
+                               return d.toFile().delete();
+                       }
+               } catch (IOException e) {
+                       LOG.error(e.getMessage(), e);
+               }
+               return false;
+       }
+
        /**
         * Deletes orphans
         * <p>