From 4eea4ea508900c35b2f200d36e3342b8198a8026 Mon Sep 17 00:00:00 2001 From: Hector Caballero Date: Mon, 5 Feb 2018 08:58:46 -0500 Subject: GC: Remove empty references folders After packaging references, the folders containing these references are not deleted. In a busy repository, this causes operations to slow down as traversing the references tree becomes longer. Delete empty reference folders after the loose references have been packed. To avoid deleting a folder that was just created by another concurrent operation, only delete folders that were not modified in the last 30 seconds. Signed-off-by: Hector Oswaldo Caballero Change-Id: Ie79447d6121271cf5e25171be377ea396c7028e0 Signed-off-by: Luca Milanesio Signed-off-by: David Pursehouse --- .../storage/file/GcDeleteEmptyRefsFoldersTest.java | 118 +++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java (limited to 'org.eclipse.jgit.test/tst/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java new file mode 100644 index 0000000000..76e1534c59 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2018 Ericsson + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.time.Instant; + +import org.junit.Before; +import org.junit.Test; + +public class GcDeleteEmptyRefsFoldersTest extends GcTestCase { + private static final String REF_FOLDER_01 = "01"; + private static final String REF_FOLDER_02 = "02"; + + private Path refsDir; + private Path heads; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + refsDir = Paths.get(repo.getDirectory().getAbsolutePath()) + .resolve("refs"); + heads = refsDir.resolve("heads"); + } + + @Test + public void emptyRefFoldersAreDeleted() throws Exception { + FileTime fileTime = FileTime.from(Instant.now().minusSeconds(31)); + Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01)); + Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02)); + Files.setLastModifiedTime(refDir01, fileTime); + Files.setLastModifiedTime(refDir02, fileTime); + assertTrue(refDir01.toFile().exists()); + assertTrue(refDir02.toFile().exists()); + gc.gc(); + + assertFalse(refDir01.toFile().exists()); + assertFalse(refDir02.toFile().exists()); + } + + @Test + public void emptyRefFoldersAreKeptIfTheyAreTooRecent() + throws Exception { + Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01)); + Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02)); + assertTrue(refDir01.toFile().exists()); + assertTrue(refDir02.toFile().exists()); + gc.gc(); + + assertTrue(refDir01.toFile().exists()); + assertTrue(refDir02.toFile().exists()); + } + + @Test + public void nonEmptyRefsFoldersAreKept() throws Exception { + Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01)); + Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02)); + Path ref01 = Files.createFile(refDir01.resolve("ref01")); + Path ref02 = Files.createFile(refDir01.resolve("ref02")); + assertTrue(refDir01.toFile().exists()); + assertTrue(refDir02.toFile().exists()); + assertTrue(ref01.toFile().exists()); + assertTrue(ref02.toFile().exists()); + gc.gc(); + assertTrue(refDir01.toFile().exists()); + assertTrue(refDir02.toFile().exists()); + assertTrue(ref01.toFile().exists()); + assertTrue(ref02.toFile().exists()); + } +} -- cgit v1.2.3 From 977726e5bb97e8aeb5d4ede69adbd8fc9801a14f Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Sat, 7 Jul 2018 23:35:20 +0100 Subject: Delete all loose refs empty directories Remove completely the empty directories under refs/ including the first level partition of the changes, when they are completely empty. Bug: 536777 Change-Id: I88304d34cc42435919c2d1480258684d993dfdca Signed-off-by: Luca Milanesio Signed-off-by: Matthias Sohn --- .../storage/file/GcDeleteEmptyRefsFoldersTest.java | 34 +++++++++++++++------- .../org/eclipse/jgit/internal/JGitText.properties | 1 + .../src/org/eclipse/jgit/internal/JGitText.java | 1 + .../org/eclipse/jgit/internal/storage/file/GC.java | 28 +++++++++++------- 4 files changed, 43 insertions(+), 21 deletions(-) (limited to 'org.eclipse.jgit.test/tst/org/eclipse/jgit') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java index 76e1534c59..b37cd7ae05 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcDeleteEmptyRefsFoldersTest.java @@ -46,6 +46,7 @@ package org.eclipse.jgit.internal.storage.file; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -56,8 +57,8 @@ import org.junit.Before; import org.junit.Test; public class GcDeleteEmptyRefsFoldersTest extends GcTestCase { - private static final String REF_FOLDER_01 = "01"; - private static final String REF_FOLDER_02 = "02"; + private static final String REF_FOLDER_01 = "A/B/01"; + private static final String REF_FOLDER_02 = "C/D/02"; private Path refsDir; private Path heads; @@ -74,23 +75,36 @@ public class GcDeleteEmptyRefsFoldersTest extends GcTestCase { @Test public void emptyRefFoldersAreDeleted() throws Exception { FileTime fileTime = FileTime.from(Instant.now().minusSeconds(31)); - Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01)); - Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02)); - Files.setLastModifiedTime(refDir01, fileTime); - Files.setLastModifiedTime(refDir02, fileTime); + Path refDir01 = Files.createDirectories(heads.resolve(REF_FOLDER_01)); + Path refDir02 = Files.createDirectories(heads.resolve(REF_FOLDER_02)); + setLastModifiedTime(fileTime, heads, REF_FOLDER_01); + setLastModifiedTime(fileTime, heads, REF_FOLDER_02); assertTrue(refDir01.toFile().exists()); assertTrue(refDir02.toFile().exists()); gc.gc(); assertFalse(refDir01.toFile().exists()); + assertFalse(refDir01.getParent().toFile().exists()); + assertFalse(refDir01.getParent().getParent().toFile().exists()); assertFalse(refDir02.toFile().exists()); + assertFalse(refDir02.getParent().toFile().exists()); + assertFalse(refDir02.getParent().getParent().toFile().exists()); + } + + private void setLastModifiedTime(FileTime fileTime, Path path, String folder) throws IOException { + long numParents = folder.chars().filter(c -> c == '/').count(); + Path folderPath = path.resolve(folder); + for(int folderLevel = 0; folderLevel <= numParents; folderLevel ++ ) { + Files.setLastModifiedTime(folderPath, fileTime); + folderPath = folderPath.getParent(); + } } @Test public void emptyRefFoldersAreKeptIfTheyAreTooRecent() throws Exception { - Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01)); - Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02)); + Path refDir01 = Files.createDirectories(heads.resolve(REF_FOLDER_01)); + Path refDir02 = Files.createDirectories(heads.resolve(REF_FOLDER_02)); assertTrue(refDir01.toFile().exists()); assertTrue(refDir02.toFile().exists()); gc.gc(); @@ -101,8 +115,8 @@ public class GcDeleteEmptyRefsFoldersTest extends GcTestCase { @Test public void nonEmptyRefsFoldersAreKept() throws Exception { - Path refDir01 = Files.createDirectory(heads.resolve(REF_FOLDER_01)); - Path refDir02 = Files.createDirectory(heads.resolve(REF_FOLDER_02)); + Path refDir01 = Files.createDirectories(heads.resolve(REF_FOLDER_01)); + Path refDir02 = Files.createDirectories(heads.resolve(REF_FOLDER_02)); Path ref01 = Files.createFile(refDir01.resolve("ref01")); Path ref02 = Files.createFile(refDir01.resolve("ref02")); assertTrue(refDir01.toFile().exists()); diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 68bb68e426..16f184babc 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -45,6 +45,7 @@ branchNameInvalid=Branch name {0} is not allowed buildingBitmaps=Building bitmaps cachedPacksPreventsIndexCreation=Using cached packs prevents index creation cachedPacksPreventsListingObjects=Using cached packs prevents listing objects +cannotAccessLastModifiedForSafeDeletion=Unable to access lastModifiedTime of file {0}, skip deletion since we cannot safely avoid race condition cannotBeCombined=Cannot be combined. cannotBeRecursiveWhenTreesAreIncluded=TreeWalk shouldn't be recursive when tree objects are included. cannotChangeActionOnComment=Cannot change action on comment line in git-rebase-todo file, old action: {0}, new action: {1}. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 56a24482be..4b7459c15b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -104,6 +104,7 @@ public class JGitText extends TranslationBundle { /***/ public String buildingBitmaps; /***/ public String cachedPacksPreventsIndexCreation; /***/ public String cachedPacksPreventsListingObjects; + /***/ public String cannotAccessLastModifiedForSafeDeletion; /***/ public String cannotBeCombined; /***/ public String cannotBeRecursiveWhenTreesAreIncluded; /***/ public String cannotChangeActionOnComment; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index eb5c1db50e..d99a485249 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -885,16 +885,31 @@ public class GC { private void deleteEmptyRefsFolders() throws IOException { Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS); + // Avoid deleting a folder that was created after the threshold so that concurrent + // operations trying to create a reference are not impacted + Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS); try (Stream entries = Files.list(refs)) { Iterator iterator = entries.iterator(); while (iterator.hasNext()) { try (Stream s = Files.list(iterator.next())) { - s.forEach(this::deleteDir); + s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir); } } } } + private boolean canBeSafelyDeleted(Path path, Instant threshold) { + try { + return Files.getLastModifiedTime(path).toInstant().isBefore(threshold); + } + catch (IOException e) { + LOG.warn(MessageFormat.format( + JGitText.get().cannotAccessLastModifiedForSafeDeletion, + path), e); + return false; + } + } + private void deleteDir(Path dir) { try (Stream dirs = Files.walk(dir)) { dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder()) @@ -910,16 +925,7 @@ public class GC { private void delete(Path d) { try { - // Avoid deleting a folder that was just created so that concurrent - // operations trying to create a reference are not impacted - Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS); - Instant lastModified = Files.getLastModifiedTime(d).toInstant(); - if (lastModified.isBefore(threshold)) { - // If the folder is not empty, the delete operation will fail - // silently. This is a cheaper alternative to filtering the - // stream in the calling method. - Files.delete(d); - } + Files.delete(d); } catch (IOException e) { LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d), e); -- cgit v1.2.3