Clean up orphan files in GC

An orphan file is either a bitmap or an idx file in pack folder,
and its corresponding pack file is missing.

Change-Id: I3c4cb1f7aa99dd7b398bdb8d513f528d7761edff
Signed-off-by: Hongkai Liu <hongkai.liu@ericsson.com>
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcOrphanFilesTest.java

@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2017 Ericsson
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class GcOrphanFilesTest extends GcTestCase {
+	private final static String PACK = "pack";
+
+	private final static String BITMAP_File_1 = PACK + "-1.bitmap";
+
+	private final static String IDX_File_2 = PACK + "-2.idx";
+
+	private final static String IDX_File_malformed = PACK + "-1234idx";
+
+	private final static String PACK_File_2 = PACK + "-2.pack";
+
+	private final static String PACK_File_3 = PACK + "-3.pack";
+
+	private File packDir;
+
+	@Before
+	public void setUp() throws Exception {
+		super.setUp();
+		packDir = new File(repo.getObjectsDirectory(), PACK);
+	}
+
+	@Test
+	public void bitmapAndIdxDeletedButPackNot() throws Exception {
+		createFileInPackFolder(BITMAP_File_1);
+		createFileInPackFolder(IDX_File_2);
+		createFileInPackFolder(PACK_File_3);
+		gc.gc();
+		assertFalse(new File(packDir, BITMAP_File_1).exists());
+		assertFalse(new File(packDir, IDX_File_2).exists());
+		assertTrue(new File(packDir, PACK_File_3).exists());
+	}
+
+	@Test
+	public void bitmapDeletedButIdxAndPackNot() throws Exception {
+		createFileInPackFolder(BITMAP_File_1);
+		createFileInPackFolder(IDX_File_2);
+		createFileInPackFolder(PACK_File_2);
+		createFileInPackFolder(PACK_File_3);
+		gc.gc();
+		assertFalse(new File(packDir, BITMAP_File_1).exists());
+		assertTrue(new File(packDir, IDX_File_2).exists());
+		assertTrue(new File(packDir, PACK_File_2).exists());
+		assertTrue(new File(packDir, PACK_File_3).exists());
+	}
+
+	@Test
+	public void malformedIdxNotDeleted() throws Exception {
+		createFileInPackFolder(IDX_File_malformed);
+		gc.gc();
+		assertTrue(new File(packDir, IDX_File_malformed).exists());
+	}
+
+	private void createFileInPackFolder(String fileName) throws IOException {
+		if (!packDir.exists() || !packDir.isDirectory()) {
+			assertTrue(packDir.mkdirs());
+		}
+		assertTrue(new File(packDir, fileName).createNewFile());
+	}
+
+	@Test
+	public void noSuchPackFolder() throws Exception {
+		assertTrue(packDir.delete());
+		gc.gc();
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java

@@ -55,6 +55,7 @@ import java.nio.channels.FileChannel;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.text.MessageFormat;
 import java.text.ParseException;
@@ -73,6 +74,8 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.eclipse.jgit.annotations.NonNull;
 import org.eclipse.jgit.dircache.DirCacheIterator;
@@ -126,6 +129,13 @@ public class GC {
 	private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
 			.compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$
 
+	private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$
+
+	private static final String BITMAP_EXT = "." //$NON-NLS-1$
+			+ PackExt.BITMAP_INDEX.getExtension();
+
+	private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$
+
 	private static final int DEFAULT_AUTOPACKLIMIT = 50;
 
 	private static final int DEFAULT_AUTOLIMIT = 6700;
@@ -654,12 +664,55 @@ public class GC {
 			throw new IOException(e);
 		}
 		prunePacked();
+		deleteOrphans();
 
 		lastPackedRefs = refsBefore;
 		lastRepackTime = time;
 		return ret;
 	}
 
+	/**
+	 * Deletes orphans
+	 * <p>
+	 * A file is considered an orphan if it is either a "bitmap" or an index
+	 * file, and its corresponding pack file is missing in the list.
+	 * </p>
+	 */
+	private void deleteOrphans() {
+		Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
+				"pack"); //$NON-NLS-1$
+		List<String> fileNames = null;
+		try (Stream<Path> files = Files.list(packDir)) {
+			fileNames = files.map(path -> path.getFileName().toString())
+					.filter(name -> {
+						return (name.endsWith(PACK_EXT)
+								|| name.endsWith(BITMAP_EXT)
+								|| name.endsWith(INDEX_EXT));
+					}).sorted(Collections.reverseOrder())
+					.collect(Collectors.toList());
+		} catch (IOException e1) {
+			// ignore
+		}
+		if (fileNames == null) {
+			return;
+		}
+
+		String base = null;
+		for (String n : fileNames) {
+			if (n.endsWith(PACK_EXT)) {
+				base = n.substring(0, n.lastIndexOf('.'));
+			} else {
+				if (base == null || !n.startsWith(base)) {
+					try {
+						Files.delete(new File(packDir.toFile(), n).toPath());
+					} catch (IOException e) {
+						LOG.error(e.getMessage(), e);
+					}
+				}
+			}
+		}
+	}
+
 	/**
 	 * @param ref
 	 *            the ref which log should be inspected