Extend FileSnapshot for packfiles to also use checksum to detect changes

If the attributes of FileSnapshot don't detect modification of a
packfile read the packfile's checksum and compare it against the
checksum cached in the loaded packfile.

Since reading the checksum needs less IO than reloading the complete
packfile this may help to reduce the overhead to detect modficiation
when a gc completes while ObjectDirectory scans for packfiles in another
thread.

Bug: 546891
Change-Id: I9811b497eb11b8a85ae689081dc5d949ca8c4be5
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

1 files changed, 143 insertions, 0 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java
new file mode 100644
index 0000000000..0e25c494e5
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.text.ParseException;
+import java.util.Collection;
+import java.util.Random;
+import java.util.zip.Deflater;
+
+import org.eclipse.jgit.api.GarbageCollectCommand;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.storage.pack.PackConfig;
+import org.junit.Test;
+
+public class PackFileSnapshotTest extends RepositoryTestCase {
+
+	@Test
+	public void testSamePackDifferentCompressionDetectChecksumChanged()
+			throws Exception {
+		Git git = Git.wrap(db);
+		File f = writeTrashFile("file", "foobar ");
+		for (int i = 0; i < 10; i++) {
+			appendRandomLine(f);
+			git.add().addFilepattern("file").call();
+			git.commit().setMessage("message" + i).call();
+		}
+
+		FileBasedConfig c = db.getConfig();
+		c.setInt(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1);
+		c.save();
+		Collection<PackFile> packs = gc(Deflater.NO_COMPRESSION);
+		assertEquals("expected 1 packfile after gc", 1, packs.size());
+		PackFile p1 = packs.iterator().next();
+		PackFileSnapshot snapshot = p1.getFileSnapshot();
+
+		packs = gc(Deflater.BEST_COMPRESSION);
+		assertEquals("expected 1 packfile after gc", 1, packs.size());
+		PackFile p2 = packs.iterator().next();
+		File pf = p2.getPackFile();
+
+		// changing compression level with aggressive gc may change size,
+		// fileKey (on *nix) and checksum. Hence FileSnapshot.isModified can
+		// return true already based on size or fileKey.
+		// So the only thing we can test here is that we ensure that checksum
+		// also changed when we read it here in this test
+		assertTrue("expected snapshot to detect modified pack",
+				snapshot.isModified(pf));
+		assertTrue("expected checksum changed", snapshot.isChecksumChanged(pf));
+	}
+
+	private void appendRandomLine(File f) throws IOException {
+		try (Writer w = Files.newBufferedWriter(f.toPath(),
+				StandardOpenOption.APPEND)) {
+			w.append(randomLine(20));
+		}
+	}
+
+	private String randomLine(int len) {
+		final int a = 97; // 'a'
+		int z = 122; // 'z'
+		Random random = new Random();
+		StringBuilder buffer = new StringBuilder(len);
+		for (int i = 0; i < len; i++) {
+			int rnd = a + (int) (random.nextFloat() * (z - a + 1));
+			buffer.append((char) rnd);
+		}
+		buffer.append('\n');
+		return buffer.toString();
+	}
+
+	private Collection<PackFile> gc(int compressionLevel)
+			throws IOException, ParseException {
+		GC gc = new GC(db);
+		PackConfig pc = new PackConfig(db.getConfig());
+		pc.setCompressionLevel(compressionLevel);
+
+		pc.setSinglePack(true);
+
+		// --aggressive
+		pc.setDeltaSearchWindowSize(
+				GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_WINDOW);
+		pc.setMaxDeltaDepth(GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_DEPTH);
+		pc.setReuseObjects(false);
+
+		gc.setPackConfig(pc);
+		gc.setExpireAgeMillis(0);
+		gc.setPackExpireAgeMillis(0);
+		return gc.gc();
+	}
+
+}