jgit/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java

/*
 * Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com>
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *   names of its contributors may be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package org.eclipse.jgit.internal.storage.file;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
//import java.nio.file.attribute.BasicFileAttributes;
import java.text.ParseException;
import java.time.Instant;
import java.util.Collection;
import java.util.Iterator;
import java.util.Random;
import java.util.zip.Deflater;

import org.eclipse.jgit.api.GarbageCollectCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.AbortedByHookException;
import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.NoFilepatternException;
import org.eclipse.jgit.api.errors.NoHeadException;
import org.eclipse.jgit.api.errors.NoMessageException;
import org.eclipse.jgit.api.errors.UnmergedPathsException;
import org.eclipse.jgit.api.errors.WrongRepositoryStateException;
import org.eclipse.jgit.junit.RepositoryTestCase;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.ConfigConstants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.storage.file.FileBasedConfig;
import org.eclipse.jgit.storage.pack.PackConfig;
import org.eclipse.jgit.util.FS;
import org.junit.Test;

public class PackFileSnapshotTest extends RepositoryTestCase {

	private static ObjectId unknownID = ObjectId
			.fromString("1234567890123456789012345678901234567890");

	@Test
	public void testSamePackDifferentCompressionDetectChecksumChanged()
			throws Exception {
		Git git = Git.wrap(db);
		File f = writeTrashFile("file", "foobar ");
		for (int i = 0; i < 10; i++) {
			appendRandomLine(f);
			git.add().addFilepattern("file").call();
			git.commit().setMessage("message" + i).call();
		}

		FileBasedConfig c = db.getConfig();
		c.setInt(ConfigConstants.CONFIG_GC_SECTION, null,
				ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1);
		c.save();
		Collection<PackFile> packs = gc(Deflater.NO_COMPRESSION);
		assertEquals("expected 1 packfile after gc", 1, packs.size());
		PackFile p1 = packs.iterator().next();
		PackFileSnapshot snapshot = p1.getFileSnapshot();

		packs = gc(Deflater.BEST_COMPRESSION);
		assertEquals("expected 1 packfile after gc", 1, packs.size());
		PackFile p2 = packs.iterator().next();
		File pf = p2.getPackFile();

		// changing compression level with aggressive gc may change size,
		// fileKey (on *nix) and checksum. Hence FileSnapshot.isModified can
		// return true already based on size or fileKey.
		// So the only thing we can test here is that we ensure that checksum
		// also changed when we read it here in this test
		assertTrue("expected snapshot to detect modified pack",
				snapshot.isModified(pf));
		assertTrue("expected checksum changed", snapshot.isChecksumChanged(pf));
	}

	private void appendRandomLine(File f, int length, Random r)
			throws IOException {
		try (Writer w = Files.newBufferedWriter(f.toPath(),
				StandardOpenOption.APPEND)) {
			appendRandomLine(w, length, r);
		}
	}

	private void appendRandomLine(File f) throws IOException {
		appendRandomLine(f, 5, new Random());
	}

	private void appendRandomLine(Writer w, int len, Random r)
			throws IOException {
		final int c1 = 32; // ' '
		int c2 = 126; // '~'
		for (int i = 0; i < len; i++) {
			w.append((char) (c1 + r.nextInt(1 + c2 - c1)));
		}
	}

	private ObjectId createTestRepo(int testDataSeed, int testDataLength)
			throws IOException, GitAPIException, NoFilepatternException,
			NoHeadException, NoMessageException, UnmergedPathsException,
			ConcurrentRefUpdateException, WrongRepositoryStateException,
			AbortedByHookException {
		// Create a repo with two commits and one file. Each commit adds
		// testDataLength number of bytes. Data are random bytes. Since the
		// seed for the random number generator is specified we will get
		// the same set of bytes for every run and for every platform
		Random r = new Random(testDataSeed);
		Git git = Git.wrap(db);
		File f = writeTrashFile("file", "foobar ");
		appendRandomLine(f, testDataLength, r);
		git.add().addFilepattern("file").call();
		git.commit().setMessage("message1").call();
		appendRandomLine(f, testDataLength, r);
		git.add().addFilepattern("file").call();
		return git.commit().setMessage("message2").call().getId();
	}

	// Try repacking so fast that you get two new packs which differ only in
	// content/chksum but have same name, size and lastmodified.
	// Since this is done with standard gc (which creates new tmp files and
	// renames them) the filekeys of the new packfiles differ helping jgit
	// to detect the fast modification
	@Test
	public void testDetectModificationAlthoughSameSizeAndModificationtime()
			throws Exception {
		int testDataSeed = 1;
		int testDataLength = 100;
		FileBasedConfig config = db.getConfig();
		// don't use mtime of the parent folder to detect pack file
		// modification.
		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
				ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, false);
		config.save();

		createTestRepo(testDataSeed, testDataLength);

		// repack to create initial packfile
		PackFile pf = repackAndCheck(5, null, null, null);
		Path packFilePath = pf.getPackFile().toPath();
		AnyObjectId chk1 = pf.getPackChecksum();
		String name = pf.getPackName();
		Long length = Long.valueOf(pf.getPackFile().length());
		FS fs = db.getFS();
		Instant m1 = fs.lastModifiedInstant(packFilePath);

		// Wait for a filesystem timer tick to enhance probability the rest of
		// this test is done before the filesystem timer ticks again.
		fsTick(packFilePath.toFile());

		// Repack to create packfile with same name, length. Lastmodified and
		// content and checksum are different since compression level differs
		AnyObjectId chk2 = repackAndCheck(6, name, length, chk1)
				.getPackChecksum();
		Instant m2 = fs.lastModifiedInstant(packFilePath);
		assumeFalse(m2.equals(m1));

		// Repack to create packfile with same name, length. Lastmodified is
		// equal to the previous one because we are in the same filesystem timer
		// slot. Content and its checksum are different
		AnyObjectId chk3 = repackAndCheck(7, name, length, chk2)
				.getPackChecksum();
		Instant m3 = fs.lastModifiedInstant(packFilePath);

		// ask for an unknown git object to force jgit to rescan the list of
		// available packs. If we would ask for a known objectid then JGit would
		// skip searching for new/modified packfiles
		db.getObjectDatabase().has(unknownID);
		assertEquals(chk3, getSinglePack(db.getObjectDatabase().getPacks())
				.getPackChecksum());
		assumeTrue(m3.equals(m2));
	}

	// Try repacking so fast that we get two new packs which differ only in
	// content and checksum but have same name, size and lastmodified.
	// To avoid that JGit detects modification by checking the filekey create
	// two new packfiles upfront and create copies of them. Then modify the
	// packfiles in-place by opening them for write and then copying the
	// content.
	@Test
	public void testDetectModificationAlthoughSameSizeAndModificationtimeAndFileKey()
			throws Exception {
		int testDataSeed = 1;
		int testDataLength = 100;
		FileBasedConfig config = db.getConfig();
		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
				ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, false);
		config.save();

		createTestRepo(testDataSeed, testDataLength);

		// Repack to create initial packfile. Make a copy of it
		PackFile pf = repackAndCheck(5, null, null, null);
		Path packFilePath = pf.getPackFile().toPath();
		Path packFileBasePath = packFilePath.resolveSibling(
				packFilePath.getFileName().toString().replaceAll(".pack", ""));
		AnyObjectId chk1 = pf.getPackChecksum();
		String name = pf.getPackName();
		Long length = Long.valueOf(pf.getPackFile().length());
		copyPack(packFileBasePath, "", ".copy1");

		// Repack to create second packfile. Make a copy of it
		AnyObjectId chk2 = repackAndCheck(6, name, length, chk1)
				.getPackChecksum();
		copyPack(packFileBasePath, "", ".copy2");

		// Repack to create third packfile
		AnyObjectId chk3 = repackAndCheck(7, name, length, chk2)
				.getPackChecksum();
		FS fs = db.getFS();
		Instant m3 = fs.lastModifiedInstant(packFilePath);
		db.getObjectDatabase().has(unknownID);
		assertEquals(chk3, getSinglePack(db.getObjectDatabase().getPacks())
				.getPackChecksum());

		// Wait for a filesystem timer tick to enhance probability the rest of
		// this test is done before the filesystem timer ticks.
		fsTick(packFilePath.toFile());

		// Copy copy2 to packfile data to force modification of packfile without
		// changing the packfile's filekey.
		copyPack(packFileBasePath, ".copy2", "");
		Instant m2 = fs.lastModifiedInstant(packFilePath);
		assumeFalse(m3.equals(m2));

		db.getObjectDatabase().has(unknownID);
		assertEquals(chk2, getSinglePack(db.getObjectDatabase().getPacks())
				.getPackChecksum());

		// Copy copy2 to packfile data to force modification of packfile without
		// changing the packfile's filekey.
		copyPack(packFileBasePath, ".copy1", "");
		Instant m1 = fs.lastModifiedInstant(packFilePath);
		assumeTrue(m2.equals(m1));
		db.getObjectDatabase().has(unknownID);
		assertEquals(chk1, getSinglePack(db.getObjectDatabase().getPacks())
				.getPackChecksum());
	}

	// Copy file from src to dst but avoid creating a new File (with new
	// FileKey) if dst already exists
	private Path copyFile(Path src, Path dst) throws IOException {
		if (Files.exists(dst)) {
			dst.toFile().setWritable(true);
			try (OutputStream dstOut = Files.newOutputStream(dst)) {
				Files.copy(src, dstOut);
				return dst;
			}
		}
		return Files.copy(src, dst, StandardCopyOption.REPLACE_EXISTING);
	}

	private Path copyPack(Path base, String srcSuffix, String dstSuffix)
			throws IOException {
		copyFile(Paths.get(base + ".idx" + srcSuffix),
				Paths.get(base + ".idx" + dstSuffix));
		copyFile(Paths.get(base + ".bitmap" + srcSuffix),
				Paths.get(base + ".bitmap" + dstSuffix));
		return copyFile(Paths.get(base + ".pack" + srcSuffix),
				Paths.get(base + ".pack" + dstSuffix));
	}

	private PackFile repackAndCheck(int compressionLevel, String oldName,
			Long oldLength, AnyObjectId oldChkSum)
			throws IOException, ParseException {
		PackFile p = getSinglePack(gc(compressionLevel));
		File pf = p.getPackFile();
		// The following two assumptions should not cause the test to fail. If
		// on a certain platform we get packfiles (containing the same git
		// objects) where the lengths differ or the checksums don't differ we
		// just skip this test. A reason for that could be that compression
		// works differently or random number generator works differently. Then
		// we have to search for more consistent test data or checkin these
		// packfiles as test resources
		assumeTrue(oldLength == null || pf.length() == oldLength.longValue());
		assumeTrue(oldChkSum == null || !p.getPackChecksum().equals(oldChkSum));
		assertTrue(oldName == null || p.getPackName().equals(oldName));
		return p;
	}

	private PackFile getSinglePack(Collection<PackFile> packs) {
		Iterator<PackFile> pIt = packs.iterator();
		PackFile p = pIt.next();
		assertFalse(pIt.hasNext());
		return p;
	}

	private Collection<PackFile> gc(int compressionLevel)
			throws IOException, ParseException {
		GC gc = new GC(db);
		PackConfig pc = new PackConfig(db.getConfig());
		pc.setCompressionLevel(compressionLevel);

		pc.setSinglePack(true);

		// --aggressive
		pc.setDeltaSearchWindowSize(
				GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_WINDOW);
		pc.setMaxDeltaDepth(GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_DEPTH);
		pc.setReuseObjects(false);

		gc.setPackConfig(pc);
		gc.setExpireAgeMillis(0);
		gc.setPackExpireAgeMillis(0);
		return gc.gc();
	}

}