jgit/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackInserterTest.java

/*
 * Copyright (C) 2017, Google Inc.
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *	 notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *	 copyright notice, this list of conditions and the following
 *	 disclaimer in the documentation and/or other materials provided
 *	 with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *	 names of its contributors may be used to endorse or promote
 *	 products derived from this software without specific prior
 *	 written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.eclipse.jgit.internal.storage.file;

import static java.util.Comparator.comparing;
import static java.util.stream.Collectors.toList;
import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.lessThan;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.fail;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Random;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.eclipse.jgit.dircache.DirCache;
import org.eclipse.jgit.dircache.DirCacheBuilder;
import org.eclipse.jgit.dircache.DirCacheEntry;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.junit.RepositoryTestCase;
import org.eclipse.jgit.lib.CommitBuilder;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.ObjectStream;
import org.eclipse.jgit.storage.file.WindowCacheConfig;
import org.eclipse.jgit.treewalk.CanonicalTreeParser;
import org.eclipse.jgit.util.IO;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

@SuppressWarnings("boxing")
public class PackInserterTest extends RepositoryTestCase {
	private WindowCacheConfig origWindowCacheConfig;

	private static final Random random = new Random(0);

	@Before
	public void setWindowCacheConfig() {
		origWindowCacheConfig = new WindowCacheConfig();
		origWindowCacheConfig.install();
	}

	@After
	public void resetWindowCacheConfig() {
		origWindowCacheConfig.install();
	}

	@Before
	public void emptyAtSetUp() throws Exception {
		assertEquals(0, listPacks().size());
		assertNoObjects();
	}

	@Test
	public void noFlush() throws Exception {
		try (PackInserter ins = newInserter()) {
			ins.insert(OBJ_BLOB, Constants.encode("foo contents"));
			// No flush.
		}
		assertNoObjects();
	}

	@Test
	public void flushEmptyPack() throws Exception {
		try (PackInserter ins = newInserter()) {
			ins.flush();
		}
		assertNoObjects();
	}

	@Test
	public void singlePack() throws Exception {
		ObjectId blobId;
		byte[] blob = Constants.encode("foo contents");
		ObjectId treeId;
		ObjectId commitId;
		byte[] commit;
		try (PackInserter ins = newInserter()) {
			blobId = ins.insert(OBJ_BLOB, blob);

			DirCache dc = DirCache.newInCore();
			DirCacheBuilder b = dc.builder();
			DirCacheEntry dce = new DirCacheEntry("foo");
			dce.setFileMode(FileMode.REGULAR_FILE);
			dce.setObjectId(blobId);
			b.add(dce);
			b.finish();
			treeId = dc.writeTree(ins);

			CommitBuilder cb = new CommitBuilder();
			cb.setTreeId(treeId);
			cb.setAuthor(author);
			cb.setCommitter(committer);
			cb.setMessage("Commit message");
			commit = cb.toByteArray();
			commitId = ins.insert(cb);
			ins.flush();
		}

		assertPacksOnly();
		List<Pack> packs = listPacks();
		assertEquals(1, packs.size());
		assertEquals(3, packs.get(0).getObjectCount());

		try (ObjectReader reader = db.newObjectReader()) {
			assertBlob(reader, blobId, blob);

			CanonicalTreeParser treeParser =
					new CanonicalTreeParser(null, reader, treeId);
			assertEquals("foo", treeParser.getEntryPathString());
			assertEquals(blobId, treeParser.getEntryObjectId());

			ObjectLoader commitLoader = reader.open(commitId);
			assertEquals(OBJ_COMMIT, commitLoader.getType());
			assertArrayEquals(commit, commitLoader.getBytes());
		}
	}

	@Test
	public void multiplePacks() throws Exception {
		ObjectId blobId1;
		ObjectId blobId2;
		byte[] blob1 = Constants.encode("blob1");
		byte[] blob2 = Constants.encode("blob2");

		try (PackInserter ins = newInserter()) {
			blobId1 = ins.insert(OBJ_BLOB, blob1);
			ins.flush();
			blobId2 = ins.insert(OBJ_BLOB, blob2);
			ins.flush();
		}

		assertPacksOnly();
		List<Pack> packs = listPacks();
		assertEquals(2, packs.size());
		assertEquals(1, packs.get(0).getObjectCount());
		assertEquals(1, packs.get(1).getObjectCount());

		try (ObjectReader reader = db.newObjectReader()) {
			assertBlob(reader, blobId1, blob1);
			assertBlob(reader, blobId2, blob2);
		}
	}

	@Test
	public void largeBlob() throws Exception {
		ObjectId blobId;
		byte[] blob = newLargeBlob();
		try (PackInserter ins = newInserter()) {
			assertThat(blob.length, greaterThan(ins.getBufferSize()));
			blobId =
					ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob));
			ins.flush();
		}

		assertPacksOnly();
		Collection<Pack> packs = listPacks();
		assertEquals(1, packs.size());
		Pack p = packs.iterator().next();
		assertEquals(1, p.getObjectCount());

		try (ObjectReader reader = db.newObjectReader()) {
			assertBlob(reader, blobId, blob);
		}
	}

	@Test
	public void overwriteExistingPack() throws Exception {
		ObjectId blobId;
		byte[] blob = Constants.encode("foo contents");

		try (PackInserter ins = newInserter()) {
			blobId = ins.insert(OBJ_BLOB, blob);
			ins.flush();
		}

		assertPacksOnly();
		List<Pack> packs = listPacks();
		assertEquals(1, packs.size());
		Pack pack = packs.get(0);
		assertEquals(1, pack.getObjectCount());

		String inode = getInode(pack.getPackFile());

		try (PackInserter ins = newInserter()) {
			ins.checkExisting(false);
			assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
			ins.flush();
		}

		assertPacksOnly();
		packs = listPacks();
		assertEquals(1, packs.size());
		pack = packs.get(0);
		assertEquals(1, pack.getObjectCount());

		if (inode != null) {
			// Old file was overwritten with new file, although objects were
			// equivalent.
			assertNotEquals(inode, getInode(pack.getPackFile()));
		}
	}

	@Test
	public void checkExisting() throws Exception {
		ObjectId blobId;
		byte[] blob = Constants.encode("foo contents");

		try (PackInserter ins = newInserter()) {
			blobId = ins.insert(OBJ_BLOB, blob);
			ins.insert(OBJ_BLOB, Constants.encode("another blob"));
			ins.flush();
		}

		assertPacksOnly();
		assertEquals(1, listPacks().size());

		try (PackInserter ins = newInserter()) {
			assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
			ins.flush();
		}

		assertPacksOnly();
		assertEquals(1, listPacks().size());

		try (PackInserter ins = newInserter()) {
			ins.checkExisting(false);
			assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
			ins.flush();
		}

		assertPacksOnly();
		assertEquals(2, listPacks().size());

		try (ObjectReader reader = db.newObjectReader()) {
			assertBlob(reader, blobId, blob);
		}
	}

	@Test
	public void insertSmallInputStreamRespectsCheckExisting() throws Exception {
		ObjectId blobId;
		byte[] blob = Constants.encode("foo contents");
		try (PackInserter ins = newInserter()) {
			assertThat(blob.length, lessThan(ins.getBufferSize()));
			blobId = ins.insert(OBJ_BLOB, blob);
			ins.insert(OBJ_BLOB, Constants.encode("another blob"));
			ins.flush();
		}

		assertPacksOnly();
		assertEquals(1, listPacks().size());

		try (PackInserter ins = newInserter()) {
			assertEquals(blobId,
					ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
			ins.flush();
		}

		assertPacksOnly();
		assertEquals(1, listPacks().size());
	}

	@Test
	public void insertLargeInputStreamBypassesCheckExisting() throws Exception {
		ObjectId blobId;
		byte[] blob = newLargeBlob();

		try (PackInserter ins = newInserter()) {
			assertThat(blob.length, greaterThan(ins.getBufferSize()));
			blobId = ins.insert(OBJ_BLOB, blob);
			ins.insert(OBJ_BLOB, Constants.encode("another blob"));
			ins.flush();
		}

		assertPacksOnly();
		assertEquals(1, listPacks().size());

		try (PackInserter ins = newInserter()) {
			assertEquals(blobId,
					ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
			ins.flush();
		}

		assertPacksOnly();
		assertEquals(2, listPacks().size());
	}

	@Test
	public void readBackSmallFiles() throws Exception {
		ObjectId blobId1;
		ObjectId blobId2;
		ObjectId blobId3;
		byte[] blob1 = Constants.encode("blob1");
		byte[] blob2 = Constants.encode("blob2");
		byte[] blob3 = Constants.encode("blob3");
		try (PackInserter ins = newInserter()) {
			assertThat(blob1.length, lessThan(ins.getBufferSize()));
			blobId1 = ins.insert(OBJ_BLOB, blob1);

			try (ObjectReader reader = ins.newReader()) {
				assertBlob(reader, blobId1, blob1);
			}

			// Read-back should not mess up the file pointer.
			blobId2 = ins.insert(OBJ_BLOB, blob2);
			ins.flush();

			blobId3 = ins.insert(OBJ_BLOB, blob3);
		}

		assertPacksOnly();
		List<Pack> packs = listPacks();
		assertEquals(1, packs.size());
		assertEquals(2, packs.get(0).getObjectCount());

		try (ObjectReader reader = db.newObjectReader()) {
			assertBlob(reader, blobId1, blob1);
			assertBlob(reader, blobId2, blob2);

			try {
				reader.open(blobId3);
				fail("Expected MissingObjectException");
			} catch (MissingObjectException expected) {
				// Expected.
			}
		}
	}

	@Test
	public void readBackLargeFile() throws Exception {
		ObjectId blobId;
		byte[] blob = newLargeBlob();

		WindowCacheConfig wcc = new WindowCacheConfig();
		wcc.setStreamFileThreshold(1024);
		wcc.install();
		try (ObjectReader reader = db.newObjectReader()) {
			assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
		}

		try (PackInserter ins = newInserter()) {
			blobId = ins.insert(OBJ_BLOB, blob);

			try (ObjectReader reader = ins.newReader()) {
				// Double-check threshold is propagated.
				assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
				assertBlob(reader, blobId, blob);
			}
		}

		assertPacksOnly();
		// Pack was streamed out to disk and read back from the temp file, but
		// ultimately rolled back and deleted.
		assertEquals(0, listPacks().size());

		try (ObjectReader reader = db.newObjectReader()) {
			try {
				reader.open(blobId);
				fail("Expected MissingObjectException");
			} catch (MissingObjectException expected) {
				// Expected.
			}
		}
	}

	@Test
	public void readBackFallsBackToRepo() throws Exception {
		ObjectId blobId;
		byte[] blob = Constants.encode("foo contents");
		try (PackInserter ins = newInserter()) {
			assertThat(blob.length, lessThan(ins.getBufferSize()));
			blobId = ins.insert(OBJ_BLOB, blob);
			ins.flush();
		}

		try (PackInserter ins = newInserter();
				ObjectReader reader = ins.newReader()) {
			assertBlob(reader, blobId, blob);
		}
	}

	@Test
	public void readBackSmallObjectBeforeLargeObject() throws Exception {
		WindowCacheConfig wcc = new WindowCacheConfig();
		wcc.setStreamFileThreshold(1024);
		wcc.install();

		ObjectId blobId1;
		ObjectId blobId2;
		ObjectId largeId;
		byte[] blob1 = Constants.encode("blob1");
		byte[] blob2 = Constants.encode("blob2");
		byte[] largeBlob = newLargeBlob();
		try (PackInserter ins = newInserter()) {
			assertThat(blob1.length, lessThan(ins.getBufferSize()));
			assertThat(largeBlob.length, greaterThan(ins.getBufferSize()));

			blobId1 = ins.insert(OBJ_BLOB, blob1);
			largeId = ins.insert(OBJ_BLOB, largeBlob);

			try (ObjectReader reader = ins.newReader()) {
				// A previous bug did not reset the file pointer to EOF after reading
				// back. We need to seek to something further back than a full buffer,
				// since the read-back code eagerly reads a full buffer's worth of data
				// from the file to pass to the inflater. If we seeked back just a small
				// amount, this step would consume the rest of the file, so the file
				// pointer would coincidentally end up back at EOF, hiding the bug.
				assertBlob(reader, blobId1, blob1);
			}

			blobId2 = ins.insert(OBJ_BLOB, blob2);

			try (ObjectReader reader = ins.newReader()) {
				assertBlob(reader, blobId1, blob1);
				assertBlob(reader, blobId2, blob2);
				assertBlob(reader, largeId, largeBlob);
			}

			ins.flush();
		}

		try (ObjectReader reader = db.newObjectReader()) {
				assertBlob(reader, blobId1, blob1);
				assertBlob(reader, blobId2, blob2);
				assertBlob(reader, largeId, largeBlob);
		}
	}

	private List<Pack> listPacks() throws Exception {
		List<Pack> fromOpenDb = listPacks(db);
		List<Pack> reopened;
		try (FileRepository db2 = new FileRepository(db.getDirectory())) {
			reopened = listPacks(db2);
		}
		assertEquals(fromOpenDb.size(), reopened.size());
		for (int i = 0 ; i < fromOpenDb.size(); i++) {
			Pack a = fromOpenDb.get(i);
			Pack b = reopened.get(i);
			assertEquals(a.getPackName(), b.getPackName());
			assertEquals(
					a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath());
			assertEquals(a.getObjectCount(), b.getObjectCount());
			a.getObjectCount();
		}
		return fromOpenDb;
	}

	private static List<Pack> listPacks(FileRepository db) throws Exception {
		return db.getObjectDatabase().getPacks().stream()
				.sorted(comparing(Pack::getPackName)).collect(toList());
	}

	private PackInserter newInserter() {
		return db.getObjectDatabase().newPackInserter();
	}

	private static byte[] newLargeBlob() {
		byte[] blob = new byte[10240];
		random.nextBytes(blob);
		return blob;
	}

	private static String getInode(File f) throws Exception {
		BasicFileAttributes attrs = Files.readAttributes(
				f.toPath(), BasicFileAttributes.class);
		Object k = attrs.fileKey();
		if (k == null) {
			return null;
		}
		Pattern p = Pattern.compile("^\\(dev=[^,]*,ino=(\\d+)\\)$");
		Matcher m = p.matcher(k.toString());
		return m.matches() ? m.group(1) : null;
	}

	private static void assertBlob(ObjectReader reader, ObjectId id,
			byte[] expected) throws Exception {
		ObjectLoader loader = reader.open(id);
		assertEquals(OBJ_BLOB, loader.getType());
		assertEquals(expected.length, loader.getSize());
		try (ObjectStream s = loader.openStream()) {
			int n = (int) s.getSize();
			byte[] actual = new byte[n];
			assertEquals(n, IO.readFully(s, actual, 0));
			assertArrayEquals(expected, actual);
		}
	}

	private void assertPacksOnly() throws Exception {
		new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx"))
				.assertNoBadFiles(db.getObjectDatabase().getDirectory());
	}

	private void assertNoObjects() throws Exception {
		new BadFileCollector(f -> true)
				.assertNoBadFiles(db.getObjectDatabase().getDirectory());
	}

	private static class BadFileCollector extends SimpleFileVisitor<Path> {
		private final Predicate<String> badName;
		private List<String> bad;

		BadFileCollector(Predicate<String> badName) {
			this.badName = badName;
		}

		void assertNoBadFiles(File f) throws IOException {
			bad = new ArrayList<>();
			Files.walkFileTree(f.toPath(), this);
			if (!bad.isEmpty()) {
				fail("unexpected files in object directory: " + bad);
			}
		}

		@Override
		public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
			Path fileName = file.getFileName();
			if (fileName != null) {
				String name = fileName.toString();
				if (!attrs.isDirectory() && badName.test(name)) {
					bad.add(name);
				}
			}
			return FileVisitResult.CONTINUE;
		}
	}
}