Add the "compression-level" option to all ArchiveCommand formats

Different archive formats support a compression level in the range
[0-9]. The value 0 is for lowest compressions and 9 for highest. Highest
levels produce output files of smaller sizes but require more memory to
do the compression.

This change allows passing a "compression-level" option to the git
archive command and implements using it for different file formats.

Change-Id: I5758f691c37ba630dbac24db67bb7da827bbc8e1
Signed-off-by: Youssef Elghareeb <ghareeb@google.com>
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

1 files changed, 106 insertions, 78 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java
index 0f98a63f5a..f2cceac4b3 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java
@@ -12,6 +12,7 @@ package org.eclipse.jgit.api;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 import java.beans.Statement;
 import java.io.BufferedInputStream;
@@ -28,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import java.util.Random;
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.ArchiveInputStream;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -55,6 +57,7 @@ import org.eclipse.jgit.util.IO;
 import org.eclipse.jgit.util.StringUtils;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 public class ArchiveCommandTest extends RepositoryTestCase {
@@ -184,88 +187,63 @@ public class ArchiveCommandTest extends RepositoryTestCase {
 
 	@Test
 	public void archiveHeadAllFilesTarTimestamps() throws Exception {
-		try (Git git = new Git(db)) {
-			createTestContent(git);
-			String fmt = "tar";
-			File archive = new File(getTemporaryDirectory(),
-					"archive." + format);
-			archive(git, archive, fmt);
-			ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive));
-
-			try (InputStream fi = Files.newInputStream(archive.toPath());
-					InputStream bi = new BufferedInputStream(fi);
-					ArchiveInputStream o = new TarArchiveInputStream(bi)) {
-				assertEntries(o);
-			}
-
-			Thread.sleep(WAIT);
-			archive(git, archive, fmt);
-			assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1,
-					ObjectId.fromRaw(IO.readFully(archive)));
-		}
+		archiveHeadAllFiles("tar");
 	}
 
 	@Test
 	public void archiveHeadAllFilesTgzTimestamps() throws Exception {
-		try (Git git = new Git(db)) {
-			createTestContent(git);
-			String fmt = "tgz";
-			File archive = new File(getTemporaryDirectory(),
-					"archive." + fmt);
-			archive(git, archive, fmt);
-			ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive));
+		archiveHeadAllFiles("tgz");
+	}
 
-			try (InputStream fi = Files.newInputStream(archive.toPath());
-					InputStream bi = new BufferedInputStream(fi);
-					InputStream gzi = new GzipCompressorInputStream(bi);
-					ArchiveInputStream o = new TarArchiveInputStream(gzi)) {
-				assertEntries(o);
-			}
+	@Test
+	public void archiveHeadAllFilesTbz2Timestamps() throws Exception {
+		archiveHeadAllFiles("tbz2");
+	}
 
-			Thread.sleep(WAIT);
-			archive(git, archive, fmt);
-			assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1,
-					ObjectId.fromRaw(IO.readFully(archive)));
-		}
+	@Test
+	public void archiveHeadAllFilesTxzTimestamps() throws Exception {
+		archiveHeadAllFiles("txz");
 	}
 
 	@Test
-	public void archiveHeadAllFilesTbz2Timestamps() throws Exception {
-		try (Git git = new Git(db)) {
-			createTestContent(git);
-			String fmt = "tbz2";
-			File archive = new File(getTemporaryDirectory(),
-					"archive." + fmt);
-			archive(git, archive, fmt);
-			ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive));
+	public void archiveHeadAllFilesZipTimestamps() throws Exception {
+		archiveHeadAllFiles("zip");
+	}
 
-			try (InputStream fi = Files.newInputStream(archive.toPath());
-					InputStream bi = new BufferedInputStream(fi);
-					InputStream gzi = new BZip2CompressorInputStream(bi);
-					ArchiveInputStream o = new TarArchiveInputStream(gzi)) {
-				assertEntries(o);
-			}
+	@Test
+	public void archiveHeadAllFilesTgzWithCompressionReducesArchiveSize() throws Exception {
+		archiveHeadAllFilesWithCompression("tgz");
+	}
 
-			Thread.sleep(WAIT);
-			archive(git, archive, fmt);
-			assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1,
-					ObjectId.fromRaw(IO.readFully(archive)));
-		}
+	@Test
+	public void archiveHeadAllFilesTbz2WithCompressionReducesArchiveSize() throws Exception {
+		archiveHeadAllFilesWithCompression("tbz2");
 	}
 
 	@Test
-	public void archiveHeadAllFilesTxzTimestamps() throws Exception {
+	@Ignore
+	public void archiveHeadAllFilesTxzWithCompressionReducesArchiveSize() throws Exception {
+		// We ignore this test because the txz format consumes a lot of memory for high level
+		// compressions.
+		archiveHeadAllFilesWithCompression("txz");
+	}
+
+	@Test
+	public void archiveHeadAllFilesZipWithCompressionReducesArchiveSize() throws Exception {
+		archiveHeadAllFilesWithCompression("zip");
+	}
+
+	private void archiveHeadAllFiles(String fmt) throws Exception {
 		try (Git git = new Git(db)) {
 			createTestContent(git);
-			String fmt = "txz";
-			File archive = new File(getTemporaryDirectory(), "archive." + fmt);
+			File archive = new File(getTemporaryDirectory(),
+					"archive." + format);
 			archive(git, archive, fmt);
 			ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive));
 
 			try (InputStream fi = Files.newInputStream(archive.toPath());
 					InputStream bi = new BufferedInputStream(fi);
-					InputStream gzi = new XZCompressorInputStream(bi);
-					ArchiveInputStream o = new TarArchiveInputStream(gzi)) {
+					ArchiveInputStream o = createArchiveInputStream(fmt, bi)) {
 				assertEntries(o);
 			}
 
@@ -276,28 +254,44 @@ public class ArchiveCommandTest extends RepositoryTestCase {
 		}
 	}
 
-	@Test
-	public void archiveHeadAllFilesZipTimestamps() throws Exception {
+	@SuppressWarnings({ "serial", "boxing" })
+	private void archiveHeadAllFilesWithCompression(String fmt) throws Exception {
 		try (Git git = new Git(db)) {
-			createTestContent(git);
-			String fmt = "zip";
-			File archive = new File(getTemporaryDirectory(), "archive." + fmt);
-			archive(git, archive, fmt);
-			ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive));
+			createLargeTestContent(git);
+			File archive = new File(getTemporaryDirectory(),
+					"archive." + format);
 
-			try (InputStream fi = Files.newInputStream(archive.toPath());
-					InputStream bi = new BufferedInputStream(fi);
-					ArchiveInputStream o = new ZipArchiveInputStream(bi)) {
-				assertEntries(o);
-			}
+			archive(git, archive, fmt, new HashMap<String, Object>() {{
+				put("compression-level", 1);
+			}});
+			int sizeCompression1 = getNumBytes(archive);
 
-			Thread.sleep(WAIT);
-			archive(git, archive, fmt);
-			assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1,
-					ObjectId.fromRaw(IO.readFully(archive)));
+			archive(git, archive, fmt, new HashMap<String, Object>() {{
+				put("compression-level", 9);
+			}});
+			int sizeCompression9 = getNumBytes(archive);
+
+			assertTrue(sizeCompression1 > sizeCompression9);
 		}
 	}
 
+	private static ArchiveInputStream createArchiveInputStream (String fmt, InputStream bi)
+			throws IOException {
+		switch (fmt) {
+			case "tar":
+				return new TarArchiveInputStream(bi);
+			case "tgz":
+				return new TarArchiveInputStream(new GzipCompressorInputStream(bi));
+			case "tbz2":
+				return new TarArchiveInputStream(new BZip2CompressorInputStream(bi));
+			case "txz":
+				return new TarArchiveInputStream(new XZCompressorInputStream(bi));
+			case "zip":
+				return new ZipArchiveInputStream(new BufferedInputStream(bi));
+		}
+		throw new IllegalArgumentException("Format " + fmt + " is not supported.");
+	}
+
 	private void createTestContent(Git git) throws IOException, GitAPIException,
 			NoFilepatternException, NoHeadException, NoMessageException,
 			UnmergedPathsException, ConcurrentRefUpdateException,
@@ -312,13 +306,40 @@ public class ArchiveCommandTest extends RepositoryTestCase {
 		git.commit().setMessage("updated file").call();
 	}
 
+	private void createLargeTestContent(Git git) throws IOException, GitAPIException,
+			NoFilepatternException, NoHeadException, NoMessageException,
+			UnmergedPathsException, ConcurrentRefUpdateException,
+			WrongRepositoryStateException, AbortedByHookException {
+		StringBuilder largeContent = new StringBuilder();
+		Random r = new Random();
+		for (int i = 0; i < 2000; i++) {
+			for (int j = 0; j < 80; j++) {
+				largeContent.append((char)(r.nextInt(26) + 'a'));
+			}
+			largeContent.append("\n");
+		}
+		writeTrashFile("large_file.txt", largeContent.toString());
+		git.add().addFilepattern("large_file.txt").call();
+		git.commit().setMessage("create file").call();
+	}
+
 	private static void archive(Git git, File archive, String fmt)
 			throws GitAPIException,
 			FileNotFoundException, AmbiguousObjectException,
 			IncorrectObjectTypeException, IOException {
+		archive(git, archive, fmt, new HashMap<>());
+	}
+
+	private static void archive(Git git, File archive, String fmt, Map<String,
+			Object> options)
+			throws GitAPIException,
+			FileNotFoundException, AmbiguousObjectException,
+			IncorrectObjectTypeException, IOException {
 		git.archive().setOutputStream(new FileOutputStream(archive))
 				.setFormat(fmt)
-				.setTree(git.getRepository().resolve("HEAD")).call();
+				.setTree(git.getRepository().resolve("HEAD"))
+				.setFormatOptions(options)
+				.call();
 	}
 
 	private static void assertEntries(ArchiveInputStream o) throws IOException {
@@ -333,6 +354,13 @@ public class ArchiveCommandTest extends RepositoryTestCase {
 		assertEquals(UNEXPECTED_ARCHIVE_SIZE, 2, n);
 	}
 
+	private static int getNumBytes(File archive) throws Exception {
+		try (InputStream fi = Files.newInputStream(archive.toPath());
+				InputStream bi = new BufferedInputStream(fi)) {
+			return bi.available();
+		}
+	}
+
 	private static class MockFormat
 			implements ArchiveCommand.Format<MockOutputStream> {