From 6f82690aaf2be783be6d77f0903788ff0832472a Mon Sep 17 00:00:00 2001 From: Youssef Elghareeb Date: Fri, 22 Jan 2021 15:30:25 +0100 Subject: Add the "compression-level" option to all ArchiveCommand formats Different archive formats support a compression level in the range [0-9]. The value 0 is for lowest compressions and 9 for highest. Highest levels produce output files of smaller sizes but require more memory to do the compression. This change allows passing a "compression-level" option to the git archive command and implements using it for different file formats. Change-Id: I5758f691c37ba630dbac24db67bb7da827bbc8e1 Signed-off-by: Youssef Elghareeb Signed-off-by: Matthias Sohn --- .../org/eclipse/jgit/api/ArchiveCommandTest.java | 184 ++++++++++++--------- 1 file changed, 106 insertions(+), 78 deletions(-) (limited to 'org.eclipse.jgit.test') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java index 0f98a63f5a..f2cceac4b3 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ArchiveCommandTest.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.api; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.beans.Statement; import java.io.BufferedInputStream; @@ -28,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; @@ -55,6 +57,7 @@ import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.StringUtils; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class ArchiveCommandTest extends RepositoryTestCase { @@ -184,88 +187,63 @@ public class ArchiveCommandTest extends RepositoryTestCase { @Test public void archiveHeadAllFilesTarTimestamps() throws Exception { - try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "tar"; - File archive = new File(getTemporaryDirectory(), - "archive." + format); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); - - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - ArchiveInputStream o = new TarArchiveInputStream(bi)) { - assertEntries(o); - } - - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); - } + archiveHeadAllFiles("tar"); } @Test public void archiveHeadAllFilesTgzTimestamps() throws Exception { - try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "tgz"; - File archive = new File(getTemporaryDirectory(), - "archive." + fmt); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + archiveHeadAllFiles("tgz"); + } - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - InputStream gzi = new GzipCompressorInputStream(bi); - ArchiveInputStream o = new TarArchiveInputStream(gzi)) { - assertEntries(o); - } + @Test + public void archiveHeadAllFilesTbz2Timestamps() throws Exception { + archiveHeadAllFiles("tbz2"); + } - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); - } + @Test + public void archiveHeadAllFilesTxzTimestamps() throws Exception { + archiveHeadAllFiles("txz"); } @Test - public void archiveHeadAllFilesTbz2Timestamps() throws Exception { - try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "tbz2"; - File archive = new File(getTemporaryDirectory(), - "archive." + fmt); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + public void archiveHeadAllFilesZipTimestamps() throws Exception { + archiveHeadAllFiles("zip"); + } - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - InputStream gzi = new BZip2CompressorInputStream(bi); - ArchiveInputStream o = new TarArchiveInputStream(gzi)) { - assertEntries(o); - } + @Test + public void archiveHeadAllFilesTgzWithCompressionReducesArchiveSize() throws Exception { + archiveHeadAllFilesWithCompression("tgz"); + } - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); - } + @Test + public void archiveHeadAllFilesTbz2WithCompressionReducesArchiveSize() throws Exception { + archiveHeadAllFilesWithCompression("tbz2"); } @Test - public void archiveHeadAllFilesTxzTimestamps() throws Exception { + @Ignore + public void archiveHeadAllFilesTxzWithCompressionReducesArchiveSize() throws Exception { + // We ignore this test because the txz format consumes a lot of memory for high level + // compressions. + archiveHeadAllFilesWithCompression("txz"); + } + + @Test + public void archiveHeadAllFilesZipWithCompressionReducesArchiveSize() throws Exception { + archiveHeadAllFilesWithCompression("zip"); + } + + private void archiveHeadAllFiles(String fmt) throws Exception { try (Git git = new Git(db)) { createTestContent(git); - String fmt = "txz"; - File archive = new File(getTemporaryDirectory(), "archive." + fmt); + File archive = new File(getTemporaryDirectory(), + "archive." + format); archive(git, archive, fmt); ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); try (InputStream fi = Files.newInputStream(archive.toPath()); InputStream bi = new BufferedInputStream(fi); - InputStream gzi = new XZCompressorInputStream(bi); - ArchiveInputStream o = new TarArchiveInputStream(gzi)) { + ArchiveInputStream o = createArchiveInputStream(fmt, bi)) { assertEntries(o); } @@ -276,28 +254,44 @@ public class ArchiveCommandTest extends RepositoryTestCase { } } - @Test - public void archiveHeadAllFilesZipTimestamps() throws Exception { + @SuppressWarnings({ "serial", "boxing" }) + private void archiveHeadAllFilesWithCompression(String fmt) throws Exception { try (Git git = new Git(db)) { - createTestContent(git); - String fmt = "zip"; - File archive = new File(getTemporaryDirectory(), "archive." + fmt); - archive(git, archive, fmt); - ObjectId hash1 = ObjectId.fromRaw(IO.readFully(archive)); + createLargeTestContent(git); + File archive = new File(getTemporaryDirectory(), + "archive." + format); - try (InputStream fi = Files.newInputStream(archive.toPath()); - InputStream bi = new BufferedInputStream(fi); - ArchiveInputStream o = new ZipArchiveInputStream(bi)) { - assertEntries(o); - } + archive(git, archive, fmt, new HashMap() {{ + put("compression-level", 1); + }}); + int sizeCompression1 = getNumBytes(archive); - Thread.sleep(WAIT); - archive(git, archive, fmt); - assertEquals(UNEXPECTED_DIFFERENT_HASH, hash1, - ObjectId.fromRaw(IO.readFully(archive))); + archive(git, archive, fmt, new HashMap() {{ + put("compression-level", 9); + }}); + int sizeCompression9 = getNumBytes(archive); + + assertTrue(sizeCompression1 > sizeCompression9); } } + private static ArchiveInputStream createArchiveInputStream (String fmt, InputStream bi) + throws IOException { + switch (fmt) { + case "tar": + return new TarArchiveInputStream(bi); + case "tgz": + return new TarArchiveInputStream(new GzipCompressorInputStream(bi)); + case "tbz2": + return new TarArchiveInputStream(new BZip2CompressorInputStream(bi)); + case "txz": + return new TarArchiveInputStream(new XZCompressorInputStream(bi)); + case "zip": + return new ZipArchiveInputStream(new BufferedInputStream(bi)); + } + throw new IllegalArgumentException("Format " + fmt + " is not supported."); + } + private void createTestContent(Git git) throws IOException, GitAPIException, NoFilepatternException, NoHeadException, NoMessageException, UnmergedPathsException, ConcurrentRefUpdateException, @@ -312,13 +306,40 @@ public class ArchiveCommandTest extends RepositoryTestCase { git.commit().setMessage("updated file").call(); } + private void createLargeTestContent(Git git) throws IOException, GitAPIException, + NoFilepatternException, NoHeadException, NoMessageException, + UnmergedPathsException, ConcurrentRefUpdateException, + WrongRepositoryStateException, AbortedByHookException { + StringBuilder largeContent = new StringBuilder(); + Random r = new Random(); + for (int i = 0; i < 2000; i++) { + for (int j = 0; j < 80; j++) { + largeContent.append((char)(r.nextInt(26) + 'a')); + } + largeContent.append("\n"); + } + writeTrashFile("large_file.txt", largeContent.toString()); + git.add().addFilepattern("large_file.txt").call(); + git.commit().setMessage("create file").call(); + } + private static void archive(Git git, File archive, String fmt) throws GitAPIException, FileNotFoundException, AmbiguousObjectException, IncorrectObjectTypeException, IOException { + archive(git, archive, fmt, new HashMap<>()); + } + + private static void archive(Git git, File archive, String fmt, Map options) + throws GitAPIException, + FileNotFoundException, AmbiguousObjectException, + IncorrectObjectTypeException, IOException { git.archive().setOutputStream(new FileOutputStream(archive)) .setFormat(fmt) - .setTree(git.getRepository().resolve("HEAD")).call(); + .setTree(git.getRepository().resolve("HEAD")) + .setFormatOptions(options) + .call(); } private static void assertEntries(ArchiveInputStream o) throws IOException { @@ -333,6 +354,13 @@ public class ArchiveCommandTest extends RepositoryTestCase { assertEquals(UNEXPECTED_ARCHIVE_SIZE, 2, n); } + private static int getNumBytes(File archive) throws Exception { + try (InputStream fi = Files.newInputStream(archive.toPath()); + InputStream bi = new BufferedInputStream(fi)) { + return bi.available(); + } + } + private static class MockFormat implements ArchiveCommand.Format { -- cgit v1.2.3