Support alternate compressed download formats (issue-174)

2 files changed, 315 insertions, 69 deletions

diff --git a/src/com/gitblit/utils/CompressionUtils.java b/src/com/gitblit/utils/CompressionUtils.java
new file mode 100644
index 00000000..7b0d0471
--- /dev/null
+++ b/src/com/gitblit/utils/CompressionUtils.java
@@ -0,0 +1,315 @@
+/*

+ * Copyright 2012 gitblit.com.

+ *

+ * Licensed under the Apache License, Version 2.0 (the "License");

+ * you may not use this file except in compliance with the License.

+ * You may obtain a copy of the License at

+ *

+ *     http://www.apache.org/licenses/LICENSE-2.0

+ *

+ * Unless required by applicable law or agreed to in writing, software

+ * distributed under the License is distributed on an "AS IS" BASIS,

+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

+ * See the License for the specific language governing permissions and

+ * limitations under the License.

+ */

+package com.gitblit.utils;

+

+import java.io.ByteArrayOutputStream;

+import java.io.IOException;

+import java.io.InputStream;

+import java.io.OutputStream;

+import java.text.MessageFormat;

+import java.util.ArrayList;

+import java.util.List;

+import java.util.zip.ZipEntry;

+import java.util.zip.ZipOutputStream;

+

+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;

+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;

+import org.apache.commons.compress.compressors.CompressorException;

+import org.apache.commons.compress.compressors.CompressorStreamFactory;

+import org.apache.commons.compress.utils.IOUtils;

+import org.eclipse.jgit.lib.Constants;

+import org.eclipse.jgit.lib.FileMode;

+import org.eclipse.jgit.lib.ObjectId;

+import org.eclipse.jgit.lib.ObjectLoader;

+import org.eclipse.jgit.lib.Repository;

+import org.eclipse.jgit.revwalk.RevBlob;

+import org.eclipse.jgit.revwalk.RevCommit;

+import org.eclipse.jgit.revwalk.RevWalk;

+import org.eclipse.jgit.treewalk.TreeWalk;

+import org.eclipse.jgit.treewalk.filter.PathFilter;

+import org.slf4j.Logger;

+import org.slf4j.LoggerFactory;

+

+/**

+ * Collection of static methods for retrieving information from a repository.

+ * 

+ * @author James Moger

+ * 

+ */

+public class CompressionUtils {

+

+	static final Logger LOGGER = LoggerFactory.getLogger(CompressionUtils.class);

+

+	/**

+	 * Log an error message and exception.

+	 * 

+	 * @param t

+	 * @param repository

+	 *            if repository is not null it MUST be the {0} parameter in the

+	 *            pattern.

+	 * @param pattern

+	 * @param objects

+	 */

+	private static void error(Throwable t, Repository repository, String pattern, Object... objects) {

+		List<Object> parameters = new ArrayList<Object>();

+		if (objects != null && objects.length > 0) {

+			for (Object o : objects) {

+				parameters.add(o);

+			}

+		}

+		if (repository != null) {

+			parameters.add(0, repository.getDirectory().getAbsolutePath());

+		}

+		LOGGER.error(MessageFormat.format(pattern, parameters.toArray()), t);

+	}

+

+	/**

+	 * Zips the contents of the tree at the (optionally) specified revision and

+	 * the (optionally) specified basepath to the supplied outputstream.

+	 * 

+	 * @param repository

+	 * @param basePath

+	 *            if unspecified, entire repository is assumed.

+	 * @param objectId

+	 *            if unspecified, HEAD is assumed.

+	 * @param os

+	 * @return true if repository was successfully zipped to supplied output

+	 *         stream

+	 */

+	public static boolean zip(Repository repository, String basePath, String objectId,

+			OutputStream os) {

+		RevCommit commit = JGitUtils.getCommit(repository, objectId);

+		if (commit == null) {

+			return false;

+		}

+		boolean success = false;

+		RevWalk rw = new RevWalk(repository);

+		TreeWalk tw = new TreeWalk(repository);

+		try {

+			tw.addTree(commit.getTree());

+			ZipOutputStream zos = new ZipOutputStream(os);

+			zos.setComment("Generated by Gitblit");

+			if (!StringUtils.isEmpty(basePath)) {

+				PathFilter f = PathFilter.create(basePath);

+				tw.setFilter(f);

+			}

+			tw.setRecursive(true);

+			while (tw.next()) {

+				if (tw.getFileMode(0) == FileMode.GITLINK) {

+					continue;

+				}

+				ZipEntry entry = new ZipEntry(tw.getPathString());

+				entry.setSize(tw.getObjectReader().getObjectSize(tw.getObjectId(0),

+						Constants.OBJ_BLOB));

+				entry.setComment(commit.getName());

+				zos.putNextEntry(entry);

+

+				ObjectId entid = tw.getObjectId(0);

+				FileMode entmode = tw.getFileMode(0);

+				RevBlob blob = (RevBlob) rw.lookupAny(entid, entmode.getObjectType());

+				rw.parseBody(blob);

+

+				ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB);

+				byte[] tmp = new byte[4096];

+				InputStream in = ldr.openStream();

+				int n;

+				while ((n = in.read(tmp)) > 0) {

+					zos.write(tmp, 0, n);

+				}

+				in.close();

+			}

+			zos.finish();

+			success = true;

+		} catch (IOException e) {

+			error(e, repository, "{0} failed to zip files from commit {1}", commit.getName());

+		} finally {

+			tw.release();

+			rw.dispose();

+		}

+		return success;

+	}

+	

+	/**

+	 * tar the contents of the tree at the (optionally) specified revision and

+	 * the (optionally) specified basepath to the supplied outputstream.

+	 * 

+	 * @param repository

+	 * @param basePath

+	 *            if unspecified, entire repository is assumed.

+	 * @param objectId

+	 *            if unspecified, HEAD is assumed.

+	 * @param os

+	 * @return true if repository was successfully zipped to supplied output

+	 *         stream

+	 */

+	public static boolean tar(Repository repository, String basePath, String objectId,

+			OutputStream os) {

+		return tar(null, repository, basePath, objectId, os);

+	}

+	

+	/**

+	 * tar.gz the contents of the tree at the (optionally) specified revision and

+	 * the (optionally) specified basepath to the supplied outputstream.

+	 * 

+	 * @param repository

+	 * @param basePath

+	 *            if unspecified, entire repository is assumed.

+	 * @param objectId

+	 *            if unspecified, HEAD is assumed.

+	 * @param os

+	 * @return true if repository was successfully zipped to supplied output

+	 *         stream

+	 */

+	public static boolean gz(Repository repository, String basePath, String objectId,

+			OutputStream os) {

+		return tar(CompressorStreamFactory.GZIP, repository, basePath, objectId, os);

+	}

+	

+	/**

+	 * tar.xz the contents of the tree at the (optionally) specified revision and

+	 * the (optionally) specified basepath to the supplied outputstream.

+	 * 

+	 * @param repository

+	 * @param basePath

+	 *            if unspecified, entire repository is assumed.

+	 * @param objectId

+	 *            if unspecified, HEAD is assumed.

+	 * @param os

+	 * @return true if repository was successfully zipped to supplied output

+	 *         stream

+	 */

+	public static boolean xz(Repository repository, String basePath, String objectId,

+			OutputStream os) {

+		return tar(CompressorStreamFactory.XZ, repository, basePath, objectId, os);

+	}

+	

+	/**

+	 * tar.bzip2 the contents of the tree at the (optionally) specified revision and

+	 * the (optionally) specified basepath to the supplied outputstream.

+	 * 

+	 * @param repository

+	 * @param basePath

+	 *            if unspecified, entire repository is assumed.

+	 * @param objectId

+	 *            if unspecified, HEAD is assumed.

+	 * @param os

+	 * @return true if repository was successfully zipped to supplied output

+	 *         stream

+	 */

+	public static boolean bzip2(Repository repository, String basePath, String objectId,

+			OutputStream os) {

+		

+		return tar(CompressorStreamFactory.BZIP2, repository, basePath, objectId, os);

+	}

+	

+	/**

+	 * Compresses/archives the contents of the tree at the (optionally)

+	 * specified revision and the (optionally) specified basepath to the

+	 * supplied outputstream.

+	 * 

+	 * @param algorithm

+	 *            compression algorithm for tar (optional)

+	 * @param repository

+	 * @param basePath

+	 *            if unspecified, entire repository is assumed.

+	 * @param objectId

+	 *            if unspecified, HEAD is assumed.

+	 * @param os

+	 * @return true if repository was successfully zipped to supplied output

+	 *         stream

+	 */

+	private static boolean tar(String algorithm, Repository repository, String basePath, String objectId,

+			OutputStream os) {

+		RevCommit commit = JGitUtils.getCommit(repository, objectId);

+		if (commit == null) {

+			return false;

+		}

+		

+		OutputStream cos = os;

+		if (!StringUtils.isEmpty(algorithm)) {

+			try {

+				cos = new CompressorStreamFactory().createCompressorOutputStream(algorithm, os);

+			} catch (CompressorException e1) {

+				error(e1, repository, "{0} failed to open {1} stream", algorithm);

+			}

+		}

+		boolean success = false;

+		RevWalk rw = new RevWalk(repository);

+		TreeWalk tw = new TreeWalk(repository);

+		try {

+			tw.addTree(commit.getTree());

+			TarArchiveOutputStream tos = new TarArchiveOutputStream(cos);

+			tos.setAddPaxHeadersForNonAsciiNames(true);

+			tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);

+			if (!StringUtils.isEmpty(basePath)) {

+				PathFilter f = PathFilter.create(basePath);

+				tw.setFilter(f);

+			}

+			tw.setRecursive(true);

+			while (tw.next()) {

+				FileMode mode = tw.getFileMode(0);

+				if (mode == FileMode.GITLINK) {

+					continue;

+				}

+				ObjectId id = tw.getObjectId(0);

+				

+				// new entry

+				TarArchiveEntry entry = new TarArchiveEntry(tw.getPathString());

+				entry.setSize(tw.getObjectReader().getObjectSize(id, Constants.OBJ_BLOB));

+				

+				if (FileMode.SYMLINK.equals(mode)) {

+					// symlink

+					entry.setMode(mode.getBits());

+					

+					// read the symlink target

+					ByteArrayOutputStream bs = new ByteArrayOutputStream();

+					RevBlob blob = (RevBlob) rw.lookupAny(id, mode.getObjectType());

+					rw.parseBody(blob);				

+					ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB);

+					IOUtils.copy(ldr.openStream(), bs);

+					entry.setLinkName(bs.toString("UTF-8"));

+				} else {

+					// regular file or executable file

+					entry.setMode(mode.getBits());

+				}				

+				entry.setModTime(commit.getAuthorIdent().getWhen());

+

+				tos.putArchiveEntry(entry);

+				

+				if (!FileMode.SYMLINK.equals(mode)) {

+					// write the blob

+					RevBlob blob = (RevBlob) rw.lookupAny(id, mode.getObjectType());

+					rw.parseBody(blob);				

+					ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB);

+					IOUtils.copy(ldr.openStream(), tos);

+				}

+				

+				// close entry

+				tos.closeArchiveEntry();

+			}

+			tos.finish();

+			tos.close();

+			cos.close();

+			success = true;

+		} catch (IOException e) {

+			error(e, repository, "{0} failed to {1} stream files from commit {2}", algorithm, commit.getName());

+		} finally {

+			tw.release();

+			rw.dispose();

+		}

+		return success;

+	}

+}

diff --git a/src/com/gitblit/utils/JGitUtils.java b/src/com/gitblit/utils/JGitUtils.java
index bc44f00f..9cfb37fd 100644
--- a/src/com/gitblit/utils/JGitUtils.java
+++ b/src/com/gitblit/utils/JGitUtils.java
@@ -19,7 +19,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.File;

 import java.io.IOException;

 import java.io.InputStream;

-import java.io.OutputStream;

 import java.text.MessageFormat;

 import java.util.ArrayList;

 import java.util.Arrays;

@@ -30,8 +29,6 @@ import java.util.List;
 import java.util.Map;

 import java.util.Map.Entry;

 import java.util.regex.Pattern;

-import java.util.zip.ZipEntry;

-import java.util.zip.ZipOutputStream;

 

 import org.eclipse.jgit.api.CloneCommand;

 import org.eclipse.jgit.api.FetchCommand;

@@ -1723,70 +1720,4 @@ public class JGitUtils {
 		}

 		return success;

 	}

-

-	/**

-	 * Zips the contents of the tree at the (optionally) specified revision and

-	 * the (optionally) specified basepath to the supplied outputstream.

-	 * 

-	 * @param repository

-	 * @param basePath

-	 *            if unspecified, entire repository is assumed.

-	 * @param objectId

-	 *            if unspecified, HEAD is assumed.

-	 * @param os

-	 * @return true if repository was successfully zipped to supplied output

-	 *         stream

-	 */

-	public static boolean zip(Repository repository, String basePath, String objectId,

-			OutputStream os) {

-		RevCommit commit = getCommit(repository, objectId);

-		if (commit == null) {

-			return false;

-		}

-		boolean success = false;

-		RevWalk rw = new RevWalk(repository);

-		TreeWalk tw = new TreeWalk(repository);

-		try {

-			tw.addTree(commit.getTree());

-			ZipOutputStream zos = new ZipOutputStream(os);

-			zos.setComment("Generated by Gitblit");

-			if (!StringUtils.isEmpty(basePath)) {

-				PathFilter f = PathFilter.create(basePath);

-				tw.setFilter(f);

-			}

-			tw.setRecursive(true);

-			while (tw.next()) {

-				if (tw.getFileMode(0) == FileMode.GITLINK) {

-					continue;

-				}

-				ZipEntry entry = new ZipEntry(tw.getPathString());

-				entry.setSize(tw.getObjectReader().getObjectSize(tw.getObjectId(0),

-						Constants.OBJ_BLOB));

-				entry.setComment(commit.getName());

-				zos.putNextEntry(entry);

-

-				ObjectId entid = tw.getObjectId(0);

-				FileMode entmode = tw.getFileMode(0);

-				RevBlob blob = (RevBlob) rw.lookupAny(entid, entmode.getObjectType());

-				rw.parseBody(blob);

-

-				ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB);

-				byte[] tmp = new byte[4096];

-				InputStream in = ldr.openStream();

-				int n;

-				while ((n = in.read(tmp)) > 0) {

-					zos.write(tmp, 0, n);

-				}

-				in.close();

-			}

-			zos.finish();

-			success = true;

-		} catch (IOException e) {

-			error(e, repository, "{0} failed to zip files from commit {1}", commit.getName());

-		} finally {

-			tw.release();

-			rw.dispose();

-		}

-		return success;

-	}

 }