From 59b817a55b04b4bd8c5950a2d97998d3af6d44e3 Mon Sep 17 00:00:00 2001 From: James Moger Date: Fri, 30 Nov 2012 18:05:35 -0500 Subject: Support alternate compressed download formats (issue-174) --- src/com/gitblit/utils/CompressionUtils.java | 315 ++++++++++++++++++++++++++++ src/com/gitblit/utils/JGitUtils.java | 69 ------ 2 files changed, 315 insertions(+), 69 deletions(-) create mode 100644 src/com/gitblit/utils/CompressionUtils.java (limited to 'src/com/gitblit/utils') diff --git a/src/com/gitblit/utils/CompressionUtils.java b/src/com/gitblit/utils/CompressionUtils.java new file mode 100644 index 00000000..7b0d0471 --- /dev/null +++ b/src/com/gitblit/utils/CompressionUtils.java @@ -0,0 +1,315 @@ +/* + * Copyright 2012 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.utils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collection of static methods for retrieving information from a repository. + * + * @author James Moger + * + */ +public class CompressionUtils { + + static final Logger LOGGER = LoggerFactory.getLogger(CompressionUtils.class); + + /** + * Log an error message and exception. + * + * @param t + * @param repository + * if repository is not null it MUST be the {0} parameter in the + * pattern. + * @param pattern + * @param objects + */ + private static void error(Throwable t, Repository repository, String pattern, Object... objects) { + List parameters = new ArrayList(); + if (objects != null && objects.length > 0) { + for (Object o : objects) { + parameters.add(o); + } + } + if (repository != null) { + parameters.add(0, repository.getDirectory().getAbsolutePath()); + } + LOGGER.error(MessageFormat.format(pattern, parameters.toArray()), t); + } + + /** + * Zips the contents of the tree at the (optionally) specified revision and + * the (optionally) specified basepath to the supplied outputstream. + * + * @param repository + * @param basePath + * if unspecified, entire repository is assumed. + * @param objectId + * if unspecified, HEAD is assumed. + * @param os + * @return true if repository was successfully zipped to supplied output + * stream + */ + public static boolean zip(Repository repository, String basePath, String objectId, + OutputStream os) { + RevCommit commit = JGitUtils.getCommit(repository, objectId); + if (commit == null) { + return false; + } + boolean success = false; + RevWalk rw = new RevWalk(repository); + TreeWalk tw = new TreeWalk(repository); + try { + tw.addTree(commit.getTree()); + ZipOutputStream zos = new ZipOutputStream(os); + zos.setComment("Generated by Gitblit"); + if (!StringUtils.isEmpty(basePath)) { + PathFilter f = PathFilter.create(basePath); + tw.setFilter(f); + } + tw.setRecursive(true); + while (tw.next()) { + if (tw.getFileMode(0) == FileMode.GITLINK) { + continue; + } + ZipEntry entry = new ZipEntry(tw.getPathString()); + entry.setSize(tw.getObjectReader().getObjectSize(tw.getObjectId(0), + Constants.OBJ_BLOB)); + entry.setComment(commit.getName()); + zos.putNextEntry(entry); + + ObjectId entid = tw.getObjectId(0); + FileMode entmode = tw.getFileMode(0); + RevBlob blob = (RevBlob) rw.lookupAny(entid, entmode.getObjectType()); + rw.parseBody(blob); + + ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB); + byte[] tmp = new byte[4096]; + InputStream in = ldr.openStream(); + int n; + while ((n = in.read(tmp)) > 0) { + zos.write(tmp, 0, n); + } + in.close(); + } + zos.finish(); + success = true; + } catch (IOException e) { + error(e, repository, "{0} failed to zip files from commit {1}", commit.getName()); + } finally { + tw.release(); + rw.dispose(); + } + return success; + } + + /** + * tar the contents of the tree at the (optionally) specified revision and + * the (optionally) specified basepath to the supplied outputstream. + * + * @param repository + * @param basePath + * if unspecified, entire repository is assumed. + * @param objectId + * if unspecified, HEAD is assumed. + * @param os + * @return true if repository was successfully zipped to supplied output + * stream + */ + public static boolean tar(Repository repository, String basePath, String objectId, + OutputStream os) { + return tar(null, repository, basePath, objectId, os); + } + + /** + * tar.gz the contents of the tree at the (optionally) specified revision and + * the (optionally) specified basepath to the supplied outputstream. + * + * @param repository + * @param basePath + * if unspecified, entire repository is assumed. + * @param objectId + * if unspecified, HEAD is assumed. + * @param os + * @return true if repository was successfully zipped to supplied output + * stream + */ + public static boolean gz(Repository repository, String basePath, String objectId, + OutputStream os) { + return tar(CompressorStreamFactory.GZIP, repository, basePath, objectId, os); + } + + /** + * tar.xz the contents of the tree at the (optionally) specified revision and + * the (optionally) specified basepath to the supplied outputstream. + * + * @param repository + * @param basePath + * if unspecified, entire repository is assumed. + * @param objectId + * if unspecified, HEAD is assumed. + * @param os + * @return true if repository was successfully zipped to supplied output + * stream + */ + public static boolean xz(Repository repository, String basePath, String objectId, + OutputStream os) { + return tar(CompressorStreamFactory.XZ, repository, basePath, objectId, os); + } + + /** + * tar.bzip2 the contents of the tree at the (optionally) specified revision and + * the (optionally) specified basepath to the supplied outputstream. + * + * @param repository + * @param basePath + * if unspecified, entire repository is assumed. + * @param objectId + * if unspecified, HEAD is assumed. + * @param os + * @return true if repository was successfully zipped to supplied output + * stream + */ + public static boolean bzip2(Repository repository, String basePath, String objectId, + OutputStream os) { + + return tar(CompressorStreamFactory.BZIP2, repository, basePath, objectId, os); + } + + /** + * Compresses/archives the contents of the tree at the (optionally) + * specified revision and the (optionally) specified basepath to the + * supplied outputstream. + * + * @param algorithm + * compression algorithm for tar (optional) + * @param repository + * @param basePath + * if unspecified, entire repository is assumed. + * @param objectId + * if unspecified, HEAD is assumed. + * @param os + * @return true if repository was successfully zipped to supplied output + * stream + */ + private static boolean tar(String algorithm, Repository repository, String basePath, String objectId, + OutputStream os) { + RevCommit commit = JGitUtils.getCommit(repository, objectId); + if (commit == null) { + return false; + } + + OutputStream cos = os; + if (!StringUtils.isEmpty(algorithm)) { + try { + cos = new CompressorStreamFactory().createCompressorOutputStream(algorithm, os); + } catch (CompressorException e1) { + error(e1, repository, "{0} failed to open {1} stream", algorithm); + } + } + boolean success = false; + RevWalk rw = new RevWalk(repository); + TreeWalk tw = new TreeWalk(repository); + try { + tw.addTree(commit.getTree()); + TarArchiveOutputStream tos = new TarArchiveOutputStream(cos); + tos.setAddPaxHeadersForNonAsciiNames(true); + tos.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + if (!StringUtils.isEmpty(basePath)) { + PathFilter f = PathFilter.create(basePath); + tw.setFilter(f); + } + tw.setRecursive(true); + while (tw.next()) { + FileMode mode = tw.getFileMode(0); + if (mode == FileMode.GITLINK) { + continue; + } + ObjectId id = tw.getObjectId(0); + + // new entry + TarArchiveEntry entry = new TarArchiveEntry(tw.getPathString()); + entry.setSize(tw.getObjectReader().getObjectSize(id, Constants.OBJ_BLOB)); + + if (FileMode.SYMLINK.equals(mode)) { + // symlink + entry.setMode(mode.getBits()); + + // read the symlink target + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + RevBlob blob = (RevBlob) rw.lookupAny(id, mode.getObjectType()); + rw.parseBody(blob); + ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB); + IOUtils.copy(ldr.openStream(), bs); + entry.setLinkName(bs.toString("UTF-8")); + } else { + // regular file or executable file + entry.setMode(mode.getBits()); + } + entry.setModTime(commit.getAuthorIdent().getWhen()); + + tos.putArchiveEntry(entry); + + if (!FileMode.SYMLINK.equals(mode)) { + // write the blob + RevBlob blob = (RevBlob) rw.lookupAny(id, mode.getObjectType()); + rw.parseBody(blob); + ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB); + IOUtils.copy(ldr.openStream(), tos); + } + + // close entry + tos.closeArchiveEntry(); + } + tos.finish(); + tos.close(); + cos.close(); + success = true; + } catch (IOException e) { + error(e, repository, "{0} failed to {1} stream files from commit {2}", algorithm, commit.getName()); + } finally { + tw.release(); + rw.dispose(); + } + return success; + } +} diff --git a/src/com/gitblit/utils/JGitUtils.java b/src/com/gitblit/utils/JGitUtils.java index bc44f00f..9cfb37fd 100644 --- a/src/com/gitblit/utils/JGitUtils.java +++ b/src/com/gitblit/utils/JGitUtils.java @@ -19,7 +19,6 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; @@ -30,8 +29,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.FetchCommand; @@ -1723,70 +1720,4 @@ public class JGitUtils { } return success; } - - /** - * Zips the contents of the tree at the (optionally) specified revision and - * the (optionally) specified basepath to the supplied outputstream. - * - * @param repository - * @param basePath - * if unspecified, entire repository is assumed. - * @param objectId - * if unspecified, HEAD is assumed. - * @param os - * @return true if repository was successfully zipped to supplied output - * stream - */ - public static boolean zip(Repository repository, String basePath, String objectId, - OutputStream os) { - RevCommit commit = getCommit(repository, objectId); - if (commit == null) { - return false; - } - boolean success = false; - RevWalk rw = new RevWalk(repository); - TreeWalk tw = new TreeWalk(repository); - try { - tw.addTree(commit.getTree()); - ZipOutputStream zos = new ZipOutputStream(os); - zos.setComment("Generated by Gitblit"); - if (!StringUtils.isEmpty(basePath)) { - PathFilter f = PathFilter.create(basePath); - tw.setFilter(f); - } - tw.setRecursive(true); - while (tw.next()) { - if (tw.getFileMode(0) == FileMode.GITLINK) { - continue; - } - ZipEntry entry = new ZipEntry(tw.getPathString()); - entry.setSize(tw.getObjectReader().getObjectSize(tw.getObjectId(0), - Constants.OBJ_BLOB)); - entry.setComment(commit.getName()); - zos.putNextEntry(entry); - - ObjectId entid = tw.getObjectId(0); - FileMode entmode = tw.getFileMode(0); - RevBlob blob = (RevBlob) rw.lookupAny(entid, entmode.getObjectType()); - rw.parseBody(blob); - - ObjectLoader ldr = repository.open(blob.getId(), Constants.OBJ_BLOB); - byte[] tmp = new byte[4096]; - InputStream in = ldr.openStream(); - int n; - while ((n = in.read(tmp)) > 0) { - zos.write(tmp, 0, n); - } - in.close(); - } - zos.finish(); - success = true; - } catch (IOException e) { - error(e, repository, "{0} failed to zip files from commit {1}", commit.getName()); - } finally { - tw.release(); - rw.dispose(); - } - return success; - } } -- cgit v1.2.3