123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600 |
- /*
- * Copyright (C) 2008-2009, Google Inc.
- * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials provided
- * with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- * names of its contributors may be used to endorse or promote
- * products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
- package org.eclipse.jgit.util;
-
- import java.io.File;
- import java.io.FileInputStream;
- import java.io.FileOutputStream;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.OutputStream;
- import java.util.ArrayList;
-
- import org.eclipse.jgit.internal.JGitText;
- import org.eclipse.jgit.lib.NullProgressMonitor;
- import org.eclipse.jgit.lib.ProgressMonitor;
- import org.eclipse.jgit.util.io.SafeBufferedOutputStream;
-
- /**
- * A fully buffered output stream.
- * <p>
- * Subclasses determine the behavior when the in-memory buffer capacity has been
- * exceeded and additional bytes are still being received for output.
- */
- public abstract class TemporaryBuffer extends OutputStream {
- /** Default limit for in-core storage. */
- protected static final int DEFAULT_IN_CORE_LIMIT = 1024 * 1024;
-
- /** Chain of data, if we are still completely in-core; otherwise null. */
- private ArrayList<Block> blocks;
-
- /**
- * Maximum number of bytes we will permit storing in memory.
- * <p>
- * When this limit is reached the data will be shifted to a file on disk,
- * preventing the JVM heap from growing out of control.
- */
- private int inCoreLimit;
-
- /** If {@link #inCoreLimit} has been reached, remainder goes here. */
- private OutputStream overflow;
-
- /**
- * Create a new empty temporary buffer.
- *
- * @param limit
- * maximum number of bytes to store in memory before entering the
- * overflow output path.
- */
- protected TemporaryBuffer(final int limit) {
- inCoreLimit = limit;
- reset();
- }
-
- @Override
- public void write(final int b) throws IOException {
- if (overflow != null) {
- overflow.write(b);
- return;
- }
-
- Block s = last();
- if (s.isFull()) {
- if (reachedInCoreLimit()) {
- overflow.write(b);
- return;
- }
-
- s = new Block();
- blocks.add(s);
- }
- s.buffer[s.count++] = (byte) b;
- }
-
- @Override
- public void write(final byte[] b, int off, int len) throws IOException {
- if (overflow == null) {
- while (len > 0) {
- Block s = last();
- if (s.isFull()) {
- if (reachedInCoreLimit())
- break;
-
- s = new Block();
- blocks.add(s);
- }
-
- final int n = Math.min(s.buffer.length - s.count, len);
- System.arraycopy(b, off, s.buffer, s.count, n);
- s.count += n;
- len -= n;
- off += n;
- }
- }
-
- if (len > 0)
- overflow.write(b, off, len);
- }
-
- /**
- * Dumps the entire buffer into the overflow stream, and flushes it.
- *
- * @throws IOException
- * the overflow stream cannot be started, or the buffer contents
- * cannot be written to it, or it failed to flush.
- */
- protected void doFlush() throws IOException {
- if (overflow == null)
- switchToOverflow();
- overflow.flush();
- }
-
- /**
- * Copy all bytes remaining on the input stream into this buffer.
- *
- * @param in
- * the stream to read from, until EOF is reached.
- * @throws IOException
- * an error occurred reading from the input stream, or while
- * writing to a local temporary file.
- */
- public void copy(final InputStream in) throws IOException {
- if (blocks != null) {
- for (;;) {
- Block s = last();
- if (s.isFull()) {
- if (reachedInCoreLimit())
- break;
- s = new Block();
- blocks.add(s);
- }
-
- int n = in.read(s.buffer, s.count, s.buffer.length - s.count);
- if (n < 1)
- return;
- s.count += n;
- }
- }
-
- final byte[] tmp = new byte[Block.SZ];
- int n;
- while ((n = in.read(tmp)) > 0)
- overflow.write(tmp, 0, n);
- }
-
- /**
- * Obtain the length (in bytes) of the buffer.
- * <p>
- * The length is only accurate after {@link #close()} has been invoked.
- *
- * @return total length of the buffer, in bytes.
- */
- public long length() {
- return inCoreLength();
- }
-
- private long inCoreLength() {
- final Block last = last();
- return ((long) blocks.size() - 1) * Block.SZ + last.count;
- }
-
- /**
- * Convert this buffer's contents into a contiguous byte array.
- * <p>
- * The buffer is only complete after {@link #close()} has been invoked.
- *
- * @return the complete byte array; length matches {@link #length()}.
- * @throws IOException
- * an error occurred reading from a local temporary file
- * @throws OutOfMemoryError
- * the buffer cannot fit in memory
- */
- public byte[] toByteArray() throws IOException {
- final long len = length();
- if (Integer.MAX_VALUE < len)
- throw new OutOfMemoryError(JGitText.get().lengthExceedsMaximumArraySize);
- final byte[] out = new byte[(int) len];
- int outPtr = 0;
- for (final Block b : blocks) {
- System.arraycopy(b.buffer, 0, out, outPtr, b.count);
- outPtr += b.count;
- }
- return out;
- }
-
- /**
- * Send this buffer to an output stream.
- * <p>
- * This method may only be invoked after {@link #close()} has completed
- * normally, to ensure all data is completely transferred.
- *
- * @param os
- * stream to send this buffer's complete content to.
- * @param pm
- * if not null progress updates are sent here. Caller should
- * initialize the task and the number of work units to <code>
- * {@link #length()}/1024</code>.
- * @throws IOException
- * an error occurred reading from a temporary file on the local
- * system, or writing to the output stream.
- */
- public void writeTo(final OutputStream os, ProgressMonitor pm)
- throws IOException {
- if (pm == null)
- pm = NullProgressMonitor.INSTANCE;
- for (final Block b : blocks) {
- os.write(b.buffer, 0, b.count);
- pm.update(b.count / 1024);
- }
- }
-
- /**
- * Open an input stream to read from the buffered data.
- * <p>
- * This method may only be invoked after {@link #close()} has completed
- * normally, to ensure all data is completely transferred.
- *
- * @return a stream to read from the buffer. The caller must close the
- * stream when it is no longer useful.
- * @throws IOException
- * an error occurred opening the temporary file.
- */
- public InputStream openInputStream() throws IOException {
- return new BlockInputStream();
- }
-
- /** Reset this buffer for reuse, purging all buffered content. */
- public void reset() {
- if (overflow != null) {
- destroy();
- }
- if (inCoreLimit < Block.SZ) {
- blocks = new ArrayList<Block>(1);
- blocks.add(new Block(inCoreLimit));
- } else {
- blocks = new ArrayList<Block>(inCoreLimit / Block.SZ);
- blocks.add(new Block());
- }
- }
-
- /**
- * Open the overflow output stream, so the remaining output can be stored.
- *
- * @return the output stream to receive the buffered content, followed by
- * the remaining output.
- * @throws IOException
- * the buffer cannot create the overflow stream.
- */
- protected abstract OutputStream overflow() throws IOException;
-
- private Block last() {
- return blocks.get(blocks.size() - 1);
- }
-
- private boolean reachedInCoreLimit() throws IOException {
- if (inCoreLength() < inCoreLimit)
- return false;
-
- switchToOverflow();
- return true;
- }
-
- private void switchToOverflow() throws IOException {
- overflow = overflow();
-
- final Block last = blocks.remove(blocks.size() - 1);
- for (final Block b : blocks)
- overflow.write(b.buffer, 0, b.count);
- blocks = null;
-
- overflow = new SafeBufferedOutputStream(overflow, Block.SZ);
- overflow.write(last.buffer, 0, last.count);
- }
-
- public void close() throws IOException {
- if (overflow != null) {
- try {
- overflow.close();
- } finally {
- overflow = null;
- }
- }
- }
-
- /** Clear this buffer so it has no data, and cannot be used again. */
- public void destroy() {
- blocks = null;
-
- if (overflow != null) {
- try {
- overflow.close();
- } catch (IOException err) {
- // We shouldn't encounter an error closing the file.
- } finally {
- overflow = null;
- }
- }
- }
-
- /**
- * A fully buffered output stream using local disk storage for large data.
- * <p>
- * Initially this output stream buffers to memory and is therefore similar
- * to ByteArrayOutputStream, but it shifts to using an on disk temporary
- * file if the output gets too large.
- * <p>
- * The content of this buffered stream may be sent to another OutputStream
- * only after this stream has been properly closed by {@link #close()}.
- */
- public static class LocalFile extends TemporaryBuffer {
- /** Directory to store the temporary file under. */
- private final File directory;
-
- /**
- * Location of our temporary file if we are on disk; otherwise null.
- * <p>
- * If we exceeded the {@link #inCoreLimit} we nulled out {@link #blocks}
- * and created this file instead. All output goes here through
- * {@link #overflow}.
- */
- private File onDiskFile;
-
- /**
- * Create a new temporary buffer.
- *
- * @deprecated Use the {@code File} overload to supply a directory.
- */
- @Deprecated
- public LocalFile() {
- this(null, DEFAULT_IN_CORE_LIMIT);
- }
-
- /**
- * Create a new temporary buffer, limiting memory usage.
- *
- * @param inCoreLimit
- * maximum number of bytes to store in memory. Storage beyond
- * this limit will use the local file.
- * @deprecated Use the {@code File,int} overload to supply a directory.
- */
- @Deprecated
- public LocalFile(final int inCoreLimit) {
- this(null, inCoreLimit);
- }
-
- /**
- * Create a new temporary buffer, limiting memory usage.
- *
- * @param directory
- * if the buffer has to spill over into a temporary file, the
- * directory where the file should be saved. If null the
- * system default temporary directory (for example /tmp) will
- * be used instead.
- */
- public LocalFile(final File directory) {
- this(directory, DEFAULT_IN_CORE_LIMIT);
- }
-
- /**
- * Create a new temporary buffer, limiting memory usage.
- *
- * @param directory
- * if the buffer has to spill over into a temporary file, the
- * directory where the file should be saved. If null the
- * system default temporary directory (for example /tmp) will
- * be used instead.
- * @param inCoreLimit
- * maximum number of bytes to store in memory. Storage beyond
- * this limit will use the local file.
- */
- public LocalFile(final File directory, final int inCoreLimit) {
- super(inCoreLimit);
- this.directory = directory;
- }
-
- protected OutputStream overflow() throws IOException {
- onDiskFile = File.createTempFile("jgit_", ".buf", directory); //$NON-NLS-1$ //$NON-NLS-2$
- return new FileOutputStream(onDiskFile);
- }
-
- public long length() {
- if (onDiskFile == null) {
- return super.length();
- }
- return onDiskFile.length();
- }
-
- public byte[] toByteArray() throws IOException {
- if (onDiskFile == null) {
- return super.toByteArray();
- }
-
- final long len = length();
- if (Integer.MAX_VALUE < len)
- throw new OutOfMemoryError(JGitText.get().lengthExceedsMaximumArraySize);
- final byte[] out = new byte[(int) len];
- final FileInputStream in = new FileInputStream(onDiskFile);
- try {
- IO.readFully(in, out, 0, (int) len);
- } finally {
- in.close();
- }
- return out;
- }
-
- public void writeTo(final OutputStream os, ProgressMonitor pm)
- throws IOException {
- if (onDiskFile == null) {
- super.writeTo(os, pm);
- return;
- }
- if (pm == null)
- pm = NullProgressMonitor.INSTANCE;
- final FileInputStream in = new FileInputStream(onDiskFile);
- try {
- int cnt;
- final byte[] buf = new byte[Block.SZ];
- while ((cnt = in.read(buf)) >= 0) {
- os.write(buf, 0, cnt);
- pm.update(cnt / 1024);
- }
- } finally {
- in.close();
- }
- }
-
- @Override
- public InputStream openInputStream() throws IOException {
- if (onDiskFile == null)
- return super.openInputStream();
- return new FileInputStream(onDiskFile);
- }
-
- @Override
- public void destroy() {
- super.destroy();
-
- if (onDiskFile != null) {
- try {
- if (!onDiskFile.delete())
- onDiskFile.deleteOnExit();
- } finally {
- onDiskFile = null;
- }
- }
- }
- }
-
- /**
- * A temporary buffer that will never exceed its in-memory limit.
- * <p>
- * If the in-memory limit is reached an IOException is thrown, rather than
- * attempting to spool to local disk.
- */
- public static class Heap extends TemporaryBuffer {
- /**
- * Create a new heap buffer with a maximum storage limit.
- *
- * @param limit
- * maximum number of bytes that can be stored in this buffer.
- * Storing beyond this many will cause an IOException to be
- * thrown during write.
- */
- public Heap(final int limit) {
- super(limit);
- }
-
- @Override
- protected OutputStream overflow() throws IOException {
- throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);
- }
- }
-
- static class Block {
- static final int SZ = 8 * 1024;
-
- final byte[] buffer;
-
- int count;
-
- Block() {
- buffer = new byte[SZ];
- }
-
- Block(int sz) {
- buffer = new byte[sz];
- }
-
- boolean isFull() {
- return count == buffer.length;
- }
- }
-
- private class BlockInputStream extends InputStream {
- private byte[] singleByteBuffer;
- private int blockIndex;
- private Block block;
- private int blockPos;
-
- BlockInputStream() {
- block = blocks.get(blockIndex);
- }
-
- @Override
- public int read() throws IOException {
- if (singleByteBuffer == null)
- singleByteBuffer = new byte[1];
- int n = read(singleByteBuffer);
- return n == 1 ? singleByteBuffer[0] & 0xff : -1;
- }
-
- @Override
- public long skip(long cnt) throws IOException {
- long skipped = 0;
- while (0 < cnt) {
- int n = (int) Math.min(block.count - blockPos, cnt);
- if (0 < n) {
- blockPos += n;
- skipped += n;
- cnt -= n;
- } else if (nextBlock())
- continue;
- else
- break;
- }
- return skipped;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- if (len == 0)
- return 0;
- int copied = 0;
- while (0 < len) {
- int c = Math.min(block.count - blockPos, len);
- if (0 < c) {
- System.arraycopy(block.buffer, blockPos, b, off, c);
- blockPos += c;
- off += c;
- len -= c;
- copied += c;
- } else if (nextBlock())
- continue;
- else
- break;
- }
- return 0 < copied ? copied : -1;
- }
-
- private boolean nextBlock() {
- if (++blockIndex < blocks.size()) {
- block = blocks.get(blockIndex);
- blockPos = 0;
- return true;
- }
- return false;
- }
- }
- }
|