123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717 |
- /*
- * Copyright (C) 2017, Google Inc.
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials provided
- * with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- * names of its contributors may be used to endorse or promote
- * products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
- package org.eclipse.jgit.internal.storage.file;
-
- import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
- import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
- import static org.eclipse.jgit.lib.Constants.OBJ_OFS_DELTA;
- import static org.eclipse.jgit.lib.Constants.OBJ_REF_DELTA;
-
- import java.io.BufferedInputStream;
- import java.io.EOFException;
- import java.io.File;
- import java.io.FileOutputStream;
- import java.io.FilterInputStream;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.OutputStream;
- import java.io.RandomAccessFile;
- import java.nio.channels.Channels;
- import java.text.MessageFormat;
- import java.util.Collection;
- import java.util.Collections;
- import java.util.HashSet;
- import java.util.List;
- import java.util.Set;
- import java.util.zip.CRC32;
- import java.util.zip.DataFormatException;
- import java.util.zip.Deflater;
- import java.util.zip.DeflaterOutputStream;
- import java.util.zip.Inflater;
- import java.util.zip.InflaterInputStream;
-
- import org.eclipse.jgit.errors.CorruptObjectException;
- import org.eclipse.jgit.errors.IncorrectObjectTypeException;
- import org.eclipse.jgit.errors.LargeObjectException;
- import org.eclipse.jgit.errors.MissingObjectException;
- import org.eclipse.jgit.internal.JGitText;
- import org.eclipse.jgit.lib.AbbreviatedObjectId;
- import org.eclipse.jgit.lib.AnyObjectId;
- import org.eclipse.jgit.lib.Constants;
- import org.eclipse.jgit.lib.InflaterCache;
- import org.eclipse.jgit.lib.ObjectId;
- import org.eclipse.jgit.lib.ObjectIdOwnerMap;
- import org.eclipse.jgit.lib.ObjectInserter;
- import org.eclipse.jgit.lib.ObjectLoader;
- import org.eclipse.jgit.lib.ObjectReader;
- import org.eclipse.jgit.lib.ObjectStream;
- import org.eclipse.jgit.transport.PackParser;
- import org.eclipse.jgit.transport.PackedObjectInfo;
- import org.eclipse.jgit.util.BlockList;
- import org.eclipse.jgit.util.FileUtils;
- import org.eclipse.jgit.util.IO;
- import org.eclipse.jgit.util.NB;
- import org.eclipse.jgit.util.io.CountingOutputStream;
- import org.eclipse.jgit.util.sha1.SHA1;
-
- /**
- * Object inserter that inserts one pack per call to {@link #flush()}, and never
- * inserts loose objects.
- */
- class PackInserter extends ObjectInserter {
- /** Always produce version 2 indexes, to get CRC data. */
- private static final int INDEX_VERSION = 2;
-
- private final ObjectDirectory db;
-
- private List<PackedObjectInfo> objectList;
- private ObjectIdOwnerMap<PackedObjectInfo> objectMap;
- private boolean rollback;
- private boolean checkExisting = true;
-
- private int compression = Deflater.BEST_COMPRESSION;
- private File tmpPack;
- private PackStream packOut;
- private Inflater cachedInflater;
-
- PackInserter(ObjectDirectory db) {
- this.db = db;
- }
-
- /**
- * Whether to check if objects exist in the repo
- *
- * @param check
- * if {@code false}, will write out possibly-duplicate objects
- * without first checking whether they exist in the repo; default
- * is true.
- */
- public void checkExisting(boolean check) {
- checkExisting = check;
- }
-
- /**
- * Set compression level for zlib deflater.
- *
- * @param compression
- * compression level for zlib deflater.
- */
- public void setCompressionLevel(int compression) {
- this.compression = compression;
- }
-
- int getBufferSize() {
- return buffer().length;
- }
-
- /** {@inheritDoc} */
- @Override
- public ObjectId insert(int type, byte[] data, int off, int len)
- throws IOException {
- ObjectId id = idFor(type, data, off, len);
- if (objectMap != null && objectMap.contains(id)) {
- return id;
- }
- // Ignore loose objects, which are potentially unreachable.
- if (checkExisting && db.hasPackedObject(id)) {
- return id;
- }
-
- long offset = beginObject(type, len);
- packOut.compress.write(data, off, len);
- packOut.compress.finish();
- return endObject(id, offset);
- }
-
- /** {@inheritDoc} */
- @Override
- public ObjectId insert(int type, long len, InputStream in)
- throws IOException {
- byte[] buf = buffer();
- if (len <= buf.length) {
- IO.readFully(in, buf, 0, (int) len);
- return insert(type, buf, 0, (int) len);
- }
-
- long offset = beginObject(type, len);
- SHA1 md = digest();
- md.update(Constants.encodedTypeString(type));
- md.update((byte) ' ');
- md.update(Constants.encodeASCII(len));
- md.update((byte) 0);
-
- while (0 < len) {
- int n = in.read(buf, 0, (int) Math.min(buf.length, len));
- if (n <= 0) {
- throw new EOFException();
- }
- md.update(buf, 0, n);
- packOut.compress.write(buf, 0, n);
- len -= n;
- }
- packOut.compress.finish();
- return endObject(md.toObjectId(), offset);
- }
-
- private long beginObject(int type, long len) throws IOException {
- if (packOut == null) {
- beginPack();
- }
- long offset = packOut.getOffset();
- packOut.beginObject(type, len);
- return offset;
- }
-
- private ObjectId endObject(ObjectId id, long offset) {
- PackedObjectInfo obj = new PackedObjectInfo(id);
- obj.setOffset(offset);
- obj.setCRC((int) packOut.crc32.getValue());
- objectList.add(obj);
- objectMap.addIfAbsent(obj);
- return id;
- }
-
- private static File idxFor(File packFile) {
- String p = packFile.getName();
- return new File(
- packFile.getParentFile(),
- p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$
- }
-
- private void beginPack() throws IOException {
- objectList = new BlockList<>();
- objectMap = new ObjectIdOwnerMap<>();
-
- rollback = true;
- tmpPack = File.createTempFile("insert_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$
- packOut = new PackStream(tmpPack);
-
- // Write the header as though it were a single object pack.
- packOut.write(packOut.hdrBuf, 0, writePackHeader(packOut.hdrBuf, 1));
- }
-
- private static int writePackHeader(byte[] buf, int objectCount) {
- System.arraycopy(Constants.PACK_SIGNATURE, 0, buf, 0, 4);
- NB.encodeInt32(buf, 4, 2); // Always use pack version 2.
- NB.encodeInt32(buf, 8, objectCount);
- return 12;
- }
-
- /** {@inheritDoc} */
- @Override
- public PackParser newPackParser(InputStream in) {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override
- public ObjectReader newReader() {
- return new Reader();
- }
-
- /** {@inheritDoc} */
- @Override
- public void flush() throws IOException {
- if (tmpPack == null) {
- return;
- }
-
- if (packOut == null) {
- throw new IOException();
- }
-
- byte[] packHash;
- try {
- packHash = packOut.finishPack();
- } finally {
- packOut = null;
- }
-
- Collections.sort(objectList);
- File tmpIdx = idxFor(tmpPack);
- writePackIndex(tmpIdx, packHash, objectList);
-
- File realPack = new File(db.getPackDirectory(),
- "pack-" + computeName(objectList).name() + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$
- db.closeAllPackHandles(realPack);
- tmpPack.setReadOnly();
- FileUtils.rename(tmpPack, realPack, ATOMIC_MOVE);
-
- File realIdx = idxFor(realPack);
- tmpIdx.setReadOnly();
- try {
- FileUtils.rename(tmpIdx, realIdx, ATOMIC_MOVE);
- } catch (IOException e) {
- File newIdx = new File(
- realIdx.getParentFile(), realIdx.getName() + ".new"); //$NON-NLS-1$
- try {
- FileUtils.rename(tmpIdx, newIdx, ATOMIC_MOVE);
- } catch (IOException e2) {
- newIdx = tmpIdx;
- e = e2;
- }
- throw new IOException(MessageFormat.format(
- JGitText.get().panicCantRenameIndexFile, newIdx,
- realIdx), e);
- }
-
- db.openPack(realPack);
- rollback = false;
- clear();
- }
-
- private static void writePackIndex(File idx, byte[] packHash,
- List<PackedObjectInfo> list) throws IOException {
- try (OutputStream os = new FileOutputStream(idx)) {
- PackIndexWriter w = PackIndexWriter.createVersion(os, INDEX_VERSION);
- w.write(list, packHash);
- }
- }
-
- private ObjectId computeName(List<PackedObjectInfo> list) {
- SHA1 md = digest().reset();
- byte[] buf = buffer();
- for (PackedObjectInfo otp : list) {
- otp.copyRawTo(buf, 0);
- md.update(buf, 0, OBJECT_ID_LENGTH);
- }
- return ObjectId.fromRaw(md.digest());
- }
-
- /** {@inheritDoc} */
- @Override
- public void close() {
- try {
- if (packOut != null) {
- try {
- packOut.close();
- } catch (IOException err) {
- // Ignore a close failure, the pack should be removed.
- }
- }
- if (rollback && tmpPack != null) {
- try {
- FileUtils.delete(tmpPack);
- } catch (IOException e) {
- // Still delete idx.
- }
- try {
- FileUtils.delete(idxFor(tmpPack));
- } catch (IOException e) {
- // Ignore error deleting temp idx.
- }
- rollback = false;
- }
- } finally {
- clear();
- try {
- InflaterCache.release(cachedInflater);
- } finally {
- cachedInflater = null;
- }
- }
- }
-
- private void clear() {
- objectList = null;
- objectMap = null;
- tmpPack = null;
- packOut = null;
- }
-
- private Inflater inflater() {
- if (cachedInflater == null) {
- cachedInflater = InflaterCache.get();
- } else {
- cachedInflater.reset();
- }
- return cachedInflater;
- }
-
- /**
- * Stream that writes to a pack file.
- * <p>
- * Backed by two views of the same open file descriptor: a random-access file,
- * and an output stream. Seeking in the file causes subsequent writes to the
- * output stream to occur wherever the file pointer is pointing, so we need to
- * take care to always seek to the end of the file before writing a new
- * object.
- * <p>
- * Callers should always use {@link #seek(long)} to seek, rather than reaching
- * into the file member. As long as this contract is followed, calls to {@link
- * #write(byte[], int, int)} are guaranteed to write at the end of the file,
- * even if there have been intermediate seeks.
- */
- private class PackStream extends OutputStream {
- final byte[] hdrBuf;
- final CRC32 crc32;
- final DeflaterOutputStream compress;
-
- private final RandomAccessFile file;
- private final CountingOutputStream out;
- private final Deflater deflater;
-
- private boolean atEnd;
-
- PackStream(File pack) throws IOException {
- file = new RandomAccessFile(pack, "rw"); //$NON-NLS-1$
- out = new CountingOutputStream(new FileOutputStream(file.getFD()));
- deflater = new Deflater(compression);
- compress = new DeflaterOutputStream(this, deflater, 8192);
- hdrBuf = new byte[32];
- crc32 = new CRC32();
- atEnd = true;
- }
-
- long getOffset() {
- // This value is accurate as long as we only ever write to the end of the
- // file, and don't seek back to overwrite any previous segments. Although
- // this is subtle, storing the stream counter this way is still preferable
- // to returning file.length() here, as it avoids a syscall and possible
- // IOException.
- return out.getCount();
- }
-
- void seek(long offset) throws IOException {
- file.seek(offset);
- atEnd = false;
- }
-
- void beginObject(int objectType, long length) throws IOException {
- crc32.reset();
- deflater.reset();
- write(hdrBuf, 0, encodeTypeSize(objectType, length));
- }
-
- private int encodeTypeSize(int type, long rawLength) {
- long nextLength = rawLength >>> 4;
- hdrBuf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (rawLength & 0x0F));
- rawLength = nextLength;
- int n = 1;
- while (rawLength > 0) {
- nextLength >>>= 7;
- hdrBuf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (rawLength & 0x7F));
- rawLength = nextLength;
- }
- return n;
- }
-
- @Override
- public void write(final int b) throws IOException {
- hdrBuf[0] = (byte) b;
- write(hdrBuf, 0, 1);
- }
-
- @Override
- public void write(byte[] data, int off, int len) throws IOException {
- crc32.update(data, off, len);
- if (!atEnd) {
- file.seek(file.length());
- atEnd = true;
- }
- out.write(data, off, len);
- }
-
- byte[] finishPack() throws IOException {
- // Overwrite placeholder header with actual object count, then hash. This
- // method intentionally uses direct seek/write calls rather than the
- // wrappers which keep track of atEnd. This leaves atEnd, the file
- // pointer, and out's counter in an inconsistent state; that's ok, since
- // this method closes the file anyway.
- try {
- file.seek(0);
- out.write(hdrBuf, 0, writePackHeader(hdrBuf, objectList.size()));
-
- byte[] buf = buffer();
- SHA1 md = digest().reset();
- file.seek(0);
- while (true) {
- int r = file.read(buf);
- if (r < 0) {
- break;
- }
- md.update(buf, 0, r);
- }
- byte[] packHash = md.digest();
- out.write(packHash, 0, packHash.length);
- return packHash;
- } finally {
- close();
- }
- }
-
- @Override
- public void close() throws IOException {
- deflater.end();
- try {
- out.close();
- } finally {
- file.close();
- }
- }
-
- byte[] inflate(long filePos, int len) throws IOException, DataFormatException {
- byte[] dstbuf;
- try {
- dstbuf = new byte[len];
- } catch (OutOfMemoryError noMemory) {
- return null; // Caller will switch to large object streaming.
- }
-
- byte[] srcbuf = buffer();
- Inflater inf = inflater();
- filePos += setInput(filePos, inf, srcbuf);
- for (int dstoff = 0;;) {
- int n = inf.inflate(dstbuf, dstoff, dstbuf.length - dstoff);
- dstoff += n;
- if (inf.finished()) {
- return dstbuf;
- }
- if (inf.needsInput()) {
- filePos += setInput(filePos, inf, srcbuf);
- } else if (n == 0) {
- throw new DataFormatException();
- }
- }
- }
-
- private int setInput(long filePos, Inflater inf, byte[] buf)
- throws IOException {
- if (file.getFilePointer() != filePos) {
- seek(filePos);
- }
- int n = file.read(buf);
- if (n < 0) {
- throw new EOFException(JGitText.get().unexpectedEofInPack);
- }
- inf.setInput(buf, 0, n);
- return n;
- }
- }
-
- private class Reader extends ObjectReader {
- private final ObjectReader ctx;
-
- private Reader() {
- ctx = db.newReader();
- setStreamFileThreshold(ctx.getStreamFileThreshold());
- }
-
- @Override
- public ObjectReader newReader() {
- return db.newReader();
- }
-
- @Override
- public ObjectInserter getCreatedFromInserter() {
- return PackInserter.this;
- }
-
- @Override
- public Collection<ObjectId> resolve(AbbreviatedObjectId id)
- throws IOException {
- Collection<ObjectId> stored = ctx.resolve(id);
- if (objectList == null) {
- return stored;
- }
-
- Set<ObjectId> r = new HashSet<>(stored.size() + 2);
- r.addAll(stored);
- for (PackedObjectInfo obj : objectList) {
- if (id.prefixCompare(obj) == 0) {
- r.add(obj.copy());
- }
- }
- return r;
- }
-
- @Override
- public ObjectLoader open(AnyObjectId objectId, int typeHint)
- throws MissingObjectException, IncorrectObjectTypeException,
- IOException {
- if (objectMap == null) {
- return ctx.open(objectId, typeHint);
- }
-
- PackedObjectInfo obj = objectMap.get(objectId);
- if (obj == null) {
- return ctx.open(objectId, typeHint);
- }
-
- byte[] buf = buffer();
- packOut.seek(obj.getOffset());
- int cnt = packOut.file.read(buf, 0, 20);
- if (cnt <= 0) {
- throw new EOFException(JGitText.get().unexpectedEofInPack);
- }
-
- int c = buf[0] & 0xff;
- int type = (c >> 4) & 7;
- if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
- throw new IOException(MessageFormat.format(
- JGitText.get().cannotReadBackDelta, Integer.toString(type)));
- }
- if (typeHint != OBJ_ANY && type != typeHint) {
- throw new IncorrectObjectTypeException(objectId.copy(), typeHint);
- }
-
- long sz = c & 0x0f;
- int ptr = 1;
- int shift = 4;
- while ((c & 0x80) != 0) {
- if (ptr >= cnt) {
- throw new EOFException(JGitText.get().unexpectedEofInPack);
- }
- c = buf[ptr++] & 0xff;
- sz += ((long) (c & 0x7f)) << shift;
- shift += 7;
- }
-
- long zpos = obj.getOffset() + ptr;
- if (sz < getStreamFileThreshold()) {
- byte[] data = inflate(obj, zpos, (int) sz);
- if (data != null) {
- return new ObjectLoader.SmallObject(type, data);
- }
- }
- return new StreamLoader(type, sz, zpos);
- }
-
- private byte[] inflate(PackedObjectInfo obj, long zpos, int sz)
- throws IOException, CorruptObjectException {
- try {
- return packOut.inflate(zpos, sz);
- } catch (DataFormatException dfe) {
- throw new CorruptObjectException(
- MessageFormat.format(
- JGitText.get().objectAtHasBadZlibStream,
- Long.valueOf(obj.getOffset()),
- tmpPack.getAbsolutePath()),
- dfe);
- }
- }
-
- @Override
- public Set<ObjectId> getShallowCommits() throws IOException {
- return ctx.getShallowCommits();
- }
-
- @Override
- public void close() {
- ctx.close();
- }
-
- private class StreamLoader extends ObjectLoader {
- private final int type;
- private final long size;
- private final long pos;
-
- StreamLoader(int type, long size, long pos) {
- this.type = type;
- this.size = size;
- this.pos = pos;
- }
-
- @Override
- public ObjectStream openStream()
- throws MissingObjectException, IOException {
- int bufsz = buffer().length;
- packOut.seek(pos);
-
- InputStream fileStream = new FilterInputStream(
- Channels.newInputStream(packOut.file.getChannel())) {
- // atEnd was already set to false by the previous seek, but it's
- // technically possible for a caller to call insert on the
- // inserter in the middle of reading from this stream. Behavior is
- // undefined in this case, so it would arguably be ok to ignore,
- // but it's not hard to at least make an attempt to not corrupt
- // the data.
- @Override
- public int read() throws IOException {
- packOut.atEnd = false;
- return super.read();
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- packOut.atEnd = false;
- return super.read(b);
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- packOut.atEnd = false;
- return super.read(b,off,len);
- }
-
- @Override
- public void close() {
- // Never close underlying RandomAccessFile, which lasts the
- // lifetime of the enclosing PackStream.
- }
- };
- return new ObjectStream.Filter(
- type, size,
- new BufferedInputStream(
- new InflaterInputStream(fileStream, inflater(), bufsz), bufsz));
- }
-
- @Override
- public int getType() {
- return type;
- }
-
- @Override
- public long getSize() {
- return size;
- }
-
- @Override
- public byte[] getCachedBytes() throws LargeObjectException {
- throw new LargeObjectException.ExceedsLimit(
- getStreamFileThreshold(), size);
- }
- }
- }
- }
|