123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497 |
- /*
- * Copyright (C) 2011, Google Inc.
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials provided
- * with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- * names of its contributors may be used to endorse or promote
- * products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
- package org.eclipse.jgit.storage.dht;
-
- import java.security.MessageDigest;
- import java.util.ArrayList;
- import java.util.Collections;
- import java.util.Comparator;
- import java.util.HashMap;
- import java.util.List;
- import java.util.Map;
- import java.util.zip.Deflater;
-
- import org.eclipse.jgit.generated.storage.dht.proto.GitStore;
- import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ChunkMeta;
- import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ChunkMeta.BaseChunk;
- import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ObjectInfo.ObjectType;
- import org.eclipse.jgit.lib.AnyObjectId;
- import org.eclipse.jgit.lib.Constants;
- import org.eclipse.jgit.lib.ObjectId;
- import org.eclipse.jgit.storage.dht.spi.Database;
- import org.eclipse.jgit.storage.dht.spi.WriteBuffer;
- import org.eclipse.jgit.transport.PackedObjectInfo;
- import org.eclipse.jgit.util.NB;
-
- /**
- * Formats one {@link PackChunk} for storage in the DHT.
- * <p>
- * Each formatter instance can be used only once.
- */
- class ChunkFormatter {
- static final int TRAILER_SIZE = 4;
-
- private final RepositoryKey repo;
-
- private final DhtInserterOptions options;
-
- private final byte[] varIntBuf;
-
- private final int maxObjects;
-
- private Map<ChunkKey, BaseChunkInfo> baseChunks;
-
- private List<StoredObject> objectList;
-
- private byte[] chunkData;
-
- private int ptr;
-
- private int mark;
-
- private int currentObjectType;
-
- private BaseChunkInfo currentObjectBase;
-
- private PackChunk.Members builder;
-
- private GitStore.ChunkInfo.Source source;
-
- private boolean fragment;
-
- private int objectType;
-
- private int objectsTotal, objectsWhole, objectsRefDelta, objectsOfsDelta;
-
- private ChunkInfo chunkInfo;
-
- ChunkFormatter(RepositoryKey repo, DhtInserterOptions options) {
- this.repo = repo;
- this.options = options;
- this.varIntBuf = new byte[32];
- this.chunkData = new byte[options.getChunkSize()];
- this.maxObjects = options.getMaxObjectCount();
- this.objectType = -1;
- }
-
- void setSource(GitStore.ChunkInfo.Source src) {
- source = src;
- }
-
- void setObjectType(int type) {
- objectType = type;
- }
-
- void setFragment() {
- fragment = true;
- }
-
- ChunkKey getChunkKey() {
- return getChunkInfo().getChunkKey();
- }
-
- ChunkInfo getChunkInfo() {
- return chunkInfo;
- }
-
- ChunkMeta getChunkMeta() {
- return builder.getMeta();
- }
-
- PackChunk getPackChunk() throws DhtException {
- return builder.build();
- }
-
- void setChunkIndex(List<PackedObjectInfo> objs) {
- builder.setChunkIndex(ChunkIndex.create(objs));
- }
-
- ChunkKey end(MessageDigest md) {
- if (md == null)
- md = Constants.newMessageDigest();
-
- // Embed a small amount of randomness into the chunk content,
- // and thus impact its name. This prevents malicious clients from
- // being able to predict what a chunk is called, which keeps them
- // from replacing an existing chunk.
- //
- chunkData = cloneArray(chunkData, ptr + TRAILER_SIZE);
- NB.encodeInt32(chunkData, ptr, options.nextChunkSalt());
- ptr += 4;
-
- md.update(chunkData, 0, ptr);
- ChunkKey key = ChunkKey.create(repo, ObjectId.fromRaw(md.digest()));
-
- GitStore.ChunkInfo.Builder info = GitStore.ChunkInfo.newBuilder();
- info.setSource(source);
- info.setObjectType(GitStore.ChunkInfo.ObjectType.valueOf(objectType));
- if (fragment)
- info.setIsFragment(true);
- info.setChunkSize(chunkData.length);
-
- GitStore.ChunkInfo.ObjectCounts.Builder cnts = info.getObjectCountsBuilder();
- cnts.setTotal(objectsTotal);
- if (objectsWhole > 0)
- cnts.setWhole(objectsWhole);
- if (objectsRefDelta > 0)
- cnts.setRefDelta(objectsRefDelta);
- if (objectsOfsDelta > 0)
- cnts.setOfsDelta(objectsOfsDelta);
-
- builder = new PackChunk.Members();
- builder.setChunkKey(key);
- builder.setChunkData(chunkData);
-
- if (baseChunks != null) {
- List<BaseChunk> list = new ArrayList<BaseChunk>(baseChunks.size());
- for (BaseChunkInfo b : baseChunks.values()) {
- if (0 < b.useCount) {
- BaseChunk.Builder c = BaseChunk.newBuilder();
- c.setRelativeStart(b.relativeStart);
- c.setChunkKey(b.key.asString());
- list.add(c.build());
- }
- }
- Collections.sort(list, new Comparator<BaseChunk>() {
- public int compare(BaseChunk a, BaseChunk b) {
- return Long.signum(a.getRelativeStart()
- - b.getRelativeStart());
- }
- });
- ChunkMeta.Builder b = ChunkMeta.newBuilder();
- b.addAllBaseChunk(list);
- ChunkMeta meta = b.build();
- builder.setMeta(meta);
- info.setMetaSize(meta.getSerializedSize());
- }
-
- if (objectList != null && !objectList.isEmpty()) {
- byte[] index = ChunkIndex.create(objectList);
- builder.setChunkIndex(index);
- info.setIndexSize(index.length);
- }
-
- chunkInfo = new ChunkInfo(key, info.build());
- return getChunkKey();
- }
-
- /**
- * Safely put the chunk to the database.
- * <p>
- * This method is slow. It first puts the chunk info, waits for success,
- * then puts the chunk itself, waits for success, and finally queues up the
- * object index with its chunk links in the supplied buffer.
- *
- * @param db
- * @param dbWriteBuffer
- * @throws DhtException
- */
- void safePut(Database db, WriteBuffer dbWriteBuffer) throws DhtException {
- WriteBuffer chunkBuf = db.newWriteBuffer();
-
- db.repository().put(repo, getChunkInfo(), chunkBuf);
- chunkBuf.flush();
-
- db.chunk().put(builder, chunkBuf);
- chunkBuf.flush();
-
- linkObjects(db, dbWriteBuffer);
- }
-
- void unsafePut(Database db, WriteBuffer dbWriteBuffer) throws DhtException {
- db.repository().put(repo, getChunkInfo(), dbWriteBuffer);
- db.chunk().put(builder, dbWriteBuffer);
- linkObjects(db, dbWriteBuffer);
- }
-
- private void linkObjects(Database db, WriteBuffer dbWriteBuffer)
- throws DhtException {
- if (objectList != null && !objectList.isEmpty()) {
- for (StoredObject obj : objectList) {
- db.objectIndex().add(ObjectIndexKey.create(repo, obj),
- obj.link(getChunkKey()), dbWriteBuffer);
- }
- }
- }
-
- boolean whole(Deflater def, int type, byte[] data, int off, final int size,
- ObjectId objId) {
- if (free() < 10 || maxObjects <= objectsTotal)
- return false;
-
- header(type, size);
- objectsWhole++;
- currentObjectType = type;
-
- int endOfHeader = ptr;
- def.setInput(data, off, size);
- def.finish();
- do {
- int left = free();
- if (left == 0) {
- rollback();
- return false;
- }
-
- int n = def.deflate(chunkData, ptr, left);
- if (n == 0) {
- rollback();
- return false;
- }
-
- ptr += n;
- } while (!def.finished());
-
- if (objectList == null)
- objectList = new ArrayList<StoredObject>();
-
- final int packedSize = ptr - endOfHeader;
- objectList.add(new StoredObject(objId, type, mark, packedSize, size));
-
- if (objectType < 0)
- objectType = type;
- else if (objectType != type)
- objectType = ChunkInfo.OBJ_MIXED;
-
- return true;
- }
-
- boolean whole(int type, long inflatedSize) {
- if (free() < 10 || maxObjects <= objectsTotal)
- return false;
-
- header(type, inflatedSize);
- objectsWhole++;
- currentObjectType = type;
- return true;
- }
-
- boolean ofsDelta(long inflatedSize, long negativeOffset) {
- final int ofsPtr = encodeVarInt(negativeOffset);
- final int ofsLen = varIntBuf.length - ofsPtr;
- if (free() < 10 + ofsLen || maxObjects <= objectsTotal)
- return false;
-
- header(Constants.OBJ_OFS_DELTA, inflatedSize);
- objectsOfsDelta++;
- currentObjectType = Constants.OBJ_OFS_DELTA;
- currentObjectBase = null;
-
- if (append(varIntBuf, ofsPtr, ofsLen))
- return true;
-
- rollback();
- return false;
- }
-
- boolean refDelta(long inflatedSize, AnyObjectId baseId) {
- if (free() < 30 || maxObjects <= objectsTotal)
- return false;
-
- header(Constants.OBJ_REF_DELTA, inflatedSize);
- objectsRefDelta++;
- currentObjectType = Constants.OBJ_REF_DELTA;
-
- baseId.copyRawTo(chunkData, ptr);
- ptr += 20;
- return true;
- }
-
- void useBaseChunk(long relativeStart, ChunkKey baseChunkKey) {
- if (baseChunks == null)
- baseChunks = new HashMap<ChunkKey, BaseChunkInfo>();
-
- BaseChunkInfo base = baseChunks.get(baseChunkKey);
- if (base == null) {
- base = new BaseChunkInfo(relativeStart, baseChunkKey);
- baseChunks.put(baseChunkKey, base);
- }
- base.useCount++;
- currentObjectBase = base;
- }
-
- void appendDeflateOutput(Deflater def) {
- while (!def.finished()) {
- int left = free();
- if (left == 0)
- return;
- int n = def.deflate(chunkData, ptr, left);
- if (n == 0)
- return;
- ptr += n;
- }
- }
-
- boolean append(byte[] data, int off, int len) {
- if (free() < len)
- return false;
-
- System.arraycopy(data, off, chunkData, ptr, len);
- ptr += len;
- return true;
- }
-
- boolean isEmpty() {
- return ptr == 0;
- }
-
- int getObjectCount() {
- return objectsTotal;
- }
-
- int position() {
- return ptr;
- }
-
- int size() {
- return ptr;
- }
-
- int free() {
- return (chunkData.length - TRAILER_SIZE) - ptr;
- }
-
- byte[] getRawChunkDataArray() {
- return chunkData;
- }
-
- int getCurrentObjectType() {
- return currentObjectType;
- }
-
- void rollback() {
- ptr = mark;
- adjustObjectCount(-1, currentObjectType);
- }
-
- void adjustObjectCount(int delta, int type) {
- objectsTotal += delta;
-
- switch (type) {
- case Constants.OBJ_COMMIT:
- case Constants.OBJ_TREE:
- case Constants.OBJ_BLOB:
- case Constants.OBJ_TAG:
- objectsWhole += delta;
- break;
-
- case Constants.OBJ_OFS_DELTA:
- objectsOfsDelta += delta;
- if (currentObjectBase != null && --currentObjectBase.useCount == 0)
- baseChunks.remove(currentObjectBase.key);
- currentObjectBase = null;
- break;
-
- case Constants.OBJ_REF_DELTA:
- objectsRefDelta += delta;
- break;
- }
- }
-
- private void header(int type, long inflatedSize) {
- mark = ptr;
- objectsTotal++;
-
- long nextLength = inflatedSize >>> 4;
- chunkData[ptr++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (inflatedSize & 0x0F));
- inflatedSize = nextLength;
- while (inflatedSize > 0) {
- nextLength >>>= 7;
- chunkData[ptr++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (inflatedSize & 0x7F));
- inflatedSize = nextLength;
- }
- }
-
- private int encodeVarInt(long value) {
- int n = varIntBuf.length - 1;
- varIntBuf[n] = (byte) (value & 0x7F);
- while ((value >>= 7) > 0)
- varIntBuf[--n] = (byte) (0x80 | (--value & 0x7F));
- return n;
- }
-
- private static byte[] cloneArray(byte[] src, int len) {
- byte[] dst = new byte[len];
- System.arraycopy(src, 0, dst, 0, len);
- return dst;
- }
-
- private static class BaseChunkInfo {
- final long relativeStart;
-
- final ChunkKey key;
-
- int useCount;
-
- BaseChunkInfo(long relativeStart, ChunkKey key) {
- this.relativeStart = relativeStart;
- this.key = key;
- }
- }
-
- private static class StoredObject extends PackedObjectInfo {
- private final int type;
-
- private final int packed;
-
- private final int inflated;
-
- StoredObject(AnyObjectId id, int type, int offset, int packed, int size) {
- super(id);
- setOffset(offset);
- this.type = type;
- this.packed = packed;
- this.inflated = size;
- }
-
- ObjectInfo link(ChunkKey key) {
- GitStore.ObjectInfo.Builder b = GitStore.ObjectInfo.newBuilder();
- b.setObjectType(ObjectType.valueOf(type));
- b.setOffset((int) getOffset());
- b.setPackedSize(packed);
- b.setInflatedSize(inflated);
- return new ObjectInfo(key, b.build());
- }
- }
- }
|