Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.

PackChunk.java 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795
  1. /*
  2. * Copyright (C) 2011, Google Inc.
  3. * and other copyright owners as documented in the project's IP log.
  4. *
  5. * This program and the accompanying materials are made available
  6. * under the terms of the Eclipse Distribution License v1.0 which
  7. * accompanies this distribution, is reproduced below, and is
  8. * available at http://www.eclipse.org/org/documents/edl-v10.php
  9. *
  10. * All rights reserved.
  11. *
  12. * Redistribution and use in source and binary forms, with or
  13. * without modification, are permitted provided that the following
  14. * conditions are met:
  15. *
  16. * - Redistributions of source code must retain the above copyright
  17. * notice, this list of conditions and the following disclaimer.
  18. *
  19. * - Redistributions in binary form must reproduce the above
  20. * copyright notice, this list of conditions and the following
  21. * disclaimer in the documentation and/or other materials provided
  22. * with the distribution.
  23. *
  24. * - Neither the name of the Eclipse Foundation, Inc. nor the
  25. * names of its contributors may be used to endorse or promote
  26. * products derived from this software without specific prior
  27. * written permission.
  28. *
  29. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  30. * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  31. * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  32. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  34. * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  35. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  36. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  37. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  38. * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  39. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  40. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  41. * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42. */
  43. package org.eclipse.jgit.storage.dht;
  44. import static org.eclipse.jgit.lib.Constants.OBJ_BAD;
  45. import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
  46. import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
  47. import static org.eclipse.jgit.lib.Constants.OBJ_OFS_DELTA;
  48. import static org.eclipse.jgit.lib.Constants.OBJ_REF_DELTA;
  49. import static org.eclipse.jgit.lib.Constants.OBJ_TAG;
  50. import static org.eclipse.jgit.lib.Constants.OBJ_TREE;
  51. import static org.eclipse.jgit.lib.Constants.newMessageDigest;
  52. import static org.eclipse.jgit.storage.dht.ChunkFormatter.TRAILER_SIZE;
  53. import java.io.IOException;
  54. import java.nio.ByteBuffer;
  55. import java.security.MessageDigest;
  56. import java.text.MessageFormat;
  57. import java.util.zip.DataFormatException;
  58. import java.util.zip.Inflater;
  59. import org.eclipse.jgit.errors.CorruptObjectException;
  60. import org.eclipse.jgit.errors.LargeObjectException;
  61. import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
  62. import org.eclipse.jgit.generated.storage.dht.proto.GitStore.ChunkMeta;
  63. import org.eclipse.jgit.lib.AnyObjectId;
  64. import org.eclipse.jgit.lib.ObjectId;
  65. import org.eclipse.jgit.lib.ObjectLoader;
  66. import org.eclipse.jgit.storage.pack.BinaryDelta;
  67. import org.eclipse.jgit.storage.pack.PackOutputStream;
  68. import org.eclipse.jgit.transport.PackParser;
  69. /**
  70. * Chunk of object data, stored under a {@link ChunkKey}.
  71. * <p>
  72. * A chunk typically contains thousands of objects, compressed in the Git native
  73. * pack file format. Its associated {@link ChunkIndex} provides offsets for each
  74. * object's header and compressed data.
  75. * <p>
  76. * Chunks (and their indexes) are opaque binary blobs meant only to be read by
  77. * the Git implementation.
  78. */
  79. public final class PackChunk {
  80. /** Constructs a {@link PackChunk} while reading from the DHT. */
  81. public static class Members {
  82. private ChunkKey chunkKey;
  83. private byte[] dataBuf;
  84. private int dataPtr;
  85. private int dataLen;
  86. private byte[] indexBuf;
  87. private int indexPtr;
  88. private int indexLen;
  89. private ChunkMeta meta;
  90. /** @return the chunk key. Never null. */
  91. public ChunkKey getChunkKey() {
  92. return chunkKey;
  93. }
  94. /**
  95. * @param key
  96. * @return {@code this}
  97. */
  98. public Members setChunkKey(ChunkKey key) {
  99. this.chunkKey = key;
  100. return this;
  101. }
  102. /** @return true if there is chunk data present. */
  103. public boolean hasChunkData() {
  104. return dataBuf != null;
  105. }
  106. /** @return the chunk data, or null if not available. */
  107. public byte[] getChunkData() {
  108. return asArray(dataBuf, dataPtr, dataLen);
  109. }
  110. /** @return the chunk data, or null if not available. */
  111. public ByteBuffer getChunkDataAsByteBuffer() {
  112. return asByteBuffer(dataBuf, dataPtr, dataLen);
  113. }
  114. private static byte[] asArray(byte[] buf, int ptr, int len) {
  115. if (buf == null)
  116. return null;
  117. if (ptr == 0 && buf.length == len)
  118. return buf;
  119. byte[] r = new byte[len];
  120. System.arraycopy(buf, ptr, r, 0, len);
  121. return r;
  122. }
  123. private static ByteBuffer asByteBuffer(byte[] buf, int ptr, int len) {
  124. return buf != null ? ByteBuffer.wrap(buf, ptr, len) : null;
  125. }
  126. /**
  127. * @param chunkData
  128. * @return {@code this}
  129. */
  130. public Members setChunkData(byte[] chunkData) {
  131. return setChunkData(chunkData, 0, chunkData.length);
  132. }
  133. /**
  134. * @param chunkData
  135. * @param ptr
  136. * @param len
  137. * @return {@code this}
  138. */
  139. public Members setChunkData(byte[] chunkData, int ptr, int len) {
  140. this.dataBuf = chunkData;
  141. this.dataPtr = ptr;
  142. this.dataLen = len;
  143. return this;
  144. }
  145. /** @return true if there is a chunk index present. */
  146. public boolean hasChunkIndex() {
  147. return indexBuf != null;
  148. }
  149. /** @return the chunk index, or null if not available. */
  150. public byte[] getChunkIndex() {
  151. return asArray(indexBuf, indexPtr, indexLen);
  152. }
  153. /** @return the chunk index, or null if not available. */
  154. public ByteBuffer getChunkIndexAsByteBuffer() {
  155. return asByteBuffer(indexBuf, indexPtr, indexLen);
  156. }
  157. /**
  158. * @param chunkIndex
  159. * @return {@code this}
  160. */
  161. public Members setChunkIndex(byte[] chunkIndex) {
  162. return setChunkIndex(chunkIndex, 0, chunkIndex.length);
  163. }
  164. /**
  165. * @param chunkIndex
  166. * @param ptr
  167. * @param len
  168. * @return {@code this}
  169. */
  170. public Members setChunkIndex(byte[] chunkIndex, int ptr, int len) {
  171. this.indexBuf = chunkIndex;
  172. this.indexPtr = ptr;
  173. this.indexLen = len;
  174. return this;
  175. }
  176. /** @return true if there is meta information present. */
  177. public boolean hasMeta() {
  178. return meta != null;
  179. }
  180. /** @return the inline meta data, or null if not available. */
  181. public ChunkMeta getMeta() {
  182. return meta;
  183. }
  184. /**
  185. * @param meta
  186. * @return {@code this}
  187. */
  188. public Members setMeta(ChunkMeta meta) {
  189. this.meta = meta;
  190. return this;
  191. }
  192. /**
  193. * @return the PackChunk instance.
  194. * @throws DhtException
  195. * if early validation indicates the chunk data is corrupt
  196. * or not recognized by this version of the library.
  197. */
  198. public PackChunk build() throws DhtException {
  199. ChunkIndex i;
  200. if (indexBuf != null)
  201. i = ChunkIndex.fromBytes(chunkKey, indexBuf, indexPtr, indexLen);
  202. else
  203. i = null;
  204. return new PackChunk(chunkKey, dataBuf, dataPtr, dataLen, i, meta);
  205. }
  206. }
  207. private static final int INFLATE_STRIDE = 512;
  208. private final ChunkKey key;
  209. private final byte[] dataBuf;
  210. private final int dataPtr;
  211. private final int dataLen;
  212. private final ChunkIndex index;
  213. private final ChunkMeta meta;
  214. private volatile Boolean valid;
  215. PackChunk(ChunkKey key, byte[] dataBuf, int dataPtr, int dataLen,
  216. ChunkIndex index, ChunkMeta meta) {
  217. this.key = key;
  218. this.dataBuf = dataBuf;
  219. this.dataPtr = dataPtr;
  220. this.dataLen = dataLen;
  221. this.index = index;
  222. this.meta = meta;
  223. }
  224. /** @return unique name of this chunk in the database. */
  225. public ChunkKey getChunkKey() {
  226. return key;
  227. }
  228. /** @return index describing the objects stored within this chunk. */
  229. public ChunkIndex getIndex() {
  230. return index;
  231. }
  232. /** @return inline meta information, or null if no data was necessary. */
  233. public ChunkMeta getMeta() {
  234. return meta;
  235. }
  236. @Override
  237. public String toString() {
  238. return "PackChunk[" + getChunkKey() + "]";
  239. }
  240. boolean hasIndex() {
  241. return index != null;
  242. }
  243. boolean isFragment() {
  244. return meta != null && 0 < meta.getFragmentCount();
  245. }
  246. int findOffset(RepositoryKey repo, AnyObjectId objId) {
  247. if (key.getRepositoryId() == repo.asInt() && index != null)
  248. return index.findOffset(objId);
  249. return -1;
  250. }
  251. boolean contains(RepositoryKey repo, AnyObjectId objId) {
  252. return 0 <= findOffset(repo, objId);
  253. }
  254. static ObjectLoader read(PackChunk pc, int pos, final DhtReader ctx,
  255. final int typeHint) throws IOException {
  256. try {
  257. return read1(pc, pos, ctx, typeHint, true /* use recentChunks */);
  258. } catch (DeltaChainCycleException cycleFound) {
  259. // A cycle can occur if recentChunks cache was used by the reader
  260. // to satisfy an OBJ_REF_DELTA, but the chunk that was chosen has
  261. // a reverse delta back onto an object already being read during
  262. // this invocation. Its not as uncommon as it sounds, as the Git
  263. // wire protocol can sometimes copy an object the repository already
  264. // has when dealing with reverts or cherry-picks.
  265. //
  266. // Work around the cycle by disabling the recentChunks cache for
  267. // this resolution only. This will force the DhtReader to re-read
  268. // OBJECT_INDEX and consider only the oldest chunk for any given
  269. // object. There cannot be a cycle if the method only walks along
  270. // the oldest chunks.
  271. try {
  272. ctx.getStatistics().deltaChainCycles++;
  273. return read1(pc, pos, ctx, typeHint, false /* no recentChunks */);
  274. } catch (DeltaChainCycleException cannotRecover) {
  275. throw new DhtException(MessageFormat.format(
  276. DhtText.get().cycleInDeltaChain, pc.getChunkKey(),
  277. Integer.valueOf(pos)));
  278. }
  279. }
  280. }
  281. @SuppressWarnings("null")
  282. private static ObjectLoader read1(PackChunk pc, int pos,
  283. final DhtReader ctx, final int typeHint, final boolean recent)
  284. throws IOException, DeltaChainCycleException {
  285. try {
  286. Delta delta = null;
  287. byte[] data = null;
  288. int type = OBJ_BAD;
  289. boolean cached = false;
  290. SEARCH: for (;;) {
  291. final byte[] dataBuf = pc.dataBuf;
  292. final int dataPtr = pc.dataPtr;
  293. final int posPtr = dataPtr + pos;
  294. int c = dataBuf[posPtr] & 0xff;
  295. int typeCode = (c >> 4) & 7;
  296. long sz = c & 15;
  297. int shift = 4;
  298. int p = 1;
  299. while ((c & 0x80) != 0) {
  300. c = dataBuf[posPtr + p++] & 0xff;
  301. sz += ((long) (c & 0x7f)) << shift;
  302. shift += 7;
  303. }
  304. switch (typeCode) {
  305. case OBJ_COMMIT:
  306. case OBJ_TREE:
  307. case OBJ_BLOB:
  308. case OBJ_TAG: {
  309. if (delta != null) {
  310. data = inflate(sz, pc, pos + p, ctx);
  311. type = typeCode;
  312. break SEARCH;
  313. }
  314. if (sz < Integer.MAX_VALUE && !pc.isFragment()) {
  315. try {
  316. data = pc.inflateOne(sz, pos + p, ctx);
  317. return new ObjectLoader.SmallObject(typeCode, data);
  318. } catch (LargeObjectException tooBig) {
  319. // Fall through and stream.
  320. }
  321. }
  322. return new LargeNonDeltaObject(typeCode, sz, pc, pos + p, ctx);
  323. }
  324. case OBJ_OFS_DELTA: {
  325. c = dataBuf[posPtr + p++] & 0xff;
  326. long base = c & 127;
  327. while ((c & 128) != 0) {
  328. base += 1;
  329. c = dataBuf[posPtr + p++] & 0xff;
  330. base <<= 7;
  331. base += (c & 127);
  332. }
  333. ChunkKey baseChunkKey;
  334. int basePosInChunk;
  335. if (base <= pos) {
  336. // Base occurs in the same chunk, just earlier.
  337. baseChunkKey = pc.getChunkKey();
  338. basePosInChunk = pos - (int) base;
  339. } else {
  340. // Long offset delta, base occurs in another chunk.
  341. // Adjust distance to be from our chunk start.
  342. base = base - pos;
  343. ChunkMeta.BaseChunk baseChunk;
  344. baseChunk = ChunkMetaUtil.getBaseChunk(
  345. pc.key,
  346. pc.meta,
  347. base);
  348. baseChunkKey = ChunkKey.fromString(baseChunk.getChunkKey());
  349. basePosInChunk = (int) (baseChunk.getRelativeStart() - base);
  350. }
  351. delta = new Delta(delta, //
  352. pc.key, pos, (int) sz, p, //
  353. baseChunkKey, basePosInChunk);
  354. if (sz != delta.deltaSize)
  355. break SEARCH;
  356. DeltaBaseCache.Entry e = delta.getBase(ctx);
  357. if (e != null) {
  358. type = e.type;
  359. data = e.data;
  360. cached = true;
  361. break SEARCH;
  362. }
  363. if (baseChunkKey != pc.getChunkKey())
  364. pc = ctx.getChunk(baseChunkKey);
  365. pos = basePosInChunk;
  366. continue SEARCH;
  367. }
  368. case OBJ_REF_DELTA: {
  369. ObjectId id = ObjectId.fromRaw(dataBuf, posPtr + p);
  370. PackChunk nc = pc;
  371. int base = pc.index.findOffset(id);
  372. if (base < 0) {
  373. DhtReader.ChunkAndOffset n;
  374. n = ctx.getChunk(id, typeHint, recent);
  375. nc = n.chunk;
  376. base = n.offset;
  377. }
  378. checkCycle(delta, pc.key, pos);
  379. delta = new Delta(delta, //
  380. pc.key, pos, (int) sz, p + 20, //
  381. nc.getChunkKey(), base);
  382. if (sz != delta.deltaSize)
  383. break SEARCH;
  384. DeltaBaseCache.Entry e = delta.getBase(ctx);
  385. if (e != null) {
  386. type = e.type;
  387. data = e.data;
  388. cached = true;
  389. break SEARCH;
  390. }
  391. pc = nc;
  392. pos = base;
  393. continue SEARCH;
  394. }
  395. default:
  396. throw new DhtException(MessageFormat.format(
  397. DhtText.get().unsupportedObjectTypeInChunk, //
  398. Integer.valueOf(typeCode), //
  399. pc.getChunkKey(), //
  400. Integer.valueOf(pos)));
  401. }
  402. }
  403. // At this point there is at least one delta to apply to data.
  404. // (Whole objects with no deltas to apply return early above.)
  405. do {
  406. if (!delta.deltaChunk.equals(pc.getChunkKey()))
  407. pc = ctx.getChunk(delta.deltaChunk);
  408. pos = delta.deltaPos;
  409. // Cache only the base immediately before desired object.
  410. if (cached)
  411. cached = false;
  412. else if (delta.next == null)
  413. delta.putBase(ctx, type, data);
  414. final byte[] cmds = delta.decompress(pc, ctx);
  415. final long sz = BinaryDelta.getResultSize(cmds);
  416. final byte[] result = newResult(sz);
  417. BinaryDelta.apply(data, cmds, result);
  418. data = result;
  419. delta = delta.next;
  420. } while (delta != null);
  421. return new ObjectLoader.SmallObject(type, data);
  422. } catch (DataFormatException dfe) {
  423. CorruptObjectException coe = new CorruptObjectException(
  424. MessageFormat.format(DhtText.get().corruptCompressedObject,
  425. pc.getChunkKey(), Integer.valueOf(pos)));
  426. coe.initCause(dfe);
  427. throw coe;
  428. }
  429. }
  430. private static byte[] inflate(long sz, PackChunk pc, int pos,
  431. DhtReader reader) throws DataFormatException, DhtException {
  432. if (pc.isFragment())
  433. return inflateFragment(sz, pc, pos, reader);
  434. return pc.inflateOne(sz, pos, reader);
  435. }
  436. private byte[] inflateOne(long sz, int pos, DhtReader reader)
  437. throws DataFormatException {
  438. // Because the chunk ends in a 4 byte CRC, there is always
  439. // more data available for input than the inflater needs.
  440. // This also helps with an optimization in libz where it
  441. // wants at least 1 extra byte of input beyond the end.
  442. final byte[] dstbuf = newResult(sz);
  443. final Inflater inf = reader.inflater();
  444. final int offset = pos;
  445. int dstoff = 0;
  446. int bs = Math.min(dataLen - pos, INFLATE_STRIDE);
  447. inf.setInput(dataBuf, dataPtr + pos, bs);
  448. pos += bs;
  449. while (dstoff < dstbuf.length) {
  450. int n = inf.inflate(dstbuf, dstoff, dstbuf.length - dstoff);
  451. if (n == 0) {
  452. if (inf.needsInput()) {
  453. bs = Math.min(dataLen - pos, INFLATE_STRIDE);
  454. inf.setInput(dataBuf, dataPtr + pos, bs);
  455. pos += bs;
  456. continue;
  457. }
  458. break;
  459. }
  460. dstoff += n;
  461. }
  462. if (dstoff != sz) {
  463. throw new DataFormatException(MessageFormat.format(
  464. DhtText.get().shortCompressedObject,
  465. getChunkKey(),
  466. Integer.valueOf(offset)));
  467. }
  468. return dstbuf;
  469. }
  470. private static byte[] inflateFragment(long sz, PackChunk pc, final int pos,
  471. DhtReader reader) throws DataFormatException, DhtException {
  472. byte[] dstbuf = newResult(sz);
  473. int dstoff = 0;
  474. final Inflater inf = reader.inflater();
  475. final ChunkMeta meta = pc.meta;
  476. int nextChunk = 1;
  477. int bs = pc.dataLen - pos - TRAILER_SIZE;
  478. inf.setInput(pc.dataBuf, pc.dataPtr + pos, bs);
  479. while (dstoff < dstbuf.length) {
  480. int n = inf.inflate(dstbuf, dstoff, dstbuf.length - dstoff);
  481. if (n == 0) {
  482. if (inf.needsInput()) {
  483. if (meta.getFragmentCount() <= nextChunk)
  484. break;
  485. pc = reader.getChunk(ChunkKey.fromString(
  486. meta.getFragment(nextChunk++)));
  487. if (meta.getFragmentCount() == nextChunk)
  488. bs = pc.dataLen; // Include trailer on last chunk.
  489. else
  490. bs = pc.dataLen - TRAILER_SIZE;
  491. inf.setInput(pc.dataBuf, pc.dataPtr, bs);
  492. continue;
  493. }
  494. break;
  495. }
  496. dstoff += n;
  497. }
  498. if (dstoff != sz) {
  499. throw new DataFormatException(MessageFormat.format(
  500. DhtText.get().shortCompressedObject,
  501. ChunkKey.fromString(meta.getFragment(0)),
  502. Integer.valueOf(pos)));
  503. }
  504. return dstbuf;
  505. }
  506. private static byte[] newResult(long sz) {
  507. if (Integer.MAX_VALUE < sz)
  508. throw new LargeObjectException.ExceedsByteArrayLimit();
  509. try {
  510. return new byte[(int) sz];
  511. } catch (OutOfMemoryError noMemory) {
  512. throw new LargeObjectException.OutOfMemory(noMemory);
  513. }
  514. }
  515. int readObjectTypeAndSize(int ptr, PackParser.ObjectTypeAndSize info) {
  516. ptr += dataPtr;
  517. int c = dataBuf[ptr++] & 0xff;
  518. int typeCode = (c >> 4) & 7;
  519. long sz = c & 15;
  520. int shift = 4;
  521. while ((c & 0x80) != 0) {
  522. c = dataBuf[ptr++] & 0xff;
  523. sz += ((long) (c & 0x7f)) << shift;
  524. shift += 7;
  525. }
  526. switch (typeCode) {
  527. case OBJ_OFS_DELTA:
  528. c = dataBuf[ptr++] & 0xff;
  529. while ((c & 128) != 0)
  530. c = dataBuf[ptr++] & 0xff;
  531. break;
  532. case OBJ_REF_DELTA:
  533. ptr += 20;
  534. break;
  535. }
  536. info.type = typeCode;
  537. info.size = sz;
  538. return ptr - dataPtr;
  539. }
  540. int read(int ptr, byte[] dst, int dstPos, int cnt) {
  541. // Do not allow readers to read the CRC-32 from the tail.
  542. int n = Math.min(cnt, (dataLen - TRAILER_SIZE) - ptr);
  543. System.arraycopy(dataBuf, dataPtr + ptr, dst, dstPos, n);
  544. return n;
  545. }
  546. void copyObjectAsIs(PackOutputStream out, DhtObjectToPack obj,
  547. boolean validate, DhtReader ctx) throws IOException,
  548. StoredObjectRepresentationNotAvailableException {
  549. if (validate && !isValid()) {
  550. StoredObjectRepresentationNotAvailableException gone;
  551. gone = new StoredObjectRepresentationNotAvailableException(obj);
  552. gone.initCause(new DhtException(MessageFormat.format(
  553. DhtText.get().corruptChunk, getChunkKey())));
  554. throw gone;
  555. }
  556. int ptr = dataPtr + obj.offset;
  557. int c = dataBuf[ptr++] & 0xff;
  558. int typeCode = (c >> 4) & 7;
  559. long inflatedSize = c & 15;
  560. int shift = 4;
  561. while ((c & 0x80) != 0) {
  562. c = dataBuf[ptr++] & 0xff;
  563. inflatedSize += ((long) (c & 0x7f)) << shift;
  564. shift += 7;
  565. }
  566. switch (typeCode) {
  567. case OBJ_OFS_DELTA:
  568. do {
  569. c = dataBuf[ptr++] & 0xff;
  570. } while ((c & 128) != 0);
  571. break;
  572. case OBJ_REF_DELTA:
  573. ptr += 20;
  574. break;
  575. }
  576. // If the size is positive, its accurate. If its -1, this is a
  577. // fragmented object that will need more handling below,
  578. // so copy all of the chunk, minus the trailer.
  579. final int maxAvail = (dataLen - TRAILER_SIZE) - (ptr - dataPtr);
  580. final int copyLen;
  581. if (0 < obj.size)
  582. copyLen = Math.min(obj.size, maxAvail);
  583. else if (-1 == obj.size)
  584. copyLen = maxAvail;
  585. else
  586. throw new DhtException(MessageFormat.format(
  587. DhtText.get().expectedObjectSizeDuringCopyAsIs, obj));
  588. out.writeHeader(obj, inflatedSize);
  589. out.write(dataBuf, ptr, copyLen);
  590. // If the object was fragmented, send all of the other fragments.
  591. if (isFragment()) {
  592. int cnt = meta.getFragmentCount();
  593. for (int fragId = 1; fragId < cnt; fragId++) {
  594. PackChunk pc = ctx.getChunk(ChunkKey.fromString(
  595. meta.getFragment(fragId)));
  596. pc.copyEntireChunkAsIs(out, obj, validate);
  597. }
  598. }
  599. }
  600. void copyEntireChunkAsIs(PackOutputStream out, DhtObjectToPack obj,
  601. boolean validate) throws IOException {
  602. if (validate && !isValid()) {
  603. if (obj != null)
  604. throw new CorruptObjectException(obj, MessageFormat.format(
  605. DhtText.get().corruptChunk, getChunkKey()));
  606. else
  607. throw new DhtException(MessageFormat.format(
  608. DhtText.get().corruptChunk, getChunkKey()));
  609. }
  610. // Do not copy the trailer onto the output stream.
  611. out.write(dataBuf, dataPtr, dataLen - TRAILER_SIZE);
  612. }
  613. @SuppressWarnings("boxing")
  614. private boolean isValid() {
  615. Boolean v = valid;
  616. if (v == null) {
  617. MessageDigest m = newMessageDigest();
  618. m.update(dataBuf, dataPtr, dataLen);
  619. v = key.getChunkHash().compareTo(m.digest(), 0) == 0;
  620. valid = v;
  621. }
  622. return v.booleanValue();
  623. }
  624. /** @return the complete size of this chunk, in memory. */
  625. int getTotalSize() {
  626. // Assume the index is part of the buffer, and report its total size..
  627. if (dataPtr != 0 || dataLen != dataBuf.length)
  628. return dataBuf.length;
  629. int sz = dataLen;
  630. if (index != null)
  631. sz += index.getIndexSize();
  632. return sz;
  633. }
  634. private static class Delta {
  635. /** Child that applies onto this object. */
  636. final Delta next;
  637. /** The chunk the delta is stored in. */
  638. final ChunkKey deltaChunk;
  639. /** Offset of the delta object. */
  640. final int deltaPos;
  641. /** Size of the inflated delta stream. */
  642. final int deltaSize;
  643. /** Total size of the delta's pack entry header (including base). */
  644. final int hdrLen;
  645. /** The chunk the base is stored in. */
  646. final ChunkKey baseChunk;
  647. /** Offset of the base object. */
  648. final int basePos;
  649. Delta(Delta next, ChunkKey dc, int ofs, int sz, int hdrLen,
  650. ChunkKey bc, int bp) {
  651. this.next = next;
  652. this.deltaChunk = dc;
  653. this.deltaPos = ofs;
  654. this.deltaSize = sz;
  655. this.hdrLen = hdrLen;
  656. this.baseChunk = bc;
  657. this.basePos = bp;
  658. }
  659. byte[] decompress(PackChunk chunk, DhtReader reader)
  660. throws DataFormatException, DhtException {
  661. return inflate(deltaSize, chunk, deltaPos + hdrLen, reader);
  662. }
  663. DeltaBaseCache.Entry getBase(DhtReader ctx) {
  664. return ctx.getDeltaBaseCache().get(baseChunk, basePos);
  665. }
  666. void putBase(DhtReader ctx, int type, byte[] data) {
  667. ctx.getDeltaBaseCache().put(baseChunk, basePos, type, data);
  668. }
  669. }
  670. private static void checkCycle(Delta delta, ChunkKey key, int ofs)
  671. throws DeltaChainCycleException {
  672. for (; delta != null; delta = delta.next) {
  673. if (delta.deltaPos == ofs && delta.deltaChunk.equals(key))
  674. throw DeltaChainCycleException.INSTANCE;
  675. }
  676. }
  677. private static class DeltaChainCycleException extends Exception {
  678. private static final long serialVersionUID = 1L;
  679. static final DeltaChainCycleException INSTANCE = new DeltaChainCycleException();
  680. }
  681. }