You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

PackFile.java 33KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207
  1. /*
  2. * Copyright (C) 2008-2009, Google Inc.
  3. * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
  4. * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org> and others
  5. *
  6. * This program and the accompanying materials are made available under the
  7. * terms of the Eclipse Distribution License v. 1.0 which is available at
  8. * https://www.eclipse.org/org/documents/edl-v10.php.
  9. *
  10. * SPDX-License-Identifier: BSD-3-Clause
  11. */
  12. package org.eclipse.jgit.internal.storage.file;
  13. import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
  14. import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
  15. import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
  16. import java.io.EOFException;
  17. import java.io.File;
  18. import java.io.FileNotFoundException;
  19. import java.io.IOException;
  20. import java.io.InterruptedIOException;
  21. import java.io.RandomAccessFile;
  22. import java.nio.MappedByteBuffer;
  23. import java.nio.channels.FileChannel.MapMode;
  24. import java.nio.file.AccessDeniedException;
  25. import java.nio.file.NoSuchFileException;
  26. import java.text.MessageFormat;
  27. import java.time.Instant;
  28. import java.util.Arrays;
  29. import java.util.Collections;
  30. import java.util.Comparator;
  31. import java.util.Iterator;
  32. import java.util.Set;
  33. import java.util.concurrent.atomic.AtomicInteger;
  34. import java.util.zip.CRC32;
  35. import java.util.zip.DataFormatException;
  36. import java.util.zip.Inflater;
  37. import org.eclipse.jgit.errors.CorruptObjectException;
  38. import org.eclipse.jgit.errors.LargeObjectException;
  39. import org.eclipse.jgit.errors.MissingObjectException;
  40. import org.eclipse.jgit.errors.NoPackSignatureException;
  41. import org.eclipse.jgit.errors.PackInvalidException;
  42. import org.eclipse.jgit.errors.PackMismatchException;
  43. import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
  44. import org.eclipse.jgit.errors.UnpackException;
  45. import org.eclipse.jgit.errors.UnsupportedPackIndexVersionException;
  46. import org.eclipse.jgit.errors.UnsupportedPackVersionException;
  47. import org.eclipse.jgit.internal.JGitText;
  48. import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
  49. import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
  50. import org.eclipse.jgit.internal.storage.pack.PackExt;
  51. import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
  52. import org.eclipse.jgit.lib.AbbreviatedObjectId;
  53. import org.eclipse.jgit.lib.AnyObjectId;
  54. import org.eclipse.jgit.lib.Constants;
  55. import org.eclipse.jgit.lib.ObjectId;
  56. import org.eclipse.jgit.lib.ObjectLoader;
  57. import org.eclipse.jgit.util.LongList;
  58. import org.eclipse.jgit.util.NB;
  59. import org.eclipse.jgit.util.RawParseUtils;
  60. import org.slf4j.Logger;
  61. import org.slf4j.LoggerFactory;
  62. /**
  63. * A Git version 2 pack file representation. A pack file contains Git objects in
  64. * delta packed format yielding high compression of lots of object where some
  65. * objects are similar.
  66. */
  67. public class PackFile implements Iterable<PackIndex.MutableEntry> {
  68. private final static Logger LOG = LoggerFactory.getLogger(PackFile.class);
  69. /**
  70. * Sorts PackFiles to be most recently created to least recently created.
  71. */
  72. public static final Comparator<PackFile> SORT = (a, b) -> b.packLastModified
  73. .compareTo(a.packLastModified);
  74. private final File packFile;
  75. private final int extensions;
  76. private File keepFile;
  77. private volatile String packName;
  78. final int hash;
  79. private RandomAccessFile fd;
  80. /** Serializes reads performed against {@link #fd}. */
  81. private final Object readLock = new Object();
  82. long length;
  83. private int activeWindows;
  84. private int activeCopyRawData;
  85. Instant packLastModified;
  86. private PackFileSnapshot fileSnapshot;
  87. private volatile boolean invalid;
  88. private volatile Exception invalidatingCause;
  89. private boolean invalidBitmap;
  90. private AtomicInteger transientErrorCount = new AtomicInteger();
  91. private byte[] packChecksum;
  92. private volatile PackIndex loadedIdx;
  93. private PackReverseIndex reverseIdx;
  94. private PackBitmapIndex bitmapIdx;
  95. /**
  96. * Objects we have tried to read, and discovered to be corrupt.
  97. * <p>
  98. * The list is allocated after the first corruption is found, and filled in
  99. * as more entries are discovered. Typically this list is never used, as
  100. * pack files do not usually contain corrupt objects.
  101. */
  102. private volatile LongList corruptObjects;
  103. /**
  104. * Construct a reader for an existing, pre-indexed packfile.
  105. *
  106. * @param packFile
  107. * path of the <code>.pack</code> file holding the data.
  108. * @param extensions
  109. * additional pack file extensions with the same base as the pack
  110. */
  111. public PackFile(File packFile, int extensions) {
  112. this.packFile = packFile;
  113. this.fileSnapshot = PackFileSnapshot.save(packFile);
  114. this.packLastModified = fileSnapshot.lastModifiedInstant();
  115. this.extensions = extensions;
  116. // Multiply by 31 here so we can more directly combine with another
  117. // value in WindowCache.hash(), without doing the multiply there.
  118. //
  119. hash = System.identityHashCode(this) * 31;
  120. length = Long.MAX_VALUE;
  121. }
  122. private PackIndex idx() throws IOException {
  123. PackIndex idx = loadedIdx;
  124. if (idx == null) {
  125. synchronized (this) {
  126. idx = loadedIdx;
  127. if (idx == null) {
  128. if (invalid) {
  129. throw new PackInvalidException(packFile, invalidatingCause);
  130. }
  131. try {
  132. long start = System.currentTimeMillis();
  133. idx = PackIndex.open(extFile(INDEX));
  134. if (LOG.isDebugEnabled()) {
  135. LOG.debug(String.format(
  136. "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$
  137. extFile(INDEX).getAbsolutePath(),
  138. Float.valueOf(extFile(INDEX).length()
  139. / (1024f * 1024)),
  140. Long.valueOf(System.currentTimeMillis()
  141. - start)));
  142. }
  143. if (packChecksum == null) {
  144. packChecksum = idx.packChecksum;
  145. fileSnapshot.setChecksum(
  146. ObjectId.fromRaw(packChecksum));
  147. } else if (!Arrays.equals(packChecksum,
  148. idx.packChecksum)) {
  149. throw new PackMismatchException(MessageFormat
  150. .format(JGitText.get().packChecksumMismatch,
  151. packFile.getPath(),
  152. ObjectId.fromRaw(packChecksum)
  153. .name(),
  154. ObjectId.fromRaw(idx.packChecksum)
  155. .name()));
  156. }
  157. loadedIdx = idx;
  158. } catch (InterruptedIOException e) {
  159. // don't invalidate the pack, we are interrupted from
  160. // another thread
  161. throw e;
  162. } catch (IOException e) {
  163. invalid = true;
  164. invalidatingCause = e;
  165. throw e;
  166. }
  167. }
  168. }
  169. }
  170. return idx;
  171. }
  172. /**
  173. * Get the File object which locates this pack on disk.
  174. *
  175. * @return the File object which locates this pack on disk.
  176. */
  177. public File getPackFile() {
  178. return packFile;
  179. }
  180. /**
  181. * Get the index for this pack file.
  182. *
  183. * @return the index for this pack file.
  184. * @throws java.io.IOException
  185. */
  186. public PackIndex getIndex() throws IOException {
  187. return idx();
  188. }
  189. /**
  190. * Get name extracted from {@code pack-*.pack} pattern.
  191. *
  192. * @return name extracted from {@code pack-*.pack} pattern.
  193. */
  194. public String getPackName() {
  195. String name = packName;
  196. if (name == null) {
  197. name = getPackFile().getName();
  198. if (name.startsWith("pack-")) //$NON-NLS-1$
  199. name = name.substring("pack-".length()); //$NON-NLS-1$
  200. if (name.endsWith(".pack")) //$NON-NLS-1$
  201. name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$
  202. packName = name;
  203. }
  204. return name;
  205. }
  206. /**
  207. * Determine if an object is contained within the pack file.
  208. * <p>
  209. * For performance reasons only the index file is searched; the main pack
  210. * content is ignored entirely.
  211. * </p>
  212. *
  213. * @param id
  214. * the object to look for. Must not be null.
  215. * @return true if the object is in this pack; false otherwise.
  216. * @throws java.io.IOException
  217. * the index file cannot be loaded into memory.
  218. */
  219. public boolean hasObject(AnyObjectId id) throws IOException {
  220. final long offset = idx().findOffset(id);
  221. return 0 < offset && !isCorrupt(offset);
  222. }
  223. /**
  224. * Determines whether a .keep file exists for this pack file.
  225. *
  226. * @return true if a .keep file exist.
  227. */
  228. public boolean shouldBeKept() {
  229. if (keepFile == null)
  230. keepFile = extFile(KEEP);
  231. return keepFile.exists();
  232. }
  233. /**
  234. * Get an object from this pack.
  235. *
  236. * @param curs
  237. * temporary working space associated with the calling thread.
  238. * @param id
  239. * the object to obtain from the pack. Must not be null.
  240. * @return the object loader for the requested object if it is contained in
  241. * this pack; null if the object was not found.
  242. * @throws IOException
  243. * the pack file or the index could not be read.
  244. */
  245. ObjectLoader get(WindowCursor curs, AnyObjectId id)
  246. throws IOException {
  247. final long offset = idx().findOffset(id);
  248. return 0 < offset && !isCorrupt(offset) ? load(curs, offset) : null;
  249. }
  250. void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit)
  251. throws IOException {
  252. idx().resolve(matches, id, matchLimit);
  253. }
  254. /**
  255. * Close the resources utilized by this repository
  256. */
  257. public void close() {
  258. WindowCache.purge(this);
  259. synchronized (this) {
  260. loadedIdx = null;
  261. reverseIdx = null;
  262. }
  263. }
  264. /**
  265. * {@inheritDoc}
  266. * <p>
  267. * Provide iterator over entries in associated pack index, that should also
  268. * exist in this pack file. Objects returned by such iterator are mutable
  269. * during iteration.
  270. * <p>
  271. * Iterator returns objects in SHA-1 lexicographical order.
  272. * </p>
  273. *
  274. * @see PackIndex#iterator()
  275. */
  276. @Override
  277. public Iterator<PackIndex.MutableEntry> iterator() {
  278. try {
  279. return idx().iterator();
  280. } catch (IOException e) {
  281. return Collections.<PackIndex.MutableEntry> emptyList().iterator();
  282. }
  283. }
  284. /**
  285. * Obtain the total number of objects available in this pack. This method
  286. * relies on pack index, giving number of effectively available objects.
  287. *
  288. * @return number of objects in index of this pack, likewise in this pack
  289. * @throws IOException
  290. * the index file cannot be loaded into memory.
  291. */
  292. long getObjectCount() throws IOException {
  293. return idx().getObjectCount();
  294. }
  295. /**
  296. * Search for object id with the specified start offset in associated pack
  297. * (reverse) index.
  298. *
  299. * @param offset
  300. * start offset of object to find
  301. * @return object id for this offset, or null if no object was found
  302. * @throws IOException
  303. * the index file cannot be loaded into memory.
  304. */
  305. ObjectId findObjectForOffset(long offset) throws IOException {
  306. return getReverseIdx().findObject(offset);
  307. }
  308. /**
  309. * Return the @{@link FileSnapshot} associated to the underlying packfile
  310. * that has been used when the object was created.
  311. *
  312. * @return the packfile @{@link FileSnapshot} that the object is loaded from.
  313. */
  314. PackFileSnapshot getFileSnapshot() {
  315. return fileSnapshot;
  316. }
  317. AnyObjectId getPackChecksum() {
  318. return ObjectId.fromRaw(packChecksum);
  319. }
  320. private final byte[] decompress(final long position, final int sz,
  321. final WindowCursor curs) throws IOException, DataFormatException {
  322. byte[] dstbuf;
  323. try {
  324. dstbuf = new byte[sz];
  325. } catch (OutOfMemoryError noMemory) {
  326. // The size may be larger than our heap allows, return null to
  327. // let the caller know allocation isn't possible and it should
  328. // use the large object streaming approach instead.
  329. //
  330. // For example, this can occur when sz is 640 MB, and JRE
  331. // maximum heap size is only 256 MB. Even if the JRE has
  332. // 200 MB free, it cannot allocate a 640 MB byte array.
  333. return null;
  334. }
  335. if (curs.inflate(this, position, dstbuf, false) != sz)
  336. throw new EOFException(MessageFormat.format(
  337. JGitText.get().shortCompressedStreamAt,
  338. Long.valueOf(position)));
  339. return dstbuf;
  340. }
  341. void copyPackAsIs(PackOutputStream out, WindowCursor curs)
  342. throws IOException {
  343. // Pin the first window, this ensures the length is accurate.
  344. curs.pin(this, 0);
  345. curs.copyPackAsIs(this, length, out);
  346. }
  347. final void copyAsIs(PackOutputStream out, LocalObjectToPack src,
  348. boolean validate, WindowCursor curs) throws IOException,
  349. StoredObjectRepresentationNotAvailableException {
  350. beginCopyAsIs(src);
  351. try {
  352. copyAsIs2(out, src, validate, curs);
  353. } finally {
  354. endCopyAsIs();
  355. }
  356. }
  357. private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
  358. boolean validate, WindowCursor curs) throws IOException,
  359. StoredObjectRepresentationNotAvailableException {
  360. final CRC32 crc1 = validate ? new CRC32() : null;
  361. final CRC32 crc2 = validate ? new CRC32() : null;
  362. final byte[] buf = out.getCopyBuffer();
  363. // Rip apart the header so we can discover the size.
  364. //
  365. readFully(src.offset, buf, 0, 20, curs);
  366. int c = buf[0] & 0xff;
  367. final int typeCode = (c >> 4) & 7;
  368. long inflatedLength = c & 15;
  369. int shift = 4;
  370. int headerCnt = 1;
  371. while ((c & 0x80) != 0) {
  372. c = buf[headerCnt++] & 0xff;
  373. inflatedLength += ((long) (c & 0x7f)) << shift;
  374. shift += 7;
  375. }
  376. if (typeCode == Constants.OBJ_OFS_DELTA) {
  377. do {
  378. c = buf[headerCnt++] & 0xff;
  379. } while ((c & 128) != 0);
  380. if (validate) {
  381. assert(crc1 != null && crc2 != null);
  382. crc1.update(buf, 0, headerCnt);
  383. crc2.update(buf, 0, headerCnt);
  384. }
  385. } else if (typeCode == Constants.OBJ_REF_DELTA) {
  386. if (validate) {
  387. assert(crc1 != null && crc2 != null);
  388. crc1.update(buf, 0, headerCnt);
  389. crc2.update(buf, 0, headerCnt);
  390. }
  391. readFully(src.offset + headerCnt, buf, 0, 20, curs);
  392. if (validate) {
  393. assert(crc1 != null && crc2 != null);
  394. crc1.update(buf, 0, 20);
  395. crc2.update(buf, 0, 20);
  396. }
  397. headerCnt += 20;
  398. } else if (validate) {
  399. assert(crc1 != null && crc2 != null);
  400. crc1.update(buf, 0, headerCnt);
  401. crc2.update(buf, 0, headerCnt);
  402. }
  403. final long dataOffset = src.offset + headerCnt;
  404. final long dataLength = src.length;
  405. final long expectedCRC;
  406. final ByteArrayWindow quickCopy;
  407. // Verify the object isn't corrupt before sending. If it is,
  408. // we report it missing instead.
  409. //
  410. try {
  411. quickCopy = curs.quickCopy(this, dataOffset, dataLength);
  412. if (validate && idx().hasCRC32Support()) {
  413. assert(crc1 != null);
  414. // Index has the CRC32 code cached, validate the object.
  415. //
  416. expectedCRC = idx().findCRC32(src);
  417. if (quickCopy != null) {
  418. quickCopy.crc32(crc1, dataOffset, (int) dataLength);
  419. } else {
  420. long pos = dataOffset;
  421. long cnt = dataLength;
  422. while (cnt > 0) {
  423. final int n = (int) Math.min(cnt, buf.length);
  424. readFully(pos, buf, 0, n, curs);
  425. crc1.update(buf, 0, n);
  426. pos += n;
  427. cnt -= n;
  428. }
  429. }
  430. if (crc1.getValue() != expectedCRC) {
  431. setCorrupt(src.offset);
  432. throw new CorruptObjectException(MessageFormat.format(
  433. JGitText.get().objectAtHasBadZlibStream,
  434. Long.valueOf(src.offset), getPackFile()));
  435. }
  436. } else if (validate) {
  437. // We don't have a CRC32 code in the index, so compute it
  438. // now while inflating the raw data to get zlib to tell us
  439. // whether or not the data is safe.
  440. //
  441. Inflater inf = curs.inflater();
  442. byte[] tmp = new byte[1024];
  443. if (quickCopy != null) {
  444. quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
  445. } else {
  446. assert(crc1 != null);
  447. long pos = dataOffset;
  448. long cnt = dataLength;
  449. while (cnt > 0) {
  450. final int n = (int) Math.min(cnt, buf.length);
  451. readFully(pos, buf, 0, n, curs);
  452. crc1.update(buf, 0, n);
  453. inf.setInput(buf, 0, n);
  454. while (inf.inflate(tmp, 0, tmp.length) > 0)
  455. continue;
  456. pos += n;
  457. cnt -= n;
  458. }
  459. }
  460. if (!inf.finished() || inf.getBytesRead() != dataLength) {
  461. setCorrupt(src.offset);
  462. throw new EOFException(MessageFormat.format(
  463. JGitText.get().shortCompressedStreamAt,
  464. Long.valueOf(src.offset)));
  465. }
  466. assert(crc1 != null);
  467. expectedCRC = crc1.getValue();
  468. } else {
  469. expectedCRC = -1;
  470. }
  471. } catch (DataFormatException dataFormat) {
  472. setCorrupt(src.offset);
  473. CorruptObjectException corruptObject = new CorruptObjectException(
  474. MessageFormat.format(
  475. JGitText.get().objectAtHasBadZlibStream,
  476. Long.valueOf(src.offset), getPackFile()),
  477. dataFormat);
  478. throw new StoredObjectRepresentationNotAvailableException(src,
  479. corruptObject);
  480. } catch (IOException ioError) {
  481. throw new StoredObjectRepresentationNotAvailableException(src,
  482. ioError);
  483. }
  484. if (quickCopy != null) {
  485. // The entire object fits into a single byte array window slice,
  486. // and we have it pinned. Write this out without copying.
  487. //
  488. out.writeHeader(src, inflatedLength);
  489. quickCopy.write(out, dataOffset, (int) dataLength);
  490. } else if (dataLength <= buf.length) {
  491. // Tiny optimization: Lots of objects are very small deltas or
  492. // deflated commits that are likely to fit in the copy buffer.
  493. //
  494. if (!validate) {
  495. long pos = dataOffset;
  496. long cnt = dataLength;
  497. while (cnt > 0) {
  498. final int n = (int) Math.min(cnt, buf.length);
  499. readFully(pos, buf, 0, n, curs);
  500. pos += n;
  501. cnt -= n;
  502. }
  503. }
  504. out.writeHeader(src, inflatedLength);
  505. out.write(buf, 0, (int) dataLength);
  506. } else {
  507. // Now we are committed to sending the object. As we spool it out,
  508. // check its CRC32 code to make sure there wasn't corruption between
  509. // the verification we did above, and us actually outputting it.
  510. //
  511. out.writeHeader(src, inflatedLength);
  512. long pos = dataOffset;
  513. long cnt = dataLength;
  514. while (cnt > 0) {
  515. final int n = (int) Math.min(cnt, buf.length);
  516. readFully(pos, buf, 0, n, curs);
  517. if (validate) {
  518. assert(crc2 != null);
  519. crc2.update(buf, 0, n);
  520. }
  521. out.write(buf, 0, n);
  522. pos += n;
  523. cnt -= n;
  524. }
  525. if (validate) {
  526. assert(crc2 != null);
  527. if (crc2.getValue() != expectedCRC) {
  528. throw new CorruptObjectException(MessageFormat.format(
  529. JGitText.get().objectAtHasBadZlibStream,
  530. Long.valueOf(src.offset), getPackFile()));
  531. }
  532. }
  533. }
  534. }
  535. boolean invalid() {
  536. return invalid;
  537. }
  538. void setInvalid() {
  539. invalid = true;
  540. }
  541. int incrementTransientErrorCount() {
  542. return transientErrorCount.incrementAndGet();
  543. }
  544. void resetTransientErrorCount() {
  545. transientErrorCount.set(0);
  546. }
  547. private void readFully(final long position, final byte[] dstbuf,
  548. int dstoff, final int cnt, final WindowCursor curs)
  549. throws IOException {
  550. if (curs.copy(this, position, dstbuf, dstoff, cnt) != cnt)
  551. throw new EOFException();
  552. }
  553. private synchronized void beginCopyAsIs(ObjectToPack otp)
  554. throws StoredObjectRepresentationNotAvailableException {
  555. if (++activeCopyRawData == 1 && activeWindows == 0) {
  556. try {
  557. doOpen();
  558. } catch (IOException thisPackNotValid) {
  559. throw new StoredObjectRepresentationNotAvailableException(otp,
  560. thisPackNotValid);
  561. }
  562. }
  563. }
  564. private synchronized void endCopyAsIs() {
  565. if (--activeCopyRawData == 0 && activeWindows == 0)
  566. doClose();
  567. }
  568. synchronized boolean beginWindowCache() throws IOException {
  569. if (++activeWindows == 1) {
  570. if (activeCopyRawData == 0)
  571. doOpen();
  572. return true;
  573. }
  574. return false;
  575. }
  576. synchronized boolean endWindowCache() {
  577. final boolean r = --activeWindows == 0;
  578. if (r && activeCopyRawData == 0)
  579. doClose();
  580. return r;
  581. }
  582. private void doOpen() throws IOException {
  583. if (invalid) {
  584. throw new PackInvalidException(packFile, invalidatingCause);
  585. }
  586. try {
  587. synchronized (readLock) {
  588. fd = new RandomAccessFile(packFile, "r"); //$NON-NLS-1$
  589. length = fd.length();
  590. onOpenPack();
  591. }
  592. } catch (InterruptedIOException e) {
  593. // don't invalidate the pack, we are interrupted from another thread
  594. openFail(false, e);
  595. throw e;
  596. } catch (FileNotFoundException fn) {
  597. // don't invalidate the pack if opening an existing file failed
  598. // since it may be related to a temporary lack of resources (e.g.
  599. // max open files)
  600. openFail(!packFile.exists(), fn);
  601. throw fn;
  602. } catch (EOFException | AccessDeniedException | NoSuchFileException
  603. | CorruptObjectException | NoPackSignatureException
  604. | PackMismatchException | UnpackException
  605. | UnsupportedPackIndexVersionException
  606. | UnsupportedPackVersionException pe) {
  607. // exceptions signaling permanent problems with a pack
  608. openFail(true, pe);
  609. throw pe;
  610. } catch (IOException | RuntimeException ge) {
  611. // generic exceptions could be transient so we should not mark the
  612. // pack invalid to avoid false MissingObjectExceptions
  613. openFail(false, ge);
  614. throw ge;
  615. }
  616. }
  617. private void openFail(boolean invalidate, Exception cause) {
  618. activeWindows = 0;
  619. activeCopyRawData = 0;
  620. invalid = invalidate;
  621. invalidatingCause = cause;
  622. doClose();
  623. }
  624. private void doClose() {
  625. synchronized (readLock) {
  626. if (fd != null) {
  627. try {
  628. fd.close();
  629. } catch (IOException err) {
  630. // Ignore a close event. We had it open only for reading.
  631. // There should not be errors related to network buffers
  632. // not flushed, etc.
  633. }
  634. fd = null;
  635. }
  636. }
  637. }
  638. ByteArrayWindow read(long pos, int size) throws IOException {
  639. synchronized (readLock) {
  640. if (invalid || fd == null) {
  641. // Due to concurrency between a read and another packfile invalidation thread
  642. // one thread could come up to this point and then fail with NPE.
  643. // Detect the situation and throw a proper exception so that can be properly
  644. // managed by the main packfile search loop and the Git client won't receive
  645. // any failures.
  646. throw new PackInvalidException(packFile, invalidatingCause);
  647. }
  648. if (length < pos + size)
  649. size = (int) (length - pos);
  650. final byte[] buf = new byte[size];
  651. fd.seek(pos);
  652. fd.readFully(buf, 0, size);
  653. return new ByteArrayWindow(this, pos, buf);
  654. }
  655. }
  656. ByteWindow mmap(long pos, int size) throws IOException {
  657. synchronized (readLock) {
  658. if (length < pos + size)
  659. size = (int) (length - pos);
  660. MappedByteBuffer map;
  661. try {
  662. map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
  663. } catch (IOException ioe1) {
  664. // The most likely reason this failed is the JVM has run out
  665. // of virtual memory. We need to discard quickly, and try to
  666. // force the GC to finalize and release any existing mappings.
  667. //
  668. System.gc();
  669. System.runFinalization();
  670. map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
  671. }
  672. if (map.hasArray())
  673. return new ByteArrayWindow(this, pos, map.array());
  674. return new ByteBufferWindow(this, pos, map);
  675. }
  676. }
  677. private void onOpenPack() throws IOException {
  678. final PackIndex idx = idx();
  679. final byte[] buf = new byte[20];
  680. fd.seek(0);
  681. fd.readFully(buf, 0, 12);
  682. if (RawParseUtils.match(buf, 0, Constants.PACK_SIGNATURE) != 4) {
  683. throw new NoPackSignatureException(JGitText.get().notAPACKFile);
  684. }
  685. final long vers = NB.decodeUInt32(buf, 4);
  686. final long packCnt = NB.decodeUInt32(buf, 8);
  687. if (vers != 2 && vers != 3) {
  688. throw new UnsupportedPackVersionException(vers);
  689. }
  690. if (packCnt != idx.getObjectCount()) {
  691. throw new PackMismatchException(MessageFormat.format(
  692. JGitText.get().packObjectCountMismatch,
  693. Long.valueOf(packCnt), Long.valueOf(idx.getObjectCount()),
  694. getPackFile()));
  695. }
  696. fd.seek(length - 20);
  697. fd.readFully(buf, 0, 20);
  698. if (!Arrays.equals(buf, packChecksum)) {
  699. throw new PackMismatchException(MessageFormat.format(
  700. JGitText.get().packChecksumMismatch,
  701. getPackFile(),
  702. ObjectId.fromRaw(buf).name(),
  703. ObjectId.fromRaw(idx.packChecksum).name()));
  704. }
  705. }
  706. ObjectLoader load(WindowCursor curs, long pos)
  707. throws IOException, LargeObjectException {
  708. try {
  709. final byte[] ib = curs.tempId;
  710. Delta delta = null;
  711. byte[] data = null;
  712. int type = Constants.OBJ_BAD;
  713. boolean cached = false;
  714. SEARCH: for (;;) {
  715. readFully(pos, ib, 0, 20, curs);
  716. int c = ib[0] & 0xff;
  717. final int typeCode = (c >> 4) & 7;
  718. long sz = c & 15;
  719. int shift = 4;
  720. int p = 1;
  721. while ((c & 0x80) != 0) {
  722. c = ib[p++] & 0xff;
  723. sz += ((long) (c & 0x7f)) << shift;
  724. shift += 7;
  725. }
  726. switch (typeCode) {
  727. case Constants.OBJ_COMMIT:
  728. case Constants.OBJ_TREE:
  729. case Constants.OBJ_BLOB:
  730. case Constants.OBJ_TAG: {
  731. if (delta != null || sz < curs.getStreamFileThreshold()) {
  732. data = decompress(pos + p, (int) sz, curs);
  733. }
  734. if (delta != null) {
  735. type = typeCode;
  736. break SEARCH;
  737. }
  738. if (data != null) {
  739. return new ObjectLoader.SmallObject(typeCode, data);
  740. }
  741. return new LargePackedWholeObject(typeCode, sz, pos, p,
  742. this, curs.db);
  743. }
  744. case Constants.OBJ_OFS_DELTA: {
  745. c = ib[p++] & 0xff;
  746. long base = c & 127;
  747. while ((c & 128) != 0) {
  748. base += 1;
  749. c = ib[p++] & 0xff;
  750. base <<= 7;
  751. base += (c & 127);
  752. }
  753. base = pos - base;
  754. delta = new Delta(delta, pos, (int) sz, p, base);
  755. if (sz != delta.deltaSize)
  756. break SEARCH;
  757. DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
  758. if (e != null) {
  759. type = e.type;
  760. data = e.data;
  761. cached = true;
  762. break SEARCH;
  763. }
  764. pos = base;
  765. continue SEARCH;
  766. }
  767. case Constants.OBJ_REF_DELTA: {
  768. readFully(pos + p, ib, 0, 20, curs);
  769. long base = findDeltaBase(ObjectId.fromRaw(ib));
  770. delta = new Delta(delta, pos, (int) sz, p + 20, base);
  771. if (sz != delta.deltaSize)
  772. break SEARCH;
  773. DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
  774. if (e != null) {
  775. type = e.type;
  776. data = e.data;
  777. cached = true;
  778. break SEARCH;
  779. }
  780. pos = base;
  781. continue SEARCH;
  782. }
  783. default:
  784. throw new IOException(MessageFormat.format(
  785. JGitText.get().unknownObjectType,
  786. Integer.valueOf(typeCode)));
  787. }
  788. }
  789. // At this point there is at least one delta to apply to data.
  790. // (Whole objects with no deltas to apply return early above.)
  791. if (data == null)
  792. throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);
  793. assert(delta != null);
  794. do {
  795. // Cache only the base immediately before desired object.
  796. if (cached)
  797. cached = false;
  798. else if (delta.next == null)
  799. curs.getDeltaBaseCache().store(this, delta.basePos, data, type);
  800. pos = delta.deltaPos;
  801. final byte[] cmds = decompress(pos + delta.hdrLen,
  802. delta.deltaSize, curs);
  803. if (cmds == null) {
  804. data = null; // Discard base in case of OutOfMemoryError
  805. throw new LargeObjectException.OutOfMemory(new OutOfMemoryError());
  806. }
  807. final long sz = BinaryDelta.getResultSize(cmds);
  808. if (Integer.MAX_VALUE <= sz)
  809. throw new LargeObjectException.ExceedsByteArrayLimit();
  810. final byte[] result;
  811. try {
  812. result = new byte[(int) sz];
  813. } catch (OutOfMemoryError tooBig) {
  814. data = null; // Discard base in case of OutOfMemoryError
  815. throw new LargeObjectException.OutOfMemory(tooBig);
  816. }
  817. BinaryDelta.apply(data, cmds, result);
  818. data = result;
  819. delta = delta.next;
  820. } while (delta != null);
  821. return new ObjectLoader.SmallObject(type, data);
  822. } catch (DataFormatException dfe) {
  823. throw new CorruptObjectException(
  824. MessageFormat.format(
  825. JGitText.get().objectAtHasBadZlibStream,
  826. Long.valueOf(pos), getPackFile()),
  827. dfe);
  828. }
  829. }
  830. private long findDeltaBase(ObjectId baseId) throws IOException,
  831. MissingObjectException {
  832. long ofs = idx().findOffset(baseId);
  833. if (ofs < 0)
  834. throw new MissingObjectException(baseId,
  835. JGitText.get().missingDeltaBase);
  836. return ofs;
  837. }
  838. private static class Delta {
  839. /** Child that applies onto this object. */
  840. final Delta next;
  841. /** Offset of the delta object. */
  842. final long deltaPos;
  843. /** Size of the inflated delta stream. */
  844. final int deltaSize;
  845. /** Total size of the delta's pack entry header (including base). */
  846. final int hdrLen;
  847. /** Offset of the base object this delta applies onto. */
  848. final long basePos;
  849. Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
  850. this.next = next;
  851. this.deltaPos = ofs;
  852. this.deltaSize = sz;
  853. this.hdrLen = hdrLen;
  854. this.basePos = baseOffset;
  855. }
  856. }
  857. byte[] getDeltaHeader(WindowCursor wc, long pos)
  858. throws IOException, DataFormatException {
  859. // The delta stream starts as two variable length integers. If we
  860. // assume they are 64 bits each, we need 16 bytes to encode them,
  861. // plus 2 extra bytes for the variable length overhead. So 18 is
  862. // the longest delta instruction header.
  863. //
  864. final byte[] hdr = new byte[18];
  865. wc.inflate(this, pos, hdr, true /* headerOnly */);
  866. return hdr;
  867. }
  868. int getObjectType(WindowCursor curs, long pos) throws IOException {
  869. final byte[] ib = curs.tempId;
  870. for (;;) {
  871. readFully(pos, ib, 0, 20, curs);
  872. int c = ib[0] & 0xff;
  873. final int type = (c >> 4) & 7;
  874. switch (type) {
  875. case Constants.OBJ_COMMIT:
  876. case Constants.OBJ_TREE:
  877. case Constants.OBJ_BLOB:
  878. case Constants.OBJ_TAG:
  879. return type;
  880. case Constants.OBJ_OFS_DELTA: {
  881. int p = 1;
  882. while ((c & 0x80) != 0)
  883. c = ib[p++] & 0xff;
  884. c = ib[p++] & 0xff;
  885. long ofs = c & 127;
  886. while ((c & 128) != 0) {
  887. ofs += 1;
  888. c = ib[p++] & 0xff;
  889. ofs <<= 7;
  890. ofs += (c & 127);
  891. }
  892. pos = pos - ofs;
  893. continue;
  894. }
  895. case Constants.OBJ_REF_DELTA: {
  896. int p = 1;
  897. while ((c & 0x80) != 0)
  898. c = ib[p++] & 0xff;
  899. readFully(pos + p, ib, 0, 20, curs);
  900. pos = findDeltaBase(ObjectId.fromRaw(ib));
  901. continue;
  902. }
  903. default:
  904. throw new IOException(
  905. MessageFormat.format(JGitText.get().unknownObjectType,
  906. Integer.valueOf(type)));
  907. }
  908. }
  909. }
  910. long getObjectSize(WindowCursor curs, AnyObjectId id)
  911. throws IOException {
  912. final long offset = idx().findOffset(id);
  913. return 0 < offset ? getObjectSize(curs, offset) : -1;
  914. }
  915. long getObjectSize(WindowCursor curs, long pos)
  916. throws IOException {
  917. final byte[] ib = curs.tempId;
  918. readFully(pos, ib, 0, 20, curs);
  919. int c = ib[0] & 0xff;
  920. final int type = (c >> 4) & 7;
  921. long sz = c & 15;
  922. int shift = 4;
  923. int p = 1;
  924. while ((c & 0x80) != 0) {
  925. c = ib[p++] & 0xff;
  926. sz += ((long) (c & 0x7f)) << shift;
  927. shift += 7;
  928. }
  929. long deltaAt;
  930. switch (type) {
  931. case Constants.OBJ_COMMIT:
  932. case Constants.OBJ_TREE:
  933. case Constants.OBJ_BLOB:
  934. case Constants.OBJ_TAG:
  935. return sz;
  936. case Constants.OBJ_OFS_DELTA:
  937. c = ib[p++] & 0xff;
  938. while ((c & 128) != 0)
  939. c = ib[p++] & 0xff;
  940. deltaAt = pos + p;
  941. break;
  942. case Constants.OBJ_REF_DELTA:
  943. deltaAt = pos + p + 20;
  944. break;
  945. default:
  946. throw new IOException(MessageFormat.format(
  947. JGitText.get().unknownObjectType, Integer.valueOf(type)));
  948. }
  949. try {
  950. return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt));
  951. } catch (DataFormatException e) {
  952. throw new CorruptObjectException(MessageFormat.format(
  953. JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
  954. getPackFile()), e);
  955. }
  956. }
  957. LocalObjectRepresentation representation(final WindowCursor curs,
  958. final AnyObjectId objectId) throws IOException {
  959. final long pos = idx().findOffset(objectId);
  960. if (pos < 0)
  961. return null;
  962. final byte[] ib = curs.tempId;
  963. readFully(pos, ib, 0, 20, curs);
  964. int c = ib[0] & 0xff;
  965. int p = 1;
  966. final int typeCode = (c >> 4) & 7;
  967. while ((c & 0x80) != 0)
  968. c = ib[p++] & 0xff;
  969. long len = (findEndOffset(pos) - pos);
  970. switch (typeCode) {
  971. case Constants.OBJ_COMMIT:
  972. case Constants.OBJ_TREE:
  973. case Constants.OBJ_BLOB:
  974. case Constants.OBJ_TAG:
  975. return LocalObjectRepresentation.newWhole(this, pos, len - p);
  976. case Constants.OBJ_OFS_DELTA: {
  977. c = ib[p++] & 0xff;
  978. long ofs = c & 127;
  979. while ((c & 128) != 0) {
  980. ofs += 1;
  981. c = ib[p++] & 0xff;
  982. ofs <<= 7;
  983. ofs += (c & 127);
  984. }
  985. ofs = pos - ofs;
  986. return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs);
  987. }
  988. case Constants.OBJ_REF_DELTA: {
  989. len -= p;
  990. len -= Constants.OBJECT_ID_LENGTH;
  991. readFully(pos + p, ib, 0, 20, curs);
  992. ObjectId id = ObjectId.fromRaw(ib);
  993. return LocalObjectRepresentation.newDelta(this, pos, len, id);
  994. }
  995. default:
  996. throw new IOException(
  997. MessageFormat.format(JGitText.get().unknownObjectType,
  998. Integer.valueOf(typeCode)));
  999. }
  1000. }
  1001. private long findEndOffset(long startOffset)
  1002. throws IOException, CorruptObjectException {
  1003. final long maxOffset = length - 20;
  1004. return getReverseIdx().findNextOffset(startOffset, maxOffset);
  1005. }
  1006. synchronized PackBitmapIndex getBitmapIndex() throws IOException {
  1007. if (invalid || invalidBitmap)
  1008. return null;
  1009. if (bitmapIdx == null && hasExt(BITMAP_INDEX)) {
  1010. final PackBitmapIndex idx;
  1011. try {
  1012. idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(),
  1013. getReverseIdx());
  1014. } catch (FileNotFoundException e) {
  1015. // Once upon a time this bitmap file existed. Now it
  1016. // has been removed. Most likely an external gc has
  1017. // removed this packfile and the bitmap
  1018. invalidBitmap = true;
  1019. return null;
  1020. }
  1021. // At this point, idx() will have set packChecksum.
  1022. if (Arrays.equals(packChecksum, idx.packChecksum))
  1023. bitmapIdx = idx;
  1024. else
  1025. invalidBitmap = true;
  1026. }
  1027. return bitmapIdx;
  1028. }
  1029. private synchronized PackReverseIndex getReverseIdx() throws IOException {
  1030. if (reverseIdx == null)
  1031. reverseIdx = new PackReverseIndex(idx());
  1032. return reverseIdx;
  1033. }
  1034. private boolean isCorrupt(long offset) {
  1035. LongList list = corruptObjects;
  1036. if (list == null)
  1037. return false;
  1038. synchronized (list) {
  1039. return list.contains(offset);
  1040. }
  1041. }
  1042. private void setCorrupt(long offset) {
  1043. LongList list = corruptObjects;
  1044. if (list == null) {
  1045. synchronized (readLock) {
  1046. list = corruptObjects;
  1047. if (list == null) {
  1048. list = new LongList();
  1049. corruptObjects = list;
  1050. }
  1051. }
  1052. }
  1053. synchronized (list) {
  1054. list.add(offset);
  1055. }
  1056. }
  1057. private File extFile(PackExt ext) {
  1058. String p = packFile.getName();
  1059. int dot = p.lastIndexOf('.');
  1060. String b = (dot < 0) ? p : p.substring(0, dot);
  1061. return new File(packFile.getParentFile(), b + '.' + ext.getExtension());
  1062. }
  1063. private boolean hasExt(PackExt ext) {
  1064. return (extensions & ext.getBit()) != 0;
  1065. }
  1066. @SuppressWarnings("nls")
  1067. @Override
  1068. public String toString() {
  1069. return "PackFile [packFileName=" + packFile.getName() + ", length="
  1070. + packFile.length() + ", packChecksum="
  1071. + ObjectId.fromRaw(packChecksum).name() + "]";
  1072. }
  1073. }