You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

PackParser.java 48KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639
  1. /*
  2. * Copyright (C) 2008-2011, Google Inc.
  3. * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
  4. * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
  5. * and other copyright owners as documented in the project's IP log.
  6. *
  7. * This program and the accompanying materials are made available
  8. * under the terms of the Eclipse Distribution License v1.0 which
  9. * accompanies this distribution, is reproduced below, and is
  10. * available at http://www.eclipse.org/org/documents/edl-v10.php
  11. *
  12. * All rights reserved.
  13. *
  14. * Redistribution and use in source and binary forms, with or
  15. * without modification, are permitted provided that the following
  16. * conditions are met:
  17. *
  18. * - Redistributions of source code must retain the above copyright
  19. * notice, this list of conditions and the following disclaimer.
  20. *
  21. * - Redistributions in binary form must reproduce the above
  22. * copyright notice, this list of conditions and the following
  23. * disclaimer in the documentation and/or other materials provided
  24. * with the distribution.
  25. *
  26. * - Neither the name of the Eclipse Foundation, Inc. nor the
  27. * names of its contributors may be used to endorse or promote
  28. * products derived from this software without specific prior
  29. * written permission.
  30. *
  31. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  32. * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  33. * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  34. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  35. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  36. * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  37. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  38. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  39. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  40. * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  41. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  42. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  43. * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44. */
  45. package org.eclipse.jgit.transport;
  46. import java.io.EOFException;
  47. import java.io.IOException;
  48. import java.io.InputStream;
  49. import java.security.MessageDigest;
  50. import java.text.MessageFormat;
  51. import java.util.ArrayList;
  52. import java.util.Arrays;
  53. import java.util.Comparator;
  54. import java.util.List;
  55. import java.util.zip.DataFormatException;
  56. import java.util.zip.Inflater;
  57. import org.eclipse.jgit.JGitText;
  58. import org.eclipse.jgit.errors.CorruptObjectException;
  59. import org.eclipse.jgit.errors.MissingObjectException;
  60. import org.eclipse.jgit.lib.AnyObjectId;
  61. import org.eclipse.jgit.lib.Constants;
  62. import org.eclipse.jgit.lib.InflaterCache;
  63. import org.eclipse.jgit.lib.MutableObjectId;
  64. import org.eclipse.jgit.lib.NullProgressMonitor;
  65. import org.eclipse.jgit.lib.ObjectChecker;
  66. import org.eclipse.jgit.lib.ObjectDatabase;
  67. import org.eclipse.jgit.lib.ObjectId;
  68. import org.eclipse.jgit.lib.ObjectIdOwnerMap;
  69. import org.eclipse.jgit.lib.ObjectIdSubclassMap;
  70. import org.eclipse.jgit.lib.ObjectInserter;
  71. import org.eclipse.jgit.lib.ObjectLoader;
  72. import org.eclipse.jgit.lib.ObjectReader;
  73. import org.eclipse.jgit.lib.ObjectStream;
  74. import org.eclipse.jgit.lib.ProgressMonitor;
  75. import org.eclipse.jgit.storage.file.PackLock;
  76. import org.eclipse.jgit.storage.pack.BinaryDelta;
  77. import org.eclipse.jgit.util.BlockList;
  78. import org.eclipse.jgit.util.IO;
  79. import org.eclipse.jgit.util.NB;
  80. /**
  81. * Parses a pack stream and imports it for an {@link ObjectInserter}.
  82. * <p>
  83. * Applications can acquire an instance of a parser from ObjectInserter's
  84. * {@link ObjectInserter#newPackParser(InputStream)} method.
  85. * <p>
  86. * Implementations of {@link ObjectInserter} should subclass this type and
  87. * provide their own logic for the various {@code on*()} event methods declared
  88. * to be abstract.
  89. */
  90. public abstract class PackParser {
  91. /** Size of the internal stream buffer. */
  92. private static final int BUFFER_SIZE = 8192;
  93. /** Location data is being obtained from. */
  94. public static enum Source {
  95. /** Data is read from the incoming stream. */
  96. INPUT,
  97. /** Data is read back from the database's buffers. */
  98. DATABASE;
  99. }
  100. /** Object database used for loading existing objects. */
  101. private final ObjectDatabase objectDatabase;
  102. private InflaterStream inflater;
  103. private byte[] tempBuffer;
  104. private byte[] hdrBuf;
  105. private final MessageDigest objectDigest;
  106. private final MutableObjectId tempObjectId;
  107. private InputStream in;
  108. private byte[] buf;
  109. /** Position in the input stream of {@code buf[0]}. */
  110. private long bBase;
  111. private int bOffset;
  112. private int bAvail;
  113. private ObjectChecker objCheck;
  114. private boolean allowThin;
  115. private boolean checkObjectCollisions;
  116. private boolean needBaseObjectIds;
  117. private boolean checkEofAfterPackFooter;
  118. private long objectCount;
  119. private PackedObjectInfo[] entries;
  120. /**
  121. * Every object contained within the incoming pack.
  122. * <p>
  123. * This is a subset of {@link #entries}, as thin packs can add additional
  124. * objects to {@code entries} by copying already existing objects from the
  125. * repository onto the end of the thin pack to make it self-contained.
  126. */
  127. private ObjectIdSubclassMap<ObjectId> newObjectIds;
  128. private int deltaCount;
  129. private int entryCount;
  130. private ObjectIdOwnerMap<DeltaChain> baseById;
  131. /**
  132. * Objects referenced by their name from deltas, that aren't in this pack.
  133. * <p>
  134. * This is the set of objects that were copied onto the end of this pack to
  135. * make it complete. These objects were not transmitted by the remote peer,
  136. * but instead were assumed to already exist in the local repository.
  137. */
  138. private ObjectIdSubclassMap<ObjectId> baseObjectIds;
  139. private LongMap<UnresolvedDelta> baseByPos;
  140. /** Blobs whose contents need to be double-checked after indexing. */
  141. private BlockList<PackedObjectInfo> deferredCheckBlobs;
  142. private MessageDigest packDigest;
  143. private ObjectReader readCurs;
  144. /** Message to protect the pack data from garbage collection. */
  145. private String lockMessage;
  146. /**
  147. * Initialize a pack parser.
  148. *
  149. * @param odb
  150. * database the parser will write its objects into.
  151. * @param src
  152. * the stream the parser will read.
  153. */
  154. protected PackParser(final ObjectDatabase odb, final InputStream src) {
  155. objectDatabase = odb.newCachedDatabase();
  156. in = src;
  157. inflater = new InflaterStream();
  158. readCurs = objectDatabase.newReader();
  159. buf = new byte[BUFFER_SIZE];
  160. tempBuffer = new byte[BUFFER_SIZE];
  161. hdrBuf = new byte[64];
  162. objectDigest = Constants.newMessageDigest();
  163. tempObjectId = new MutableObjectId();
  164. packDigest = Constants.newMessageDigest();
  165. checkObjectCollisions = true;
  166. }
  167. /** @return true if a thin pack (missing base objects) is permitted. */
  168. public boolean isAllowThin() {
  169. return allowThin;
  170. }
  171. /**
  172. * Configure this index pack instance to allow a thin pack.
  173. * <p>
  174. * Thin packs are sometimes used during network transfers to allow a delta
  175. * to be sent without a base object. Such packs are not permitted on disk.
  176. *
  177. * @param allow
  178. * true to enable a thin pack.
  179. */
  180. public void setAllowThin(final boolean allow) {
  181. allowThin = allow;
  182. }
  183. /** @return if true received objects are verified to prevent collisions. */
  184. public boolean isCheckObjectCollisions() {
  185. return checkObjectCollisions;
  186. }
  187. /**
  188. * Enable checking for collisions with existing objects.
  189. * <p>
  190. * By default PackParser looks for each received object in the repository.
  191. * If the object already exists, the existing object is compared
  192. * byte-for-byte with the newly received copy to ensure they are identical.
  193. * The receive is aborted with an exception if any byte differs. This check
  194. * is necessary to prevent an evil attacker from supplying a replacement
  195. * object into this repository in the event that a discovery enabling SHA-1
  196. * collisions is made.
  197. * <p>
  198. * This check may be very costly to perform, and some repositories may have
  199. * other ways to segregate newly received object data. The check is enabled
  200. * by default, but can be explicitly disabled if the implementation can
  201. * provide the same guarantee, or is willing to accept the risks associated
  202. * with bypassing the check.
  203. *
  204. * @param check
  205. * true to enable collision checking (strongly encouraged).
  206. */
  207. public void setCheckObjectCollisions(boolean check) {
  208. checkObjectCollisions = check;
  209. }
  210. /**
  211. * Configure this index pack instance to keep track of new objects.
  212. * <p>
  213. * By default an index pack doesn't save the new objects that were created
  214. * when it was instantiated. Setting this flag to {@code true} allows the
  215. * caller to use {@link #getNewObjectIds()} to retrieve that list.
  216. *
  217. * @param b
  218. * {@code true} to enable keeping track of new objects.
  219. */
  220. public void setNeedNewObjectIds(boolean b) {
  221. if (b)
  222. newObjectIds = new ObjectIdSubclassMap<ObjectId>();
  223. else
  224. newObjectIds = null;
  225. }
  226. private boolean needNewObjectIds() {
  227. return newObjectIds != null;
  228. }
  229. /**
  230. * Configure this index pack instance to keep track of the objects assumed
  231. * for delta bases.
  232. * <p>
  233. * By default an index pack doesn't save the objects that were used as delta
  234. * bases. Setting this flag to {@code true} will allow the caller to use
  235. * {@link #getBaseObjectIds()} to retrieve that list.
  236. *
  237. * @param b
  238. * {@code true} to enable keeping track of delta bases.
  239. */
  240. public void setNeedBaseObjectIds(boolean b) {
  241. this.needBaseObjectIds = b;
  242. }
  243. /** @return true if the EOF should be read from the input after the footer. */
  244. public boolean isCheckEofAfterPackFooter() {
  245. return checkEofAfterPackFooter;
  246. }
  247. /**
  248. * Ensure EOF is read from the input stream after the footer.
  249. *
  250. * @param b
  251. * true if the EOF should be read; false if it is not checked.
  252. */
  253. public void setCheckEofAfterPackFooter(boolean b) {
  254. checkEofAfterPackFooter = b;
  255. }
  256. /** @return the new objects that were sent by the user */
  257. public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
  258. if (newObjectIds != null)
  259. return newObjectIds;
  260. return new ObjectIdSubclassMap<ObjectId>();
  261. }
  262. /** @return set of objects the incoming pack assumed for delta purposes */
  263. public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
  264. if (baseObjectIds != null)
  265. return baseObjectIds;
  266. return new ObjectIdSubclassMap<ObjectId>();
  267. }
  268. /**
  269. * Configure the checker used to validate received objects.
  270. * <p>
  271. * Usually object checking isn't necessary, as Git implementations only
  272. * create valid objects in pack files. However, additional checking may be
  273. * useful if processing data from an untrusted source.
  274. *
  275. * @param oc
  276. * the checker instance; null to disable object checking.
  277. */
  278. public void setObjectChecker(final ObjectChecker oc) {
  279. objCheck = oc;
  280. }
  281. /**
  282. * Configure the checker used to validate received objects.
  283. * <p>
  284. * Usually object checking isn't necessary, as Git implementations only
  285. * create valid objects in pack files. However, additional checking may be
  286. * useful if processing data from an untrusted source.
  287. * <p>
  288. * This is shorthand for:
  289. *
  290. * <pre>
  291. * setObjectChecker(on ? new ObjectChecker() : null);
  292. * </pre>
  293. *
  294. * @param on
  295. * true to enable the default checker; false to disable it.
  296. */
  297. public void setObjectChecking(final boolean on) {
  298. setObjectChecker(on ? new ObjectChecker() : null);
  299. }
  300. /** @return the message to record with the pack lock. */
  301. public String getLockMessage() {
  302. return lockMessage;
  303. }
  304. /**
  305. * Set the lock message for the incoming pack data.
  306. *
  307. * @param msg
  308. * if not null, the message to associate with the incoming data
  309. * while it is locked to prevent garbage collection.
  310. */
  311. public void setLockMessage(String msg) {
  312. lockMessage = msg;
  313. }
  314. /**
  315. * Get the number of objects in the stream.
  316. * <p>
  317. * The object count is only available after {@link #parse(ProgressMonitor)}
  318. * has returned. The count may have been increased if the stream was a thin
  319. * pack, and missing bases objects were appending onto it by the subclass.
  320. *
  321. * @return number of objects parsed out of the stream.
  322. */
  323. public int getObjectCount() {
  324. return entryCount;
  325. }
  326. /***
  327. * Get the information about the requested object.
  328. * <p>
  329. * The object information is only available after
  330. * {@link #parse(ProgressMonitor)} has returned.
  331. *
  332. * @param nth
  333. * index of the object in the stream. Must be between 0 and
  334. * {@link #getObjectCount()}-1.
  335. * @return the object information.
  336. */
  337. public PackedObjectInfo getObject(int nth) {
  338. return entries[nth];
  339. }
  340. /**
  341. * Get all of the objects, sorted by their name.
  342. * <p>
  343. * The object information is only available after
  344. * {@link #parse(ProgressMonitor)} has returned.
  345. * <p>
  346. * To maintain lower memory usage and good runtime performance, this method
  347. * sorts the objects in-place and therefore impacts the ordering presented
  348. * by {@link #getObject(int)}.
  349. *
  350. * @param cmp
  351. * comparison function, if null objects are stored by ObjectId.
  352. * @return sorted list of objects in this pack stream.
  353. */
  354. public List<PackedObjectInfo> getSortedObjectList(
  355. Comparator<PackedObjectInfo> cmp) {
  356. Arrays.sort(entries, 0, entryCount, cmp);
  357. List<PackedObjectInfo> list = Arrays.asList(entries);
  358. if (entryCount < entries.length)
  359. list = list.subList(0, entryCount);
  360. return list;
  361. }
  362. /**
  363. * Parse the pack stream.
  364. *
  365. * @param progress
  366. * callback to provide progress feedback during parsing. If null,
  367. * {@link NullProgressMonitor} will be used.
  368. * @return the pack lock, if one was requested by setting
  369. * {@link #setLockMessage(String)}.
  370. * @throws IOException
  371. * the stream is malformed, or contains corrupt objects.
  372. */
  373. public final PackLock parse(ProgressMonitor progress) throws IOException {
  374. return parse(progress, progress);
  375. }
  376. /**
  377. * Parse the pack stream.
  378. *
  379. * @param receiving
  380. * receives progress feedback during the initial receiving
  381. * objects phase. If null, {@link NullProgressMonitor} will be
  382. * used.
  383. * @param resolving
  384. * receives progress feedback during the resolving objects phase.
  385. * @return the pack lock, if one was requested by setting
  386. * {@link #setLockMessage(String)}.
  387. * @throws IOException
  388. * the stream is malformed, or contains corrupt objects.
  389. */
  390. public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
  391. throws IOException {
  392. if (receiving == null)
  393. receiving = NullProgressMonitor.INSTANCE;
  394. if (resolving == null)
  395. resolving = NullProgressMonitor.INSTANCE;
  396. if (receiving == resolving)
  397. receiving.start(2 /* tasks */);
  398. try {
  399. readPackHeader();
  400. entries = new PackedObjectInfo[(int) objectCount];
  401. baseById = new ObjectIdOwnerMap<DeltaChain>();
  402. baseByPos = new LongMap<UnresolvedDelta>();
  403. deferredCheckBlobs = new BlockList<PackedObjectInfo>();
  404. receiving.beginTask(JGitText.get().receivingObjects,
  405. (int) objectCount);
  406. try {
  407. for (int done = 0; done < objectCount; done++) {
  408. indexOneObject();
  409. receiving.update(1);
  410. if (receiving.isCancelled())
  411. throw new IOException(JGitText.get().downloadCancelled);
  412. }
  413. readPackFooter();
  414. endInput();
  415. } finally {
  416. receiving.endTask();
  417. }
  418. if (!deferredCheckBlobs.isEmpty())
  419. doDeferredCheckBlobs();
  420. if (deltaCount > 0) {
  421. resolveDeltas(resolving);
  422. if (entryCount < objectCount) {
  423. if (!isAllowThin()) {
  424. throw new IOException(MessageFormat.format(JGitText
  425. .get().packHasUnresolvedDeltas,
  426. (objectCount - entryCount)));
  427. }
  428. resolveDeltasWithExternalBases(resolving);
  429. if (entryCount < objectCount) {
  430. throw new IOException(MessageFormat.format(JGitText
  431. .get().packHasUnresolvedDeltas,
  432. (objectCount - entryCount)));
  433. }
  434. }
  435. }
  436. packDigest = null;
  437. baseById = null;
  438. baseByPos = null;
  439. } finally {
  440. try {
  441. if (readCurs != null)
  442. readCurs.release();
  443. } finally {
  444. readCurs = null;
  445. }
  446. try {
  447. inflater.release();
  448. } finally {
  449. inflater = null;
  450. objectDatabase.close();
  451. }
  452. }
  453. return null; // By default there is no locking.
  454. }
  455. private void resolveDeltas(final ProgressMonitor progress)
  456. throws IOException {
  457. progress.beginTask(JGitText.get().resolvingDeltas, deltaCount);
  458. final int last = entryCount;
  459. for (int i = 0; i < last; i++) {
  460. final int before = entryCount;
  461. resolveDeltas(entries[i]);
  462. progress.update(entryCount - before);
  463. if (progress.isCancelled())
  464. throw new IOException(
  465. JGitText.get().downloadCancelledDuringIndexing);
  466. }
  467. progress.endTask();
  468. }
  469. private void resolveDeltas(final PackedObjectInfo oe) throws IOException {
  470. UnresolvedDelta children = firstChildOf(oe);
  471. if (children == null)
  472. return;
  473. DeltaVisit visit = new DeltaVisit();
  474. visit.nextChild = children;
  475. ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
  476. switch (info.type) {
  477. case Constants.OBJ_COMMIT:
  478. case Constants.OBJ_TREE:
  479. case Constants.OBJ_BLOB:
  480. case Constants.OBJ_TAG:
  481. visit.data = inflateAndReturn(Source.DATABASE, info.size);
  482. visit.id = oe;
  483. break;
  484. default:
  485. throw new IOException(MessageFormat.format(
  486. JGitText.get().unknownObjectType, info.type));
  487. }
  488. if (!checkCRC(oe.getCRC())) {
  489. throw new IOException(MessageFormat.format(
  490. JGitText.get().corruptionDetectedReReadingAt, oe
  491. .getOffset()));
  492. }
  493. resolveDeltas(visit.next(), info.type, info);
  494. }
  495. private void resolveDeltas(DeltaVisit visit, final int type,
  496. ObjectTypeAndSize info) throws IOException {
  497. do {
  498. info = openDatabase(visit.delta, info);
  499. switch (info.type) {
  500. case Constants.OBJ_OFS_DELTA:
  501. case Constants.OBJ_REF_DELTA:
  502. break;
  503. default:
  504. throw new IOException(MessageFormat.format(
  505. JGitText.get().unknownObjectType, info.type));
  506. }
  507. visit.data = BinaryDelta.apply(visit.parent.data, //
  508. inflateAndReturn(Source.DATABASE, info.size));
  509. if (!checkCRC(visit.delta.crc))
  510. throw new IOException(MessageFormat.format(
  511. JGitText.get().corruptionDetectedReReadingAt,
  512. visit.delta.position));
  513. objectDigest.update(Constants.encodedTypeString(type));
  514. objectDigest.update((byte) ' ');
  515. objectDigest.update(Constants.encodeASCII(visit.data.length));
  516. objectDigest.update((byte) 0);
  517. objectDigest.update(visit.data);
  518. tempObjectId.fromRaw(objectDigest.digest(), 0);
  519. verifySafeObject(tempObjectId, type, visit.data);
  520. PackedObjectInfo oe;
  521. oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
  522. oe.setOffset(visit.delta.position);
  523. onInflatedObjectData(oe, type, visit.data);
  524. addObjectAndTrack(oe);
  525. visit.id = oe;
  526. visit.nextChild = firstChildOf(oe);
  527. visit = visit.next();
  528. } while (visit != null);
  529. }
  530. /**
  531. * Read the header of the current object.
  532. * <p>
  533. * After the header has been parsed, this method automatically invokes
  534. * {@link #onObjectHeader(Source, byte[], int, int)} to allow the
  535. * implementation to update its internal checksums for the bytes read.
  536. * <p>
  537. * When this method returns the database will be positioned on the first
  538. * byte of the deflated data stream.
  539. *
  540. * @param info
  541. * the info object to populate.
  542. * @return {@code info}, after populating.
  543. * @throws IOException
  544. * the size cannot be read.
  545. */
  546. protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
  547. throws IOException {
  548. int hdrPtr = 0;
  549. int c = readFrom(Source.DATABASE);
  550. hdrBuf[hdrPtr++] = (byte) c;
  551. info.type = (c >> 4) & 7;
  552. long sz = c & 15;
  553. int shift = 4;
  554. while ((c & 0x80) != 0) {
  555. c = readFrom(Source.DATABASE);
  556. hdrBuf[hdrPtr++] = (byte) c;
  557. sz += (c & 0x7f) << shift;
  558. shift += 7;
  559. }
  560. info.size = sz;
  561. switch (info.type) {
  562. case Constants.OBJ_COMMIT:
  563. case Constants.OBJ_TREE:
  564. case Constants.OBJ_BLOB:
  565. case Constants.OBJ_TAG:
  566. onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  567. break;
  568. case Constants.OBJ_OFS_DELTA:
  569. c = readFrom(Source.DATABASE);
  570. hdrBuf[hdrPtr++] = (byte) c;
  571. while ((c & 128) != 0) {
  572. c = readFrom(Source.DATABASE);
  573. hdrBuf[hdrPtr++] = (byte) c;
  574. }
  575. onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  576. break;
  577. case Constants.OBJ_REF_DELTA:
  578. System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
  579. hdrPtr += 20;
  580. use(20);
  581. onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  582. break;
  583. default:
  584. throw new IOException(MessageFormat.format(
  585. JGitText.get().unknownObjectType, info.type));
  586. }
  587. return info;
  588. }
  589. private UnresolvedDelta removeBaseById(final AnyObjectId id) {
  590. final DeltaChain d = baseById.get(id);
  591. return d != null ? d.remove() : null;
  592. }
  593. private static UnresolvedDelta reverse(UnresolvedDelta c) {
  594. UnresolvedDelta tail = null;
  595. while (c != null) {
  596. final UnresolvedDelta n = c.next;
  597. c.next = tail;
  598. tail = c;
  599. c = n;
  600. }
  601. return tail;
  602. }
  603. private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
  604. UnresolvedDelta a = reverse(removeBaseById(oe));
  605. UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));
  606. if (a == null)
  607. return b;
  608. if (b == null)
  609. return a;
  610. UnresolvedDelta first = null;
  611. UnresolvedDelta last = null;
  612. while (a != null || b != null) {
  613. UnresolvedDelta curr;
  614. if (b == null || (a != null && a.position < b.position)) {
  615. curr = a;
  616. a = a.next;
  617. } else {
  618. curr = b;
  619. b = b.next;
  620. }
  621. if (last != null)
  622. last.next = curr;
  623. else
  624. first = curr;
  625. last = curr;
  626. curr.next = null;
  627. }
  628. return first;
  629. }
  630. private void resolveDeltasWithExternalBases(final ProgressMonitor progress)
  631. throws IOException {
  632. growEntries(baseById.size());
  633. if (needBaseObjectIds)
  634. baseObjectIds = new ObjectIdSubclassMap<ObjectId>();
  635. final List<DeltaChain> missing = new ArrayList<DeltaChain>(64);
  636. for (final DeltaChain baseId : baseById) {
  637. if (baseId.head == null)
  638. continue;
  639. if (needBaseObjectIds)
  640. baseObjectIds.add(baseId);
  641. final ObjectLoader ldr;
  642. try {
  643. ldr = readCurs.open(baseId);
  644. } catch (MissingObjectException notFound) {
  645. missing.add(baseId);
  646. continue;
  647. }
  648. final DeltaVisit visit = new DeltaVisit();
  649. visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
  650. visit.id = baseId;
  651. final int typeCode = ldr.getType();
  652. final PackedObjectInfo oe = newInfo(baseId, null, null);
  653. if (onAppendBase(typeCode, visit.data, oe))
  654. entries[entryCount++] = oe;
  655. visit.nextChild = firstChildOf(oe);
  656. resolveDeltas(visit.next(), typeCode, new ObjectTypeAndSize());
  657. if (progress.isCancelled())
  658. throw new IOException(
  659. JGitText.get().downloadCancelledDuringIndexing);
  660. }
  661. for (final DeltaChain base : missing) {
  662. if (base.head != null)
  663. throw new MissingObjectException(base, "delta base");
  664. }
  665. onEndThinPack();
  666. }
  667. private void growEntries(int extraObjects) {
  668. final PackedObjectInfo[] ne;
  669. ne = new PackedObjectInfo[(int) objectCount + extraObjects];
  670. System.arraycopy(entries, 0, ne, 0, entryCount);
  671. entries = ne;
  672. }
  673. private void readPackHeader() throws IOException {
  674. final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
  675. final int p = fill(Source.INPUT, hdrln);
  676. for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
  677. if (buf[p + k] != Constants.PACK_SIGNATURE[k])
  678. throw new IOException(JGitText.get().notAPACKFile);
  679. final long vers = NB.decodeUInt32(buf, p + 4);
  680. if (vers != 2 && vers != 3)
  681. throw new IOException(MessageFormat.format(
  682. JGitText.get().unsupportedPackVersion, vers));
  683. objectCount = NB.decodeUInt32(buf, p + 8);
  684. use(hdrln);
  685. onPackHeader(objectCount);
  686. }
  687. private void readPackFooter() throws IOException {
  688. sync();
  689. final byte[] actHash = packDigest.digest();
  690. final int c = fill(Source.INPUT, 20);
  691. final byte[] srcHash = new byte[20];
  692. System.arraycopy(buf, c, srcHash, 0, 20);
  693. use(20);
  694. // The input stream should be at EOF at this point. We do not support
  695. // yielding back any remaining buffered data after the pack footer, so
  696. // protocols that embed a pack stream are required to either end their
  697. // stream with the pack, or embed the pack with a framing system like
  698. // the SideBandInputStream does.
  699. if (bAvail != 0)
  700. throw new CorruptObjectException(MessageFormat.format(
  701. JGitText.get().expectedEOFReceived,
  702. "\\x" + Integer.toHexString(buf[bOffset] & 0xff)));
  703. if (isCheckEofAfterPackFooter()) {
  704. int eof = in.read();
  705. if (0 <= eof)
  706. throw new CorruptObjectException(MessageFormat.format(
  707. JGitText.get().expectedEOFReceived,
  708. "\\x" + Integer.toHexString(eof)));
  709. }
  710. if (!Arrays.equals(actHash, srcHash))
  711. throw new CorruptObjectException(
  712. JGitText.get().corruptObjectPackfileChecksumIncorrect);
  713. onPackFooter(srcHash);
  714. }
  715. // Cleanup all resources associated with our input parsing.
  716. private void endInput() {
  717. in = null;
  718. }
  719. // Read one entire object or delta from the input.
  720. private void indexOneObject() throws IOException {
  721. final long streamPosition = streamPosition();
  722. int hdrPtr = 0;
  723. int c = readFrom(Source.INPUT);
  724. hdrBuf[hdrPtr++] = (byte) c;
  725. final int typeCode = (c >> 4) & 7;
  726. long sz = c & 15;
  727. int shift = 4;
  728. while ((c & 0x80) != 0) {
  729. c = readFrom(Source.INPUT);
  730. hdrBuf[hdrPtr++] = (byte) c;
  731. sz += (c & 0x7f) << shift;
  732. shift += 7;
  733. }
  734. switch (typeCode) {
  735. case Constants.OBJ_COMMIT:
  736. case Constants.OBJ_TREE:
  737. case Constants.OBJ_BLOB:
  738. case Constants.OBJ_TAG:
  739. onBeginWholeObject(streamPosition, typeCode, sz);
  740. onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  741. whole(streamPosition, typeCode, sz);
  742. break;
  743. case Constants.OBJ_OFS_DELTA: {
  744. c = readFrom(Source.INPUT);
  745. hdrBuf[hdrPtr++] = (byte) c;
  746. long ofs = c & 127;
  747. while ((c & 128) != 0) {
  748. ofs += 1;
  749. c = readFrom(Source.INPUT);
  750. hdrBuf[hdrPtr++] = (byte) c;
  751. ofs <<= 7;
  752. ofs += (c & 127);
  753. }
  754. final long base = streamPosition - ofs;
  755. onBeginOfsDelta(streamPosition, base, sz);
  756. onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  757. inflateAndSkip(Source.INPUT, sz);
  758. UnresolvedDelta n = onEndDelta();
  759. n.position = streamPosition;
  760. n.next = baseByPos.put(base, n);
  761. deltaCount++;
  762. break;
  763. }
  764. case Constants.OBJ_REF_DELTA: {
  765. c = fill(Source.INPUT, 20);
  766. final ObjectId base = ObjectId.fromRaw(buf, c);
  767. System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
  768. hdrPtr += 20;
  769. use(20);
  770. DeltaChain r = baseById.get(base);
  771. if (r == null) {
  772. r = new DeltaChain(base);
  773. baseById.add(r);
  774. }
  775. onBeginRefDelta(streamPosition, base, sz);
  776. onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  777. inflateAndSkip(Source.INPUT, sz);
  778. UnresolvedDelta n = onEndDelta();
  779. n.position = streamPosition;
  780. r.add(n);
  781. deltaCount++;
  782. break;
  783. }
  784. default:
  785. throw new IOException(MessageFormat.format(
  786. JGitText.get().unknownObjectType, typeCode));
  787. }
  788. }
  789. private void whole(final long pos, final int type, final long sz)
  790. throws IOException {
  791. objectDigest.update(Constants.encodedTypeString(type));
  792. objectDigest.update((byte) ' ');
  793. objectDigest.update(Constants.encodeASCII(sz));
  794. objectDigest.update((byte) 0);
  795. final byte[] data;
  796. boolean checkContentLater = false;
  797. if (type == Constants.OBJ_BLOB) {
  798. byte[] readBuffer = buffer();
  799. InputStream inf = inflate(Source.INPUT, sz);
  800. long cnt = 0;
  801. while (cnt < sz) {
  802. int r = inf.read(readBuffer);
  803. if (r <= 0)
  804. break;
  805. objectDigest.update(readBuffer, 0, r);
  806. cnt += r;
  807. }
  808. inf.close();
  809. tempObjectId.fromRaw(objectDigest.digest(), 0);
  810. checkContentLater = isCheckObjectCollisions()
  811. && readCurs.has(tempObjectId);
  812. data = null;
  813. } else {
  814. data = inflateAndReturn(Source.INPUT, sz);
  815. objectDigest.update(data);
  816. tempObjectId.fromRaw(objectDigest.digest(), 0);
  817. verifySafeObject(tempObjectId, type, data);
  818. }
  819. PackedObjectInfo obj = newInfo(tempObjectId, null, null);
  820. obj.setOffset(pos);
  821. onEndWholeObject(obj);
  822. if (data != null)
  823. onInflatedObjectData(obj, type, data);
  824. addObjectAndTrack(obj);
  825. if (checkContentLater)
  826. deferredCheckBlobs.add(obj);
  827. }
  828. private void verifySafeObject(final AnyObjectId id, final int type,
  829. final byte[] data) throws IOException {
  830. if (objCheck != null) {
  831. try {
  832. objCheck.check(type, data);
  833. } catch (CorruptObjectException e) {
  834. throw new IOException(MessageFormat.format(
  835. JGitText.get().invalidObject, Constants
  836. .typeString(type), id.name(), e.getMessage()));
  837. }
  838. }
  839. if (isCheckObjectCollisions()) {
  840. try {
  841. final ObjectLoader ldr = readCurs.open(id, type);
  842. final byte[] existingData = ldr.getCachedBytes(data.length);
  843. if (!Arrays.equals(data, existingData)) {
  844. throw new IOException(MessageFormat.format(
  845. JGitText.get().collisionOn, id.name()));
  846. }
  847. } catch (MissingObjectException notLocal) {
  848. // This is OK, we don't have a copy of the object locally
  849. // but the API throws when we try to read it as usually its
  850. // an error to read something that doesn't exist.
  851. }
  852. }
  853. }
  854. private void doDeferredCheckBlobs() throws IOException {
  855. final byte[] readBuffer = buffer();
  856. final byte[] curBuffer = new byte[readBuffer.length];
  857. ObjectTypeAndSize info = new ObjectTypeAndSize();
  858. for (PackedObjectInfo obj : deferredCheckBlobs) {
  859. info = openDatabase(obj, info);
  860. if (info.type != Constants.OBJ_BLOB)
  861. throw new IOException(MessageFormat.format(
  862. JGitText.get().unknownObjectType, info.type));
  863. ObjectStream cur = readCurs.open(obj, info.type).openStream();
  864. try {
  865. long sz = info.size;
  866. if (cur.getSize() != sz)
  867. throw new IOException(MessageFormat.format(
  868. JGitText.get().collisionOn, obj.name()));
  869. InputStream pck = inflate(Source.DATABASE, sz);
  870. while (0 < sz) {
  871. int n = (int) Math.min(readBuffer.length, sz);
  872. IO.readFully(cur, curBuffer, 0, n);
  873. IO.readFully(pck, readBuffer, 0, n);
  874. for (int i = 0; i < n; i++) {
  875. if (curBuffer[i] != readBuffer[i])
  876. throw new IOException(MessageFormat.format(JGitText
  877. .get().collisionOn, obj.name()));
  878. }
  879. sz -= n;
  880. }
  881. pck.close();
  882. } finally {
  883. cur.close();
  884. }
  885. }
  886. }
  887. /** @return current position of the input stream being parsed. */
  888. private long streamPosition() {
  889. return bBase + bOffset;
  890. }
  891. private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
  892. ObjectTypeAndSize info) throws IOException {
  893. bOffset = 0;
  894. bAvail = 0;
  895. return seekDatabase(obj, info);
  896. }
  897. private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
  898. ObjectTypeAndSize info) throws IOException {
  899. bOffset = 0;
  900. bAvail = 0;
  901. return seekDatabase(delta, info);
  902. }
  903. // Consume exactly one byte from the buffer and return it.
  904. private int readFrom(final Source src) throws IOException {
  905. if (bAvail == 0)
  906. fill(src, 1);
  907. bAvail--;
  908. return buf[bOffset++] & 0xff;
  909. }
  910. // Consume cnt bytes from the buffer.
  911. private void use(final int cnt) {
  912. bOffset += cnt;
  913. bAvail -= cnt;
  914. }
  915. // Ensure at least need bytes are available in in {@link #buf}.
  916. private int fill(final Source src, final int need) throws IOException {
  917. while (bAvail < need) {
  918. int next = bOffset + bAvail;
  919. int free = buf.length - next;
  920. if (free + bAvail < need) {
  921. switch (src) {
  922. case INPUT:
  923. sync();
  924. break;
  925. case DATABASE:
  926. if (bAvail > 0)
  927. System.arraycopy(buf, bOffset, buf, 0, bAvail);
  928. bOffset = 0;
  929. break;
  930. }
  931. next = bAvail;
  932. free = buf.length - next;
  933. }
  934. switch (src) {
  935. case INPUT:
  936. next = in.read(buf, next, free);
  937. break;
  938. case DATABASE:
  939. next = readDatabase(buf, next, free);
  940. break;
  941. }
  942. if (next <= 0)
  943. throw new EOFException(JGitText.get().packfileIsTruncated);
  944. bAvail += next;
  945. }
  946. return bOffset;
  947. }
  948. // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
  949. private void sync() throws IOException {
  950. packDigest.update(buf, 0, bOffset);
  951. onStoreStream(buf, 0, bOffset);
  952. if (bAvail > 0)
  953. System.arraycopy(buf, bOffset, buf, 0, bAvail);
  954. bBase += bOffset;
  955. bOffset = 0;
  956. }
  957. /** @return a temporary byte array for use by the caller. */
  958. protected byte[] buffer() {
  959. return tempBuffer;
  960. }
  961. /**
  962. * Construct a PackedObjectInfo instance for this parser.
  963. *
  964. * @param id
  965. * identity of the object to be tracked.
  966. * @param delta
  967. * if the object was previously an unresolved delta, this is the
  968. * delta object that was tracking it. Otherwise null.
  969. * @param deltaBase
  970. * if the object was previously an unresolved delta, this is the
  971. * ObjectId of the base of the delta. The base may be outside of
  972. * the pack stream if the stream was a thin-pack.
  973. * @return info object containing this object's data.
  974. */
  975. protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
  976. ObjectId deltaBase) {
  977. PackedObjectInfo oe = new PackedObjectInfo(id);
  978. if (delta != null)
  979. oe.setCRC(delta.crc);
  980. return oe;
  981. }
  982. /**
  983. * Store bytes received from the raw stream.
  984. * <p>
  985. * This method is invoked during {@link #parse(ProgressMonitor)} as data is
  986. * consumed from the incoming stream. Implementors may use this event to
  987. * archive the raw incoming stream to the destination repository in large
  988. * chunks, without paying attention to object boundaries.
  989. * <p>
  990. * The only component of the pack not supplied to this method is the last 20
  991. * bytes of the pack that comprise the trailing SHA-1 checksum. Those are
  992. * passed to {@link #onPackFooter(byte[])}.
  993. *
  994. * @param raw
  995. * buffer to copy data out of.
  996. * @param pos
  997. * first offset within the buffer that is valid.
  998. * @param len
  999. * number of bytes in the buffer that are valid.
  1000. * @throws IOException
  1001. * the stream cannot be archived.
  1002. */
  1003. protected abstract void onStoreStream(byte[] raw, int pos, int len)
  1004. throws IOException;
  1005. /**
  1006. * Store (and/or checksum) an object header.
  1007. * <p>
  1008. * Invoked after any of the {@code onBegin()} events. The entire header is
  1009. * supplied in a single invocation, before any object data is supplied.
  1010. *
  1011. * @param src
  1012. * where the data came from
  1013. * @param raw
  1014. * buffer to read data from.
  1015. * @param pos
  1016. * first offset within buffer that is valid.
  1017. * @param len
  1018. * number of bytes in buffer that are valid.
  1019. * @throws IOException
  1020. * the stream cannot be archived.
  1021. */
  1022. protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
  1023. int len) throws IOException;
  1024. /**
  1025. * Store (and/or checksum) a portion of an object's data.
  1026. * <p>
  1027. * This method may be invoked multiple times per object, depending on the
  1028. * size of the object, the size of the parser's internal read buffer, and
  1029. * the alignment of the object relative to the read buffer.
  1030. * <p>
  1031. * Invoked after {@link #onObjectHeader(Source, byte[], int, int)}.
  1032. *
  1033. * @param src
  1034. * where the data came from
  1035. * @param raw
  1036. * buffer to read data from.
  1037. * @param pos
  1038. * first offset within buffer that is valid.
  1039. * @param len
  1040. * number of bytes in buffer that are valid.
  1041. * @throws IOException
  1042. * the stream cannot be archived.
  1043. */
  1044. protected abstract void onObjectData(Source src, byte[] raw, int pos,
  1045. int len) throws IOException;
  1046. /**
  1047. * Invoked for commits, trees, tags, and small blobs.
  1048. *
  1049. * @param obj
  1050. * the object info, populated.
  1051. * @param typeCode
  1052. * the type of the object.
  1053. * @param data
  1054. * inflated data for the object.
  1055. * @throws IOException
  1056. * the object cannot be archived.
  1057. */
  1058. protected abstract void onInflatedObjectData(PackedObjectInfo obj,
  1059. int typeCode, byte[] data) throws IOException;
  1060. /**
  1061. * Provide the implementation with the original stream's pack header.
  1062. *
  1063. * @param objCnt
  1064. * number of objects expected in the stream.
  1065. * @throws IOException
  1066. * the implementation refuses to work with this many objects.
  1067. */
  1068. protected abstract void onPackHeader(long objCnt) throws IOException;
  1069. /**
  1070. * Provide the implementation with the original stream's pack footer.
  1071. *
  1072. * @param hash
  1073. * the trailing 20 bytes of the pack, this is a SHA-1 checksum of
  1074. * all of the pack data.
  1075. * @throws IOException
  1076. * the stream cannot be archived.
  1077. */
  1078. protected abstract void onPackFooter(byte[] hash) throws IOException;
  1079. /**
  1080. * Provide the implementation with a base that was outside of the pack.
  1081. * <p>
  1082. * This event only occurs on a thin pack for base objects that were outside
  1083. * of the pack and came from the local repository. Usually an implementation
  1084. * uses this event to compress the base and append it onto the end of the
  1085. * pack, so the pack stays self-contained.
  1086. *
  1087. * @param typeCode
  1088. * type of the base object.
  1089. * @param data
  1090. * complete content of the base object.
  1091. * @param info
  1092. * packed object information for this base. Implementors must
  1093. * populate the CRC and offset members if returning true.
  1094. * @return true if the {@code info} should be included in the object list
  1095. * returned by {@link #getSortedObjectList(Comparator)}, false if it
  1096. * should not be included.
  1097. * @throws IOException
  1098. * the base could not be included into the pack.
  1099. */
  1100. protected abstract boolean onAppendBase(int typeCode, byte[] data,
  1101. PackedObjectInfo info) throws IOException;
  1102. /**
  1103. * Event indicating a thin pack has been completely processed.
  1104. * <p>
  1105. * This event is invoked only if a thin pack has delta references to objects
  1106. * external from the pack. The event is called after all of those deltas
  1107. * have been resolved.
  1108. *
  1109. * @throws IOException
  1110. * the pack cannot be archived.
  1111. */
  1112. protected abstract void onEndThinPack() throws IOException;
  1113. /**
  1114. * Reposition the database to re-read a previously stored object.
  1115. * <p>
  1116. * If the database is computing CRC-32 checksums for object data, it should
  1117. * reset its internal CRC instance during this method call.
  1118. *
  1119. * @param obj
  1120. * the object position to begin reading from. This is from
  1121. * {@link #newInfo(AnyObjectId, UnresolvedDelta, ObjectId)}.
  1122. * @param info
  1123. * object to populate with type and size.
  1124. * @return the {@code info} object.
  1125. * @throws IOException
  1126. * the database cannot reposition to this location.
  1127. */
  1128. protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
  1129. ObjectTypeAndSize info) throws IOException;
  1130. /**
  1131. * Reposition the database to re-read a previously stored object.
  1132. * <p>
  1133. * If the database is computing CRC-32 checksums for object data, it should
  1134. * reset its internal CRC instance during this method call.
  1135. *
  1136. * @param delta
  1137. * the object position to begin reading from. This is an instance
  1138. * previously returned by {@link #onEndDelta()}.
  1139. * @param info
  1140. * object to populate with type and size.
  1141. * @return the {@code info} object.
  1142. * @throws IOException
  1143. * the database cannot reposition to this location.
  1144. */
  1145. protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
  1146. ObjectTypeAndSize info) throws IOException;
  1147. /**
  1148. * Read from the database's current position into the buffer.
  1149. *
  1150. * @param dst
  1151. * the buffer to copy read data into.
  1152. * @param pos
  1153. * position within {@code dst} to start copying data into.
  1154. * @param cnt
  1155. * ideal target number of bytes to read. Actual read length may
  1156. * be shorter.
  1157. * @return number of bytes stored.
  1158. * @throws IOException
  1159. * the database cannot be accessed.
  1160. */
  1161. protected abstract int readDatabase(byte[] dst, int pos, int cnt)
  1162. throws IOException;
  1163. /**
  1164. * Check the current CRC matches the expected value.
  1165. * <p>
  1166. * This method is invoked when an object is read back in from the database
  1167. * and its data is used during delta resolution. The CRC is validated after
  1168. * the object has been fully read, allowing the parser to verify there was
  1169. * no silent data corruption.
  1170. * <p>
  1171. * Implementations are free to ignore this check by always returning true if
  1172. * they are performing other data integrity validations at a lower level.
  1173. *
  1174. * @param oldCRC
  1175. * the prior CRC that was recorded during the first scan of the
  1176. * object from the pack stream.
  1177. * @return true if the CRC matches; false if it does not.
  1178. */
  1179. protected abstract boolean checkCRC(int oldCRC);
  1180. /**
  1181. * Event notifying the start of an object stored whole (not as a delta).
  1182. *
  1183. * @param streamPosition
  1184. * position of this object in the incoming stream.
  1185. * @param type
  1186. * type of the object; one of {@link Constants#OBJ_COMMIT},
  1187. * {@link Constants#OBJ_TREE}, {@link Constants#OBJ_BLOB}, or
  1188. * {@link Constants#OBJ_TAG}.
  1189. * @param inflatedSize
  1190. * size of the object when fully inflated. The size stored within
  1191. * the pack may be larger or smaller, and is not yet known.
  1192. * @throws IOException
  1193. * the object cannot be recorded.
  1194. */
  1195. protected abstract void onBeginWholeObject(long streamPosition, int type,
  1196. long inflatedSize) throws IOException;
  1197. /**
  1198. * Event notifying the the current object.
  1199. *
  1200. *@param info
  1201. * object information.
  1202. * @throws IOException
  1203. * the object cannot be recorded.
  1204. */
  1205. protected abstract void onEndWholeObject(PackedObjectInfo info)
  1206. throws IOException;
  1207. /**
  1208. * Event notifying start of a delta referencing its base by offset.
  1209. *
  1210. * @param deltaStreamPosition
  1211. * position of this object in the incoming stream.
  1212. * @param baseStreamPosition
  1213. * position of the base object in the incoming stream. The base
  1214. * must be before the delta, therefore {@code baseStreamPosition
  1215. * &lt; deltaStreamPosition}. This is <b>not</b> the position
  1216. * returned by a prior end object event.
  1217. * @param inflatedSize
  1218. * size of the delta when fully inflated. The size stored within
  1219. * the pack may be larger or smaller, and is not yet known.
  1220. * @throws IOException
  1221. * the object cannot be recorded.
  1222. */
  1223. protected abstract void onBeginOfsDelta(long deltaStreamPosition,
  1224. long baseStreamPosition, long inflatedSize) throws IOException;
  1225. /**
  1226. * Event notifying start of a delta referencing its base by ObjectId.
  1227. *
  1228. * @param deltaStreamPosition
  1229. * position of this object in the incoming stream.
  1230. * @param baseId
  1231. * name of the base object. This object may be later in the
  1232. * stream, or might not appear at all in the stream (in the case
  1233. * of a thin-pack).
  1234. * @param inflatedSize
  1235. * size of the delta when fully inflated. The size stored within
  1236. * the pack may be larger or smaller, and is not yet known.
  1237. * @throws IOException
  1238. * the object cannot be recorded.
  1239. */
  1240. protected abstract void onBeginRefDelta(long deltaStreamPosition,
  1241. AnyObjectId baseId, long inflatedSize) throws IOException;
  1242. /**
  1243. * Event notifying the the current object.
  1244. *
  1245. *@return object information that must be populated with at least the
  1246. * offset.
  1247. * @throws IOException
  1248. * the object cannot be recorded.
  1249. */
  1250. protected UnresolvedDelta onEndDelta() throws IOException {
  1251. return new UnresolvedDelta();
  1252. }
  1253. /** Type and size information about an object in the database buffer. */
  1254. public static class ObjectTypeAndSize {
  1255. /** The type of the object. */
  1256. public int type;
  1257. /** The inflated size of the object. */
  1258. public long size;
  1259. }
  1260. private void inflateAndSkip(final Source src, final long inflatedSize)
  1261. throws IOException {
  1262. final InputStream inf = inflate(src, inflatedSize);
  1263. IO.skipFully(inf, inflatedSize);
  1264. inf.close();
  1265. }
  1266. private byte[] inflateAndReturn(final Source src, final long inflatedSize)
  1267. throws IOException {
  1268. final byte[] dst = new byte[(int) inflatedSize];
  1269. final InputStream inf = inflate(src, inflatedSize);
  1270. IO.readFully(inf, dst, 0, dst.length);
  1271. inf.close();
  1272. return dst;
  1273. }
  1274. private InputStream inflate(final Source src, final long inflatedSize)
  1275. throws IOException {
  1276. inflater.open(src, inflatedSize);
  1277. return inflater;
  1278. }
  1279. private static class DeltaChain extends ObjectIdOwnerMap.Entry {
  1280. UnresolvedDelta head;
  1281. DeltaChain(final AnyObjectId id) {
  1282. super(id);
  1283. }
  1284. UnresolvedDelta remove() {
  1285. final UnresolvedDelta r = head;
  1286. if (r != null)
  1287. head = null;
  1288. return r;
  1289. }
  1290. void add(final UnresolvedDelta d) {
  1291. d.next = head;
  1292. head = d;
  1293. }
  1294. }
  1295. /** Information about an unresolved delta in this pack stream. */
  1296. public static class UnresolvedDelta {
  1297. long position;
  1298. int crc;
  1299. UnresolvedDelta next;
  1300. /** @return offset within the input stream. */
  1301. public long getOffset() {
  1302. return position;
  1303. }
  1304. /** @return the CRC-32 checksum of the stored delta data. */
  1305. public int getCRC() {
  1306. return crc;
  1307. }
  1308. /**
  1309. * @param crc32
  1310. * the CRC-32 checksum of the stored delta data.
  1311. */
  1312. public void setCRC(int crc32) {
  1313. crc = crc32;
  1314. }
  1315. }
  1316. private static class DeltaVisit {
  1317. final UnresolvedDelta delta;
  1318. ObjectId id;
  1319. byte[] data;
  1320. DeltaVisit parent;
  1321. UnresolvedDelta nextChild;
  1322. DeltaVisit() {
  1323. this.delta = null; // At the root of the stack we have a base.
  1324. }
  1325. DeltaVisit(DeltaVisit parent) {
  1326. this.parent = parent;
  1327. this.delta = parent.nextChild;
  1328. parent.nextChild = delta.next;
  1329. }
  1330. DeltaVisit next() {
  1331. // If our parent has no more children, discard it.
  1332. if (parent != null && parent.nextChild == null) {
  1333. parent.data = null;
  1334. parent = parent.parent;
  1335. }
  1336. if (nextChild != null)
  1337. return new DeltaVisit(this);
  1338. // If we have no child ourselves, our parent must (if it exists),
  1339. // due to the discard rule above. With no parent, we are done.
  1340. if (parent != null)
  1341. return new DeltaVisit(parent);
  1342. return null;
  1343. }
  1344. }
  1345. private void addObjectAndTrack(PackedObjectInfo oe) {
  1346. entries[entryCount++] = oe;
  1347. if (needNewObjectIds())
  1348. newObjectIds.add(oe);
  1349. }
  1350. private class InflaterStream extends InputStream {
  1351. private final Inflater inf;
  1352. private final byte[] skipBuffer;
  1353. private Source src;
  1354. private long expectedSize;
  1355. private long actualSize;
  1356. private int p;
  1357. InflaterStream() {
  1358. inf = InflaterCache.get();
  1359. skipBuffer = new byte[512];
  1360. }
  1361. void release() {
  1362. inf.reset();
  1363. InflaterCache.release(inf);
  1364. }
  1365. void open(Source source, long inflatedSize) throws IOException {
  1366. src = source;
  1367. expectedSize = inflatedSize;
  1368. actualSize = 0;
  1369. p = fill(src, 1);
  1370. inf.setInput(buf, p, bAvail);
  1371. }
  1372. @Override
  1373. public long skip(long toSkip) throws IOException {
  1374. long n = 0;
  1375. while (n < toSkip) {
  1376. final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
  1377. final int r = read(skipBuffer, 0, cnt);
  1378. if (r <= 0)
  1379. break;
  1380. n += r;
  1381. }
  1382. return n;
  1383. }
  1384. @Override
  1385. public int read() throws IOException {
  1386. int n = read(skipBuffer, 0, 1);
  1387. return n == 1 ? skipBuffer[0] & 0xff : -1;
  1388. }
  1389. @Override
  1390. public int read(byte[] dst, int pos, int cnt) throws IOException {
  1391. try {
  1392. int n = 0;
  1393. while (n < cnt) {
  1394. int r = inf.inflate(dst, pos + n, cnt - n);
  1395. if (r == 0) {
  1396. if (inf.finished())
  1397. break;
  1398. if (inf.needsInput()) {
  1399. onObjectData(src, buf, p, bAvail);
  1400. use(bAvail);
  1401. p = fill(src, 1);
  1402. inf.setInput(buf, p, bAvail);
  1403. } else {
  1404. throw new CorruptObjectException(
  1405. MessageFormat
  1406. .format(
  1407. JGitText.get().packfileCorruptionDetected,
  1408. JGitText.get().unknownZlibError));
  1409. }
  1410. } else {
  1411. n += r;
  1412. }
  1413. }
  1414. actualSize += n;
  1415. return 0 < n ? n : -1;
  1416. } catch (DataFormatException dfe) {
  1417. throw new CorruptObjectException(MessageFormat.format(JGitText
  1418. .get().packfileCorruptionDetected, dfe.getMessage()));
  1419. }
  1420. }
  1421. @Override
  1422. public void close() throws IOException {
  1423. // We need to read here to enter the loop above and pump the
  1424. // trailing checksum into the Inflater. It should return -1 as the
  1425. // caller was supposed to consume all content.
  1426. //
  1427. if (read(skipBuffer) != -1 || actualSize != expectedSize) {
  1428. throw new CorruptObjectException(MessageFormat.format(JGitText
  1429. .get().packfileCorruptionDetected,
  1430. JGitText.get().wrongDecompressedLength));
  1431. }
  1432. int used = bAvail - inf.getRemaining();
  1433. if (0 < used) {
  1434. onObjectData(src, buf, p, used);
  1435. use(used);
  1436. }
  1437. inf.reset();
  1438. }
  1439. }
  1440. }