Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.

DfsReader.java 22KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792
  1. /*
  2. * Copyright (C) 2008-2011, Google Inc.
  3. * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org>
  4. * and other copyright owners as documented in the project's IP log.
  5. *
  6. * This program and the accompanying materials are made available
  7. * under the terms of the Eclipse Distribution License v1.0 which
  8. * accompanies this distribution, is reproduced below, and is
  9. * available at http://www.eclipse.org/org/documents/edl-v10.php
  10. *
  11. * All rights reserved.
  12. *
  13. * Redistribution and use in source and binary forms, with or
  14. * without modification, are permitted provided that the following
  15. * conditions are met:
  16. *
  17. * - Redistributions of source code must retain the above copyright
  18. * notice, this list of conditions and the following disclaimer.
  19. *
  20. * - Redistributions in binary form must reproduce the above
  21. * copyright notice, this list of conditions and the following
  22. * disclaimer in the documentation and/or other materials provided
  23. * with the distribution.
  24. *
  25. * - Neither the name of the Eclipse Foundation, Inc. nor the
  26. * names of its contributors may be used to endorse or promote
  27. * products derived from this software without specific prior
  28. * written permission.
  29. *
  30. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  31. * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  32. * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  33. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  34. * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  35. * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  37. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  38. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  39. * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  40. * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  41. * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  42. * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  43. */
  44. package org.eclipse.jgit.internal.storage.dfs;
  45. import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
  46. import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
  47. import java.io.IOException;
  48. import java.util.ArrayList;
  49. import java.util.Arrays;
  50. import java.util.Collection;
  51. import java.util.Collections;
  52. import java.util.Comparator;
  53. import java.util.HashSet;
  54. import java.util.Iterator;
  55. import java.util.LinkedList;
  56. import java.util.List;
  57. import java.util.Set;
  58. import java.util.zip.DataFormatException;
  59. import java.util.zip.Inflater;
  60. import org.eclipse.jgit.errors.IncorrectObjectTypeException;
  61. import org.eclipse.jgit.errors.MissingObjectException;
  62. import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
  63. import org.eclipse.jgit.internal.JGitText;
  64. import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackList;
  65. import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
  66. import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl;
  67. import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
  68. import org.eclipse.jgit.internal.storage.file.PackIndex;
  69. import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
  70. import org.eclipse.jgit.internal.storage.pack.CachedPack;
  71. import org.eclipse.jgit.internal.storage.pack.ObjectReuseAsIs;
  72. import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
  73. import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
  74. import org.eclipse.jgit.internal.storage.pack.PackWriter;
  75. import org.eclipse.jgit.lib.AbbreviatedObjectId;
  76. import org.eclipse.jgit.lib.AnyObjectId;
  77. import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
  78. import org.eclipse.jgit.lib.AsyncObjectSizeQueue;
  79. import org.eclipse.jgit.lib.BitmapIndex;
  80. import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
  81. import org.eclipse.jgit.lib.InflaterCache;
  82. import org.eclipse.jgit.lib.ObjectId;
  83. import org.eclipse.jgit.lib.ObjectLoader;
  84. import org.eclipse.jgit.lib.ObjectReader;
  85. import org.eclipse.jgit.lib.ProgressMonitor;
  86. import org.eclipse.jgit.util.BlockList;
  87. /**
  88. * Reader to access repository content through.
  89. * <p>
  90. * See the base {@link ObjectReader} documentation for details. Notably, a
  91. * reader is not thread safe.
  92. */
  93. public class DfsReader extends ObjectReader implements ObjectReuseAsIs {
  94. private static final int MAX_RESOLVE_MATCHES = 256;
  95. /** Temporary buffer large enough for at least one raw object id. */
  96. final byte[] tempId = new byte[OBJECT_ID_LENGTH];
  97. /** Database this reader loads objects from. */
  98. final DfsObjDatabase db;
  99. final DfsReaderIoStats.Accumulator stats = new DfsReaderIoStats.Accumulator();
  100. private Inflater inf;
  101. private DfsBlock block;
  102. private DeltaBaseCache baseCache;
  103. private DfsPackFile last;
  104. private boolean avoidUnreachable;
  105. /**
  106. * Initialize a new DfsReader
  107. *
  108. * @param db
  109. * parent DfsObjDatabase.
  110. */
  111. protected DfsReader(DfsObjDatabase db) {
  112. this.db = db;
  113. this.streamFileThreshold = db.getReaderOptions().getStreamFileThreshold();
  114. }
  115. DfsReaderOptions getOptions() {
  116. return db.getReaderOptions();
  117. }
  118. DeltaBaseCache getDeltaBaseCache() {
  119. if (baseCache == null)
  120. baseCache = new DeltaBaseCache(this);
  121. return baseCache;
  122. }
  123. @Override
  124. public ObjectReader newReader() {
  125. return db.newReader();
  126. }
  127. @Override
  128. public void setAvoidUnreachableObjects(boolean avoid) {
  129. avoidUnreachable = avoid;
  130. }
  131. @Override
  132. public BitmapIndex getBitmapIndex() throws IOException {
  133. for (DfsPackFile pack : db.getPacks()) {
  134. PackBitmapIndex bitmapIndex = pack.getBitmapIndex(this);
  135. if (bitmapIndex != null)
  136. return new BitmapIndexImpl(bitmapIndex);
  137. }
  138. return null;
  139. }
  140. @Override
  141. public Collection<CachedPack> getCachedPacksAndUpdate(
  142. BitmapBuilder needBitmap) throws IOException {
  143. for (DfsPackFile pack : db.getPacks()) {
  144. PackBitmapIndex bitmapIndex = pack.getBitmapIndex(this);
  145. if (needBitmap.removeAllOrNone(bitmapIndex))
  146. return Collections.<CachedPack> singletonList(
  147. new DfsCachedPack(pack));
  148. }
  149. return Collections.emptyList();
  150. }
  151. @Override
  152. public Collection<ObjectId> resolve(AbbreviatedObjectId id)
  153. throws IOException {
  154. if (id.isComplete())
  155. return Collections.singleton(id.toObjectId());
  156. HashSet<ObjectId> matches = new HashSet<>(4);
  157. PackList packList = db.getPackList();
  158. resolveImpl(packList, id, matches);
  159. if (matches.size() < MAX_RESOLVE_MATCHES && packList.dirty()) {
  160. stats.scanPacks++;
  161. resolveImpl(db.scanPacks(packList), id, matches);
  162. }
  163. return matches;
  164. }
  165. private void resolveImpl(PackList packList, AbbreviatedObjectId id,
  166. HashSet<ObjectId> matches) throws IOException {
  167. for (DfsPackFile pack : packList.packs) {
  168. if (skipGarbagePack(pack)) {
  169. continue;
  170. }
  171. pack.resolve(this, matches, id, MAX_RESOLVE_MATCHES);
  172. if (matches.size() >= MAX_RESOLVE_MATCHES) {
  173. break;
  174. }
  175. }
  176. }
  177. @Override
  178. public boolean has(AnyObjectId objectId) throws IOException {
  179. if (last != null
  180. && !skipGarbagePack(last)
  181. && last.hasObject(this, objectId))
  182. return true;
  183. PackList packList = db.getPackList();
  184. if (hasImpl(packList, objectId)) {
  185. return true;
  186. } else if (packList.dirty()) {
  187. stats.scanPacks++;
  188. return hasImpl(db.scanPacks(packList), objectId);
  189. }
  190. return false;
  191. }
  192. private boolean hasImpl(PackList packList, AnyObjectId objectId)
  193. throws IOException {
  194. for (DfsPackFile pack : packList.packs) {
  195. if (pack == last || skipGarbagePack(pack))
  196. continue;
  197. if (pack.hasObject(this, objectId)) {
  198. last = pack;
  199. return true;
  200. }
  201. }
  202. return false;
  203. }
  204. @Override
  205. public ObjectLoader open(AnyObjectId objectId, int typeHint)
  206. throws MissingObjectException, IncorrectObjectTypeException,
  207. IOException {
  208. ObjectLoader ldr;
  209. if (last != null && !skipGarbagePack(last)) {
  210. ldr = last.get(this, objectId);
  211. if (ldr != null) {
  212. return checkType(ldr, objectId, typeHint);
  213. }
  214. }
  215. PackList packList = db.getPackList();
  216. ldr = openImpl(packList, objectId);
  217. if (ldr != null) {
  218. return checkType(ldr, objectId, typeHint);
  219. }
  220. if (packList.dirty()) {
  221. stats.scanPacks++;
  222. ldr = openImpl(db.scanPacks(packList), objectId);
  223. if (ldr != null) {
  224. return checkType(ldr, objectId, typeHint);
  225. }
  226. }
  227. if (typeHint == OBJ_ANY)
  228. throw new MissingObjectException(objectId.copy(),
  229. JGitText.get().unknownObjectType2);
  230. throw new MissingObjectException(objectId.copy(), typeHint);
  231. }
  232. private static ObjectLoader checkType(ObjectLoader ldr, AnyObjectId id,
  233. int typeHint) throws IncorrectObjectTypeException {
  234. if (typeHint != OBJ_ANY && ldr.getType() != typeHint) {
  235. throw new IncorrectObjectTypeException(id.copy(), typeHint);
  236. }
  237. return ldr;
  238. }
  239. private ObjectLoader openImpl(PackList packList, AnyObjectId objectId)
  240. throws IOException {
  241. for (DfsPackFile pack : packList.packs) {
  242. if (pack == last || skipGarbagePack(pack)) {
  243. continue;
  244. }
  245. ObjectLoader ldr = pack.get(this, objectId);
  246. if (ldr != null) {
  247. last = pack;
  248. return ldr;
  249. }
  250. }
  251. return null;
  252. }
  253. @Override
  254. public Set<ObjectId> getShallowCommits() {
  255. return Collections.emptySet();
  256. }
  257. private static final Comparator<FoundObject<?>> FOUND_OBJECT_SORT = new Comparator<FoundObject<?>>() {
  258. @Override
  259. public int compare(FoundObject<?> a, FoundObject<?> b) {
  260. int cmp = a.packIndex - b.packIndex;
  261. if (cmp == 0)
  262. cmp = Long.signum(a.offset - b.offset);
  263. return cmp;
  264. }
  265. };
  266. private static class FoundObject<T extends ObjectId> {
  267. final T id;
  268. final DfsPackFile pack;
  269. final long offset;
  270. final int packIndex;
  271. FoundObject(T objectId, int packIdx, DfsPackFile pack, long offset) {
  272. this.id = objectId;
  273. this.pack = pack;
  274. this.offset = offset;
  275. this.packIndex = packIdx;
  276. }
  277. FoundObject(T objectId) {
  278. this.id = objectId;
  279. this.pack = null;
  280. this.offset = 0;
  281. this.packIndex = 0;
  282. }
  283. }
  284. private <T extends ObjectId> Iterable<FoundObject<T>> findAll(
  285. Iterable<T> objectIds) throws IOException {
  286. Collection<T> pending = new LinkedList<>();
  287. for (T id : objectIds) {
  288. pending.add(id);
  289. }
  290. PackList packList = db.getPackList();
  291. List<FoundObject<T>> r = new ArrayList<>();
  292. findAllImpl(packList, pending, r);
  293. if (!pending.isEmpty() && packList.dirty()) {
  294. stats.scanPacks++;
  295. findAllImpl(db.scanPacks(packList), pending, r);
  296. }
  297. for (T t : pending) {
  298. r.add(new FoundObject<>(t));
  299. }
  300. Collections.sort(r, FOUND_OBJECT_SORT);
  301. return r;
  302. }
  303. private <T extends ObjectId> void findAllImpl(PackList packList,
  304. Collection<T> pending, List<FoundObject<T>> r) {
  305. DfsPackFile[] packs = packList.packs;
  306. if (packs.length == 0) {
  307. return;
  308. }
  309. int lastIdx = 0;
  310. DfsPackFile lastPack = packs[lastIdx];
  311. OBJECT_SCAN: for (Iterator<T> it = pending.iterator(); it.hasNext();) {
  312. T t = it.next();
  313. if (!skipGarbagePack(lastPack)) {
  314. try {
  315. long p = lastPack.findOffset(this, t);
  316. if (0 < p) {
  317. r.add(new FoundObject<>(t, lastIdx, lastPack, p));
  318. it.remove();
  319. continue;
  320. }
  321. } catch (IOException e) {
  322. // Fall though and try to examine other packs.
  323. }
  324. }
  325. for (int i = 0; i < packs.length; i++) {
  326. if (i == lastIdx)
  327. continue;
  328. DfsPackFile pack = packs[i];
  329. if (skipGarbagePack(pack))
  330. continue;
  331. try {
  332. long p = pack.findOffset(this, t);
  333. if (0 < p) {
  334. r.add(new FoundObject<>(t, i, pack, p));
  335. it.remove();
  336. lastIdx = i;
  337. lastPack = pack;
  338. continue OBJECT_SCAN;
  339. }
  340. } catch (IOException e) {
  341. // Examine other packs.
  342. }
  343. }
  344. }
  345. last = lastPack;
  346. }
  347. private boolean skipGarbagePack(DfsPackFile pack) {
  348. return avoidUnreachable && pack.isGarbage();
  349. }
  350. @Override
  351. public <T extends ObjectId> AsyncObjectLoaderQueue<T> open(
  352. Iterable<T> objectIds, final boolean reportMissing) {
  353. Iterable<FoundObject<T>> order;
  354. IOException error = null;
  355. try {
  356. order = findAll(objectIds);
  357. } catch (IOException e) {
  358. order = Collections.emptyList();
  359. error = e;
  360. }
  361. final Iterator<FoundObject<T>> idItr = order.iterator();
  362. final IOException findAllError = error;
  363. return new AsyncObjectLoaderQueue<T>() {
  364. private FoundObject<T> cur;
  365. @Override
  366. public boolean next() throws MissingObjectException, IOException {
  367. if (idItr.hasNext()) {
  368. cur = idItr.next();
  369. return true;
  370. } else if (findAllError != null) {
  371. throw findAllError;
  372. } else {
  373. return false;
  374. }
  375. }
  376. @Override
  377. public T getCurrent() {
  378. return cur.id;
  379. }
  380. @Override
  381. public ObjectId getObjectId() {
  382. return cur.id;
  383. }
  384. @Override
  385. public ObjectLoader open() throws IOException {
  386. if (cur.pack == null)
  387. throw new MissingObjectException(cur.id,
  388. JGitText.get().unknownObjectType2);
  389. return cur.pack.load(DfsReader.this, cur.offset);
  390. }
  391. @Override
  392. public boolean cancel(boolean mayInterruptIfRunning) {
  393. return true;
  394. }
  395. @Override
  396. public void release() {
  397. // Nothing to clean up.
  398. }
  399. };
  400. }
  401. @Override
  402. public <T extends ObjectId> AsyncObjectSizeQueue<T> getObjectSize(
  403. Iterable<T> objectIds, final boolean reportMissing) {
  404. Iterable<FoundObject<T>> order;
  405. IOException error = null;
  406. try {
  407. order = findAll(objectIds);
  408. } catch (IOException e) {
  409. order = Collections.emptyList();
  410. error = e;
  411. }
  412. final Iterator<FoundObject<T>> idItr = order.iterator();
  413. final IOException findAllError = error;
  414. return new AsyncObjectSizeQueue<T>() {
  415. private FoundObject<T> cur;
  416. private long sz;
  417. @Override
  418. public boolean next() throws MissingObjectException, IOException {
  419. if (idItr.hasNext()) {
  420. cur = idItr.next();
  421. if (cur.pack == null)
  422. throw new MissingObjectException(cur.id,
  423. JGitText.get().unknownObjectType2);
  424. sz = cur.pack.getObjectSize(DfsReader.this, cur.offset);
  425. return true;
  426. } else if (findAllError != null) {
  427. throw findAllError;
  428. } else {
  429. return false;
  430. }
  431. }
  432. @Override
  433. public T getCurrent() {
  434. return cur.id;
  435. }
  436. @Override
  437. public ObjectId getObjectId() {
  438. return cur.id;
  439. }
  440. @Override
  441. public long getSize() {
  442. return sz;
  443. }
  444. @Override
  445. public boolean cancel(boolean mayInterruptIfRunning) {
  446. return true;
  447. }
  448. @Override
  449. public void release() {
  450. // Nothing to clean up.
  451. }
  452. };
  453. }
  454. @Override
  455. public long getObjectSize(AnyObjectId objectId, int typeHint)
  456. throws MissingObjectException, IncorrectObjectTypeException,
  457. IOException {
  458. if (last != null && !skipGarbagePack(last)) {
  459. long sz = last.getObjectSize(this, objectId);
  460. if (0 <= sz) {
  461. return sz;
  462. }
  463. }
  464. PackList packList = db.getPackList();
  465. long sz = getObjectSizeImpl(packList, objectId);
  466. if (0 <= sz) {
  467. return sz;
  468. }
  469. if (packList.dirty()) {
  470. sz = getObjectSizeImpl(packList, objectId);
  471. if (0 <= sz) {
  472. return sz;
  473. }
  474. }
  475. if (typeHint == OBJ_ANY) {
  476. throw new MissingObjectException(objectId.copy(),
  477. JGitText.get().unknownObjectType2);
  478. }
  479. throw new MissingObjectException(objectId.copy(), typeHint);
  480. }
  481. private long getObjectSizeImpl(PackList packList, AnyObjectId objectId)
  482. throws IOException {
  483. for (DfsPackFile pack : packList.packs) {
  484. if (pack == last || skipGarbagePack(pack)) {
  485. continue;
  486. }
  487. long sz = pack.getObjectSize(this, objectId);
  488. if (0 <= sz) {
  489. last = pack;
  490. return sz;
  491. }
  492. }
  493. return -1;
  494. }
  495. @Override
  496. public DfsObjectToPack newObjectToPack(AnyObjectId objectId, int type) {
  497. return new DfsObjectToPack(objectId, type);
  498. }
  499. private static final Comparator<DfsObjectToPack> OFFSET_SORT = new Comparator<DfsObjectToPack>() {
  500. @Override
  501. public int compare(DfsObjectToPack a, DfsObjectToPack b) {
  502. return Long.signum(a.getOffset() - b.getOffset());
  503. }
  504. };
  505. @Override
  506. public void selectObjectRepresentation(PackWriter packer,
  507. ProgressMonitor monitor, Iterable<ObjectToPack> objects)
  508. throws IOException, MissingObjectException {
  509. // Don't check dirty bit on PackList; assume ObjectToPacks all came from the
  510. // current list.
  511. for (DfsPackFile pack : sortPacksForSelectRepresentation()) {
  512. List<DfsObjectToPack> tmp = findAllFromPack(pack, objects);
  513. if (tmp.isEmpty())
  514. continue;
  515. Collections.sort(tmp, OFFSET_SORT);
  516. PackReverseIndex rev = pack.getReverseIdx(this);
  517. DfsObjectRepresentation rep = new DfsObjectRepresentation(pack);
  518. for (DfsObjectToPack otp : tmp) {
  519. pack.representation(rep, otp.getOffset(), this, rev);
  520. otp.setOffset(0);
  521. packer.select(otp, rep);
  522. if (!otp.isFound()) {
  523. otp.setFound();
  524. monitor.update(1);
  525. }
  526. }
  527. }
  528. }
  529. private static final Comparator<DfsPackFile> PACK_SORT_FOR_REUSE = new Comparator<DfsPackFile>() {
  530. @Override
  531. public int compare(DfsPackFile af, DfsPackFile bf) {
  532. DfsPackDescription ad = af.getPackDescription();
  533. DfsPackDescription bd = bf.getPackDescription();
  534. PackSource as = ad.getPackSource();
  535. PackSource bs = bd.getPackSource();
  536. if (as != null && as == bs && DfsPackDescription.isGC(as)) {
  537. // Push smaller GC files last; these likely have higher quality
  538. // delta compression and the contained representation should be
  539. // favored over other files.
  540. return Long.signum(bd.getFileSize(PACK) - ad.getFileSize(PACK));
  541. }
  542. // DfsPackDescription.compareTo already did a reasonable sort.
  543. // Rely on Arrays.sort being stable, leaving equal elements.
  544. return 0;
  545. }
  546. };
  547. private DfsPackFile[] sortPacksForSelectRepresentation()
  548. throws IOException {
  549. DfsPackFile[] packs = db.getPacks();
  550. DfsPackFile[] sorted = new DfsPackFile[packs.length];
  551. System.arraycopy(packs, 0, sorted, 0, packs.length);
  552. Arrays.sort(sorted, PACK_SORT_FOR_REUSE);
  553. return sorted;
  554. }
  555. private List<DfsObjectToPack> findAllFromPack(DfsPackFile pack,
  556. Iterable<ObjectToPack> objects) throws IOException {
  557. List<DfsObjectToPack> tmp = new BlockList<>();
  558. PackIndex idx = pack.getPackIndex(this);
  559. for (ObjectToPack otp : objects) {
  560. long p = idx.findOffset(otp);
  561. if (0 < p && !pack.isCorrupt(p)) {
  562. otp.setOffset(p);
  563. tmp.add((DfsObjectToPack) otp);
  564. }
  565. }
  566. return tmp;
  567. }
  568. @Override
  569. public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp,
  570. boolean validate) throws IOException,
  571. StoredObjectRepresentationNotAvailableException {
  572. DfsObjectToPack src = (DfsObjectToPack) otp;
  573. src.pack.copyAsIs(out, src, validate, this);
  574. }
  575. @Override
  576. public void writeObjects(PackOutputStream out, List<ObjectToPack> list)
  577. throws IOException {
  578. for (ObjectToPack otp : list)
  579. out.writeObject(otp);
  580. }
  581. @Override
  582. public void copyPackAsIs(PackOutputStream out, CachedPack pack)
  583. throws IOException {
  584. ((DfsCachedPack) pack).copyAsIs(out, this);
  585. }
  586. /**
  587. * Copy bytes from the window to a caller supplied buffer.
  588. *
  589. * @param pack
  590. * the file the desired window is stored within.
  591. * @param position
  592. * position within the file to read from.
  593. * @param dstbuf
  594. * destination buffer to copy into.
  595. * @param dstoff
  596. * offset within <code>dstbuf</code> to start copying into.
  597. * @param cnt
  598. * number of bytes to copy. This value may exceed the number of
  599. * bytes remaining in the window starting at offset
  600. * <code>pos</code>.
  601. * @return number of bytes actually copied; this may be less than
  602. * <code>cnt</code> if <code>cnt</code> exceeded the number of bytes
  603. * available.
  604. * @throws IOException
  605. * this cursor does not match the provider or id and the proper
  606. * window could not be acquired through the provider's cache.
  607. */
  608. int copy(DfsPackFile pack, long position, byte[] dstbuf, int dstoff, int cnt)
  609. throws IOException {
  610. if (cnt == 0)
  611. return 0;
  612. long length = pack.length;
  613. if (0 <= length && length <= position)
  614. return 0;
  615. int need = cnt;
  616. do {
  617. pin(pack, position);
  618. int r = block.copy(position, dstbuf, dstoff, need);
  619. position += r;
  620. dstoff += r;
  621. need -= r;
  622. if (length < 0)
  623. length = pack.length;
  624. } while (0 < need && position < length);
  625. return cnt - need;
  626. }
  627. /**
  628. * Inflate a region of the pack starting at {@code position}.
  629. *
  630. * @param pack
  631. * the file the desired window is stored within.
  632. * @param position
  633. * position within the file to read from.
  634. * @param dstbuf
  635. * destination buffer the inflater should output decompressed
  636. * data to. Must be large enough to store the entire stream,
  637. * unless headerOnly is true.
  638. * @param headerOnly
  639. * if true the caller wants only {@code dstbuf.length} bytes.
  640. * @return number of bytes inflated into <code>dstbuf</code>.
  641. * @throws IOException
  642. * this cursor does not match the provider or id and the proper
  643. * window could not be acquired through the provider's cache.
  644. * @throws DataFormatException
  645. * the inflater encountered an invalid chunk of data. Data
  646. * stream corruption is likely.
  647. */
  648. int inflate(DfsPackFile pack, long position, byte[] dstbuf,
  649. boolean headerOnly) throws IOException, DataFormatException {
  650. prepareInflater();
  651. pin(pack, position);
  652. position += block.setInput(position, inf);
  653. for (int dstoff = 0;;) {
  654. int n = inf.inflate(dstbuf, dstoff, dstbuf.length - dstoff);
  655. dstoff += n;
  656. if (inf.finished() || (headerOnly && dstoff == dstbuf.length)) {
  657. stats.inflatedBytes += dstoff;
  658. return dstoff;
  659. } else if (inf.needsInput()) {
  660. pin(pack, position);
  661. position += block.setInput(position, inf);
  662. } else if (n == 0)
  663. throw new DataFormatException();
  664. }
  665. }
  666. DfsBlock quickCopy(DfsPackFile p, long pos, long cnt)
  667. throws IOException {
  668. pin(p, pos);
  669. if (block.contains(p.key, pos + (cnt - 1)))
  670. return block;
  671. return null;
  672. }
  673. Inflater inflater() {
  674. prepareInflater();
  675. return inf;
  676. }
  677. private void prepareInflater() {
  678. if (inf == null)
  679. inf = InflaterCache.get();
  680. else
  681. inf.reset();
  682. }
  683. void pin(DfsPackFile pack, long position) throws IOException {
  684. DfsBlock b = block;
  685. if (b == null || !b.contains(pack.key, position)) {
  686. // If memory is low, we may need what is in our window field to
  687. // be cleaned up by the GC during the get for the next window.
  688. // So we always clear it, even though we are just going to set
  689. // it again.
  690. block = null;
  691. block = pack.getOrLoadBlock(position, this);
  692. }
  693. }
  694. void unpin() {
  695. block = null;
  696. }
  697. /** @return IO statistics accumulated by this reader. */
  698. public DfsReaderIoStats getIoStats() {
  699. return new DfsReaderIoStats(stats);
  700. }
  701. /** Release the current window cursor. */
  702. @Override
  703. public void close() {
  704. last = null;
  705. block = null;
  706. baseCache = null;
  707. try {
  708. InflaterCache.release(inf);
  709. } finally {
  710. inf = null;
  711. }
  712. }
  713. }