jgit/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java

/*
 * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *   names of its contributors may be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.eclipse.jgit.transport;

import static org.eclipse.jgit.lib.RefDatabase.ALL;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.eclipse.jgit.errors.CompoundException;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.file.ObjectDirectory;
import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.file.PackLock;
import org.eclipse.jgit.internal.storage.file.UnpackedObject;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.MutableObjectId;
import org.eclipse.jgit.lib.ObjectChecker;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.DateRevQueue;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevFlag;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevTag;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.FileUtils;

/**
 * Generic fetch support for dumb transport protocols.
 * <p>
 * Since there are no Git-specific smarts on the remote side of the connection
 * the client side must determine which objects it needs to copy in order to
 * completely fetch the requested refs and their history. The generic walk
 * support in this class parses each individual object (once it has been copied
 * to the local repository) and examines the list of objects that must also be
 * copied to create a complete history. Objects which are already available
 * locally are retained (and not copied), saving bandwidth for incremental
 * fetches. Pack files are copied from the remote repository only as a last
 * resort, as the entire pack must be copied locally in order to access any
 * single object.
 * <p>
 * This fetch connection does not actually perform the object data transfer.
 * Instead it delegates the transfer to a {@link WalkRemoteObjectDatabase},
 * which knows how to read individual files from the remote repository and
 * supply the data as a standard Java InputStream.
 *
 * @see WalkRemoteObjectDatabase
 */
class WalkFetchConnection extends BaseFetchConnection {
	/** The repository this transport fetches into, or pushes out of. */
	final Repository local;

	/** If not null the validator for received objects. */
	final ObjectChecker objCheck;

	/**
	 * List of all remote repositories we may need to get objects out of.
	 * <p>
	 * The first repository in the list is the one we were asked to fetch from;
	 * the remaining repositories point to the alternate locations we can fetch
	 * objects through.
	 */
	private final List<WalkRemoteObjectDatabase> remotes;

	/** Most recently used item in {@link #remotes}. */
	private int lastRemoteIdx;

	private final RevWalk revWalk;

	private final TreeWalk treeWalk;

	/** Objects whose direct dependents we know we have (or will have). */
	private final RevFlag COMPLETE;

	/** Objects that have already entered {@link #workQueue}. */
	private final RevFlag IN_WORK_QUEUE;

	/** Commits that have already entered {@link #localCommitQueue}. */
	private final RevFlag LOCALLY_SEEN;

	/** Commits already reachable from all local refs. */
	private final DateRevQueue localCommitQueue;

	/** Objects we need to copy from the remote repository. */
	private LinkedList<ObjectId> workQueue;

	/** Databases we have not yet obtained the list of packs from. */
	private final LinkedList<WalkRemoteObjectDatabase> noPacksYet;

	/** Databases we have not yet obtained the alternates from. */
	private final LinkedList<WalkRemoteObjectDatabase> noAlternatesYet;

	/** Packs we have discovered, but have not yet fetched locally. */
	private final LinkedList<RemotePack> unfetchedPacks;

	/**
	 * Packs whose indexes we have looked at in {@link #unfetchedPacks}.
	 * <p>
	 * We try to avoid getting duplicate copies of the same pack through
	 * multiple alternates by only looking at packs whose names are not yet in
	 * this collection.
	 */
	private final Set<String> packsConsidered;

	private final MutableObjectId idBuffer = new MutableObjectId();

	/**
	 * Errors received while trying to obtain an object.
	 * <p>
	 * If the fetch winds up failing because we cannot locate a specific object
	 * then we need to report all errors related to that object back to the
	 * caller as there may be cascading failures.
	 */
	private final HashMap<ObjectId, List<Throwable>> fetchErrors;

	String lockMessage;

	final List<PackLock> packLocks;

	/** Inserter to write objects onto {@link #local}. */
	final ObjectInserter inserter;

	/** Inserter to read objects from {@link #local}. */
	private final ObjectReader reader;

	WalkFetchConnection(final WalkTransport t, final WalkRemoteObjectDatabase w) {
		Transport wt = (Transport)t;
		local = wt.local;
		objCheck = wt.getObjectChecker();
		inserter = local.newObjectInserter();
		reader = inserter.newReader();

		remotes = new ArrayList<>();
		remotes.add(w);

		unfetchedPacks = new LinkedList<>();
		packsConsidered = new HashSet<>();

		noPacksYet = new LinkedList<>();
		noPacksYet.add(w);

		noAlternatesYet = new LinkedList<>();
		noAlternatesYet.add(w);

		fetchErrors = new HashMap<>();
		packLocks = new ArrayList<>(4);

		revWalk = new RevWalk(reader);
		revWalk.setRetainBody(false);
		treeWalk = new TreeWalk(reader);
		COMPLETE = revWalk.newFlag("COMPLETE"); //$NON-NLS-1$
		IN_WORK_QUEUE = revWalk.newFlag("IN_WORK_QUEUE"); //$NON-NLS-1$
		LOCALLY_SEEN = revWalk.newFlag("LOCALLY_SEEN"); //$NON-NLS-1$

		localCommitQueue = new DateRevQueue();
		workQueue = new LinkedList<>();
	}

	@Override
	public boolean didFetchTestConnectivity() {
		return true;
	}

	@Override
	protected void doFetch(final ProgressMonitor monitor,
			final Collection<Ref> want, final Set<ObjectId> have)
			throws TransportException {
		markLocalRefsComplete(have);
		queueWants(want);

		while (!monitor.isCancelled() && !workQueue.isEmpty()) {
			final ObjectId id = workQueue.removeFirst();
			if (!(id instanceof RevObject) || !((RevObject) id).has(COMPLETE))
				downloadObject(monitor, id);
			process(id);
		}

		try {
			inserter.flush();
		} catch (IOException e) {
			throw new TransportException(e.getMessage(), e);
		}
	}

	@Override
	public Collection<PackLock> getPackLocks() {
		return packLocks;
	}

	@Override
	public void setPackLockMessage(final String message) {
		lockMessage = message;
	}

	@Override
	public void close() {
		inserter.close();
		reader.close();
		for (final RemotePack p : unfetchedPacks) {
			if (p.tmpIdx != null)
				p.tmpIdx.delete();
		}
		for (final WalkRemoteObjectDatabase r : remotes)
			r.close();
	}

	private void queueWants(final Collection<Ref> want)
			throws TransportException {
		final HashSet<ObjectId> inWorkQueue = new HashSet<>();
		for (final Ref r : want) {
			final ObjectId id = r.getObjectId();
			if (id == null) {
				throw new NullPointerException(MessageFormat.format(
						JGitText.get().transportProvidedRefWithNoObjectId, r.getName()));
			}
			try {
				final RevObject obj = revWalk.parseAny(id);
				if (obj.has(COMPLETE))
					continue;
				if (inWorkQueue.add(id)) {
					obj.add(IN_WORK_QUEUE);
					workQueue.add(obj);
				}
			} catch (MissingObjectException e) {
				if (inWorkQueue.add(id))
					workQueue.add(id);
			} catch (IOException e) {
				throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
			}
		}
	}

	private void process(final ObjectId id) throws TransportException {
		final RevObject obj;
		try {
			if (id instanceof RevObject) {
				obj = (RevObject) id;
				if (obj.has(COMPLETE))
					return;
				revWalk.parseHeaders(obj);
			} else {
				obj = revWalk.parseAny(id);
				if (obj.has(COMPLETE))
					return;
			}
		} catch (IOException e) {
			throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
		}

		switch (obj.getType()) {
		case Constants.OBJ_BLOB:
			processBlob(obj);
			break;
		case Constants.OBJ_TREE:
			processTree(obj);
			break;
		case Constants.OBJ_COMMIT:
			processCommit(obj);
			break;
		case Constants.OBJ_TAG:
			processTag(obj);
			break;
		default:
			throw new TransportException(MessageFormat.format(JGitText.get().unknownObjectType, id.name()));
		}

		// If we had any prior errors fetching this object they are
		// now resolved, as the object was parsed successfully.
		//
		fetchErrors.remove(id);
	}

	private void processBlob(final RevObject obj) throws TransportException {
		try {
			if (reader.has(obj, Constants.OBJ_BLOB))
				obj.add(COMPLETE);
			else
				throw new TransportException(MessageFormat.format(JGitText
						.get().cannotReadBlob, obj.name()),
						new MissingObjectException(obj, Constants.TYPE_BLOB));
		} catch (IOException error) {
			throw new TransportException(MessageFormat.format(
					JGitText.get().cannotReadBlob, obj.name()), error);
		}
	}

	private void processTree(final RevObject obj) throws TransportException {
		try {
			treeWalk.reset(obj);
			while (treeWalk.next()) {
				final FileMode mode = treeWalk.getFileMode(0);
				final int sType = mode.getObjectType();

				switch (sType) {
				case Constants.OBJ_BLOB:
				case Constants.OBJ_TREE:
					treeWalk.getObjectId(idBuffer, 0);
					needs(revWalk.lookupAny(idBuffer, sType));
					continue;

				default:
					if (FileMode.GITLINK.equals(mode))
						continue;
					treeWalk.getObjectId(idBuffer, 0);
					throw new CorruptObjectException(MessageFormat.format(JGitText.get().invalidModeFor
							, mode, idBuffer.name(), treeWalk.getPathString(), obj.getId().name()));
				}
			}
		} catch (IOException ioe) {
			throw new TransportException(MessageFormat.format(JGitText.get().cannotReadTree, obj.name()), ioe);
		}
		obj.add(COMPLETE);
	}

	private void processCommit(final RevObject obj) throws TransportException {
		final RevCommit commit = (RevCommit) obj;
		markLocalCommitsComplete(commit.getCommitTime());
		needs(commit.getTree());
		for (final RevCommit p : commit.getParents())
			needs(p);
		obj.add(COMPLETE);
	}

	private void processTag(final RevObject obj) {
		final RevTag tag = (RevTag) obj;
		needs(tag.getObject());
		obj.add(COMPLETE);
	}

	private void needs(final RevObject obj) {
		if (obj.has(COMPLETE))
			return;
		if (!obj.has(IN_WORK_QUEUE)) {
			obj.add(IN_WORK_QUEUE);
			workQueue.add(obj);
		}
	}

	private void downloadObject(final ProgressMonitor pm, final AnyObjectId id)
			throws TransportException {
		if (alreadyHave(id))
			return;

		for (;;) {
			// Try a pack file we know about, but don't have yet. Odds are
			// that if it has this object, it has others related to it so
			// getting the pack is a good bet.
			//
			if (downloadPackedObject(pm, id))
				return;

			// Search for a loose object over all alternates, starting
			// from the one we last successfully located an object through.
			//
			final String idStr = id.name();
			final String subdir = idStr.substring(0, 2);
			final String file = idStr.substring(2);
			final String looseName = subdir + "/" + file; //$NON-NLS-1$

			for (int i = lastRemoteIdx; i < remotes.size(); i++) {
				if (downloadLooseObject(id, looseName, remotes.get(i))) {
					lastRemoteIdx = i;
					return;
				}
			}
			for (int i = 0; i < lastRemoteIdx; i++) {
				if (downloadLooseObject(id, looseName, remotes.get(i))) {
					lastRemoteIdx = i;
					return;
				}
			}

			// Try to obtain more pack information and search those.
			//
			while (!noPacksYet.isEmpty()) {
				final WalkRemoteObjectDatabase wrr = noPacksYet.removeFirst();
				final Collection<String> packNameList;
				try {
					pm.beginTask(JGitText.get().listingPacks,
							ProgressMonitor.UNKNOWN);
					packNameList = wrr.getPackNames();
				} catch (IOException e) {
					// Try another repository.
					//
					recordError(id, e);
					continue;
				} finally {
					pm.endTask();
				}

				if (packNameList == null || packNameList.isEmpty())
					continue;
				for (final String packName : packNameList) {
					if (packsConsidered.add(packName))
						unfetchedPacks.add(new RemotePack(wrr, packName));
				}
				if (downloadPackedObject(pm, id))
					return;
			}

			// Try to expand the first alternate we haven't expanded yet.
			//
			Collection<WalkRemoteObjectDatabase> al = expandOneAlternate(id, pm);
			if (al != null && !al.isEmpty()) {
				for (final WalkRemoteObjectDatabase alt : al) {
					remotes.add(alt);
					noPacksYet.add(alt);
					noAlternatesYet.add(alt);
				}
				continue;
			}

			// We could not obtain the object. There may be reasons why.
			//
			List<Throwable> failures = fetchErrors.get(id);
			final TransportException te;

			te = new TransportException(MessageFormat.format(JGitText.get().cannotGet, id.name()));
			if (failures != null && !failures.isEmpty()) {
				if (failures.size() == 1)
					te.initCause(failures.get(0));
				else
					te.initCause(new CompoundException(failures));
			}
			throw te;
		}
	}

	private boolean alreadyHave(final AnyObjectId id) throws TransportException {
		try {
			return reader.has(id);
		} catch (IOException error) {
			throw new TransportException(MessageFormat.format(
					JGitText.get().cannotReadObject, id.name()), error);
		}
	}

	private boolean downloadPackedObject(final ProgressMonitor monitor,
			final AnyObjectId id) throws TransportException {
		// Search for the object in a remote pack whose index we have,
		// but whose pack we do not yet have.
		//
		final Iterator<RemotePack> packItr = unfetchedPacks.iterator();
		while (packItr.hasNext() && !monitor.isCancelled()) {
			final RemotePack pack = packItr.next();
			try {
				pack.openIndex(monitor);
			} catch (IOException err) {
				// If the index won't open its either not found or
				// its a format we don't recognize. In either case
				// we may still be able to obtain the object from
				// another source, so don't consider it a failure.
				//
				recordError(id, err);
				packItr.remove();
				continue;
			}

			if (monitor.isCancelled()) {
				// If we were cancelled while the index was opening
				// the open may have aborted. We can't search an
				// unopen index.
				//
				return false;
			}

			if (!pack.index.hasObject(id)) {
				// Not in this pack? Try another.
				//
				continue;
			}

			// It should be in the associated pack. Download that
			// and attach it to the local repository so we can use
			// all of the contained objects.
			//
			try {
				pack.downloadPack(monitor);
			} catch (IOException err) {
				// If the pack failed to download, index correctly,
				// or open in the local repository we may still be
				// able to obtain this object from another pack or
				// an alternate.
				//
				recordError(id, err);
				continue;
			} finally {
				// If the pack was good its in the local repository
				// and Repository.hasObject(id) will succeed in the
				// future, so we do not need this data anymore. If
				// it failed the index and pack are unusable and we
				// shouldn't consult them again.
				//
				try {
					if (pack.tmpIdx != null)
						FileUtils.delete(pack.tmpIdx);
				} catch (IOException e) {
					throw new TransportException(e.getMessage(), e);
				}
				packItr.remove();
			}

			if (!alreadyHave(id)) {
				// What the hell? This pack claimed to have
				// the object, but after indexing we didn't
				// actually find it in the pack.
				//
				recordError(id, new FileNotFoundException(MessageFormat.format(
						JGitText.get().objectNotFoundIn, id.name(), pack.packName)));
				continue;
			}

			// Complete any other objects that we can.
			//
			final Iterator<ObjectId> pending = swapFetchQueue();
			while (pending.hasNext()) {
				final ObjectId p = pending.next();
				if (pack.index.hasObject(p)) {
					pending.remove();
					process(p);
				} else {
					workQueue.add(p);
				}
			}
			return true;

		}
		return false;
	}

	private Iterator<ObjectId> swapFetchQueue() {
		final Iterator<ObjectId> r = workQueue.iterator();
		workQueue = new LinkedList<>();
		return r;
	}

	private boolean downloadLooseObject(final AnyObjectId id,
			final String looseName, final WalkRemoteObjectDatabase remote)
			throws TransportException {
		try {
			final byte[] compressed = remote.open(looseName).toArray();
			verifyAndInsertLooseObject(id, compressed);
			return true;
		} catch (FileNotFoundException e) {
			// Not available in a loose format from this alternate?
			// Try another strategy to get the object.
			//
			recordError(id, e);
			return false;
		} catch (IOException e) {
			throw new TransportException(MessageFormat.format(JGitText.get().cannotDownload, id.name()), e);
		}
	}

	private void verifyAndInsertLooseObject(final AnyObjectId id,
			final byte[] compressed) throws IOException {
		final ObjectLoader uol;
		try {
			uol = UnpackedObject.parse(compressed, id);
		} catch (CorruptObjectException parsingError) {
			// Some HTTP servers send back a "200 OK" status with an HTML
			// page that explains the requested file could not be found.
			// These servers are most certainly misconfigured, but many
			// of them exist in the world, and many of those are hosting
			// Git repositories.
			//
			// Since an HTML page is unlikely to hash to one of our loose
			// objects we treat this condition as a FileNotFoundException
			// and attempt to recover by getting the object from another
			// source.
			//
			final FileNotFoundException e;
			e = new FileNotFoundException(id.name());
			e.initCause(parsingError);
			throw e;
		}

		final int type = uol.getType();
		final byte[] raw = uol.getCachedBytes();
		if (objCheck != null) {
			try {
				objCheck.check(id, type, raw);
			} catch (CorruptObjectException e) {
				throw new TransportException(MessageFormat.format(
						JGitText.get().transportExceptionInvalid,
						Constants.typeString(type), id.name(), e.getMessage()));
			}
		}

		ObjectId act = inserter.insert(type, raw);
		if (!AnyObjectId.equals(id, act)) {
			throw new TransportException(MessageFormat.format(
					JGitText.get().incorrectHashFor, id.name(), act.name(),
					Constants.typeString(type),
					Integer.valueOf(compressed.length)));
		}
	}

	private Collection<WalkRemoteObjectDatabase> expandOneAlternate(
			final AnyObjectId id, final ProgressMonitor pm) {
		while (!noAlternatesYet.isEmpty()) {
			final WalkRemoteObjectDatabase wrr = noAlternatesYet.removeFirst();
			try {
				pm.beginTask(JGitText.get().listingAlternates, ProgressMonitor.UNKNOWN);
				Collection<WalkRemoteObjectDatabase> altList = wrr
						.getAlternates();
				if (altList != null && !altList.isEmpty())
					return altList;
			} catch (IOException e) {
				// Try another repository.
				//
				recordError(id, e);
			} finally {
				pm.endTask();
			}
		}
		return null;
	}

	private void markLocalRefsComplete(final Set<ObjectId> have) throws TransportException {
		Map<String, Ref> refs;
		try {
			refs = local.getRefDatabase().getRefs(ALL);
		} catch (IOException e) {
			throw new TransportException(e.getMessage(), e);
		}
		for (final Ref r : refs.values()) {
			try {
				markLocalObjComplete(revWalk.parseAny(r.getObjectId()));
			} catch (IOException readError) {
				throw new TransportException(MessageFormat.format(JGitText.get().localRefIsMissingObjects, r.getName()), readError);
			}
		}
		for (final ObjectId id : have) {
			try {
				markLocalObjComplete(revWalk.parseAny(id));
			} catch (IOException readError) {
				throw new TransportException(MessageFormat.format(JGitText.get().transportExceptionMissingAssumed, id.name()), readError);
			}
		}
	}

	private void markLocalObjComplete(RevObject obj) throws IOException {
		while (obj.getType() == Constants.OBJ_TAG) {
			obj.add(COMPLETE);
			obj = ((RevTag) obj).getObject();
			revWalk.parseHeaders(obj);
		}

		switch (obj.getType()) {
		case Constants.OBJ_BLOB:
			obj.add(COMPLETE);
			break;
		case Constants.OBJ_COMMIT:
			pushLocalCommit((RevCommit) obj);
			break;
		case Constants.OBJ_TREE:
			markTreeComplete((RevTree) obj);
			break;
		}
	}

	private void markLocalCommitsComplete(final int until)
			throws TransportException {
		try {
			for (;;) {
				final RevCommit c = localCommitQueue.peek();
				if (c == null || c.getCommitTime() < until)
					return;
				localCommitQueue.next();

				markTreeComplete(c.getTree());
				for (final RevCommit p : c.getParents())
					pushLocalCommit(p);
			}
		} catch (IOException err) {
			throw new TransportException(JGitText.get().localObjectsIncomplete, err);
		}
	}

	private void pushLocalCommit(final RevCommit p)
			throws MissingObjectException, IOException {
		if (p.has(LOCALLY_SEEN))
			return;
		revWalk.parseHeaders(p);
		p.add(LOCALLY_SEEN);
		p.add(COMPLETE);
		p.carry(COMPLETE);
		localCommitQueue.add(p);
	}

	private void markTreeComplete(final RevTree tree) throws IOException {
		if (tree.has(COMPLETE))
			return;
		tree.add(COMPLETE);
		treeWalk.reset(tree);
		while (treeWalk.next()) {
			final FileMode mode = treeWalk.getFileMode(0);
			final int sType = mode.getObjectType();

			switch (sType) {
			case Constants.OBJ_BLOB:
				treeWalk.getObjectId(idBuffer, 0);
				revWalk.lookupAny(idBuffer, sType).add(COMPLETE);
				continue;

			case Constants.OBJ_TREE: {
				treeWalk.getObjectId(idBuffer, 0);
				final RevObject o = revWalk.lookupAny(idBuffer, sType);
				if (!o.has(COMPLETE)) {
					o.add(COMPLETE);
					treeWalk.enterSubtree();
				}
				continue;
			}
			default:
				if (FileMode.GITLINK.equals(mode))
					continue;
				treeWalk.getObjectId(idBuffer, 0);
				throw new CorruptObjectException(MessageFormat.format(JGitText.get().corruptObjectInvalidMode3
						, mode, idBuffer.name(), treeWalk.getPathString(), tree.name()));
			}
		}
	}

	private void recordError(final AnyObjectId id, final Throwable what) {
		final ObjectId objId = id.copy();
		List<Throwable> errors = fetchErrors.get(objId);
		if (errors == null) {
			errors = new ArrayList<>(2);
			fetchErrors.put(objId, errors);
		}
		errors.add(what);
	}

	private class RemotePack {
		final WalkRemoteObjectDatabase connection;

		final String packName;

		final String idxName;

		File tmpIdx;

		PackIndex index;

		RemotePack(final WalkRemoteObjectDatabase c, final String pn) {
			connection = c;
			packName = pn;
			idxName = packName.substring(0, packName.length() - 5) + ".idx"; //$NON-NLS-1$

			String tn = idxName;
			if (tn.startsWith("pack-")) //$NON-NLS-1$
				tn = tn.substring(5);
			if (tn.endsWith(".idx")) //$NON-NLS-1$
				tn = tn.substring(0, tn.length() - 4);

			if (local.getObjectDatabase() instanceof ObjectDirectory) {
				tmpIdx = new File(((ObjectDirectory) local.getObjectDatabase())
								.getDirectory(),
						"walk-" + tn + ".walkidx"); //$NON-NLS-1$ //$NON-NLS-2$
			}
		}

		void openIndex(final ProgressMonitor pm) throws IOException {
			if (index != null)
				return;
			if (tmpIdx == null)
				tmpIdx = File.createTempFile("jgit-walk-", ".idx"); //$NON-NLS-1$ //$NON-NLS-2$
			else if (tmpIdx.isFile()) {
				try {
					index = PackIndex.open(tmpIdx);
					return;
				} catch (FileNotFoundException err) {
					// Fall through and get the file.
				}
			}

			final WalkRemoteObjectDatabase.FileStream s;
			s = connection.open("pack/" + idxName); //$NON-NLS-1$
			pm.beginTask("Get " + idxName.substring(0, 12) + "..idx", //$NON-NLS-1$ //$NON-NLS-2$
					s.length < 0 ? ProgressMonitor.UNKNOWN
							: (int) (s.length / 1024));
			try {
				final FileOutputStream fos = new FileOutputStream(tmpIdx);
				try {
					final byte[] buf = new byte[2048];
					int cnt;
					while (!pm.isCancelled() && (cnt = s.in.read(buf)) >= 0) {
						fos.write(buf, 0, cnt);
						pm.update(cnt / 1024);
					}
				} finally {
					fos.close();
				}
			} catch (IOException err) {
				FileUtils.delete(tmpIdx);
				throw err;
			} finally {
				s.in.close();
			}
			pm.endTask();

			if (pm.isCancelled()) {
				FileUtils.delete(tmpIdx);
				return;
			}

			try {
				index = PackIndex.open(tmpIdx);
			} catch (IOException e) {
				FileUtils.delete(tmpIdx);
				throw e;
			}
		}

		void downloadPack(final ProgressMonitor monitor) throws IOException {
			String name = "pack/" + packName; //$NON-NLS-1$
			WalkRemoteObjectDatabase.FileStream s = connection.open(name);
			try {
				PackParser parser = inserter.newPackParser(s.in);
				parser.setAllowThin(false);
				parser.setObjectChecker(objCheck);
				parser.setLockMessage(lockMessage);
				PackLock lock = parser.parse(monitor);
				if (lock != null)
					packLocks.add(lock);
			} finally {
				s.in.close();
			}
		}
	}
}