jgit/org.eclipse.jgit.storage.dht/src/org/eclipse/jgit/storage/dht/OpenQueue.java

/*
 * Copyright (C) 2011, Google Inc.
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *   names of its contributors may be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package org.eclipse.jgit.storage.dht;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;

import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;

/**
 * Locates objects in large batches, then opens them clustered by chunk.
 * <p>
 * To simplify the implementation this method performs lookups for the
 * {@link ObjectInfo} in large batches, clusters those by ChunkKey, and loads
 * the chunks with a {@link Prefetcher}.
 * <p>
 * The lookup queue is completely spun out during the first invocation of
 * {@link #next()}, ensuring all chunks are known before any single chunk is
 * accessed. This is necessary to improve access locality and prevent thrashing
 * of the local ChunkCache. It also causes {@link MissingObjectException} to be
 * thrown at the start of traversal, until the lookup queue is exhausted.
 *
 * @param <T>
 *            type of object to associate with the loader.
 */
final class OpenQueue<T extends ObjectId> extends QueueObjectLookup<T>
		implements AsyncObjectLoaderQueue<T> {
	private Map<ChunkKey, Collection<ObjectWithInfo<T>>> byChunk;

	private Iterator<Collection<ObjectWithInfo<T>>> chunkItr;

	private Iterator<ObjectWithInfo<T>> objectItr;

	private Prefetcher prefetcher;

	private ObjectWithInfo<T> current;

	private PackChunk currChunk;

	OpenQueue(DhtReader reader, Iterable<T> objectIds, boolean reportMissing) {
		super(reader, reportMissing);
		setCacheLoadedInfo(true);
		setNeedChunkOnly(true);
		init(objectIds);

		byChunk = new LinkedHashMap<ChunkKey, Collection<ObjectWithInfo<T>>>();
		objectItr = Collections.<ObjectWithInfo<T>> emptyList().iterator();
	}

	public boolean next() throws MissingObjectException, IOException {
		if (chunkItr == null)
			init();

		if (!objectItr.hasNext()) {
			currChunk = null;
			if (!chunkItr.hasNext()) {
				release();
				return false;
			}
			objectItr = chunkItr.next().iterator();
		}

		current = objectItr.next();
		return true;
	}

	public T getCurrent() {
		return current.object;
	}

	public ObjectId getObjectId() {
		return getCurrent();
	}

	public ObjectLoader open() throws IOException {
		ChunkKey chunkKey = current.chunkKey;

		// Objects returned by the queue are clustered by chunk. This object
		// is either in the current chunk, or are the next chunk ready on the
		// prefetcher. Anything else is a programming error.
		//
		PackChunk chunk;
		if (currChunk != null && chunkKey.equals(currChunk.getChunkKey()))
			chunk = currChunk;
		else {
			chunk = prefetcher.get(chunkKey);
			if (chunk == null)
				throw new DhtMissingChunkException(chunkKey);
			currChunk = chunk;
			reader.recentChunk(chunk);
		}

		if (current.info != null) {
			int ptr = current.info.getOffset();
			int type = current.info.getType();
			return PackChunk.read(chunk, ptr, reader, type);
		} else {
			int ptr = chunk.findOffset(repo, current.object);
			if (ptr < 0)
				throw DhtReader.missing(current.object, ObjectReader.OBJ_ANY);
			return PackChunk.read(chunk, ptr, reader, ObjectReader.OBJ_ANY);
		}
	}

	@Override
	public boolean cancel(boolean mayInterruptIfRunning) {
		release();
		return true;
	}

	@Override
	public void release() {
		reader.getRecentChunks().setMaxBytes(reader.getOptions().getChunkLimit());
		prefetcher = null;
		currChunk = null;
	}

	private void init() throws IOException {
		ObjectWithInfo<T> c;

		while ((c = nextObjectWithInfo()) != null) {
			ChunkKey chunkKey = c.chunkKey;
			Collection<ObjectWithInfo<T>> list = byChunk.get(chunkKey);
			if (list == null) {
				list = new ArrayList<ObjectWithInfo<T>>();
				byChunk.put(chunkKey, list);

				if (prefetcher == null) {
					int limit = reader.getOptions().getChunkLimit();
					int ratio = reader.getOptions().getOpenQueuePrefetchRatio();
					int prefetchLimit = (int) (limit * (ratio / 100.0));
					reader.getRecentChunks().setMaxBytes(limit - prefetchLimit);
					prefetcher = new Prefetcher(reader, 0, prefetchLimit);
				}
				prefetcher.push(chunkKey);
			}
			list.add(c);
		}

		chunkItr = byChunk.values().iterator();
	}
}