123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191 |
- /*
- * Copyright (C) 2011, Google Inc.
- * and other copyright owners as documented in the project's IP log.
- *
- * This program and the accompanying materials are made available
- * under the terms of the Eclipse Distribution License v1.0 which
- * accompanies this distribution, is reproduced below, and is
- * available at http://www.eclipse.org/org/documents/edl-v10.php
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials provided
- * with the distribution.
- *
- * - Neither the name of the Eclipse Foundation, Inc. nor the
- * names of its contributors may be used to endorse or promote
- * products derived from this software without specific prior
- * written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
- package org.eclipse.jgit.storage.dht;
-
- import java.io.IOException;
- import java.util.ArrayList;
- import java.util.Collection;
- import java.util.Collections;
- import java.util.Iterator;
- import java.util.LinkedHashMap;
- import java.util.Map;
-
- import org.eclipse.jgit.errors.MissingObjectException;
- import org.eclipse.jgit.lib.AsyncObjectLoaderQueue;
- import org.eclipse.jgit.lib.ObjectId;
- import org.eclipse.jgit.lib.ObjectLoader;
- import org.eclipse.jgit.lib.ObjectReader;
-
- /**
- * Locates objects in large batches, then opens them clustered by chunk.
- * <p>
- * To simplify the implementation this method performs lookups for the
- * {@link ObjectInfo} in large batches, clusters those by ChunkKey, and loads
- * the chunks with a {@link Prefetcher}.
- * <p>
- * The lookup queue is completely spun out during the first invocation of
- * {@link #next()}, ensuring all chunks are known before any single chunk is
- * accessed. This is necessary to improve access locality and prevent thrashing
- * of the local ChunkCache. It also causes {@link MissingObjectException} to be
- * thrown at the start of traversal, until the lookup queue is exhausted.
- *
- * @param <T>
- * type of object to associate with the loader.
- */
- final class OpenQueue<T extends ObjectId> extends QueueObjectLookup<T>
- implements AsyncObjectLoaderQueue<T> {
- private Map<ChunkKey, Collection<ObjectWithInfo<T>>> byChunk;
-
- private Iterator<Collection<ObjectWithInfo<T>>> chunkItr;
-
- private Iterator<ObjectWithInfo<T>> objectItr;
-
- private Prefetcher prefetcher;
-
- private ObjectWithInfo<T> current;
-
- private PackChunk currChunk;
-
- OpenQueue(DhtReader reader, Iterable<T> objectIds, boolean reportMissing) {
- super(reader, reportMissing);
- setCacheLoadedInfo(true);
- setNeedChunkOnly(true);
- init(objectIds);
-
- byChunk = new LinkedHashMap<ChunkKey, Collection<ObjectWithInfo<T>>>();
- objectItr = Collections.<ObjectWithInfo<T>> emptyList().iterator();
- }
-
- public boolean next() throws MissingObjectException, IOException {
- if (chunkItr == null)
- init();
-
- if (!objectItr.hasNext()) {
- currChunk = null;
- if (!chunkItr.hasNext()) {
- release();
- return false;
- }
- objectItr = chunkItr.next().iterator();
- }
-
- current = objectItr.next();
- return true;
- }
-
- public T getCurrent() {
- return current.object;
- }
-
- public ObjectId getObjectId() {
- return getCurrent();
- }
-
- public ObjectLoader open() throws IOException {
- ChunkKey chunkKey = current.chunkKey;
-
- // Objects returned by the queue are clustered by chunk. This object
- // is either in the current chunk, or are the next chunk ready on the
- // prefetcher. Anything else is a programming error.
- //
- PackChunk chunk;
- if (currChunk != null && chunkKey.equals(currChunk.getChunkKey()))
- chunk = currChunk;
- else {
- chunk = prefetcher.get(chunkKey);
- if (chunk == null)
- throw new DhtMissingChunkException(chunkKey);
- currChunk = chunk;
- reader.recentChunk(chunk);
- }
-
- if (current.info != null) {
- int ptr = current.info.getOffset();
- int type = current.info.getType();
- return PackChunk.read(chunk, ptr, reader, type);
- } else {
- int ptr = chunk.findOffset(repo, current.object);
- if (ptr < 0)
- throw DhtReader.missing(current.object, ObjectReader.OBJ_ANY);
- return PackChunk.read(chunk, ptr, reader, ObjectReader.OBJ_ANY);
- }
- }
-
- @Override
- public boolean cancel(boolean mayInterruptIfRunning) {
- release();
- return true;
- }
-
- @Override
- public void release() {
- reader.getRecentChunks().setMaxBytes(reader.getOptions().getChunkLimit());
- prefetcher = null;
- currChunk = null;
- }
-
- private void init() throws IOException {
- ObjectWithInfo<T> c;
-
- while ((c = nextObjectWithInfo()) != null) {
- ChunkKey chunkKey = c.chunkKey;
- Collection<ObjectWithInfo<T>> list = byChunk.get(chunkKey);
- if (list == null) {
- list = new ArrayList<ObjectWithInfo<T>>();
- byChunk.put(chunkKey, list);
-
- if (prefetcher == null) {
- int limit = reader.getOptions().getChunkLimit();
- int ratio = reader.getOptions().getOpenQueuePrefetchRatio();
- int prefetchLimit = (int) (limit * (ratio / 100.0));
- reader.getRecentChunks().setMaxBytes(limit - prefetchLimit);
- prefetcher = new Prefetcher(reader, 0, prefetchLimit);
- }
- prefetcher.push(chunkKey);
- }
- list.add(c);
- }
-
- chunkItr = byChunk.values().iterator();
- }
- }
|