diff options
4 files changed, 197 insertions, 33 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index bb51fc4ca7..c48a49da7a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -44,18 +44,18 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; +import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; -import static org.eclipse.jgit.lib.RefDatabase.ALL; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import org.eclipse.jgit.internal.JGitText; @@ -63,6 +63,7 @@ import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -94,14 +95,11 @@ public class DfsGarbageCollector { private long coalesceGarbageLimit = 50 << 20; - private Map<String, Ref> refsBefore; - private List<DfsPackFile> packsBefore; private Set<ObjectId> allHeads; - private Set<ObjectId> nonHeads; - + private Set<ObjectId> txnHeads; private Set<ObjectId> tagTargets; /** @@ -197,19 +195,22 @@ public class DfsGarbageCollector { refdb.refresh(); objdb.clearCache(); - refsBefore = refdb.getRefs(ALL); + Collection<Ref> refsBefore = RefTreeNames.allRefs(refdb); packsBefore = packsToRebuild(); if (packsBefore.isEmpty()) return true; allHeads = new HashSet<ObjectId>(); nonHeads = new HashSet<ObjectId>(); + txnHeads = new HashSet<ObjectId>(); tagTargets = new HashSet<ObjectId>(); - for (Ref ref : refsBefore.values()) { + for (Ref ref : refsBefore) { if (ref.isSymbolic() || ref.getObjectId() == null) continue; if (isHead(ref)) allHeads.add(ref.getObjectId()); + else if (RefTreeNames.isRefTree(refdb, ref.getName())) + txnHeads.add(ref.getObjectId()); else nonHeads.add(ref.getObjectId()); if (ref.getPeeledObjectId() != null) @@ -221,6 +222,7 @@ public class DfsGarbageCollector { try { packHeads(pm); packRest(pm); + packRefTreeGraph(pm); packGarbage(pm); objdb.commitPack(newPackDesc, toPrune()); rollback = false; @@ -276,12 +278,11 @@ public class DfsGarbageCollector { try (PackWriter pw = newPackWriter()) { pw.setTagTargets(tagTargets); - pw.preparePack(pm, allHeads, Collections.<ObjectId> emptySet()); + pw.preparePack(pm, allHeads, none()); if (0 < pw.getObjectCount()) writePack(GC, pw, pm); } } - private void packRest(ProgressMonitor pm) throws IOException { if (nonHeads.isEmpty()) return; @@ -295,6 +296,23 @@ public class DfsGarbageCollector { } } + private void packRefTreeGraph(ProgressMonitor pm) throws IOException { + if (txnHeads.isEmpty()) + return; + + try (PackWriter pw = newPackWriter()) { + for (ObjectIdSet packedObjs : newPackObj) + pw.excludeObjects(packedObjs); + pw.preparePack(pm, txnHeads, none()); + if (0 < pw.getObjectCount()) + writePack(GC_TXN, pw, pm); + } + } + + private static Set<ObjectId> none() { + return Collections.<ObjectId> emptySet(); + } + private void packGarbage(ProgressMonitor pm) throws IOException { // TODO(sop) This is ugly. The garbage pack needs to be deleted. PackConfig cfg = new PackConfig(packConfig); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 5f491ff2fd..3641560ee9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -91,6 +91,13 @@ public abstract class DfsObjDatabase extends ObjectDatabase { GC(1), /** + * RefTreeGraph pack was created by Git garbage collection. + * + * @see DfsGarbageCollector + */ + GC_TXN(1), + + /** * The pack was created by compacting multiple packs together. * <p> * Packs created by compacting multiple packs together aren't nearly as diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index a5c95b3bca..8677164a67 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -45,7 +45,6 @@ package org.eclipse.jgit.internal.storage.file; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; -import static org.eclipse.jgit.lib.RefDatabase.ALL; import java.io.File; import java.io.FileOutputStream; @@ -63,11 +62,9 @@ import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.TreeMap; @@ -80,6 +77,7 @@ import org.eclipse.jgit.errors.NoWorkTreeException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; @@ -128,7 +126,7 @@ public class GC { * difference between the current refs and the refs which existed during * last {@link #repack()}. */ - private Map<String, Ref> lastPackedRefs; + private Collection<Ref> lastPackedRefs; /** * Holds the starting time of the last repack() execution. This is needed in @@ -362,17 +360,20 @@ public class GC { // during last repack(). Only those refs will survive which have been // added or modified since the last repack. Only these can save existing // loose refs from being pruned. - Map<String, Ref> newRefs; + Collection<Ref> newRefs; if (lastPackedRefs == null || lastPackedRefs.isEmpty()) newRefs = getAllRefs(); else { - newRefs = new HashMap<String, Ref>(); - for (Iterator<Map.Entry<String, Ref>> i = getAllRefs().entrySet() - .iterator(); i.hasNext();) { - Entry<String, Ref> newEntry = i.next(); - Ref old = lastPackedRefs.get(newEntry.getKey()); - if (!equals(newEntry.getValue(), old)) - newRefs.put(newEntry.getKey(), newEntry.getValue()); + Map<String, Ref> last = new HashMap<>(); + for (Ref r : lastPackedRefs) { + last.put(r.getName(), r); + } + newRefs = new ArrayList<>(); + for (Ref r : getAllRefs()) { + Ref old = last.get(r.getName()); + if (!equals(r, old)) { + newRefs.add(r); + } } } @@ -384,10 +385,10 @@ public class GC { // leave this method. ObjectWalk w = new ObjectWalk(repo); try { - for (Ref cr : newRefs.values()) + for (Ref cr : newRefs) w.markStart(w.parseAny(cr.getObjectId())); if (lastPackedRefs != null) - for (Ref lpr : lastPackedRefs.values()) + for (Ref lpr : lastPackedRefs) w.markUninteresting(w.parseAny(lpr.getObjectId())); removeReferenced(deletionCandidates, w); } finally { @@ -405,11 +406,11 @@ public class GC { // additional reflog entries not handled during last repack() ObjectWalk w = new ObjectWalk(repo); try { - for (Ref ar : getAllRefs().values()) + for (Ref ar : getAllRefs()) for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) w.markStart(w.parseAny(id)); if (lastPackedRefs != null) - for (Ref lpr : lastPackedRefs.values()) + for (Ref lpr : lastPackedRefs) w.markUninteresting(w.parseAny(lpr.getObjectId())); removeReferenced(deletionCandidates, w); } finally { @@ -530,19 +531,23 @@ public class GC { Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks(); long time = System.currentTimeMillis(); - Map<String, Ref> refsBefore = getAllRefs(); + Collection<Ref> refsBefore = getAllRefs(); Set<ObjectId> allHeads = new HashSet<ObjectId>(); Set<ObjectId> nonHeads = new HashSet<ObjectId>(); + Set<ObjectId> txnHeads = new HashSet<ObjectId>(); Set<ObjectId> tagTargets = new HashSet<ObjectId>(); Set<ObjectId> indexObjects = listNonHEADIndexObjects(); + RefDatabase refdb = repo.getRefDatabase(); - for (Ref ref : refsBefore.values()) { + for (Ref ref : refsBefore) { nonHeads.addAll(listRefLogObjects(ref, 0)); if (ref.isSymbolic() || ref.getObjectId() == null) continue; if (ref.getName().startsWith(Constants.R_HEADS)) allHeads.add(ref.getObjectId()); + else if (RefTreeNames.isRefTree(refdb, ref.getName())) + txnHeads.add(ref.getObjectId()); else nonHeads.add(ref.getObjectId()); if (ref.getPeeledObjectId() != null) @@ -572,6 +577,11 @@ public class GC { if (rest != null) ret.add(rest); } + if (!txnHeads.isEmpty()) { + PackFile txn = writePack(txnHeads, null, null, excluded); + if (txn != null) + ret.add(txn); + } try { deleteOldPacks(toBeDeleted, ret); } catch (ParseException e) { @@ -624,11 +634,16 @@ public class GC { * @return a map where names of refs point to ref objects * @throws IOException */ - private Map<String, Ref> getAllRefs() throws IOException { - Map<String, Ref> ret = repo.getRefDatabase().getRefs(ALL); - for (Ref ref : repo.getRefDatabase().getAdditionalRefs()) - ret.put(ref.getName(), ref); - return ret; + private Collection<Ref> getAllRefs() throws IOException { + Collection<Ref> refs = RefTreeNames.allRefs(repo.getRefDatabase()); + List<Ref> addl = repo.getRefDatabase().getAdditionalRefs(); + if (!addl.isEmpty()) { + List<Ref> all = new ArrayList<>(refs.size() + addl.size()); + all.addAll(refs); + all.addAll(addl); + return all; + } + return refs; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java new file mode 100644 index 0000000000..239a745277 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2016, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.internal.storage.reftree; + +import static org.eclipse.jgit.lib.RefDatabase.ALL; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; + +/** Magic reference name logic for RefTrees. */ +public class RefTreeNames { + /** + * Suffix used on a {@link RefTreeDatabase#getTxnNamespace()} for user data. + * <p> + * A {@link RefTreeDatabase}'s namespace may include a subspace (e.g. + * {@code "refs/txn/stage/"}) containing commit objects from the usual user + * portion of the repository (e.g. {@code "refs/heads/"}). These should be + * packed by the garbage collector alongside other user content rather than + * with the RefTree. + */ + private static final String STAGE = "stage/"; //$NON-NLS-1$ + + /** + * Determine if the reference is likely to be a RefTree. + * + * @param refdb + * database instance. + * @param ref + * reference name. + * @return {@code true} if the reference is a RefTree. + */ + public static boolean isRefTree(RefDatabase refdb, String ref) { + if (refdb instanceof RefTreeDatabase) { + RefTreeDatabase b = (RefTreeDatabase) refdb; + if (ref.equals(b.getTxnCommitted())) { + return true; + } + + String namespace = b.getTxnNamespace(); + if (namespace != null + && ref.startsWith(namespace) + && !ref.startsWith(namespace + STAGE)) { + return true; + } + } + return false; + } + + /** + * Snapshot all references from a RefTreeDatabase and its bootstrap. + * <p> + * There may be name conflicts with multiple {@link Ref} objects containing + * the same name in the returned collection. + * + * @param refdb + * database instance. + * @return all known references. + * @throws IOException + * references cannot be enumerated. + */ + public static Collection<Ref> allRefs(RefDatabase refdb) + throws IOException { + Collection<Ref> refs = refdb.getRefs(ALL).values(); + if (!(refdb instanceof RefTreeDatabase)) { + return refs; + } + + RefDatabase bootstrap = ((RefTreeDatabase) refdb).getBootstrap(); + Collection<Ref> br = bootstrap.getRefs(ALL).values(); + List<Ref> all = new ArrayList<>(refs.size() + br.size()); + all.addAll(refs); + all.addAll(br); + return all; + } + + private RefTreeNames() { + } +} |