diff options
7 files changed, 933 insertions, 2 deletions
diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF index e73b352729..c59f636a51 100644 --- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF @@ -37,8 +37,11 @@ Import-Package: javax.servlet;version="[3.1.0,4.0.0)", org.eclipse.jgit.errors;version="[4.9.0,4.10.0)", org.eclipse.jgit.gitrepo;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.ketch;version="[4.9.0,4.10.0)", + org.eclipse.jgit.internal.storage.dfs;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.storage.file;version="[4.9.0,4.10.0)", + org.eclipse.jgit.internal.storage.io;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.storage.pack;version="[4.9.0,4.10.0)", + org.eclipse.jgit.internal.storage.reftable;version="[4.9.0,4.10.0)", org.eclipse.jgit.internal.storage.reftree;version="[4.9.0,4.10.0)", org.eclipse.jgit.lfs;version="[4.9.0,4.10.0)", org.eclipse.jgit.lfs.lib;version="[4.9.0,4.10.0)", diff --git a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin index 5495be6f88..902547305d 100644 --- a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin +++ b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin @@ -38,10 +38,12 @@ org.eclipse.jgit.pgm.Tag org.eclipse.jgit.pgm.UploadPack org.eclipse.jgit.pgm.Version +org.eclipse.jgit.pgm.debug.BenchmarkReftable org.eclipse.jgit.pgm.debug.DiffAlgorithms org.eclipse.jgit.pgm.debug.LfsStore org.eclipse.jgit.pgm.debug.MakeCacheTree org.eclipse.jgit.pgm.debug.ReadDirCache +org.eclipse.jgit.pgm.debug.ReadReftable org.eclipse.jgit.pgm.debug.RebuildCommitGraph org.eclipse.jgit.pgm.debug.RebuildRefTree org.eclipse.jgit.pgm.debug.ShowCacheTree @@ -49,5 +51,6 @@ org.eclipse.jgit.pgm.debug.ShowCommands org.eclipse.jgit.pgm.debug.ShowDirCache org.eclipse.jgit.pgm.debug.ShowPackDelta org.eclipse.jgit.pgm.debug.TextHashFunctions -org.eclipse.jgit.pgm.debug.WriteDirCache - +org.eclipse.jgit.pgm.debug.VerifyReftable +org.eclipse.jgit.pgm.debug.WriteReftable +org.eclipse.jgit.pgm.debug.WriteReftable diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java new file mode 100644 index 0000000000..71c8db8e45 --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java @@ -0,0 +1,315 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.pgm.debug; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.lib.Constants.HEAD; +import static org.eclipse.jgit.lib.Constants.MASTER; +import static org.eclipse.jgit.lib.Constants.R_HEADS; +import static org.eclipse.jgit.lib.Ref.Storage.NEW; +import static org.eclipse.jgit.lib.Ref.Storage.PACKED; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; + +import org.eclipse.jgit.internal.storage.io.BlockSource; +import org.eclipse.jgit.internal.storage.reftable.RefCursor; +import org.eclipse.jgit.internal.storage.reftable.ReftableReader; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.SymbolicRef; +import org.eclipse.jgit.pgm.Command; +import org.eclipse.jgit.pgm.TextBuiltin; +import org.eclipse.jgit.util.RefList; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.Option; + +@Command +class BenchmarkReftable extends TextBuiltin { + enum Test { + SCAN, + SEEK_COLD, SEEK_HOT, + BY_ID_COLD, BY_ID_HOT; + } + + @Option(name = "--tries") + private int tries = 10; + + @Option(name = "--test") + private Test test = Test.SCAN; + + @Option(name = "--ref") + private String ref; + + @Option(name = "--object-id") + private String objectId; + + @Argument(index = 0) + private String lsRemotePath; + + @Argument(index = 1) + private String reftablePath; + + @Override + protected void run() throws Exception { + switch (test) { + case SCAN: + scan(); + break; + + case SEEK_COLD: + seekCold(ref); + break; + case SEEK_HOT: + seekHot(ref); + break; + + case BY_ID_COLD: + byIdCold(ObjectId.fromString(objectId)); + break; + case BY_ID_HOT: + byIdHot(ObjectId.fromString(objectId)); + break; + } + } + + private void printf(String fmt, Object... args) throws IOException { + errw.println(String.format(fmt, args)); + } + + @SuppressWarnings({ "nls", "boxing" }) + private void scan() throws Exception { + long start, tot; + + start = System.currentTimeMillis(); + for (int i = 0; i < tries; i++) { + readLsRemote(); + } + tot = System.currentTimeMillis() - start; + printf("%12s %10d ms %6d ms/run", "packed-refs", tot, tot / tries); + + start = System.currentTimeMillis(); + for (int i = 0; i < tries; i++) { + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + try (RefCursor rc = reader.allRefs()) { + while (rc.next()) { + rc.getRef(); + } + } + } + } + tot = System.currentTimeMillis() - start; + printf("%12s %10d ms %6d ms/run", "reftable", tot, tot / tries); + } + + private RefList<Ref> readLsRemote() + throws IOException, FileNotFoundException { + RefList.Builder<Ref> list = new RefList.Builder<>(); + try (BufferedReader br = new BufferedReader(new InputStreamReader( + new FileInputStream(lsRemotePath), UTF_8))) { + Ref last = null; + String line; + while ((line = br.readLine()) != null) { + ObjectId id = ObjectId.fromString(line.substring(0, 40)); + String name = line.substring(41, line.length()); + if (last != null && name.endsWith("^{}")) { //$NON-NLS-1$ + last = new ObjectIdRef.PeeledTag(PACKED, last.getName(), + last.getObjectId(), id); + list.set(list.size() - 1, last); + continue; + } + + if (name.equals(HEAD)) { + last = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW, + R_HEADS + MASTER, null)); + } else { + last = new ObjectIdRef.PeeledNonTag(PACKED, name, id); + } + list.add(last); + } + } + list.sort(); + return list.toRefList(); + } + + @SuppressWarnings({ "nls", "boxing" }) + private void seekCold(String refName) throws Exception { + long start, tot; + + int lsTries = Math.min(tries, 64); + start = System.nanoTime(); + for (int i = 0; i < lsTries; i++) { + readLsRemote().get(refName); + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs", + tot / 1000, + (((double) tot) / lsTries) / 1000, + lsTries); + + start = System.nanoTime(); + for (int i = 0; i < tries; i++) { + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + try (RefCursor rc = reader.seekRef(refName)) { + while (rc.next()) { + rc.getRef(); + } + } + } + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable", + tot / 1000, + (((double) tot) / tries) / 1000, + tries); + } + + @SuppressWarnings({ "nls", "boxing" }) + private void seekHot(String refName) throws Exception { + long start, tot; + + int lsTries = Math.min(tries, 64); + start = System.nanoTime(); + RefList<Ref> lsRemote = readLsRemote(); + for (int i = 0; i < lsTries; i++) { + lsRemote.get(refName); + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs", + tot / 1000, (((double) tot) / lsTries) / 1000, lsTries); + + start = System.nanoTime(); + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + for (int i = 0; i < tries; i++) { + try (RefCursor rc = reader.seekRef(refName)) { + while (rc.next()) { + rc.getRef(); + } + } + } + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable", + tot / 1000, (((double) tot) / tries) / 1000, tries); + } + + @SuppressWarnings({ "nls", "boxing" }) + private void byIdCold(ObjectId id) throws Exception { + long start, tot; + + int lsTries = Math.min(tries, 64); + start = System.nanoTime(); + for (int i = 0; i < lsTries; i++) { + for (Ref r : readLsRemote()) { + if (id.equals(r.getObjectId())) { + continue; + } + } + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs", + tot / 1000, (((double) tot) / lsTries) / 1000, lsTries); + + start = System.nanoTime(); + for (int i = 0; i < tries; i++) { + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + try (RefCursor rc = reader.byObjectId(id)) { + while (rc.next()) { + rc.getRef(); + } + } + } + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable", + tot / 1000, (((double) tot) / tries) / 1000, tries); + } + + @SuppressWarnings({ "nls", "boxing" }) + private void byIdHot(ObjectId id) throws Exception { + long start, tot; + + int lsTries = Math.min(tries, 64); + start = System.nanoTime(); + RefList<Ref> lsRemote = readLsRemote(); + for (int i = 0; i < lsTries; i++) { + for (Ref r : lsRemote) { + if (id.equals(r.getObjectId())) { + continue; + } + } + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs", + tot / 1000, (((double) tot) / lsTries) / 1000, lsTries); + + start = System.nanoTime(); + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + for (int i = 0; i < tries; i++) { + try (RefCursor rc = reader.byObjectId(id)) { + while (rc.next()) { + rc.getRef(); + } + } + } + } + tot = System.nanoTime() - start; + printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable", + tot / 1000, (((double) tot) / tries) / 1000, tries); + } +} diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java new file mode 100644 index 0000000000..9b8db3e0f8 --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.pgm.debug; + +import java.io.FileInputStream; +import java.io.IOException; + +import org.eclipse.jgit.internal.storage.io.BlockSource; +import org.eclipse.jgit.internal.storage.reftable.RefCursor; +import org.eclipse.jgit.internal.storage.reftable.ReftableReader; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.pgm.Command; +import org.eclipse.jgit.pgm.TextBuiltin; +import org.kohsuke.args4j.Argument; + +@Command +class ReadReftable extends TextBuiltin { + @Argument(index = 0) + private String input; + + @Argument(index = 1, required = false) + private String ref; + + @Override + protected void run() throws Exception { + try (FileInputStream in = new FileInputStream(input); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + try (RefCursor rc = ref != null + ? reader.seekRef(ref) + : reader.allRefs()) { + while (rc.next()) { + write(rc.getRef()); + } + } + } + } + + private void write(Ref r) throws IOException { + if (r.isSymbolic()) { + outw.println(r.getTarget().getName() + '\t' + r.getName()); + return; + } + + ObjectId id1 = r.getObjectId(); + if (id1 != null) { + outw.println(id1.name() + '\t' + r.getName()); + } + + ObjectId id2 = r.getPeeledObjectId(); + if (id2 != null) { + outw.println('^' + id2.name()); + } + } +} diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java new file mode 100644 index 0000000000..dffb579895 --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java @@ -0,0 +1,211 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.pgm.debug; + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.eclipse.jgit.internal.storage.io.BlockSource; +import org.eclipse.jgit.internal.storage.reftable.RefCursor; +import org.eclipse.jgit.internal.storage.reftable.ReftableReader; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefComparator; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.eclipse.jgit.pgm.Command; +import org.eclipse.jgit.pgm.TextBuiltin; +import org.kohsuke.args4j.Argument; + +@Command +class VerifyReftable extends TextBuiltin { + private static final long SEED1 = 0xaba8bb4de4caf86cL; + private static final long SEED2 = 0x28bb5c25ad43ecb5L; + + @Argument(index = 0) + private String lsRemotePath; + + @Argument(index = 1) + private String reftablePath; + + @Override + protected void run() throws Exception { + List<Ref> refs = WriteReftable.readRefs(lsRemotePath); + + try (FileInputStream in = new FileInputStream(reftablePath); + BlockSource src = BlockSource.from(in); + ReftableReader reader = new ReftableReader(src)) { + scan(refs, reader); + seek(refs, reader); + byId(refs, reader); + } + } + + @SuppressWarnings("nls") + private void scan(List<Ref> refs, ReftableReader reader) + throws IOException { + errw.print(String.format("%-20s", "sequential scan...")); + errw.flush(); + try (RefCursor rc = reader.allRefs()) { + for (Ref exp : refs) { + verify(exp, rc); + } + if (rc.next()) { + throw die("expected end of table"); + } + } + errw.println(" OK"); + } + + @SuppressWarnings("nls") + private void seek(List<Ref> refs, ReftableReader reader) + throws IOException { + List<Ref> rnd = new ArrayList<>(refs); + Collections.shuffle(rnd, new Random(SEED1)); + + TextProgressMonitor pm = new TextProgressMonitor(errw); + pm.beginTask("random seek", rnd.size()); + for (Ref exp : rnd) { + try (RefCursor rc = reader.seekRef(exp.getName())) { + verify(exp, rc); + if (rc.next()) { + throw die("should not have ref after " + exp.getName()); + } + } + pm.update(1); + } + pm.endTask(); + } + + @SuppressWarnings("nls") + private void byId(List<Ref> refs, ReftableReader reader) + throws IOException { + Map<ObjectId, List<Ref>> want = groupById(refs); + List<List<Ref>> rnd = new ArrayList<>(want.values()); + Collections.shuffle(rnd, new Random(SEED2)); + + TextProgressMonitor pm = new TextProgressMonitor(errw); + pm.beginTask("byObjectId", rnd.size()); + for (List<Ref> exp : rnd) { + Collections.sort(exp, RefComparator.INSTANCE); + ObjectId id = exp.get(0).getObjectId(); + try (RefCursor rc = reader.byObjectId(id)) { + for (Ref r : exp) { + verify(r, rc); + } + } + pm.update(1); + } + pm.endTask(); + } + + private static Map<ObjectId, List<Ref>> groupById(List<Ref> refs) { + Map<ObjectId, List<Ref>> m = new HashMap<>(); + for (Ref r : refs) { + ObjectId id = r.getObjectId(); + if (id != null) { + List<Ref> c = m.get(id); + if (c == null) { + c = new ArrayList<>(2); + m.put(id, c); + } + c.add(r); + } + } + return m; + } + + @SuppressWarnings("nls") + private void verify(Ref exp, RefCursor rc) throws IOException { + if (!rc.next()) { + throw die("ended before " + exp.getName()); + } + + Ref act = rc.getRef(); + if (!exp.getName().equals(act.getName())) { + throw die(String.format("expected %s, found %s", + exp.getName(), + act.getName())); + } + + if (exp.isSymbolic()) { + if (!act.isSymbolic()) { + throw die("expected " + act.getName() + " to be symbolic"); + } + if (!exp.getTarget().getName().equals(act.getTarget().getName())) { + throw die(String.format("expected %s to be %s, found %s", + exp.getName(), + exp.getLeaf().getName(), + act.getLeaf().getName())); + } + return; + } + + if (!AnyObjectId.equals(exp.getObjectId(), act.getObjectId())) { + throw die(String.format("expected %s to be %s, found %s", + exp.getName(), + id(exp.getObjectId()), + id(act.getObjectId()))); + } + + if (exp.getPeeledObjectId() != null + && !AnyObjectId.equals(exp.getPeeledObjectId(), act.getPeeledObjectId())) { + throw die(String.format("expected %s to be %s, found %s", + exp.getName(), + id(exp.getPeeledObjectId()), + id(act.getPeeledObjectId()))); + } + } + + @SuppressWarnings("nls") + private static String id(ObjectId id) { + return id != null ? id.name() : "<null>"; + } +} diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java new file mode 100644 index 0000000000..76ffa194a3 --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java @@ -0,0 +1,301 @@ +/* + * Copyright (C) 2017, Google Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package org.eclipse.jgit.pgm.debug; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.lib.Constants.HEAD; +import static org.eclipse.jgit.lib.Constants.MASTER; +import static org.eclipse.jgit.lib.Constants.R_HEADS; +import static org.eclipse.jgit.lib.Ref.Storage.NEW; +import static org.eclipse.jgit.lib.Ref.Storage.PACKED; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; +import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.SymbolicRef; +import org.eclipse.jgit.pgm.Command; +import org.eclipse.jgit.pgm.TextBuiltin; +import org.kohsuke.args4j.Argument; +import org.kohsuke.args4j.Option; + +@Command +class WriteReftable extends TextBuiltin { + private static final int KIB = 1 << 10; + private static final int MIB = 1 << 20; + + @Option(name = "--block-size") + private int refBlockSize; + + @Option(name = "--log-block-size") + private int logBlockSize; + + @Option(name = "--restart-interval") + private int restartInterval; + + @Option(name = "--index-levels") + private int indexLevels; + + @Option(name = "--reflog-in") + private String reflogIn; + + @Option(name = "--no-index-objects") + private boolean noIndexObjects; + + @Argument(index = 0) + private String in; + + @Argument(index = 1) + private String out; + + @SuppressWarnings({ "nls", "boxing" }) + @Override + protected void run() throws Exception { + List<Ref> refs = readRefs(in); + List<LogEntry> logs = readLog(reflogIn); + + ReftableWriter.Stats stats; + try (OutputStream os = new FileOutputStream(out)) { + ReftableConfig cfg = new ReftableConfig(); + cfg.setIndexObjects(!noIndexObjects); + if (refBlockSize > 0) { + cfg.setRefBlockSize(refBlockSize); + } + if (logBlockSize > 0) { + cfg.setLogBlockSize(logBlockSize); + } + if (restartInterval > 0) { + cfg.setRestartInterval(restartInterval); + } + if (indexLevels > 0) { + cfg.setMaxIndexLevels(indexLevels); + } + + ReftableWriter w = new ReftableWriter(cfg); + w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs)); + w.begin(os); + w.sortAndWriteRefs(refs); + for (LogEntry e : logs) { + w.writeLog(e.ref, e.updateIndex, e.who, + e.oldId, e.newId, e.message); + } + stats = w.finish().getStats(); + } + + double fileMiB = ((double) stats.totalBytes()) / MIB; + printf("Summary:"); + printf(" file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes()); + printf(" padding : %d KiB", stats.paddingBytes() / KIB); + errw.println(); + + printf("Refs:"); + printf(" ref blk : %d", stats.refBlockSize()); + printf(" restarts: %d", stats.restartInterval()); + printf(" refs : %d", stats.refCount()); + if (stats.refIndexLevels() > 0) { + int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB); + printf(" idx sz : %d KiB", idxSize); + printf(" idx lvl : %d", stats.refIndexLevels()); + } + printf(" avg ref : %d bytes", stats.refBytes() / refs.size()); + errw.println(); + + if (stats.objCount() > 0) { + int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB); + int idLen = stats.objIdLength(); + printf("Objects:"); + printf(" obj blk : %d", stats.refBlockSize()); + printf(" restarts: %d", stats.restartInterval()); + printf(" objects : %d", stats.objCount()); + printf(" obj sz : %d MiB (%d bytes)", objMiB, stats.objBytes()); + if (stats.objIndexSize() > 0) { + int s = (int) Math.round(((double) stats.objIndexSize()) / KIB); + printf(" idx sz : %d KiB", s); + printf(" idx lvl : %d", stats.objIndexLevels()); + } + printf(" id len : %d bytes (%d hex digits)", idLen, 2 * idLen); + printf(" avg obj : %d bytes", stats.objBytes() / stats.objCount()); + errw.println(); + } + if (stats.logCount() > 0) { + int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB); + printf("Log:"); + printf(" log blk : %d", stats.logBlockSize()); + printf(" logs : %d", stats.logCount()); + printf(" log sz : %d MiB (%d bytes)", logMiB, stats.logBytes()); + printf(" avg log : %d bytes", stats.logBytes() / logs.size()); + errw.println(); + } + } + + private void printf(String fmt, Object... args) throws IOException { + errw.println(String.format(fmt, args)); + } + + static List<Ref> readRefs(String inputFile) throws IOException { + List<Ref> refs = new ArrayList<>(); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(inputFile), UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + ObjectId id = ObjectId.fromString(line.substring(0, 40)); + String name = line.substring(41, line.length()); + if (name.endsWith("^{}")) { //$NON-NLS-1$ + int lastIdx = refs.size() - 1; + Ref last = refs.get(lastIdx); + refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED, + last.getName(), last.getObjectId(), id)); + continue; + } + + Ref ref; + if (name.equals(HEAD)) { + ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW, + R_HEADS + MASTER, null)); + } else { + ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id); + } + refs.add(ref); + } + } + Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName())); + return refs; + } + + private static List<LogEntry> readLog(String logPath) + throws FileNotFoundException, IOException { + if (logPath == null) { + return Collections.emptyList(); + } + + List<LogEntry> log = new ArrayList<>(); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(logPath), UTF_8))) { + @SuppressWarnings("nls") + Pattern pattern = Pattern.compile("([^,]+)" // 1: ref + + ",([0-9]+(?:[.][0-9]+)?)" // 2: time + + ",([^,]+)" // 3: who + + ",([^,]+)" // 4: old + + ",([^,]+)" // 5: new + + ",(.*)"); // 6: msg + String line; + while ((line = br.readLine()) != null) { + Matcher m = pattern.matcher(line); + if (!m.matches()) { + throw new IOException("unparsed line: " + line); //$NON-NLS-1$ + } + String ref = m.group(1); + double t = Double.parseDouble(m.group(2)); + long time = ((long) t) * 1000L; + long index = (long) (t * 1e6); + String user = m.group(3); + ObjectId oldId = parseId(m.group(4)); + ObjectId newId = parseId(m.group(5)); + String msg = m.group(6); + String email = user + "@gerrit"; //$NON-NLS-1$ + PersonIdent who = new PersonIdent(user, email, time, -480); + log.add(new LogEntry(ref, index, who, oldId, newId, msg)); + } + } + Collections.sort(log, LogEntry::compare); + return log; + } + + private static long min(List<LogEntry> log) { + return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0); + } + + private static long max(List<LogEntry> log) { + return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0); + } + + private static ObjectId parseId(String s) { + if ("NULL".equals(s)) { //$NON-NLS-1$ + return ObjectId.zeroId(); + } + return ObjectId.fromString(s); + } + + private static class LogEntry { + static int compare(LogEntry a, LogEntry b) { + int cmp = a.ref.compareTo(b.ref); + if (cmp == 0) { + cmp = Long.signum(b.updateIndex - a.updateIndex); + } + return cmp; + } + + final String ref; + final long updateIndex; + final PersonIdent who; + final ObjectId oldId; + final ObjectId newId; + final String message; + + LogEntry(String ref, long updateIndex, PersonIdent who, + ObjectId oldId, ObjectId newId, String message) { + this.ref = ref; + this.updateIndex = updateIndex; + this.who = who; + this.oldId = oldId; + this.newId = newId; + this.message = message; + } + } +} diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index bbc500fe35..e3260812de 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -74,6 +74,7 @@ Export-Package: org.eclipse.jgit.annotations;version="4.9.0", org.eclipse.jgit.lfs, org.eclipse.jgit.pgm, org.eclipse.jgit.pgm.test", + org.eclipse.jgit.internal.storage.io;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", org.eclipse.jgit.internal.storage.pack;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", org.eclipse.jgit.internal.storage.reftable;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", org.eclipse.jgit.internal.storage.reftree;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm", |