From: Han-Wen Nienhuys Date: Sun, 17 Nov 2019 17:51:14 +0000 (-0800) Subject: Remove reftree and ketch X-Git-Tag: v5.11.0.202102031030-m2~73 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=7386f73bb0a5754f0b1dfc38aae37d68461a8cd8;p=jgit.git Remove reftree and ketch This was experimental code and never used in production. Change-Id: Ia3da7f2b82d9e365cec2ccf9397cbc47439cd150 Signed-off-by: Han-Wen Nienhuys Signed-off-by: Matthias Sohn --- diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF index 6a20396d71..1dfb418010 100644 --- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF @@ -22,12 +22,10 @@ Import-Package: javax.servlet;version="[3.1.0,4.0.0)", org.eclipse.jgit.dircache;version="[5.11.0,5.12.0)", org.eclipse.jgit.errors;version="[5.11.0,5.12.0)", org.eclipse.jgit.gitrepo;version="[5.11.0,5.12.0)", - org.eclipse.jgit.internal.ketch;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.storage.file;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.storage.io;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.storage.pack;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.storage.reftable;version="[5.11.0,5.12.0)", - org.eclipse.jgit.internal.storage.reftree;version="[5.11.0,5.12.0)", org.eclipse.jgit.lfs;version="[5.11.0,5.12.0)", org.eclipse.jgit.lfs.server;version="[5.11.0,5.12.0)", org.eclipse.jgit.lfs.server.fs;version="[5.11.0,5.12.0)", diff --git a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin index 062b9643a3..e645255e96 100644 --- a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin +++ b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin @@ -47,7 +47,6 @@ org.eclipse.jgit.pgm.debug.MakeCacheTree org.eclipse.jgit.pgm.debug.ReadDirCache org.eclipse.jgit.pgm.debug.ReadReftable org.eclipse.jgit.pgm.debug.RebuildCommitGraph -org.eclipse.jgit.pgm.debug.RebuildRefTree org.eclipse.jgit.pgm.debug.ShowCacheTree org.eclipse.jgit.pgm.debug.ShowCommands org.eclipse.jgit.pgm.debug.ShowDirCache diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index bf2455283e..afa253eeb5 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -118,7 +118,6 @@ metaVar_file=FILE metaVar_filepattern=filepattern metaVar_gitDir=GIT_DIR metaVar_hostName=HOSTNAME -metaVar_ketchServerType=SERVERTYPE metaVar_lfsStorage=STORAGE metaVar_linesOfContext=lines metaVar_message=message @@ -247,7 +246,6 @@ usage_DisplayTheVersionOfJgit=Display the version of jgit usage_Gc=Cleanup unnecessary files and optimize the local repository usage_Glog=View commit history as a graph usage_IndexPack=Build pack index file for an existing packed archive -usage_ketchServerType=Ketch server type usage_LFSDirectory=Directory to store large objects usage_LFSPort=Server http port usage_LFSRunStore=Store (fs | s3), store lfs objects in file system or Amazon S3 @@ -267,8 +265,6 @@ usage_PreserveOldPacks=Preserve old pack files by moving them into the preserved usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files usage_ReadDirCache= Read the DirCache 100 times usage_RebuildCommitGraph=Recreate a repository from another one's commit graph -usage_RebuildRefTree=Copy references into a RefTree -usage_RebuildRefTreeEnable=set extensions.refStorage = reftree usage_Remote=Manage set of tracked repositories usage_RepositoryToReadFrom=Repository to read from usage_RepositoryToReceiveInto=Repository to receive into diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java index bf9102552c..f987f2c806 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Daemon.java @@ -13,19 +13,12 @@ package org.eclipse.jgit.pgm; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URISyntaxException; import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executors; import org.eclipse.jgit.errors.ConfigInvalidException; -import org.eclipse.jgit.internal.ketch.KetchLeader; -import org.eclipse.jgit.internal.ketch.KetchLeaderCache; -import org.eclipse.jgit.internal.ketch.KetchPreReceive; -import org.eclipse.jgit.internal.ketch.KetchSystem; -import org.eclipse.jgit.internal.ketch.KetchText; -import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.pgm.internal.CLIText; import org.eclipse.jgit.storage.file.FileBasedConfig; @@ -33,10 +26,7 @@ import org.eclipse.jgit.storage.file.WindowCacheConfig; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.DaemonClient; import org.eclipse.jgit.transport.DaemonService; -import org.eclipse.jgit.transport.ReceivePack; import org.eclipse.jgit.transport.resolver.FileResolver; -import org.eclipse.jgit.transport.resolver.ReceivePackFactory; -import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.SystemReader; import org.kohsuke.args4j.Argument; @@ -71,13 +61,6 @@ class Daemon extends TextBuiltin { @Option(name = "--export-all", usage = "usage_exportWithoutGitDaemonExportOk") boolean exportAll; - @Option(name = "--ketch", metaVar = "metaVar_ketchServerType", usage = "usage_ketchServerType") - KetchServerType ketchServerType; - - enum KetchServerType { - LEADER; - } - @Argument(required = true, metaVar = "metaVar_directory", usage = "usage_directoriesToExport") List directory = new ArrayList<>(); @@ -102,9 +85,9 @@ class Daemon extends TextBuiltin { } cfg = new FileBasedConfig(configFile, FS.DETECTED); } - cfg.load(); - new WindowCacheConfig().fromConfig(cfg).install(); - packConfig.fromConfig(cfg); + cfg.load(); + new WindowCacheConfig().fromConfig(cfg).install(); + packConfig.fromConfig(cfg); int threads = packConfig.getThreads(); if (threads <= 0) @@ -137,9 +120,6 @@ class Daemon extends TextBuiltin { service(d, n).setOverridable(true); for (String n : forbidOverride) service(d, n).setOverridable(false); - if (ketchServerType == KetchServerType.LEADER) { - startKetchLeader(d); - } d.start(); outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress())); } @@ -162,24 +142,4 @@ class Daemon extends TextBuiltin { throw die(MessageFormat.format(CLIText.get().serviceNotSupported, n)); return svc; } - - private void startKetchLeader(org.eclipse.jgit.transport.Daemon daemon) { - KetchSystem system = new KetchSystem(); - final KetchLeaderCache leaders = new KetchLeaderCache(system); - final ReceivePackFactory factory; - - factory = daemon.getReceivePackFactory(); - daemon.setReceivePackFactory((DaemonClient req, Repository repo) -> { - ReceivePack rp = factory.create(req, repo); - KetchLeader leader; - try { - leader = leaders.get(repo); - } catch (URISyntaxException err) { - throw new ServiceNotEnabledException( - KetchText.get().invalidFollowerUri, err); - } - rp.setPreReceiveHook(new KetchPreReceive(leader)); - return rp; - }); - } } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/RebuildRefTree.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/RebuildRefTree.java deleted file mode 100644 index 38951ba428..0000000000 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/RebuildRefTree.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2015, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.pgm.debug; - -import static org.eclipse.jgit.lib.Constants.HEAD; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ConfigConstants; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.StoredConfig; -import org.eclipse.jgit.pgm.Command; -import org.eclipse.jgit.pgm.TextBuiltin; -import org.eclipse.jgit.revwalk.RevWalk; -import org.kohsuke.args4j.Option; - -@Command(usage = "usage_RebuildRefTree") -class RebuildRefTree extends TextBuiltin { - @Option(name = "--enable", usage = "usage_RebuildRefTreeEnable") - boolean enable; - - private String txnNamespace; - private String txnCommitted; - - /** {@inheritDoc} */ - @Override - protected void run() throws Exception { - try (ObjectReader reader = db.newObjectReader(); - RevWalk rw = new RevWalk(reader); - ObjectInserter inserter = db.newObjectInserter()) { - RefDatabase refDb = db.getRefDatabase(); - if (refDb instanceof RefTreeDatabase) { - RefTreeDatabase d = (RefTreeDatabase) refDb; - refDb = d.getBootstrap(); - txnNamespace = d.getTxnNamespace(); - txnCommitted = d.getTxnCommitted(); - } else { - RefTreeDatabase d = new RefTreeDatabase(db, refDb); - txnNamespace = d.getTxnNamespace(); - txnCommitted = d.getTxnCommitted(); - } - - errw.format("Rebuilding %s from %s", //$NON-NLS-1$ - txnCommitted, refDb.getClass().getSimpleName()); - errw.println(); - errw.flush(); - - CommitBuilder b = new CommitBuilder(); - Ref ref = refDb.exactRef(txnCommitted); - RefUpdate update = refDb.newUpdate(txnCommitted, true); - ObjectId oldTreeId; - - if (ref != null && ref.getObjectId() != null) { - ObjectId oldId = ref.getObjectId(); - update.setExpectedOldObjectId(oldId); - b.setParentId(oldId); - oldTreeId = rw.parseCommit(oldId).getTree(); - } else { - update.setExpectedOldObjectId(ObjectId.zeroId()); - oldTreeId = ObjectId.zeroId(); - } - - RefTree tree = rebuild(refDb); - b.setTreeId(tree.writeTree(inserter)); - b.setAuthor(new PersonIdent(db)); - b.setCommitter(b.getAuthor()); - if (b.getTreeId().equals(oldTreeId)) { - return; - } - - update.setNewObjectId(inserter.insert(b)); - inserter.flush(); - - RefUpdate.Result result = update.update(rw); - switch (result) { - case NEW: - case FAST_FORWARD: - break; - default: - throw die(String.format("%s: %s", update.getName(), result)); //$NON-NLS-1$ - } - - if (enable && !(db.getRefDatabase() instanceof RefTreeDatabase)) { - StoredConfig cfg = db.getConfig(); - cfg.setInt(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION, 1); - cfg.setString(ConfigConstants.CONFIG_EXTENSIONS_SECTION, null, - ConfigConstants.CONFIG_KEY_REFSTORAGE, - ConfigConstants.CONFIG_REFSTORAGE_REFTREE); - cfg.save(); - errw.println("Enabled reftree."); //$NON-NLS-1$ - errw.flush(); - } - } - } - - private RefTree rebuild(RefDatabase refdb) throws IOException { - RefTree tree = RefTree.newEmptyTree(); - List cmds - = new ArrayList<>(); - - Ref head = refdb.exactRef(HEAD); - if (head != null) { - cmds.add(new org.eclipse.jgit.internal.storage.reftree.Command( - null, - head)); - } - - for (Ref r : refdb.getRefs()) { - if (r.getName().equals(txnCommitted) || r.getName().equals(HEAD) - || r.getName().startsWith(txnNamespace)) { - continue; - } - cmds.add(new org.eclipse.jgit.internal.storage.reftree.Command( - null, - db.getRefDatabase().peel(r))); - } - tree.apply(cmds); - return tree; - } -} diff --git a/org.eclipse.jgit.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.test/META-INF/MANIFEST.MF index 148456d1cf..a1239468db 100644 --- a/org.eclipse.jgit.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.test/META-INF/MANIFEST.MF @@ -39,7 +39,6 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", org.eclipse.jgit.internal.storage.io;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.storage.pack;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.storage.reftable;version="[5.11.0,5.12.0)", - org.eclipse.jgit.internal.storage.reftree;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.transport.connectivity;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.transport.http;version="[5.11.0,5.12.0)", org.eclipse.jgit.internal.transport.parser;version="[5.11.0,5.12.0)", diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java index 18cf117174..b2c8ad5e7f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackDescriptionTest.java @@ -13,7 +13,6 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; -import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; @@ -82,7 +81,7 @@ public final class DfsPackDescriptionTest { DfsPackDescription.objectLookupComparator( new PackSource.ComparatorBuilder() .add(GC) - .add(INSERT, RECEIVE, GC_REST, GC_TXN, UNREACHABLE_GARBAGE) + .add(INSERT, RECEIVE, GC_REST, UNREACHABLE_GARBAGE) .add(COMPACT) .build()), a, b); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java index 6fcd4ac051..dfd112976c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackSourceTest.java @@ -14,7 +14,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.CO import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.DEFAULT_COMPARATOR; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; -import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; @@ -30,7 +29,6 @@ public class PackSourceTest { assertEquals(0, DEFAULT_COMPARATOR.compare(COMPACT, COMPACT)); assertEquals(0, DEFAULT_COMPARATOR.compare(GC, GC)); assertEquals(0, DEFAULT_COMPARATOR.compare(GC_REST, GC_REST)); - assertEquals(0, DEFAULT_COMPARATOR.compare(GC_TXN, GC_TXN)); assertEquals(0, DEFAULT_COMPARATOR.compare(UNREACHABLE_GARBAGE, UNREACHABLE_GARBAGE)); assertEquals(0, DEFAULT_COMPARATOR.compare(INSERT, RECEIVE)); @@ -47,11 +45,5 @@ public class PackSourceTest { assertEquals(-1, DEFAULT_COMPARATOR.compare(GC, GC_REST)); assertEquals(1, DEFAULT_COMPARATOR.compare(GC_REST, GC)); - - assertEquals(-1, DEFAULT_COMPARATOR.compare(GC_REST, GC_TXN)); - assertEquals(1, DEFAULT_COMPARATOR.compare(GC_TXN, GC_REST)); - - assertEquals(-1, DEFAULT_COMPARATOR.compare(GC_TXN, UNREACHABLE_GARBAGE)); - assertEquals(1, DEFAULT_COMPARATOR.compare(UNREACHABLE_GARBAGE, GC_TXN)); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/LocalDiskRefTreeDatabaseTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/LocalDiskRefTreeDatabaseTest.java deleted file mode 100644 index 86016d8f76..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/LocalDiskRefTreeDatabaseTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2016 Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.MASTER; -import static org.eclipse.jgit.lib.Constants.ORIG_HEAD; -import static org.eclipse.jgit.lib.Constants.R_HEADS; -import static org.eclipse.jgit.lib.RefDatabase.ALL; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import org.eclipse.jgit.internal.storage.file.FileRepository; -import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.storage.file.FileBasedConfig; -import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.junit.Before; -import org.junit.Test; - -public class LocalDiskRefTreeDatabaseTest extends LocalDiskRepositoryTestCase { - private FileRepository repo; - private RefTreeDatabase refdb; - private RefDatabase bootstrap; - - private TestRepository testRepo; - private RevCommit A; - private RevCommit B; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - FileRepository init = createWorkRepository(); - FileBasedConfig cfg = init.getConfig(); - cfg.setInt("core", null, "repositoryformatversion", 1); - cfg.setString("extensions", null, "refStorage", "reftree"); - cfg.save(); - - repo = (FileRepository) new FileRepositoryBuilder() - .setGitDir(init.getDirectory()) - .build(); - refdb = (RefTreeDatabase) repo.getRefDatabase(); - bootstrap = refdb.getBootstrap(); - addRepoToClose(repo); - - RefUpdate head = refdb.newUpdate(HEAD, true); - head.link(R_HEADS + MASTER); - - testRepo = new TestRepository<>(init); - A = testRepo.commit().create(); - B = testRepo.commit(testRepo.getRevWalk().parseCommit(A)); - } - - @Test - public void testHeadOrigHead() throws IOException { - RefUpdate master = refdb.newUpdate(HEAD, false); - master.setExpectedOldObjectId(ObjectId.zeroId()); - master.setNewObjectId(A); - assertEquals(RefUpdate.Result.NEW, master.update()); - assertEquals(A, refdb.exactRef(HEAD).getObjectId()); - - RefUpdate orig = refdb.newUpdate(ORIG_HEAD, true); - orig.setNewObjectId(B); - assertEquals(RefUpdate.Result.NEW, orig.update()); - - File origFile = new File(repo.getDirectory(), ORIG_HEAD); - assertEquals(B.name() + '\n', read(origFile)); - assertEquals(B, bootstrap.exactRef(ORIG_HEAD).getObjectId()); - assertEquals(B, refdb.exactRef(ORIG_HEAD).getObjectId()); - assertFalse(refdb.getRefs(ALL).containsKey(ORIG_HEAD)); - - List addl = refdb.getAdditionalRefs(); - assertEquals(2, addl.size()); - assertEquals(ORIG_HEAD, addl.get(1).getName()); - assertEquals(B, addl.get(1).getObjectId()); - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java deleted file mode 100644 index ecee5e5d2b..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabaseTest.java +++ /dev/null @@ -1,689 +0,0 @@ -/* - * Copyright (C) 2010, 2013, 2016 Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.ORIG_HEAD; -import static org.eclipse.jgit.lib.Constants.R_HEADS; -import static org.eclipse.jgit.lib.Constants.R_TAGS; -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.ALL; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; -import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.junit.Before; -import org.junit.Test; - -public class RefTreeDatabaseTest { - private InMemRefTreeRepo repo; - private RefTreeDatabase refdb; - private RefDatabase bootstrap; - - private TestRepository testRepo; - private RevCommit A; - private RevCommit B; - private RevTag v1_0; - - @Before - public void setUp() throws Exception { - repo = new InMemRefTreeRepo(new DfsRepositoryDescription("test")); - bootstrap = refdb.getBootstrap(); - - testRepo = new TestRepository<>(repo); - A = testRepo.commit().create(); - B = testRepo.commit(testRepo.getRevWalk().parseCommit(A)); - v1_0 = testRepo.tag("v1_0", B); - testRepo.getRevWalk().parseBody(v1_0); - } - - @Test - public void testSupportsAtomic() { - assertTrue(refdb.performsAtomicTransactions()); - } - - @Test - public void testGetRefs_EmptyDatabase() throws IOException { - assertTrue("no references", refdb.getRefs(ALL).isEmpty()); - assertTrue("no references", refdb.getRefs(R_HEADS).isEmpty()); - assertTrue("no references", refdb.getRefs(R_TAGS).isEmpty()); - assertTrue("no references", refdb.getAdditionalRefs().isEmpty()); - } - - @Test - public void testGetAdditionalRefs() throws IOException { - update("refs/heads/master", A); - - List addl = refdb.getAdditionalRefs(); - assertEquals(1, addl.size()); - assertEquals("refs/txn/committed", addl.get(0).getName()); - assertEquals(getTxnCommitted(), addl.get(0).getObjectId()); - } - - @Test - public void testGetRefs_HeadOnOneBranch() throws IOException { - symref(HEAD, "refs/heads/master"); - update("refs/heads/master", A); - - Map all = refdb.getRefs(ALL); - assertEquals(2, all.size()); - assertTrue("has HEAD", all.containsKey(HEAD)); - assertTrue("has master", all.containsKey("refs/heads/master")); - - Ref head = all.get(HEAD); - Ref master = all.get("refs/heads/master"); - - assertEquals(HEAD, head.getName()); - assertTrue(head.isSymbolic()); - assertSame(LOOSE, head.getStorage()); - assertSame("uses same ref as target", master, head.getTarget()); - - assertEquals("refs/heads/master", master.getName()); - assertFalse(master.isSymbolic()); - assertSame(PACKED, master.getStorage()); - assertEquals(A, master.getObjectId()); - } - - @Test - public void testGetRefs_DetachedHead() throws IOException { - update(HEAD, A); - - Map all = refdb.getRefs(ALL); - assertEquals(1, all.size()); - assertTrue("has HEAD", all.containsKey(HEAD)); - - Ref head = all.get(HEAD); - assertEquals(HEAD, head.getName()); - assertFalse(head.isSymbolic()); - assertSame(PACKED, head.getStorage()); - assertEquals(A, head.getObjectId()); - } - - @Test - public void testGetRefs_DeeplyNestedBranch() throws IOException { - String name = "refs/heads/a/b/c/d/e/f/g/h/i/j/k"; - update(name, A); - - Map all = refdb.getRefs(ALL); - assertEquals(1, all.size()); - - Ref r = all.get(name); - assertEquals(name, r.getName()); - assertFalse(r.isSymbolic()); - assertSame(PACKED, r.getStorage()); - assertEquals(A, r.getObjectId()); - } - - @Test - public void testGetRefs_HeadBranchNotBorn() throws IOException { - update("refs/heads/A", A); - update("refs/heads/B", B); - - Map all = refdb.getRefs(ALL); - assertEquals(2, all.size()); - assertFalse("no HEAD", all.containsKey(HEAD)); - - Ref a = all.get("refs/heads/A"); - Ref b = all.get("refs/heads/B"); - - assertEquals(A, a.getObjectId()); - assertEquals(B, b.getObjectId()); - - assertEquals("refs/heads/A", a.getName()); - assertEquals("refs/heads/B", b.getName()); - } - - @Test - public void testGetRefs_HeadsOnly() throws IOException { - update("refs/heads/A", A); - update("refs/heads/B", B); - update("refs/tags/v1.0", v1_0); - - Map heads = refdb.getRefs(R_HEADS); - assertEquals(2, heads.size()); - - Ref a = heads.get("A"); - Ref b = heads.get("B"); - - assertEquals("refs/heads/A", a.getName()); - assertEquals("refs/heads/B", b.getName()); - - assertEquals(A, a.getObjectId()); - assertEquals(B, b.getObjectId()); - } - - @Test - public void testGetRefs_TagsOnly() throws IOException { - update("refs/heads/A", A); - update("refs/heads/B", B); - update("refs/tags/v1.0", v1_0); - - Map tags = refdb.getRefs(R_TAGS); - assertEquals(1, tags.size()); - - Ref a = tags.get("v1.0"); - assertEquals("refs/tags/v1.0", a.getName()); - assertEquals(v1_0, a.getObjectId()); - assertTrue(a.isPeeled()); - assertEquals(v1_0.getObject(), a.getPeeledObjectId()); - } - - @Test - public void testGetRefs_HeadsSymref() throws IOException { - symref("refs/heads/other", "refs/heads/master"); - update("refs/heads/master", A); - - Map heads = refdb.getRefs(R_HEADS); - assertEquals(2, heads.size()); - - Ref master = heads.get("master"); - Ref other = heads.get("other"); - - assertEquals("refs/heads/master", master.getName()); - assertEquals(A, master.getObjectId()); - - assertEquals("refs/heads/other", other.getName()); - assertEquals(A, other.getObjectId()); - assertSame(master, other.getTarget()); - } - - @Test - public void testGetRefs_InvalidPrefixes() throws IOException { - update("refs/heads/A", A); - - assertTrue("empty refs/heads", refdb.getRefs("refs/heads").isEmpty()); - assertTrue("empty objects", refdb.getRefs("objects").isEmpty()); - assertTrue("empty objects/", refdb.getRefs("objects/").isEmpty()); - } - - @Test - public void testGetRefs_DiscoversNew() throws IOException { - update("refs/heads/master", A); - Map orig = refdb.getRefs(ALL); - - update("refs/heads/next", B); - Map next = refdb.getRefs(ALL); - - assertEquals(1, orig.size()); - assertEquals(2, next.size()); - - assertFalse(orig.containsKey("refs/heads/next")); - assertTrue(next.containsKey("refs/heads/next")); - - assertEquals(A, next.get("refs/heads/master").getObjectId()); - assertEquals(B, next.get("refs/heads/next").getObjectId()); - } - - @Test - public void testGetRefs_DiscoversModified() throws IOException { - symref(HEAD, "refs/heads/master"); - update("refs/heads/master", A); - - Map all = refdb.getRefs(ALL); - assertEquals(A, all.get(HEAD).getObjectId()); - - update("refs/heads/master", B); - all = refdb.getRefs(ALL); - assertEquals(B, all.get(HEAD).getObjectId()); - assertEquals(B, refdb.exactRef(HEAD).getObjectId()); - } - - @Test - public void testGetRefs_CycleInSymbolicRef() throws IOException { - symref("refs/1", "refs/2"); - symref("refs/2", "refs/3"); - symref("refs/3", "refs/4"); - symref("refs/4", "refs/5"); - symref("refs/5", "refs/end"); - update("refs/end", A); - - Map all = refdb.getRefs(ALL); - Ref r = all.get("refs/1"); - assertNotNull("has 1", r); - - assertEquals("refs/1", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/2", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/3", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/4", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/5", r.getName()); - assertEquals(A, r.getObjectId()); - assertTrue(r.isSymbolic()); - - r = r.getTarget(); - assertEquals("refs/end", r.getName()); - assertEquals(A, r.getObjectId()); - assertFalse(r.isSymbolic()); - - symref("refs/5", "refs/6"); - symref("refs/6", "refs/end"); - all = refdb.getRefs(ALL); - assertNull("mising 1 due to cycle", all.get("refs/1")); - assertEquals(A, all.get("refs/2").getObjectId()); - assertEquals(A, all.get("refs/3").getObjectId()); - assertEquals(A, all.get("refs/4").getObjectId()); - assertEquals(A, all.get("refs/5").getObjectId()); - assertEquals(A, all.get("refs/6").getObjectId()); - assertEquals(A, all.get("refs/end").getObjectId()); - } - - @Test - public void testGetRef_NonExistingBranchConfig() throws IOException { - assertNull("find branch config", refdb.findRef("config")); - assertNull("find branch config", refdb.findRef("refs/heads/config")); - } - - @Test - public void testGetRef_FindBranchConfig() throws IOException { - update("refs/heads/config", A); - - for (String t : new String[] { "config", "refs/heads/config" }) { - Ref r = refdb.findRef(t); - assertNotNull("find branch config (" + t + ")", r); - assertEquals("for " + t, "refs/heads/config", r.getName()); - assertEquals("for " + t, A, r.getObjectId()); - } - } - - @Test - public void testFirstExactRef() throws IOException { - update("refs/heads/A", A); - update("refs/tags/v1.0", v1_0); - - Ref a = refdb.firstExactRef("refs/heads/A", "refs/tags/v1.0"); - Ref one = refdb.firstExactRef("refs/tags/v1.0", "refs/heads/A"); - - assertEquals("refs/heads/A", a.getName()); - assertEquals("refs/tags/v1.0", one.getName()); - - assertEquals(A, a.getObjectId()); - assertEquals(v1_0, one.getObjectId()); - } - - @Test - public void testExactRef_DiscoversModified() throws IOException { - symref(HEAD, "refs/heads/master"); - update("refs/heads/master", A); - assertEquals(A, refdb.exactRef(HEAD).getObjectId()); - - update("refs/heads/master", B); - assertEquals(B, refdb.exactRef(HEAD).getObjectId()); - } - - @Test - public void testIsNameConflicting() throws IOException { - update("refs/heads/a/b", A); - update("refs/heads/q", B); - - // new references cannot replace an existing container - assertTrue(refdb.isNameConflicting("refs")); - assertTrue(refdb.isNameConflicting("refs/heads")); - assertTrue(refdb.isNameConflicting("refs/heads/a")); - - // existing reference is not conflicting - assertFalse(refdb.isNameConflicting("refs/heads/a/b")); - - // new references are not conflicting - assertFalse(refdb.isNameConflicting("refs/heads/a/d")); - assertFalse(refdb.isNameConflicting("refs/heads/master")); - - // existing reference must not be used as a container - assertTrue(refdb.isNameConflicting("refs/heads/a/b/c")); - assertTrue(refdb.isNameConflicting("refs/heads/q/master")); - - // refs/txn/ names always conflict. - assertTrue(refdb.isNameConflicting(refdb.getTxnCommitted())); - assertTrue(refdb.isNameConflicting("refs/txn/foo")); - } - - @Test - public void testUpdate_RefusesRefsTxnNamespace() throws IOException { - ObjectId txnId = getTxnCommitted(); - - RefUpdate u = refdb.newUpdate("refs/txn/tmp", false); - u.setNewObjectId(B); - assertEquals(RefUpdate.Result.LOCK_FAILURE, u.update()); - assertEquals(txnId, getTxnCommitted()); - - ReceiveCommand cmd = command(null, B, "refs/txn/tmp"); - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addCommand(cmd); - try (RevWalk rw = new RevWalk(repo)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); - assertEquals(MessageFormat.format(JGitText.get().invalidRefName, - "refs/txn/tmp"), cmd.getMessage()); - assertEquals(txnId, getTxnCommitted()); - } - - @Test - public void testUpdate_RefusesDotLockInRefName() throws IOException { - ObjectId txnId = getTxnCommitted(); - - RefUpdate u = refdb.newUpdate("refs/heads/pu.lock", false); - u.setNewObjectId(B); - assertEquals(RefUpdate.Result.REJECTED, u.update()); - assertEquals(txnId, getTxnCommitted()); - - ReceiveCommand cmd = command(null, B, "refs/heads/pu.lock"); - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addCommand(cmd); - try (RevWalk rw = new RevWalk(repo)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); - assertEquals(JGitText.get().funnyRefname, cmd.getMessage()); - assertEquals(txnId, getTxnCommitted()); - } - - @Test - public void testUpdate_RefusesOrigHeadOnBare() throws IOException { - assertTrue(refdb.getRepository().isBare()); - ObjectId txnId = getTxnCommitted(); - - RefUpdate orig = refdb.newUpdate(ORIG_HEAD, true); - orig.setNewObjectId(B); - assertEquals(RefUpdate.Result.LOCK_FAILURE, orig.update()); - assertEquals(txnId, getTxnCommitted()); - - ReceiveCommand cmd = command(null, B, ORIG_HEAD); - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addCommand(cmd); - try (RevWalk rw = new RevWalk(repo)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); - assertEquals( - MessageFormat.format(JGitText.get().invalidRefName, ORIG_HEAD), - cmd.getMessage()); - assertEquals(txnId, getTxnCommitted()); - } - - @Test - public void testBatchRefUpdate_NonFastForwardAborts() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(B, A, "refs/heads/masters")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(txnId, getTxnCommitted()); - - assertEquals(REJECTED_NONFASTFORWARD, - commands.get(1).getResult()); - assertEquals(REJECTED_OTHER_REASON, - commands.get(0).getResult()); - assertEquals(JGitText.get().transactionAborted, - commands.get(0).getMessage()); - } - - @Test - public void testBatchRefUpdate_ForceUpdate() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(B, A, "refs/heads/masters")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertNotEquals(txnId, getTxnCommitted()); - - Map refs = refdb.getRefs(ALL); - assertEquals(OK, commands.get(0).getResult()); - assertEquals(OK, commands.get(1).getResult()); - assertEquals( - "[refs/heads/master, refs/heads/masters]", - refs.keySet().toString()); - assertEquals(B.getId(), refs.get("refs/heads/master").getObjectId()); - assertEquals(A.getId(), refs.get("refs/heads/masters").getObjectId()); - } - - @Test - public void testBatchRefUpdate_NonFastForwardDoesNotDoExpensiveMergeCheck() - throws IOException { - update("refs/heads/master", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(B, A, "refs/heads/master")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo) { - @Override - public boolean isMergedInto(RevCommit base, RevCommit tip) { - fail("isMergedInto() should not be called"); - return false; - } - }) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertNotEquals(txnId, getTxnCommitted()); - - Map refs = refdb.getRefs(ALL); - assertEquals(OK, commands.get(0).getResult()); - assertEquals(A.getId(), refs.get("refs/heads/master").getObjectId()); - } - - @Test - public void testBatchRefUpdate_ConflictCausesAbort() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(null, A, "refs/heads/master/x"), - command(null, A, "refs/heads")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertEquals(txnId, getTxnCommitted()); - - assertEquals(LOCK_FAILURE, commands.get(0).getResult()); - - assertEquals(REJECTED_OTHER_REASON, commands.get(1).getResult()); - assertEquals(JGitText.get().transactionAborted, - commands.get(1).getMessage()); - - assertEquals(REJECTED_OTHER_REASON, commands.get(2).getResult()); - assertEquals(JGitText.get().transactionAborted, - commands.get(2).getMessage()); - } - - @Test - public void testBatchRefUpdate_NoConflictIfDeleted() throws IOException { - update("refs/heads/master", A); - update("refs/heads/masters", B); - ObjectId txnId = getTxnCommitted(); - - List commands = Arrays.asList( - command(A, B, "refs/heads/master"), - command(null, A, "refs/heads/masters/x"), - command(B, null, "refs/heads/masters")); - BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); - batchUpdate.setAllowNonFastForwards(true); - batchUpdate.addCommand(commands); - try (RevWalk rw = new RevWalk(repo)) { - batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); - } - assertNotEquals(txnId, getTxnCommitted()); - - assertEquals(OK, commands.get(0).getResult()); - assertEquals(OK, commands.get(1).getResult()); - assertEquals(OK, commands.get(2).getResult()); - - Map refs = refdb.getRefs(ALL); - assertEquals( - "[refs/heads/master, refs/heads/masters/x]", - refs.keySet().toString()); - assertEquals(A.getId(), refs.get("refs/heads/masters/x").getObjectId()); - } - - private ObjectId getTxnCommitted() throws IOException { - Ref r = bootstrap.exactRef(refdb.getTxnCommitted()); - if (r != null && r.getObjectId() != null) { - return r.getObjectId(); - } - return ObjectId.zeroId(); - } - - private static ReceiveCommand command(AnyObjectId a, AnyObjectId b, - String name) { - return new ReceiveCommand( - a != null ? a.copy() : ObjectId.zeroId(), - b != null ? b.copy() : ObjectId.zeroId(), - name); - } - - private void symref(String name, String dst) - throws IOException { - commit((ObjectReader reader, RefTree tree) -> { - Ref old = tree.exactRef(reader, name); - Command n = new Command(old, new SymbolicRef(name, - new ObjectIdRef.Unpeeled(Ref.Storage.NEW, dst, null))); - return tree.apply(Collections.singleton(n)); - }); - } - - private void update(String name, ObjectId id) - throws IOException { - commit((ObjectReader reader, RefTree tree) -> { - Ref old = tree.exactRef(reader, name); - Command n; - try (RevWalk rw = new RevWalk(repo)) { - n = new Command(old, Command.toRef(rw, id, null, name, true)); - } - return tree.apply(Collections.singleton(n)); - }); - } - - interface Function { - boolean apply(ObjectReader reader, RefTree tree) throws IOException; - } - - private void commit(Function fun) throws IOException { - try (ObjectReader reader = repo.newObjectReader(); - ObjectInserter inserter = repo.newObjectInserter(); - RevWalk rw = new RevWalk(reader)) { - RefUpdate u = bootstrap.newUpdate(refdb.getTxnCommitted(), false); - CommitBuilder cb = new CommitBuilder(); - testRepo.setAuthorAndCommitter(cb); - - Ref ref = bootstrap.exactRef(refdb.getTxnCommitted()); - RefTree tree; - if (ref != null && ref.getObjectId() != null) { - tree = RefTree.read(reader, rw.parseTree(ref.getObjectId())); - cb.setParentId(ref.getObjectId()); - u.setExpectedOldObjectId(ref.getObjectId()); - } else { - tree = RefTree.newEmptyTree(); - u.setExpectedOldObjectId(ObjectId.zeroId()); - } - - assertTrue(fun.apply(reader, tree)); - cb.setTreeId(tree.writeTree(inserter)); - u.setNewObjectId(inserter.insert(cb)); - inserter.flush(); - switch (u.update(rw)) { - case NEW: - case FAST_FORWARD: - break; - default: - fail("Expected " + u.getName() + " to update"); - } - } - } - - private class InMemRefTreeRepo extends InMemoryRepository { - private final RefTreeDatabase refs; - - InMemRefTreeRepo(DfsRepositoryDescription repoDesc) { - super(repoDesc); - refs = new RefTreeDatabase(this, super.getRefDatabase(), - "refs/txn/committed"); - RefTreeDatabaseTest.this.refdb = refs; - } - - @Override - public RefDatabase getRefDatabase() { - return refs; - } - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeTest.java deleted file mode 100644 index a5b01900cd..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/reftree/RefTreeTest.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.R_HEADS; -import static org.eclipse.jgit.lib.Constants.R_TAGS; -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; - -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; -import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevBlob; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.junit.Before; -import org.junit.Test; - -public class RefTreeTest { - private static final String R_MASTER = R_HEADS + "master"; - private InMemoryRepository repo; - private TestRepository git; - - @Before - public void setUp() throws IOException { - repo = new InMemoryRepository(new DfsRepositoryDescription("RefTree")); - git = new TestRepository<>(repo); - } - - @Test - public void testEmptyTree() throws IOException { - RefTree tree = RefTree.newEmptyTree(); - try (ObjectReader reader = repo.newObjectReader()) { - assertNull(HEAD, tree.exactRef(reader, HEAD)); - assertNull("master", tree.exactRef(reader, R_MASTER)); - } - } - - @Test - public void testApplyThenReadMaster() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob id = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, id)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - assertSame(NOT_ATTEMPTED, cmd.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, R_MASTER); - assertNotNull(R_MASTER, m); - assertEquals(R_MASTER, m.getName()); - assertEquals(id, m.getObjectId()); - assertTrue("peeled", m.isPeeled()); - } - } - - @Test - public void testUpdateMaster() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob id1 = git.blob("A"); - Command cmd1 = new Command(null, ref(R_MASTER, id1)); - assertTrue(tree.apply(Collections.singletonList(cmd1))); - assertSame(NOT_ATTEMPTED, cmd1.getResult()); - - RevBlob id2 = git.blob("B"); - Command cmd2 = new Command(ref(R_MASTER, id1), ref(R_MASTER, id2)); - assertTrue(tree.apply(Collections.singletonList(cmd2))); - assertSame(NOT_ATTEMPTED, cmd2.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, R_MASTER); - assertNotNull(R_MASTER, m); - assertEquals(R_MASTER, m.getName()); - assertEquals(id2, m.getObjectId()); - assertTrue("peeled", m.isPeeled()); - } - } - - @Test - public void testHeadSymref() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob id = git.blob("A"); - Command cmd1 = new Command(null, ref(R_MASTER, id)); - Command cmd2 = new Command(null, symref(HEAD, R_MASTER)); - assertTrue(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); - assertSame(NOT_ATTEMPTED, cmd1.getResult()); - assertSame(NOT_ATTEMPTED, cmd2.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, HEAD); - assertNotNull(HEAD, m); - assertEquals(HEAD, m.getName()); - assertTrue("symbolic", m.isSymbolic()); - assertNotNull(m.getTarget()); - assertEquals(R_MASTER, m.getTarget().getName()); - assertEquals(id, m.getTarget().getObjectId()); - } - - // Writing flushes some buffers, re-read from blob. - ObjectId newId = write(tree); - try (ObjectReader reader = repo.newObjectReader(); - RevWalk rw = new RevWalk(reader)) { - tree = RefTree.read(reader, rw.parseTree(newId)); - Ref m = tree.exactRef(reader, HEAD); - assertEquals(R_MASTER, m.getTarget().getName()); - } - } - - @Test - public void testTagIsPeeled() throws Exception { - String name = "v1.0"; - RefTree tree = RefTree.newEmptyTree(); - RevBlob id = git.blob("A"); - RevTag tag = git.tag(name, id); - - String ref = R_TAGS + name; - Command cmd = create(ref, tag); - assertTrue(tree.apply(Collections.singletonList(cmd))); - assertSame(NOT_ATTEMPTED, cmd.getResult()); - - try (ObjectReader reader = repo.newObjectReader()) { - Ref m = tree.exactRef(reader, ref); - assertNotNull(ref, m); - assertEquals(ref, m.getName()); - assertEquals(tag, m.getObjectId()); - assertTrue("peeled", m.isPeeled()); - assertEquals(id, m.getPeeledObjectId()); - } - } - - @Test - public void testApplyAlreadyExists() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - Command cmd1 = create(R_MASTER, b); - Command cmd2 = create(R_MASTER, b); - assertFalse(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertSame(REJECTED_OTHER_REASON, cmd2.getResult()); - assertEquals(JGitText.get().transactionAborted, cmd2.getMessage()); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyWrongOldId() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - RevBlob c = git.blob("C"); - Command cmd1 = update(R_MASTER, b, c); - Command cmd2 = create(R_MASTER, b); - assertFalse(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertSame(REJECTED_OTHER_REASON, cmd2.getResult()); - assertEquals(JGitText.get().transactionAborted, cmd2.getMessage()); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyWrongOldIdButAlreadyCurrentIsNoOp() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - cmd = update(R_MASTER, b, a); - assertTrue(tree.apply(Collections.singletonList(cmd))); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyCannotCreateSubdirectory() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - Command cmd1 = create(R_MASTER + "/fail", b); - assertFalse(tree.apply(Collections.singletonList(cmd1))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertEquals(treeId, write(tree)); - } - - @Test - public void testApplyCannotCreateParentRef() throws Exception { - RefTree tree = RefTree.newEmptyTree(); - RevBlob a = git.blob("A"); - Command cmd = new Command(null, ref(R_MASTER, a)); - assertTrue(tree.apply(Collections.singletonList(cmd))); - ObjectId treeId = write(tree); - - RevBlob b = git.blob("B"); - Command cmd1 = create("refs/heads", b); - assertFalse(tree.apply(Collections.singletonList(cmd1))); - assertSame(LOCK_FAILURE, cmd1.getResult()); - assertEquals(treeId, write(tree)); - } - - private static Ref ref(String name, ObjectId id) { - return new ObjectIdRef.PeeledNonTag(LOOSE, name, id); - } - - private static Ref symref(String name, String dest) { - Ref d = new ObjectIdRef.PeeledNonTag(NEW, dest, null); - return new SymbolicRef(name, d); - } - - private Command create(String name, ObjectId id) - throws MissingObjectException, IOException { - return update(name, ObjectId.zeroId(), id); - } - - private Command update(String name, ObjectId oldId, ObjectId newId) - throws MissingObjectException, IOException { - try (RevWalk rw = new RevWalk(repo)) { - return new Command(rw, new ReceiveCommand(oldId, newId, name)); - } - } - - private ObjectId write(RefTree tree) throws IOException { - try (ObjectInserter ins = repo.newObjectInserter()) { - ObjectId id = tree.writeTree(ins); - ins.flush(); - return id; - } - } -} diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 035ed37c10..cc3ffb24d1 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,6 +1,12 @@ + + + + + + diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index c85cb8936b..9dc26ecb49 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -72,10 +72,6 @@ Export-Package: org.eclipse.jgit.annotations;version="5.11.0", org.eclipse.jgit.http.test", org.eclipse.jgit.internal.fsck;version="5.11.0"; x-friends:="org.eclipse.jgit.test", - org.eclipse.jgit.internal.ketch;version="5.11.0"; - x-friends:="org.eclipse.jgit.junit, - org.eclipse.jgit.test, - org.eclipse.jgit.pgm", org.eclipse.jgit.internal.revwalk;version="5.11.0";x-internal:=true, org.eclipse.jgit.internal.storage.dfs;version="5.11.0"; x-friends:="org.eclipse.jgit.test, @@ -104,10 +100,6 @@ Export-Package: org.eclipse.jgit.annotations;version="5.11.0", org.eclipse.jgit.junit, org.eclipse.jgit.test, org.eclipse.jgit.pgm", - org.eclipse.jgit.internal.storage.reftree;version="5.11.0"; - x-friends:="org.eclipse.jgit.junit, - org.eclipse.jgit.test, - org.eclipse.jgit.pgm", org.eclipse.jgit.internal.submodule;version="5.11.0";x-internal:=true, org.eclipse.jgit.internal.transport.connectivity;version="5.11.0"; x-friends:="org.eclipse.jgit.test", diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties deleted file mode 100644 index 1fbb7cb3b5..0000000000 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/ketch/KetchText.properties +++ /dev/null @@ -1,13 +0,0 @@ -accepted=accepted. -cannotFetchFromLocalReplica=cannot fetch from LocalReplica -failed=failed! -invalidFollowerUri=invalid follower URI -leaderFailedToStore=leader failed to store -localReplicaRequired=LocalReplica instance is required -mismatchedTxnNamespace=mismatched txnNamespace; expected {0} found {1} -outsideTxnNamespace=ref {0} is outside of txnNamespace {1} -proposingUpdates=Proposing updates -queuedProposalFailedToApply=queued proposal failed to apply -starting=starting! -unsupportedVoterCount=unsupported voter count {0}, expected one of {1} -waitingForQueue=Waiting for queue diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java deleted file mode 100644 index 5ddbcbd0ed..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ElectionRound.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.KetchConstants.TERM; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeoutException; - -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.TreeFormatter; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.time.ProposedTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The initial {@link Round} for a leaderless repository, used to establish a - * leader. - */ -class ElectionRound extends Round { - private static final Logger log = LoggerFactory.getLogger(ElectionRound.class); - - private long term; - - ElectionRound(KetchLeader leader, LogIndex head) { - super(leader, head); - } - - @Override - void start() throws IOException { - ObjectId id; - try (Repository git = leader.openRepository(); - ProposedTimestamp ts = getSystem().getClock().propose(); - ObjectInserter inserter = git.newObjectInserter()) { - id = bumpTerm(git, ts, inserter); - inserter.flush(); - blockUntil(ts); - } - runAsync(id); - } - - @Override - void success() { - // Do nothing upon election, KetchLeader will copy the term. - } - - long getTerm() { - return term; - } - - private ObjectId bumpTerm(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException { - CommitBuilder b = new CommitBuilder(); - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(acceptedOldIndex); - if (getSystem().requireMonotonicLeaderElections()) { - if (ts.read(SECONDS) < c.getCommitTime()) { - throw new TimeIsUncertainException(); - } - } - b.setTreeId(c.getTree()); - b.setParentId(acceptedOldIndex); - term = parseTerm(c.getFooterLines(TERM)) + 1; - } - } else { - term = 1; - b.setTreeId(inserter.insert(new TreeFormatter())); - } - - StringBuilder msg = new StringBuilder(); - msg.append(KetchConstants.TERM.getName()) - .append(": ") //$NON-NLS-1$ - .append(term); - - String tag = leader.getSystem().newLeaderTag(); - if (tag != null && !tag.isEmpty()) { - msg.append(' ').append(tag); - } - - b.setAuthor(leader.getSystem().newCommitter(ts)); - b.setCommitter(b.getAuthor()); - b.setMessage(msg.toString()); - - if (log.isDebugEnabled()) { - log.debug("Trying to elect myself " + b.getMessage()); //$NON-NLS-1$ - } - return inserter.insert(b); - } - - private static long parseTerm(List footer) { - if (footer.isEmpty()) { - return 0; - } - - String s = footer.get(0); - int p = s.indexOf(' '); - if (p > 0) { - s = s.substring(0, p); - } - return Long.parseLong(s, 10); - } - - private void blockUntil(ProposedTimestamp ts) throws IOException { - try { - ts.blockUntil(getSystem().getMaxWaitForMonotonicClock()); - } catch (InterruptedException | TimeoutException e) { - throw new TimeIsUncertainException(e); - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java deleted file mode 100644 index f4a7f592f0..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchConstants.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.revwalk.FooterKey; - -/** - * Frequently used constants in a Ketch system. - */ -public class KetchConstants { - /** - * Default reference namespace holding {@link #ACCEPTED} and - * {@link #COMMITTED} references and the {@link #STAGE} sub-namespace. - */ - public static final String DEFAULT_TXN_NAMESPACE = "refs/txn/"; //$NON-NLS-1$ - - /** Reference name holding the RefTree accepted by a follower. */ - public static final String ACCEPTED = "accepted"; //$NON-NLS-1$ - - /** Reference name holding the RefTree known to be committed. */ - public static final String COMMITTED = "committed"; //$NON-NLS-1$ - - /** Reference subdirectory holding proposed heads. */ - public static final String STAGE = "stage/"; //$NON-NLS-1$ - - /** Footer containing the current term. */ - public static final FooterKey TERM = new FooterKey("Term"); //$NON-NLS-1$ - - /** Section for Ketch configuration ({@code ketch}). */ - public static final String CONFIG_SECTION_KETCH = "ketch"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-type}) */ - public static final String CONFIG_KEY_TYPE = "ketch-type"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-commit}) */ - public static final String CONFIG_KEY_COMMIT = "ketch-commit"; //$NON-NLS-1$ - - /** Behavior for a replica ({@code remote.$name.ketch-speed}) */ - public static final String CONFIG_KEY_SPEED = "ketch-speed"; //$NON-NLS-1$ - - private KetchConstants() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java deleted file mode 100644 index 743d1939c8..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeader.java +++ /dev/null @@ -1,604 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.CANDIDATE; -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.LEADER; -import static org.eclipse.jgit.internal.ketch.KetchLeader.State.SHUTDOWN; -import static org.eclipse.jgit.internal.ketch.KetchReplica.Participation.FOLLOWER_ONLY; -import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A leader managing consensus across remote followers. - *

- * A leader instance starts up in - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#CANDIDATE} and tries - * to begin a new term by sending an - * {@link org.eclipse.jgit.internal.ketch.ElectionRound} to all replicas. Its - * term starts if a majority of replicas have accepted this leader instance for - * the term. - *

- * Once elected by a majority the instance enters - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER} and runs - * proposals offered to {@link #queueProposal(Proposal)}. This continues until - * the leader is timed out for inactivity, or is deposed by a competing leader - * gaining its own majority. - *

- * Once timed out or deposed this {@code KetchLeader} instance should be - * discarded, and a new instance takes over. - *

- * Each leader instance coordinates a group of - * {@link org.eclipse.jgit.internal.ketch.KetchReplica}s. Replica instances are - * owned by the leader instance and must be discarded when the leader is - * discarded. - *

- * In Ketch all push requests are issued through the leader. The steps are as - * follows (see {@link org.eclipse.jgit.internal.ketch.KetchPreReceive} for an - * example): - *

    - *
  • Create a {@link org.eclipse.jgit.internal.ketch.Proposal} with the - * {@link org.eclipse.jgit.transport.ReceiveCommand}s that represent the push. - *
  • Invoke {@link #queueProposal(Proposal)} on the leader instance. - *
  • Wait for consensus with - * {@link org.eclipse.jgit.internal.ketch.Proposal#await()}. - *
  • To examine the status of the push, check - * {@link org.eclipse.jgit.internal.ketch.Proposal#getCommands()}, looking at - * {@link org.eclipse.jgit.internal.storage.reftree.Command#getResult()}. - *
- *

- * The leader gains consensus by first pushing the needed objects and a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree} representing the - * desired target repository state to the {@code refs/txn/accepted} branch on - * each of the replicas. Once a majority has succeeded, the leader commits the - * state by either pushing the {@code refs/txn/accepted} value to - * {@code refs/txn/committed} (for Ketch-aware replicas) or by pushing updates - * to {@code refs/heads/master}, etc. for stock Git replicas. - *

- * Internally, the actual transport to replicas is performed on background - * threads via the {@link org.eclipse.jgit.internal.ketch.KetchSystem}'s - * executor service. For performance, the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}, - * {@link org.eclipse.jgit.internal.ketch.KetchReplica} and - * {@link org.eclipse.jgit.internal.ketch.Proposal} objects share some state, - * and may invoke each other's methods on different threads. This access is - * protected by the leader's {@link #lock} object. Care must be taken to prevent - * concurrent access by correctly obtaining the leader's lock. - */ -public abstract class KetchLeader { - private static final Logger log = LoggerFactory.getLogger(KetchLeader.class); - - /** Current state of the leader instance. */ - public enum State { - /** Newly created instance trying to elect itself leader. */ - CANDIDATE, - - /** Leader instance elected by a majority. */ - LEADER, - - /** Instance has been deposed by another with a more recent term. */ - DEPOSED, - - /** Leader has been gracefully shutdown, e.g. due to inactivity. */ - SHUTDOWN; - } - - private final KetchSystem system; - - /** Leader's knowledge of replicas for this repository. */ - private KetchReplica[] voters; - private KetchReplica[] followers; - private LocalReplica self; - - /** - * Lock protecting all data within this leader instance. - *

- * This lock extends into the {@link KetchReplica} instances used by the - * leader. They share the same lock instance to simplify concurrency. - */ - final Lock lock; - - private State state = CANDIDATE; - - /** Term of this leader, once elected. */ - private long term; - - /** - * Pending proposals accepted into the queue in FIFO order. - *

- * These proposals were preflighted and do not contain any conflicts with - * each other and their expectations matched the leader's local view of the - * agreed upon {@code refs/txn/accepted} tree. - */ - private final List queued; - - /** - * State of the repository's RefTree after applying all entries in - * {@link #queued}. New proposals must be consistent with this tree to be - * appended to the end of {@link #queued}. - *

- * Must be deep-copied with {@link RefTree#copy()} if - * {@link #roundHoldsReferenceToRefTree} is {@code true}. - */ - private RefTree refTree; - - /** - * If {@code true} {@link #refTree} must be duplicated before queuing the - * next proposal. The {@link #refTree} was passed into the constructor of a - * {@link ProposalRound}, and that external reference to the {@link RefTree} - * object is held by the proposal until it materializes the tree object in - * the object store. This field is set {@code true} when the proposal begins - * execution and set {@code false} once tree objects are persisted in the - * local repository's object store or {@link #refTree} is replaced with a - * copy to isolate it from any running rounds. - *

- * If proposals arrive less frequently than the {@code RefTree} is written - * out to the repository the {@link #roundHoldsReferenceToRefTree} behavior - * avoids duplicating {@link #refTree}, reducing both time and memory used. - * However if proposals arrive more frequently {@link #refTree} must be - * duplicated to prevent newly queued proposals from corrupting the - * {@link #runningRound}. - */ - volatile boolean roundHoldsReferenceToRefTree; - - /** End of the leader's log. */ - private LogIndex headIndex; - - /** Leader knows this (and all prior) states are committed. */ - private LogIndex committedIndex; - - /** - * Is the leader idle with no work pending? If {@code true} there is no work - * for the leader (normal state). This field is {@code false} when the - * leader thread is scheduled for execution, or while {@link #runningRound} - * defines a round in progress. - */ - private boolean idle; - - /** Current round the leader is preparing and waiting for a vote on. */ - private Round runningRound; - - /** - * Construct a leader for a Ketch instance. - * - * @param system - * Ketch system configuration the leader must adhere to. - */ - protected KetchLeader(KetchSystem system) { - this.system = system; - this.lock = new ReentrantLock(true /* fair */); - this.queued = new ArrayList<>(4); - this.idle = true; - } - - /** @return system configuration. */ - KetchSystem getSystem() { - return system; - } - - /** - * Configure the replicas used by this Ketch instance. - *

- * Replicas should be configured once at creation before any proposals are - * executed. Once elections happen, reconfiguration is a complicated - * concept that is not currently supported. - * - * @param replicas - * members participating with the same repository. - */ - public void setReplicas(Collection replicas) { - List v = new ArrayList<>(5); - List f = new ArrayList<>(5); - for (KetchReplica r : replicas) { - switch (r.getParticipation()) { - case FULL: - v.add(r); - break; - - case FOLLOWER_ONLY: - f.add(r); - break; - } - } - - Collection validVoters = validVoterCounts(); - if (!validVoters.contains(Integer.valueOf(v.size()))) { - throw new IllegalArgumentException(MessageFormat.format( - KetchText.get().unsupportedVoterCount, - Integer.valueOf(v.size()), - validVoters)); - } - - LocalReplica me = findLocal(v); - if (me == null) { - throw new IllegalArgumentException( - KetchText.get().localReplicaRequired); - } - - lock.lock(); - try { - voters = v.toArray(new KetchReplica[0]); - followers = f.toArray(new KetchReplica[0]); - self = me; - } finally { - lock.unlock(); - } - } - - private static Collection validVoterCounts() { - @SuppressWarnings("boxing") - Integer[] valid = { - // An odd number of voting replicas is required. - 1, 3, 5, 7, 9 }; - return Arrays.asList(valid); - } - - private static LocalReplica findLocal(Collection voters) { - for (KetchReplica r : voters) { - if (r instanceof LocalReplica) { - return (LocalReplica) r; - } - } - return null; - } - - /** - * Get an instance of the repository for use by a leader thread. - *

- * The caller will close the repository. - * - * @return opened repository for use by the leader thread. - * @throws java.io.IOException - * cannot reopen the repository for the leader. - */ - protected abstract Repository openRepository() throws IOException; - - /** - * Queue a reference update proposal for consensus. - *

- * This method does not wait for consensus to be reached. The proposal is - * checked to look for risks of conflicts, and then submitted into the queue - * for distribution as soon as possible. - *

- * Callers must use {@link org.eclipse.jgit.internal.ketch.Proposal#await()} - * to see if the proposal is done. - * - * @param proposal - * the proposed reference updates to queue for consideration. - * Once execution is complete the individual reference result - * fields will be populated with the outcome. - * @throws java.lang.InterruptedException - * current thread was interrupted. The proposal may have been - * aborted if it was not yet queued for execution. - * @throws java.io.IOException - * unrecoverable error preventing proposals from being attempted - * by this leader. - */ - public void queueProposal(Proposal proposal) - throws InterruptedException, IOException { - try { - lock.lockInterruptibly(); - } catch (InterruptedException e) { - proposal.abort(); - throw e; - } - try { - if (refTree == null) { - initialize(); - for (Proposal p : queued) { - refTree.apply(p.getCommands()); - } - } else if (roundHoldsReferenceToRefTree) { - refTree = refTree.copy(); - roundHoldsReferenceToRefTree = false; - } - - if (!refTree.apply(proposal.getCommands())) { - // A conflict exists so abort the proposal. - proposal.abort(); - return; - } - - queued.add(proposal); - proposal.notifyState(QUEUED); - - if (idle) { - scheduleLeader(); - } - } finally { - lock.unlock(); - } - } - - private void initialize() throws IOException { - try (Repository git = openRepository(); RevWalk rw = new RevWalk(git)) { - self.initialize(git); - - ObjectId accepted = self.getTxnAccepted(); - if (!ObjectId.zeroId().equals(accepted)) { - RevCommit c = rw.parseCommit(accepted); - headIndex = LogIndex.unknown(accepted); - refTree = RefTree.read(rw.getObjectReader(), c.getTree()); - } else { - headIndex = LogIndex.unknown(ObjectId.zeroId()); - refTree = RefTree.newEmptyTree(); - } - } - } - - private void scheduleLeader() { - idle = false; - system.getExecutor().execute(this::runLeader); - } - - private void runLeader() { - Round round; - lock.lock(); - try { - switch (state) { - case CANDIDATE: - round = new ElectionRound(this, headIndex); - break; - - case LEADER: - round = newProposalRound(); - break; - - case DEPOSED: - case SHUTDOWN: - default: - log.warn("Leader cannot run {}", state); //$NON-NLS-1$ - // TODO(sop): Redirect proposals. - return; - } - } finally { - lock.unlock(); - } - - try { - round.start(); - } catch (IOException e) { - // TODO(sop) Depose leader if it cannot use its repository. - log.error(KetchText.get().leaderFailedToStore, e); - lock.lock(); - try { - nextRound(); - } finally { - lock.unlock(); - } - } - } - - private ProposalRound newProposalRound() { - List todo = new ArrayList<>(queued); - queued.clear(); - roundHoldsReferenceToRefTree = true; - return new ProposalRound(this, headIndex, todo, refTree); - } - - /** @return term of this leader's reign. */ - long getTerm() { - return term; - } - - /** @return end of the leader's log. */ - LogIndex getHead() { - return headIndex; - } - - /** - * @return state leader knows it has committed across a quorum of replicas. - */ - LogIndex getCommitted() { - return committedIndex; - } - - boolean isIdle() { - return idle; - } - - void runAsync(Round round) { - lock.lock(); - try { - // End of the log is this round. Once transport begins it is - // reasonable to assume at least one replica will eventually get - // this, and there is reasonable probability it commits. - headIndex = round.acceptedNewIndex; - runningRound = round; - - for (KetchReplica replica : voters) { - replica.pushTxnAcceptedAsync(round); - } - for (KetchReplica replica : followers) { - replica.pushTxnAcceptedAsync(round); - } - } finally { - lock.unlock(); - } - } - - /** - * Asynchronous signal from a replica after completion. - *

- * Must be called while {@link #lock} is held by the replica. - * - * @param replica - * replica posting a completion event. - */ - void onReplicaUpdate(KetchReplica replica) { - if (log.isDebugEnabled()) { - log.debug("Replica {} finished:\n{}", //$NON-NLS-1$ - replica.describeForLog(), snapshot()); - } - - if (replica.getParticipation() == FOLLOWER_ONLY) { - // Followers cannot vote, so votes haven't changed. - return; - } else if (runningRound == null) { - // No round running, no need to tally votes. - return; - } - - assert headIndex.equals(runningRound.acceptedNewIndex); - int matching = 0; - for (KetchReplica r : voters) { - if (r.hasAccepted(headIndex)) { - matching++; - } - } - - int quorum = voters.length / 2 + 1; - boolean success = matching >= quorum; - if (!success) { - return; - } - - switch (state) { - case CANDIDATE: - term = ((ElectionRound) runningRound).getTerm(); - state = LEADER; - if (log.isDebugEnabled()) { - log.debug("Won election, running term " + term); //$NON-NLS-1$ - } - - //$FALL-THROUGH$ - case LEADER: - committedIndex = headIndex; - if (log.isDebugEnabled()) { - log.debug("Committed {} in term {}", //$NON-NLS-1$ - committedIndex.describeForLog(), - Long.valueOf(term)); - } - nextRound(); - commitAsync(replica); - notifySuccess(runningRound); - if (log.isDebugEnabled()) { - log.debug("Leader state:\n{}", snapshot()); //$NON-NLS-1$ - } - break; - - default: - log.debug("Leader ignoring replica while in {}", state); //$NON-NLS-1$ - break; - } - } - - private void notifySuccess(Round round) { - // Drop the leader lock while notifying Proposal listeners. - lock.unlock(); - try { - round.success(); - } finally { - lock.lock(); - } - } - - private void commitAsync(KetchReplica caller) { - for (KetchReplica r : voters) { - if (r == caller) { - continue; - } - if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { - r.pushCommitAsync(committedIndex); - } - } - for (KetchReplica r : followers) { - if (r == caller) { - continue; - } - if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { - r.pushCommitAsync(committedIndex); - } - } - } - - /** Schedule the next round; invoked while {@link #lock} is held. */ - void nextRound() { - runningRound = null; - - if (queued.isEmpty()) { - idle = true; - } else { - // Caller holds lock. Reschedule leader on a new thread so - // the call stack can unwind and lock is not held unexpectedly - // during prepare for the next round. - scheduleLeader(); - } - } - - /** - * Snapshot this leader - * - * @return snapshot of this leader - */ - public LeaderSnapshot snapshot() { - lock.lock(); - try { - LeaderSnapshot s = new LeaderSnapshot(); - s.state = state; - s.term = term; - s.headIndex = headIndex; - s.committedIndex = committedIndex; - s.idle = isIdle(); - for (KetchReplica r : voters) { - s.replicas.add(r.snapshot()); - } - for (KetchReplica r : followers) { - s.replicas.add(r.snapshot()); - } - return s; - } finally { - lock.unlock(); - } - } - - /** - * Gracefully shutdown this leader and cancel outstanding operations. - */ - public void shutdown() { - lock.lock(); - try { - if (state != SHUTDOWN) { - state = SHUTDOWN; - for (KetchReplica r : voters) { - r.shutdown(); - } - for (KetchReplica r : followers) { - r.shutdown(); - } - } - } finally { - lock.unlock(); - } - } - - /** {@inheritDoc} */ - @Override - public String toString() { - return snapshot().toString(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java deleted file mode 100644 index e01fb3ae53..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchLeaderCache.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.net.URISyntaxException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.eclipse.jgit.internal.storage.dfs.DfsRepository; -import org.eclipse.jgit.lib.Repository; - -/** - * A cache of live leader instances, keyed by repository. - *

- * Ketch only assigns a leader to a repository when needed. If - * {@link #get(Repository)} is called for a repository that does not have a - * leader, the leader is created and added to the cache. - */ -public class KetchLeaderCache { - private final KetchSystem system; - private final ConcurrentMap leaders; - private final Lock startLock; - - /** - * Initialize a new leader cache. - * - * @param system - * system configuration for the leaders - */ - public KetchLeaderCache(KetchSystem system) { - this.system = system; - leaders = new ConcurrentHashMap<>(); - startLock = new ReentrantLock(true /* fair */); - } - - /** - * Lookup the leader instance for a given repository. - * - * @param repo - * repository to get the leader for. - * @return the leader instance for the repository. - * @throws java.net.URISyntaxException - * remote configuration contains an invalid URL. - */ - public KetchLeader get(Repository repo) - throws URISyntaxException { - String key = computeKey(repo); - KetchLeader leader = leaders.get(key); - if (leader != null) { - return leader; - } - return startLeader(key, repo); - } - - private KetchLeader startLeader(String key, Repository repo) - throws URISyntaxException { - startLock.lock(); - try { - KetchLeader leader = leaders.get(key); - if (leader != null) { - return leader; - } - leader = system.createLeader(repo); - leaders.put(key, leader); - return leader; - } finally { - startLock.unlock(); - } - } - - private static String computeKey(Repository repo) { - if (repo instanceof DfsRepository) { - DfsRepository dfs = (DfsRepository) repo; - return dfs.getDescription().getRepositoryName(); - } - - if (repo.getDirectory() != null) { - return repo.getDirectory().toURI().toString(); - } - - throw new IllegalArgumentException(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java deleted file mode 100644 index 1c9535f7be..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchPreReceive.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.Collection; - -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.transport.PreReceiveHook; -import org.eclipse.jgit.transport.ProgressSpinner; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.ReceivePack; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * PreReceiveHook for handling push traffic in a Ketch system. - *

- * Install an instance on {@link org.eclipse.jgit.transport.ReceivePack} to - * capture the commands and other connection state and relay them through the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}, allowing the leader to - * gain consensus about the new reference state. - */ -public class KetchPreReceive implements PreReceiveHook { - private static final Logger log = LoggerFactory.getLogger(KetchPreReceive.class); - - private final KetchLeader leader; - - /** - * Construct a hook executing updates through a - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. - * - * @param leader - * leader for this repository. - */ - public KetchPreReceive(KetchLeader leader) { - this.leader = leader; - } - - /** {@inheritDoc} */ - @Override - public void onPreReceive(ReceivePack rp, Collection cmds) { - cmds = ReceiveCommand.filter(cmds, NOT_ATTEMPTED); - if (cmds.isEmpty()) { - return; - } - - try { - Proposal proposal = new Proposal(rp.getRevWalk(), cmds) - .setPushCertificate(rp.getPushCertificate()) - .setAuthor(rp.getRefLogIdent()) - .setMessage("push"); //$NON-NLS-1$ - leader.queueProposal(proposal); - if (proposal.isDone()) { - // This failed fast, e.g. conflict or bad precondition. - return; - } - - ProgressSpinner spinner = new ProgressSpinner( - rp.getMessageOutputStream()); - if (proposal.getState() == QUEUED) { - waitForQueue(proposal, spinner); - } - if (!proposal.isDone()) { - waitForPropose(proposal, spinner); - } - } catch (IOException | InterruptedException e) { - String msg = JGitText.get().transactionAborted; - for (ReceiveCommand cmd : cmds) { - if (cmd.getResult() == NOT_ATTEMPTED) { - cmd.setResult(REJECTED_OTHER_REASON, msg); - } - } - log.error(msg, e); - } - } - - private void waitForQueue(Proposal proposal, ProgressSpinner spinner) - throws InterruptedException { - spinner.beginTask(KetchText.get().waitingForQueue, 1, SECONDS); - while (!proposal.awaitStateChange(QUEUED, 250, MILLISECONDS)) { - spinner.update(); - } - switch (proposal.getState()) { - case RUNNING: - default: - spinner.endTask(KetchText.get().starting); - break; - - case EXECUTED: - spinner.endTask(KetchText.get().accepted); - break; - - case ABORTED: - spinner.endTask(KetchText.get().failed); - break; - } - } - - private void waitForPropose(Proposal proposal, ProgressSpinner spinner) - throws InterruptedException { - spinner.beginTask(KetchText.get().proposingUpdates, 2, SECONDS); - while (!proposal.await(250, MILLISECONDS)) { - spinner.update(); - } - spinner.endTask(proposal.getState() == EXECUTED - ? KetchText.get().accepted - : KetchText.get().failed); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java deleted file mode 100644 index a9a694a565..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchReplica.java +++ /dev/null @@ -1,758 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.BATCHED; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.FAST; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.CURRENT; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.CREATE; - -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.Future; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.SystemReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A Ketch replica, either {@link org.eclipse.jgit.internal.ketch.LocalReplica} - * or {@link org.eclipse.jgit.internal.ketch.RemoteGitReplica}. - *

- * Replicas can be either a stock Git replica, or a Ketch-aware replica. - *

- * A stock Git replica has no special knowledge of Ketch and simply stores - * objects and references. Ketch communicates with the stock Git replica using - * the Git push wire protocol. The - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} commits an agreed upon - * state by pushing all references to the Git replica, for example - * {@code "refs/heads/master"} is pushed during commit. Stock Git replicas use - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS} to - * record the final state. - *

- * Ketch-aware replicas understand the {@code RefTree} sent during the proposal - * and during commit are able to update their own reference space to match the - * state represented by the {@code RefTree}. Ketch-aware replicas typically use - * a {@link org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase} and - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#TXN_COMMITTED} - * to record the final state. - *

- * KetchReplica instances are tightly coupled with a single - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. Some state may be - * accessed by the leader thread and uses the leader's own - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} to protect shared - * data. - */ -public abstract class KetchReplica { - static final Logger log = LoggerFactory.getLogger(KetchReplica.class); - private static final byte[] PEEL = { ' ', '^' }; - - /** Participation of a replica in establishing consensus. */ - public enum Participation { - /** Replica can vote. */ - FULL, - - /** Replica does not vote, but tracks leader. */ - FOLLOWER_ONLY; - } - - /** How this replica wants to receive Ketch commit operations. */ - public enum CommitMethod { - /** All references are pushed to the peer as standard Git. */ - ALL_REFS, - - /** Only {@code refs/txn/committed} is written/updated. */ - TXN_COMMITTED; - } - - /** Delay before committing to a replica. */ - public enum CommitSpeed { - /** - * Send the commit immediately, even if it could be batched with the - * next proposal. - */ - FAST, - - /** - * If the next proposal is available, batch the commit with it, - * otherwise just send the commit. This generates less network use, but - * may provide slower consistency on the replica. - */ - BATCHED; - } - - /** Current state of a replica. */ - public enum State { - /** Leader has not yet contacted the replica. */ - UNKNOWN, - - /** Replica is behind the consensus. */ - LAGGING, - - /** Replica matches the consensus. */ - CURRENT, - - /** Replica has a different (or unknown) history. */ - DIVERGENT, - - /** Replica's history contains the leader's history. */ - AHEAD, - - /** Replica can not be contacted. */ - OFFLINE; - } - - private final KetchLeader leader; - private final String replicaName; - private final Participation participation; - private final CommitMethod commitMethod; - private final CommitSpeed commitSpeed; - private final long minRetryMillis; - private final long maxRetryMillis; - private final Map> staged; - private final Map running; - private final Map waiting; - private final List queued; - - /** - * Value known for {@code "refs/txn/accepted"}. - *

- * Raft literature refers to this as {@code matchIndex}. - */ - private ObjectId txnAccepted; - - /** - * Value known for {@code "refs/txn/committed"}. - *

- * Raft literature refers to this as {@code commitIndex}. In traditional - * Raft this is a state variable inside the follower implementation, but - * Ketch keeps it in the leader. - */ - private ObjectId txnCommitted; - - /** What is happening with this replica. */ - private State state = UNKNOWN; - private String error; - - /** Scheduled retry due to communication failure. */ - private Future retryFuture; - private long lastRetryMillis; - private long retryAtMillis; - - /** - * Configure a replica representation. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this replica for debugging. - * @param cfg - * how Ketch should treat the replica. - */ - protected KetchReplica(KetchLeader leader, String name, ReplicaConfig cfg) { - this.leader = leader; - this.replicaName = name; - this.participation = cfg.getParticipation(); - this.commitMethod = cfg.getCommitMethod(); - this.commitSpeed = cfg.getCommitSpeed(); - this.minRetryMillis = cfg.getMinRetry(MILLISECONDS); - this.maxRetryMillis = cfg.getMaxRetry(MILLISECONDS); - this.staged = new HashMap<>(); - this.running = new HashMap<>(); - this.waiting = new HashMap<>(); - this.queued = new ArrayList<>(4); - } - - /** - * Get system configuration. - * - * @return system configuration. - */ - public KetchSystem getSystem() { - return getLeader().getSystem(); - } - - /** - * Get leader instance this replica follows. - * - * @return leader instance this replica follows. - */ - public KetchLeader getLeader() { - return leader; - } - - /** - * Get unique-ish name for debugging. - * - * @return unique-ish name for debugging. - */ - public String getName() { - return replicaName; - } - - /** - * Get description of this replica for error/debug logging purposes. - * - * @return description of this replica for error/debug logging purposes. - */ - protected String describeForLog() { - return getName(); - } - - /** - * Get how the replica participates in this Ketch system. - * - * @return how the replica participates in this Ketch system. - */ - public Participation getParticipation() { - return participation; - } - - /** - * Get how Ketch will commit to the repository. - * - * @return how Ketch will commit to the repository. - */ - public CommitMethod getCommitMethod() { - return commitMethod; - } - - /** - * Get when Ketch will commit to the repository. - * - * @return when Ketch will commit to the repository. - */ - public CommitSpeed getCommitSpeed() { - return commitSpeed; - } - - /** - * Called by leader to perform graceful shutdown. - *

- * Default implementation cancels any scheduled retry. Subclasses may add - * additional logic before or after calling {@code super.shutdown()}. - *

- * Called with {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held - * by caller. - */ - protected void shutdown() { - Future f = retryFuture; - if (f != null) { - retryFuture = null; - f.cancel(true); - } - } - - ReplicaSnapshot snapshot() { - ReplicaSnapshot s = new ReplicaSnapshot(this); - s.accepted = txnAccepted; - s.committed = txnCommitted; - s.state = state; - s.error = error; - s.retryAtMillis = waitingForRetry() ? retryAtMillis : 0; - return s; - } - - /** - * Update the leader's view of the replica after a poll. - *

- * Called with {@link KetchLeader#lock} held by caller. - * - * @param refs - * map of refs from the replica. - */ - void initialize(Map refs) { - if (txnAccepted == null) { - txnAccepted = getId(refs.get(getSystem().getTxnAccepted())); - } - if (txnCommitted == null) { - txnCommitted = getId(refs.get(getSystem().getTxnCommitted())); - } - } - - ObjectId getTxnAccepted() { - return txnAccepted; - } - - boolean hasAccepted(LogIndex id) { - return equals(txnAccepted, id); - } - - private static boolean equals(@Nullable ObjectId a, LogIndex b) { - return a != null && b != null && AnyObjectId.isEqual(a, b); - } - - /** - * Schedule a proposal round with the replica. - *

- * Called with {@link KetchLeader#lock} held by caller. - * - * @param round - * current round being run by the leader. - */ - void pushTxnAcceptedAsync(Round round) { - List cmds = new ArrayList<>(); - if (commitSpeed == BATCHED) { - LogIndex committedIndex = leader.getCommitted(); - if (equals(txnAccepted, committedIndex) - && !equals(txnCommitted, committedIndex)) { - prepareTxnCommitted(cmds, committedIndex); - } - } - - // TODO(sop) Lagging replicas should build accept on the fly. - if (round.stageCommands != null) { - for (ReceiveCommand cmd : round.stageCommands) { - // TODO(sop): Do not send certain object graphs to replica. - cmds.add(copy(cmd)); - } - } - cmds.add(new ReceiveCommand( - round.acceptedOldIndex, round.acceptedNewIndex, - getSystem().getTxnAccepted())); - pushAsync(new ReplicaPushRequest(this, cmds)); - } - - private static ReceiveCommand copy(ReceiveCommand c) { - return new ReceiveCommand(c.getOldId(), c.getNewId(), c.getRefName()); - } - - boolean shouldPushUnbatchedCommit(LogIndex committed, boolean leaderIdle) { - return (leaderIdle || commitSpeed == FAST) && hasAccepted(committed); - } - - void pushCommitAsync(LogIndex committed) { - List cmds = new ArrayList<>(); - prepareTxnCommitted(cmds, committed); - pushAsync(new ReplicaPushRequest(this, cmds)); - } - - private void prepareTxnCommitted(List cmds, - ObjectId committed) { - removeStaged(cmds, committed); - cmds.add(new ReceiveCommand( - txnCommitted, committed, - getSystem().getTxnCommitted())); - } - - private void removeStaged(List cmds, ObjectId committed) { - List a = staged.remove(committed); - if (a != null) { - delete(cmds, a); - } - if (staged.isEmpty() || !(committed instanceof LogIndex)) { - return; - } - - LogIndex committedIndex = (LogIndex) committed; - Iterator>> itr = staged - .entrySet().iterator(); - while (itr.hasNext()) { - Map.Entry> e = itr.next(); - if (e.getKey() instanceof LogIndex) { - LogIndex stagedIndex = (LogIndex) e.getKey(); - if (stagedIndex.isBefore(committedIndex)) { - delete(cmds, e.getValue()); - itr.remove(); - } - } - } - } - - private static void delete(List cmds, - List createCmds) { - for (ReceiveCommand cmd : createCmds) { - ObjectId id = cmd.getNewId(); - String name = cmd.getRefName(); - cmds.add(new ReceiveCommand(id, ObjectId.zeroId(), name)); - } - } - - /** - * Determine the next push for this replica (if any) and start it. - *

- * If the replica has successfully accepted the committed state of the - * leader, this method will push all references to the replica using the - * configured {@link CommitMethod}. - *

- * If the replica is {@link State#LAGGING} this method will begin catch up - * by sending a more recent {@code refs/txn/accepted}. - *

- * Must be invoked with {@link KetchLeader#lock} held by caller. - */ - private void runNextPushRequest() { - LogIndex committed = leader.getCommitted(); - if (!equals(txnCommitted, committed) - && shouldPushUnbatchedCommit(committed, leader.isIdle())) { - pushCommitAsync(committed); - } - - if (queued.isEmpty() || !running.isEmpty() || waitingForRetry()) { - return; - } - - // Collapse all queued requests into a single request. - Map cmdMap = new HashMap<>(); - for (ReplicaPushRequest req : queued) { - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - ReceiveCommand old = cmdMap.remove(name); - if (old != null) { - cmd = new ReceiveCommand( - old.getOldId(), cmd.getNewId(), - name); - } - cmdMap.put(name, cmd); - } - } - queued.clear(); - waiting.clear(); - - List next = new ArrayList<>(cmdMap.values()); - for (ReceiveCommand cmd : next) { - running.put(cmd.getRefName(), cmd); - } - startPush(new ReplicaPushRequest(this, next)); - } - - private void pushAsync(ReplicaPushRequest req) { - if (defer(req)) { - // TODO(sop) Collapse during long retry outage. - for (ReceiveCommand cmd : req.getCommands()) { - waiting.put(cmd.getRefName(), cmd); - } - queued.add(req); - } else { - for (ReceiveCommand cmd : req.getCommands()) { - running.put(cmd.getRefName(), cmd); - } - startPush(req); - } - } - - private boolean defer(ReplicaPushRequest req) { - if (waitingForRetry()) { - // Prior communication failure; everything is deferred. - return true; - } - - for (ReceiveCommand nextCmd : req.getCommands()) { - ReceiveCommand priorCmd = waiting.get(nextCmd.getRefName()); - if (priorCmd == null) { - priorCmd = running.get(nextCmd.getRefName()); - } - if (priorCmd != null) { - // Another request pending on same ref; that must go first. - // Verify priorCmd.newId == nextCmd.oldId? - return true; - } - } - return false; - } - - private boolean waitingForRetry() { - Future f = retryFuture; - return f != null && !f.isDone(); - } - - private void retryLater(ReplicaPushRequest req) { - Collection cmds = req.getCommands(); - for (ReceiveCommand cmd : cmds) { - cmd.setResult(NOT_ATTEMPTED, null); - if (!waiting.containsKey(cmd.getRefName())) { - waiting.put(cmd.getRefName(), cmd); - } - } - queued.add(0, new ReplicaPushRequest(this, cmds)); - - if (!waitingForRetry()) { - long delay = FileUtils - .delay(lastRetryMillis, minRetryMillis, maxRetryMillis); - if (log.isDebugEnabled()) { - log.debug("Retrying {} after {} ms", //$NON-NLS-1$ - describeForLog(), Long.valueOf(delay)); - } - lastRetryMillis = delay; - retryAtMillis = SystemReader.getInstance().getCurrentTime() + delay; - retryFuture = getSystem().getExecutor() - .schedule(new WeakRetryPush(this), delay, MILLISECONDS); - } - } - - /** Weakly holds a retrying replica, allowing it to garbage collect. */ - static class WeakRetryPush extends WeakReference - implements Callable { - WeakRetryPush(KetchReplica r) { - super(r); - } - - @Override - public Void call() throws Exception { - KetchReplica r = get(); - if (r != null) { - r.doRetryPush(); - } - return null; - } - } - - private void doRetryPush() { - leader.lock.lock(); - try { - retryFuture = null; - runNextPushRequest(); - } finally { - leader.lock.unlock(); - } - } - - /** - * Begin executing a single push. - *

- * This method must move processing onto another thread. Called with - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held by caller. - * - * @param req - * the request to send to the replica. - */ - protected abstract void startPush(ReplicaPushRequest req); - - /** - * Callback from {@link ReplicaPushRequest} upon success or failure. - *

- * Acquires the {@link KetchLeader#lock} and updates the leader's internal - * knowledge about this replica to reflect what has been learned during a - * push to the replica. In some cases of divergence this method may take - * some time to determine how the replica has diverged; to reduce contention - * this is evaluated before acquiring the leader lock. - * - * @param repo - * local repository instance used by the push thread. - * @param req - * push request just attempted. - */ - void afterPush(@Nullable Repository repo, ReplicaPushRequest req) { - ReceiveCommand acceptCmd = null; - ReceiveCommand commitCmd = null; - List stages = null; - - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - if (name.equals(getSystem().getTxnAccepted())) { - acceptCmd = cmd; - } else if (name.equals(getSystem().getTxnCommitted())) { - commitCmd = cmd; - } else if (cmd.getResult() == OK && cmd.getType() == CREATE - && name.startsWith(getSystem().getTxnStage())) { - if (stages == null) { - stages = new ArrayList<>(); - } - stages.add(cmd); - } - } - - State newState = null; - ObjectId acceptId = readId(req, acceptCmd); - if (repo != null && acceptCmd != null && acceptCmd.getResult() != OK - && req.getException() == null) { - try (LagCheck lag = new LagCheck(this, repo)) { - newState = lag.check(acceptId, acceptCmd); - acceptId = lag.getRemoteId(); - } - } - - leader.lock.lock(); - try { - for (ReceiveCommand cmd : req.getCommands()) { - running.remove(cmd.getRefName()); - } - - Throwable err = req.getException(); - if (err != null) { - state = OFFLINE; - error = err.toString(); - retryLater(req); - leader.onReplicaUpdate(this); - return; - } - - lastRetryMillis = 0; - error = null; - updateView(req, acceptId, commitCmd); - - if (acceptCmd != null && acceptCmd.getResult() == OK) { - state = hasAccepted(leader.getHead()) ? CURRENT : LAGGING; - if (stages != null) { - staged.put(acceptCmd.getNewId(), stages); - } - } else if (newState != null) { - state = newState; - } - - leader.onReplicaUpdate(this); - runNextPushRequest(); - } finally { - leader.lock.unlock(); - } - } - - private void updateView(ReplicaPushRequest req, @Nullable ObjectId acceptId, - ReceiveCommand commitCmd) { - if (acceptId != null) { - txnAccepted = acceptId; - } - - ObjectId committed = readId(req, commitCmd); - if (committed != null) { - txnCommitted = committed; - } else if (acceptId != null && txnCommitted == null) { - // Initialize during first conversation. - Map adv = req.getRefs(); - if (adv != null) { - Ref refs = adv.get(getSystem().getTxnCommitted()); - txnCommitted = getId(refs); - } - } - } - - @Nullable - private static ObjectId readId(ReplicaPushRequest req, - @Nullable ReceiveCommand cmd) { - if (cmd == null) { - // Ref was not in the command list, do not trust advertisement. - return null; - - } else if (cmd.getResult() == OK) { - // Currently at newId. - return cmd.getNewId(); - } - - Map refs = req.getRefs(); - return refs != null ? getId(refs.get(cmd.getRefName())) : null; - } - - /** - * Fetch objects from the remote using the calling thread. - *

- * Called without {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock}. - * - * @param repo - * local repository to fetch objects into. - * @param req - * the request to fetch from a replica. - * @throws java.io.IOException - * communication with the replica was not possible. - */ - protected abstract void blockingFetch(Repository repo, - ReplicaFetchRequest req) throws IOException; - - /** - * Build a list of commands to commit - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS}. - * - * @param git - * local leader repository to read committed state from. - * @param current - * all known references in the replica's repository. Typically - * this comes from a push advertisement. - * @param committed - * state being pushed to {@code refs/txn/committed}. - * @return commands to update during commit. - * @throws java.io.IOException - * cannot read the committed state. - */ - protected Collection prepareCommit(Repository git, - Map current, ObjectId committed) throws IOException { - List delta = new ArrayList<>(); - Map remote = new HashMap<>(current); - try (RevWalk rw = new RevWalk(git); - TreeWalk tw = new TreeWalk(rw.getObjectReader())) { - tw.setRecursive(true); - tw.addTree(rw.parseCommit(committed).getTree()); - while (tw.next()) { - if (tw.getRawMode(0) != TYPE_GITLINK - || tw.isPathSuffix(PEEL, 2)) { - // Symbolic references cannot be pushed. - // Caching peeled values is handled remotely. - continue; - } - - // TODO(sop) Do not send certain ref names to replica. - String name = RefTree.refName(tw.getPathString()); - Ref oldRef = remote.remove(name); - ObjectId oldId = getId(oldRef); - ObjectId newId = tw.getObjectId(0); - if (!AnyObjectId.isEqual(oldId, newId)) { - delta.add(new ReceiveCommand(oldId, newId, name)); - } - } - } - - // Delete any extra references not in the committed state. - for (Ref ref : remote.values()) { - if (canDelete(ref)) { - delta.add(new ReceiveCommand( - ref.getObjectId(), ObjectId.zeroId(), - ref.getName())); - } - } - return delta; - } - - boolean canDelete(Ref ref) { - String name = ref.getName(); - if (HEAD.equals(name)) { - return false; - } - if (name.startsWith(getSystem().getTxnNamespace())) { - return false; - } - // TODO(sop) Do not delete precious names from replica. - return true; - } - - @NonNull - static ObjectId getId(@Nullable Ref ref) { - if (ref != null) { - ObjectId id = ref.getObjectId(); - if (id != null) { - return id; - } - } - return ObjectId.zeroId(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java deleted file mode 100644 index 8ad1d6033a..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchSystem.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchConstants.ACCEPTED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.COMMITTED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_SECTION_KETCH; -import static org.eclipse.jgit.internal.ketch.KetchConstants.DEFAULT_TXN_NAMESPACE; -import static org.eclipse.jgit.internal.ketch.KetchConstants.STAGE; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_NAME; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; - -import java.net.URISyntaxException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.URIish; -import org.eclipse.jgit.util.time.MonotonicClock; -import org.eclipse.jgit.util.time.MonotonicSystemClock; -import org.eclipse.jgit.util.time.ProposedTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Ketch system-wide configuration. - *

- * This class provides useful defaults for testing and small proof of concepts. - * Full scale installations are expected to subclass and override methods to - * provide consistent configuration across all managed repositories. - *

- * Servers should configure their own - * {@link java.util.concurrent.ScheduledExecutorService}. - */ -public class KetchSystem { - private static final Random RNG = new Random(); - - /** - * Get default executor, one thread per available processor. - * - * @return default executor, one thread per available processor. - */ - public static ScheduledExecutorService defaultExecutor() { - return DefaultExecutorHolder.I; - } - - private final ScheduledExecutorService executor; - private final MonotonicClock clock; - private final String txnNamespace; - private final String txnAccepted; - private final String txnCommitted; - private final String txnStage; - - /** - * Create a default system with a thread pool of 1 thread per CPU. - */ - public KetchSystem() { - this(defaultExecutor(), new MonotonicSystemClock(), DEFAULT_TXN_NAMESPACE); - } - - /** - * Create a Ketch system with the provided executor service. - * - * @param executor - * thread pool to run background operations. - * @param clock - * clock to create timestamps. - * @param txnNamespace - * reference namespace for the RefTree graph and associated - * transaction state. Must begin with {@code "refs/"} and end - * with {@code '/'}, for example {@code "refs/txn/"}. - */ - public KetchSystem(ScheduledExecutorService executor, MonotonicClock clock, - String txnNamespace) { - this.executor = executor; - this.clock = clock; - this.txnNamespace = txnNamespace; - this.txnAccepted = txnNamespace + ACCEPTED; - this.txnCommitted = txnNamespace + COMMITTED; - this.txnStage = txnNamespace + STAGE; - } - - /** - * Get executor to perform background operations. - * - * @return executor to perform background operations. - */ - public ScheduledExecutorService getExecutor() { - return executor; - } - - /** - * Get clock to obtain timestamps from. - * - * @return clock to obtain timestamps from. - */ - public MonotonicClock getClock() { - return clock; - } - - /** - * Get how long the leader will wait for the {@link #getClock()}'s - * {@code ProposedTimestamp} used in commits proposed to the RefTree graph - * ({@link #getTxnAccepted()}) - * - * @return how long the leader will wait for the {@link #getClock()}'s - * {@code ProposedTimestamp} used in commits proposed to the RefTree - * graph ({@link #getTxnAccepted()}). Defaults to 5 seconds. - */ - public Duration getMaxWaitForMonotonicClock() { - return Duration.ofSeconds(5); - } - - /** - * Whether elections should require monotonically increasing commit - * timestamps - * - * @return {@code true} if elections should require monotonically increasing - * commit timestamps. This requires a very good - * {@link org.eclipse.jgit.util.time.MonotonicClock}. - */ - public boolean requireMonotonicLeaderElections() { - return false; - } - - /** - * Get the namespace used for the RefTree graph and transaction management. - * - * @return reference namespace such as {@code "refs/txn/"}. - */ - public String getTxnNamespace() { - return txnNamespace; - } - - /** - * Get name of the accepted RefTree graph. - * - * @return name of the accepted RefTree graph. - */ - public String getTxnAccepted() { - return txnAccepted; - } - - /** - * Get name of the committed RefTree graph. - * - * @return name of the committed RefTree graph. - */ - public String getTxnCommitted() { - return txnCommitted; - } - - /** - * Get prefix for staged objects, e.g. {@code "refs/txn/stage/"}. - * - * @return prefix for staged objects, e.g. {@code "refs/txn/stage/"}. - */ - public String getTxnStage() { - return txnStage; - } - - /** - * Create new committer {@code PersonIdent} for ketch system - * - * @param time - * timestamp for the committer. - * @return identity line for the committer header of a RefTreeGraph. - */ - public PersonIdent newCommitter(ProposedTimestamp time) { - String name = "ketch"; //$NON-NLS-1$ - String email = "ketch@system"; //$NON-NLS-1$ - return new PersonIdent(name, email, time); - } - - /** - * Construct a random tag to identify a candidate during leader election. - *

- * Multiple processes trying to elect themselves leaders at exactly the same - * time (rounded to seconds) using the same - * {@link #newCommitter(ProposedTimestamp)} identity strings, for the same - * term, may generate the same ObjectId for the election commit and falsely - * assume they have both won. - *

- * Candidates add this tag to their election ballot commit to disambiguate - * the election. The tag only needs to be unique for a given triplet of - * {@link #newCommitter(ProposedTimestamp)}, system time (rounded to - * seconds), and term. If every replica in the system uses a unique - * {@code newCommitter} (such as including the host name after the - * {@code "@"} in the email address) the tag could be the empty string. - *

- * The default implementation generates a few bytes of random data. - * - * @return unique tag; null or empty string if {@code newCommitter()} is - * sufficiently unique to identify the leader. - */ - @Nullable - public String newLeaderTag() { - int n = RNG.nextInt(1 << (6 * 4)); - return String.format("%06x", Integer.valueOf(n)); //$NON-NLS-1$ - } - - /** - * Construct the KetchLeader instance of a repository. - * - * @param repo - * local repository stored by the leader. - * @return leader instance. - * @throws java.net.URISyntaxException - * a follower configuration contains an unsupported URI. - */ - public KetchLeader createLeader(Repository repo) - throws URISyntaxException { - KetchLeader leader = new KetchLeader(this) { - @Override - protected Repository openRepository() { - repo.incrementOpen(); - return repo; - } - }; - leader.setReplicas(createReplicas(leader, repo)); - return leader; - } - - /** - * Get the collection of replicas for a repository. - *

- * The collection of replicas must include the local repository. - * - * @param leader - * the leader driving these replicas. - * @param repo - * repository to get the replicas of. - * @return collection of replicas for the specified repository. - * @throws java.net.URISyntaxException - * a configured URI is invalid. - */ - protected List createReplicas(KetchLeader leader, - Repository repo) throws URISyntaxException { - List replicas = new ArrayList<>(); - Config cfg = repo.getConfig(); - String localName = getLocalName(cfg); - for (String name : cfg.getSubsections(CONFIG_KEY_REMOTE)) { - if (!hasParticipation(cfg, name)) { - continue; - } - - ReplicaConfig kc = ReplicaConfig.newFromConfig(cfg, name); - if (name.equals(localName)) { - replicas.add(new LocalReplica(leader, name, kc)); - continue; - } - - RemoteConfig rc = new RemoteConfig(cfg, name); - List uris = rc.getPushURIs(); - if (uris.isEmpty()) { - uris = rc.getURIs(); - } - for (URIish uri : uris) { - String n = uris.size() == 1 ? name : uri.getHost(); - replicas.add(new RemoteGitReplica(leader, n, uri, kc, rc)); - } - } - return replicas; - } - - private static boolean hasParticipation(Config cfg, String name) { - return cfg.getString(CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE) != null; - } - - private static String getLocalName(Config cfg) { - return cfg.getString(CONFIG_SECTION_KETCH, null, CONFIG_KEY_NAME); - } - - static class DefaultExecutorHolder { - private static final Logger log = LoggerFactory.getLogger(KetchSystem.class); - static final ScheduledExecutorService I = create(); - - private static ScheduledExecutorService create() { - int cores = Runtime.getRuntime().availableProcessors(); - int threads = Math.max(5, cores); - log.info("Using {} threads", Integer.valueOf(threads)); //$NON-NLS-1$ - return Executors.newScheduledThreadPool( - threads, - new ThreadFactory() { - private final AtomicInteger threadCnt = new AtomicInteger(); - - @Override - public Thread newThread(Runnable r) { - int id = threadCnt.incrementAndGet(); - Thread thr = new Thread(r); - thr.setName("KetchExecutor-" + id); //$NON-NLS-1$ - return thr; - } - }); - } - - private DefaultExecutorHolder() { - } - } - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java deleted file mode 100644 index 6f9038bbdf..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/KetchText.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.nls.NLS; -import org.eclipse.jgit.nls.TranslationBundle; - -/** - * Translation bundle for the Ketch implementation. - */ -public class KetchText extends TranslationBundle { - /** - * Get an instance of this translation bundle. - * - * @return instance of this translation bundle. - */ - public static KetchText get() { - return NLS.getBundleFor(KetchText.class); - } - - // @formatter:off - /***/ public String accepted; - /***/ public String cannotFetchFromLocalReplica; - /***/ public String failed; - /***/ public String invalidFollowerUri; - /***/ public String leaderFailedToStore; - /***/ public String localReplicaRequired; - /***/ public String mismatchedTxnNamespace; - /***/ public String outsideTxnNamespace; - /***/ public String proposingUpdates; - /***/ public String queuedProposalFailedToApply; - /***/ public String starting; - /***/ public String unsupportedVoterCount; - /***/ public String waitingForQueue; -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java deleted file mode 100644 index 1f8384ff76..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LagCheck.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.AHEAD; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.DIVERGENT; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; -import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * A helper to check if a {@link KetchReplica} is ahead or behind the leader. - */ -class LagCheck implements AutoCloseable { - private final KetchReplica replica; - private final Repository repo; - private RevWalk rw; - private ObjectId remoteId; - - LagCheck(KetchReplica replica, Repository repo) { - this.replica = replica; - this.repo = repo; - initRevWalk(); - } - - private void initRevWalk() { - if (rw != null) { - rw.close(); - } - - rw = new RevWalk(repo); - rw.setRetainBody(false); - } - - /** {@inheritDoc} */ - @Override - public void close() { - if (rw != null) { - rw.close(); - rw = null; - } - } - - ObjectId getRemoteId() { - return remoteId; - } - - KetchReplica.State check(ObjectId acceptId, ReceiveCommand acceptCmd) { - remoteId = acceptId; - if (remoteId == null) { - // Nothing advertised by the replica, value is unknown. - return UNKNOWN; - } - - if (AnyObjectId.isEqual(remoteId, ObjectId.zeroId())) { - // Replica does not have the txnAccepted reference. - return LAGGING; - } - - try { - RevCommit remote; - try { - remote = parseRemoteCommit(acceptCmd.getRefName()); - } catch (RefGoneException gone) { - // Replica does not have the txnAccepted reference. - return LAGGING; - } catch (MissingObjectException notFound) { - // Local repository does not know this commit so it cannot - // be including the replica's log. - return DIVERGENT; - } - - RevCommit head = rw.parseCommit(acceptCmd.getNewId()); - if (rw.isMergedInto(remote, head)) { - return LAGGING; - } - - // TODO(sop) Check term to see if my leader was deposed. - if (rw.isMergedInto(head, remote)) { - return AHEAD; - } - return DIVERGENT; - } catch (IOException err) { - KetchReplica.log.error(String.format( - "Cannot compare %s", //$NON-NLS-1$ - acceptCmd.getRefName()), err); - return UNKNOWN; - } - } - - private RevCommit parseRemoteCommit(String refName) - throws IOException, MissingObjectException, RefGoneException { - try { - return rw.parseCommit(remoteId); - } catch (MissingObjectException notLocal) { - // Fall through and try to acquire the object by fetching it. - } - - ReplicaFetchRequest fetch = new ReplicaFetchRequest( - Collections.singleton(refName), - Collections. emptySet()); - try { - replica.blockingFetch(repo, fetch); - } catch (IOException fetchErr) { - KetchReplica.log.error(String.format( - "Cannot fetch %s (%s) from %s", //$NON-NLS-1$ - remoteId.abbreviate(8).name(), refName, - replica.describeForLog()), fetchErr); - throw new MissingObjectException(remoteId, OBJ_COMMIT); - } - - Map adv = fetch.getRefs(); - if (adv == null) { - throw new MissingObjectException(remoteId, OBJ_COMMIT); - } - - Ref ref = adv.get(refName); - if (ref == null || ref.getObjectId() == null) { - throw new RefGoneException(); - } - - initRevWalk(); - remoteId = ref.getObjectId(); - return rw.parseCommit(remoteId); - } - - private static class RefGoneException extends Exception { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java deleted file mode 100644 index ce0672c168..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LeaderSnapshot.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; - -/** - * A snapshot of a leader and its view of the world. - */ -public class LeaderSnapshot { - final List replicas = new ArrayList<>(); - KetchLeader.State state; - long term; - LogIndex headIndex; - LogIndex committedIndex; - boolean idle; - - LeaderSnapshot() { - } - - /** - * Get unmodifiable view of configured replicas. - * - * @return unmodifiable view of configured replicas. - */ - public Collection getReplicas() { - return Collections.unmodifiableList(replicas); - } - - /** - * Get current state of the leader. - * - * @return current state of the leader. - */ - public KetchLeader.State getState() { - return state; - } - - /** - * Whether the leader is not running a round to reach consensus, and has no - * rounds queued. - * - * @return {@code true} if the leader is not running a round to reach - * consensus, and has no rounds queued. - */ - public boolean isIdle() { - return idle; - } - - /** - * Get term of this leader - * - * @return term of this leader. Valid only if {@link #getState()} is - * currently - * {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER}. - */ - public long getTerm() { - return term; - } - - /** - * Get end of the leader's log - * - * @return end of the leader's log; null if leader hasn't started up enough - * to begin its own election. - */ - @Nullable - public LogIndex getHead() { - return headIndex; - } - - /** - * Get state the leader knows is committed on a majority of participant - * replicas - * - * @return state the leader knows is committed on a majority of participant - * replicas. Null until the leader instance has committed a log - * index within its own term. - */ - @Nullable - public LogIndex getCommitted() { - return committedIndex; - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - s.append(isIdle() ? "IDLE" : "RUNNING"); //$NON-NLS-1$ //$NON-NLS-2$ - s.append(" state ").append(getState()); //$NON-NLS-1$ - if (getTerm() > 0) { - s.append(" term ").append(getTerm()); //$NON-NLS-1$ - } - s.append('\n'); - s.append(String.format( - "%-10s %12s %12s\n", //$NON-NLS-1$ - "Replica", "Accepted", "Committed")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - s.append("------------------------------------\n"); //$NON-NLS-1$ - debug(s, "(leader)", getHead(), getCommitted()); //$NON-NLS-1$ - s.append('\n'); - for (ReplicaSnapshot r : getReplicas()) { - debug(s, r); - s.append('\n'); - } - s.append('\n'); - return s.toString(); - } - - private static void debug(StringBuilder b, ReplicaSnapshot s) { - KetchReplica replica = s.getReplica(); - debug(b, replica.getName(), s.getAccepted(), s.getCommitted()); - b.append(String.format(" %-8s %s", //$NON-NLS-1$ - replica.getParticipation(), s.getState())); - if (s.getState() == OFFLINE) { - String err = s.getErrorMessage(); - if (err != null) { - b.append(" (").append(err).append(')'); //$NON-NLS-1$ - } - } - } - - private static void debug(StringBuilder s, String name, - ObjectId accepted, ObjectId committed) { - s.append(String.format( - "%-10s %-12s %-12s", //$NON-NLS-1$ - name, str(accepted), str(committed))); - } - - static String str(ObjectId c) { - if (c instanceof LogIndex) { - return ((LogIndex) c).describeForLog(); - } else if (c != null) { - return c.abbreviate(8).name(); - } - return "-"; //$NON-NLS-1$ - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java deleted file mode 100644 index b2d59d77da..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LocalReplica.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.TXN_COMMITTED; -import static org.eclipse.jgit.lib.RefDatabase.ALL; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.MonotonicClock; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** - * Ketch replica running on the same system as the - * {@link org.eclipse.jgit.internal.ketch.KetchLeader}. - */ -public class LocalReplica extends KetchReplica { - /** - * Configure a local replica. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this replica for debugging. - * @param cfg - * how Ketch should treat the local system. - */ - public LocalReplica(KetchLeader leader, String name, ReplicaConfig cfg) { - super(leader, name, cfg); - } - - /** {@inheritDoc} */ - @Override - protected String describeForLog() { - return String.format("%s (leader)", getName()); //$NON-NLS-1$ - } - - /** - * Initializes local replica by reading accepted and committed references. - *

- * Loads accepted and committed references from the reference database of - * the local replica and stores their current ObjectIds in memory. - * - * @param repo - * repository to initialize state from. - * @throws IOException - * cannot read repository state. - */ - void initialize(Repository repo) throws IOException { - RefDatabase refdb = repo.getRefDatabase(); - if (refdb instanceof RefTreeDatabase) { - RefTreeDatabase treeDb = (RefTreeDatabase) refdb; - String txnNamespace = getSystem().getTxnNamespace(); - if (!txnNamespace.equals(treeDb.getTxnNamespace())) { - throw new IOException(MessageFormat.format( - KetchText.get().mismatchedTxnNamespace, - txnNamespace, treeDb.getTxnNamespace())); - } - refdb = treeDb.getBootstrap(); - } - initialize(refdb.exactRef( - getSystem().getTxnAccepted(), - getSystem().getTxnCommitted())); - } - - /** {@inheritDoc} */ - @Override - protected void startPush(ReplicaPushRequest req) { - getSystem().getExecutor().execute(() -> { - MonotonicClock clk = getSystem().getClock(); - try (Repository git = getLeader().openRepository(); - ProposedTimestamp ts = clk.propose()) { - try { - update(git, req, ts); - req.done(git); - } catch (Throwable err) { - req.setException(git, err); - } - } catch (IOException err) { - req.setException(null, err); - } - }); - } - - /** {@inheritDoc} */ - @Override - protected void blockingFetch(Repository repo, ReplicaFetchRequest req) - throws IOException { - throw new IOException(KetchText.get().cannotFetchFromLocalReplica); - } - - private void update(Repository git, ReplicaPushRequest req, - ProposedTimestamp ts) throws IOException { - RefDatabase refdb = git.getRefDatabase(); - CommitMethod method = getCommitMethod(); - - // Local replica probably uses RefTreeDatabase, the request should - // be only for the txnNamespace, so drop to the bootstrap layer. - if (refdb instanceof RefTreeDatabase) { - if (!isOnlyTxnNamespace(req.getCommands())) { - return; - } - - refdb = ((RefTreeDatabase) refdb).getBootstrap(); - method = TXN_COMMITTED; - } - - BatchRefUpdate batch = refdb.newBatchUpdate(); - batch.addProposedTimestamp(ts); - batch.setRefLogIdent(getSystem().newCommitter(ts)); - batch.setRefLogMessage("ketch", false); //$NON-NLS-1$ - batch.setAllowNonFastForwards(true); - - // RefDirectory updates multiple references sequentially. - // Run everything else first, then accepted (if present), - // then committed (if present). This ensures an earlier - // failure will not update these critical references. - ReceiveCommand accepted = null; - ReceiveCommand committed = null; - for (ReceiveCommand cmd : req.getCommands()) { - String name = cmd.getRefName(); - if (name.equals(getSystem().getTxnAccepted())) { - accepted = cmd; - } else if (name.equals(getSystem().getTxnCommitted())) { - committed = cmd; - } else { - batch.addCommand(cmd); - } - } - if (committed != null && method == ALL_REFS) { - Map refs = refdb.getRefs(ALL); - batch.addCommand(prepareCommit(git, refs, committed.getNewId())); - } - if (accepted != null) { - batch.addCommand(accepted); - } - if (committed != null) { - batch.addCommand(committed); - } - - try (RevWalk rw = new RevWalk(git)) { - batch.execute(rw, NullProgressMonitor.INSTANCE); - } - - // KetchReplica only cares about accepted and committed in - // advertisement. If they failed, store the current values - // back in the ReplicaPushRequest. - List failed = new ArrayList<>(2); - checkFailed(failed, accepted); - checkFailed(failed, committed); - if (!failed.isEmpty()) { - String[] arr = failed.toArray(new String[0]); - req.setRefs(refdb.exactRef(arr)); - } - } - - private static void checkFailed(List failed, ReceiveCommand cmd) { - if (cmd != null && cmd.getResult() != OK) { - failed.add(cmd.getRefName()); - } - } - - private boolean isOnlyTxnNamespace(Collection cmdList) { - // Be paranoid and reject non txnNamespace names, this - // is a programming error in Ketch that should not occur. - - String txnNamespace = getSystem().getTxnNamespace(); - for (ReceiveCommand cmd : cmdList) { - if (!cmd.getRefName().startsWith(txnNamespace)) { - cmd.setResult(REJECTED_OTHER_REASON, - MessageFormat.format( - KetchText.get().outsideTxnNamespace, - cmd.getRefName(), txnNamespace)); - ReceiveCommand.abort(cmdList); - return false; - } - } - return true; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java deleted file mode 100644 index ed65c06fae..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/LogIndex.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; - -/** - * An ObjectId for a commit extended with incrementing log index. - *

- * For any two LogIndex instances, {@code A} is an ancestor of {@code C} - * reachable through parent edges in the graph if {@code A.index < C.index}. - * LogIndex provides a performance optimization for Ketch, the same information - * can be obtained from {@link org.eclipse.jgit.revwalk.RevWalk}. - *

- * Index values are only valid within a single - * {@link org.eclipse.jgit.internal.ketch.KetchLeader} instance after it has won - * an election. By restricting scope to a single leader new leaders do not need - * to traverse the entire history to determine the next {@code index} for new - * proposals. This differs from Raft, where leader election uses the log index - * and the term number to determine which replica holds a sufficiently - * up-to-date log. Since Ketch uses Git objects for storage of its replicated - * log, it keeps the term number as Raft does but uses standard Git operations - * to imply the log index. - *

- * {@link org.eclipse.jgit.internal.ketch.Round#runAsync(AnyObjectId)} bumps the - * index as each new round is constructed. - */ -public class LogIndex extends ObjectId { - static LogIndex unknown(AnyObjectId id) { - return new LogIndex(id, 0); - } - - private final long index; - - private LogIndex(AnyObjectId id, long index) { - super(id); - this.index = index; - } - - LogIndex nextIndex(AnyObjectId id) { - return new LogIndex(id, index + 1); - } - - /** - * Get index provided by the current leader instance. - * - * @return index provided by the current leader instance. - */ - public long getIndex() { - return index; - } - - /** - * Check if this log position committed before another log position. - *

- * Only valid for log positions in memory for the current leader. - * - * @param c - * other (more recent) log position. - * @return true if this log position was before {@code c} or equal to c and - * therefore any agreement of {@code c} implies agreement on this - * log position. - */ - boolean isBefore(LogIndex c) { - return index <= c.index; - } - - /** - * Create string suitable for debug logging containing the log index and - * abbreviated ObjectId. - * - * @return string suitable for debug logging containing the log index and - * abbreviated ObjectId. - */ - @SuppressWarnings("boxing") - public String describeForLog() { - return String.format("%5d/%s", index, abbreviate(6).name()); //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @SuppressWarnings("boxing") - @Override - public String toString() { - return String.format("LogId[%5d/%s]", index, name()); //$NON-NLS-1$ - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java deleted file mode 100644 index ca27281a8e..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Proposal.java +++ /dev/null @@ -1,415 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.Proposal.State.ABORTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; -import static org.eclipse.jgit.internal.ketch.Proposal.State.NEW; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.storage.reftree.Command; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.PushCertificate; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** - * A proposal to be applied in a Ketch system. - *

- * Pushing to a Ketch leader results in the leader making a proposal. The - * proposal includes the list of reference updates. The leader attempts to send - * the proposal to a quorum of replicas by pushing the proposal to a "staging" - * area under the {@code refs/txn/stage/} namespace. If the proposal succeeds - * then the changes are durable and the leader can commit the proposal. - *

- * Proposals are executed by - * {@link org.eclipse.jgit.internal.ketch.KetchLeader#queueProposal(Proposal)}, - * which runs them asynchronously in the background. Proposals are thread-safe - * futures allowing callers to {@link #await()} for results or be notified by - * callback using {@link #addListener(Runnable)}. - */ -public class Proposal { - /** Current state of the proposal. */ - public enum State { - /** Proposal has not yet been given to a {@link KetchLeader}. */ - NEW(false), - - /** - * Proposal was validated and has entered the queue, but a round - * containing this proposal has not started yet. - */ - QUEUED(false), - - /** Round containing the proposal has begun and is in progress. */ - RUNNING(false), - - /** - * Proposal was executed through a round. Individual results from - * {@link Proposal#getCommands()}, {@link Command#getResult()} explain - * the success or failure outcome. - */ - EXECUTED(true), - - /** Proposal was aborted and did not reach consensus. */ - ABORTED(true); - - private final boolean done; - - private State(boolean done) { - this.done = done; - } - - /** @return true if this is a terminal state. */ - public boolean isDone() { - return done; - } - } - - private final List commands; - private PersonIdent author; - private String message; - private PushCertificate pushCert; - - private List timestamps; - private final List listeners = new CopyOnWriteArrayList<>(); - private final AtomicReference state = new AtomicReference<>(NEW); - - /** - * Create a proposal from a list of Ketch commands. - * - * @param cmds - * prepared list of commands. - */ - public Proposal(List cmds) { - commands = Collections.unmodifiableList(new ArrayList<>(cmds)); - } - - /** - * Create a proposal from a collection of received commands. - * - * @param rw - * walker to assist in preparing commands. - * @param cmds - * list of pending commands. - * @throws org.eclipse.jgit.errors.MissingObjectException - * newId of a command is not found locally. - * @throws java.io.IOException - * local objects cannot be accessed. - */ - public Proposal(RevWalk rw, Collection cmds) - throws MissingObjectException, IOException { - commands = asCommandList(rw, cmds); - } - - private static List asCommandList(RevWalk rw, - Collection cmds) - throws MissingObjectException, IOException { - List commands = new ArrayList<>(cmds.size()); - for (ReceiveCommand cmd : cmds) { - commands.add(new Command(rw, cmd)); - } - return Collections.unmodifiableList(commands); - } - - /** - * Get commands from this proposal. - * - * @return commands from this proposal. - */ - public Collection getCommands() { - return commands; - } - - /** - * Get optional author of the proposal. - * - * @return optional author of the proposal. - */ - @Nullable - public PersonIdent getAuthor() { - return author; - } - - /** - * Set the author for the proposal. - * - * @param who - * optional identity of the author of the proposal. - * @return {@code this} - */ - public Proposal setAuthor(@Nullable PersonIdent who) { - author = who; - return this; - } - - /** - * Get optional message for the commit log of the RefTree. - * - * @return optional message for the commit log of the RefTree. - */ - @Nullable - public String getMessage() { - return message; - } - - /** - * Set the message to appear in the commit log of the RefTree. - * - * @param msg - * message text for the commit. - * @return {@code this} - */ - public Proposal setMessage(@Nullable String msg) { - message = msg != null && !msg.isEmpty() ? msg : null; - return this; - } - - /** - * Get optional certificate signing the references. - * - * @return optional certificate signing the references. - */ - @Nullable - public PushCertificate getPushCertificate() { - return pushCert; - } - - /** - * Set the push certificate signing the references. - * - * @param cert - * certificate, may be null. - * @return {@code this} - */ - public Proposal setPushCertificate(@Nullable PushCertificate cert) { - pushCert = cert; - return this; - } - - /** - * Get timestamps that Ketch must block for. - * - * @return timestamps that Ketch must block for. These may have been used as - * commit times inside the objects involved in the proposal. - */ - public List getProposedTimestamps() { - if (timestamps != null) { - return timestamps; - } - return Collections.emptyList(); - } - - /** - * Request the proposal to wait for the affected timestamps to resolve. - * - * @param ts - * a {@link org.eclipse.jgit.util.time.ProposedTimestamp} object. - * @return {@code this}. - */ - public Proposal addProposedTimestamp(ProposedTimestamp ts) { - if (timestamps == null) { - timestamps = new ArrayList<>(4); - } - timestamps.add(ts); - return this; - } - - /** - * Add a callback to be invoked when the proposal is done. - *

- * A proposal is done when it has entered either - * {@link org.eclipse.jgit.internal.ketch.Proposal.State#EXECUTED} or - * {@link org.eclipse.jgit.internal.ketch.Proposal.State#ABORTED} state. If - * the proposal is already done {@code callback.run()} is immediately - * invoked on the caller's thread. - * - * @param callback - * method to run after the proposal is done. The callback may be - * run on a Ketch system thread and should be completed quickly. - */ - public void addListener(Runnable callback) { - boolean runNow = false; - synchronized (state) { - if (state.get().isDone()) { - runNow = true; - } else { - listeners.add(callback); - } - } - if (runNow) { - callback.run(); - } - } - - /** Set command result as OK. */ - void success() { - for (Command c : commands) { - if (c.getResult() == NOT_ATTEMPTED) { - c.setResult(OK); - } - } - notifyState(EXECUTED); - } - - /** Mark commands as "transaction aborted". */ - void abort() { - Command.abort(commands, null); - notifyState(ABORTED); - } - - /** - * Read the current state of the proposal. - * - * @return read the current state of the proposal. - */ - public State getState() { - return state.get(); - } - - /** - * Whether the proposal was attempted - * - * @return {@code true} if the proposal was attempted. A true value does not - * mean consensus was reached, only that the proposal was considered - * and will not be making any more progress beyond its current - * state. - */ - public boolean isDone() { - return state.get().isDone(); - } - - /** - * Wait for the proposal to be attempted and {@link #isDone()} to be true. - * - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public void await() throws InterruptedException { - synchronized (state) { - while (!state.get().isDone()) { - state.wait(); - } - } - } - - /** - * Wait for the proposal to be attempted and {@link #isDone()} to be true. - * - * @param wait - * how long to wait. - * @param unit - * unit describing the wait time. - * @return true if the proposal is done; false if the method timed out. - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public boolean await(long wait, TimeUnit unit) throws InterruptedException { - synchronized (state) { - if (state.get().isDone()) { - return true; - } - state.wait(unit.toMillis(wait)); - return state.get().isDone(); - } - } - - /** - * Wait for the proposal to exit a state. - * - * @param notIn - * state the proposal should not be in to return. - * @param wait - * how long to wait. - * @param unit - * unit describing the wait time. - * @return true if the proposal exited the state; false on time out. - * @throws java.lang.InterruptedException - * caller was interrupted before proposal executed. - */ - public boolean awaitStateChange(State notIn, long wait, TimeUnit unit) - throws InterruptedException { - synchronized (state) { - if (state.get() != notIn) { - return true; - } - state.wait(unit.toMillis(wait)); - return state.get() != notIn; - } - } - - void notifyState(State s) { - synchronized (state) { - state.set(s); - state.notifyAll(); - } - if (s.isDone()) { - for (Runnable callback : listeners) { - callback.run(); - } - listeners.clear(); - } - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - s.append("Ketch Proposal {\n"); //$NON-NLS-1$ - s.append(" ").append(state.get()).append('\n'); //$NON-NLS-1$ - if (author != null) { - s.append(" author ").append(author).append('\n'); //$NON-NLS-1$ - } - if (message != null) { - s.append(" message ").append(message).append('\n'); //$NON-NLS-1$ - } - for (Command c : commands) { - s.append(" "); //$NON-NLS-1$ - format(s, c.getOldRef(), "CREATE"); //$NON-NLS-1$ - s.append(' '); - format(s, c.getNewRef(), "DELETE"); //$NON-NLS-1$ - s.append(' ').append(c.getRefName()); - if (c.getResult() != ReceiveCommand.Result.NOT_ATTEMPTED) { - s.append(' ').append(c.getResult()); // $NON-NLS-1$ - } - s.append('\n'); - } - s.append('}'); - return s.toString(); - } - - private static void format(StringBuilder s, @Nullable Ref r, String n) { - if (r == null) { - s.append(n); - } else if (r.isSymbolic()) { - s.append(r.getTarget().getName()); - } else { - ObjectId id = r.getObjectId(); - if (id != null) { - s.append(id.abbreviate(8).name()); - } - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java deleted file mode 100644 index b73183abd0..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ProposalRound.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.Proposal.State.RUNNING; - -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.storage.reftree.Command; -import org.eclipse.jgit.internal.storage.reftree.RefTree; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.util.time.ProposedTimestamp; - -/** A {@link Round} that aggregates and sends user {@link Proposal}s. */ -class ProposalRound extends Round { - private final List todo; - private RefTree queuedTree; - - ProposalRound(KetchLeader leader, LogIndex head, List todo, - @Nullable RefTree tree) { - super(leader, head); - this.todo = todo; - - if (tree != null && canCombine(todo)) { - this.queuedTree = tree; - } else { - leader.roundHoldsReferenceToRefTree = false; - } - } - - private static boolean canCombine(List todo) { - Proposal first = todo.get(0); - for (int i = 1; i < todo.size(); i++) { - if (!canCombine(first, todo.get(i))) { - return false; - } - } - return true; - } - - private static boolean canCombine(Proposal a, Proposal b) { - String aMsg = nullToEmpty(a.getMessage()); - String bMsg = nullToEmpty(b.getMessage()); - return aMsg.equals(bMsg) && canCombine(a.getAuthor(), b.getAuthor()); - } - - private static String nullToEmpty(@Nullable String str) { - return str != null ? str : ""; //$NON-NLS-1$ - } - - private static boolean canCombine(@Nullable PersonIdent a, - @Nullable PersonIdent b) { - if (a != null && b != null) { - // Same name and email address. Combine timestamp as the two - // proposals are running concurrently and appear together or - // not at all from the point of view of an outside reader. - return a.getName().equals(b.getName()) - && a.getEmailAddress().equals(b.getEmailAddress()); - } - - // If a and b are null, both will be the system identity. - return a == null && b == null; - } - - @Override - void start() throws IOException { - for (Proposal p : todo) { - p.notifyState(RUNNING); - } - try { - ObjectId id; - try (Repository git = leader.openRepository(); - ProposedTimestamp ts = getSystem().getClock().propose()) { - id = insertProposals(git, ts); - blockUntil(ts); - } - runAsync(id); - } catch (NoOp e) { - for (Proposal p : todo) { - p.success(); - } - leader.lock.lock(); - try { - leader.nextRound(); - } finally { - leader.lock.unlock(); - } - } catch (IOException e) { - abort(); - throw e; - } - } - - private ObjectId insertProposals(Repository git, ProposedTimestamp ts) - throws IOException, NoOp { - ObjectId id; - try (ObjectInserter inserter = git.newObjectInserter()) { - // TODO(sop) Process signed push certificates. - - if (queuedTree != null) { - id = insertSingleProposal(git, ts, inserter); - } else { - id = insertMultiProposal(git, ts, inserter); - } - - stageCommands = makeStageList(git, inserter); - inserter.flush(); - } - return id; - } - - private ObjectId insertSingleProposal(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException, NoOp { - // Fast path: tree is passed in with all proposals applied. - ObjectId treeId = queuedTree.writeTree(inserter); - queuedTree = null; - leader.roundHoldsReferenceToRefTree = false; - - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(acceptedOldIndex); - if (treeId.equals(c.getTree())) { - throw new NoOp(); - } - } - } - - Proposal p = todo.get(0); - CommitBuilder b = new CommitBuilder(); - b.setTreeId(treeId); - if (!ObjectId.zeroId().equals(acceptedOldIndex)) { - b.setParentId(acceptedOldIndex); - } - b.setCommitter(leader.getSystem().newCommitter(ts)); - b.setAuthor(p.getAuthor() != null ? p.getAuthor() : b.getCommitter()); - b.setMessage(message(p)); - return inserter.insert(b); - } - - private ObjectId insertMultiProposal(Repository git, ProposedTimestamp ts, - ObjectInserter inserter) throws IOException, NoOp { - // The tree was not passed in, or there are multiple proposals - // each needing their own commit. Reset the tree and replay each - // proposal in order as individual commits. - ObjectId lastIndex = acceptedOldIndex; - ObjectId oldTreeId; - RefTree tree; - if (ObjectId.zeroId().equals(lastIndex)) { - oldTreeId = ObjectId.zeroId(); - tree = RefTree.newEmptyTree(); - } else { - try (RevWalk rw = new RevWalk(git)) { - RevCommit c = rw.parseCommit(lastIndex); - oldTreeId = c.getTree(); - tree = RefTree.read(rw.getObjectReader(), c.getTree()); - } - } - - PersonIdent committer = leader.getSystem().newCommitter(ts); - for (Proposal p : todo) { - if (!tree.apply(p.getCommands())) { - // This should not occur, previously during queuing the - // commands were successfully applied to the pending tree. - // Abort the entire round. - throw new IOException( - KetchText.get().queuedProposalFailedToApply); - } - - ObjectId treeId = tree.writeTree(inserter); - if (treeId.equals(oldTreeId)) { - continue; - } - - CommitBuilder b = new CommitBuilder(); - b.setTreeId(treeId); - if (!ObjectId.zeroId().equals(lastIndex)) { - b.setParentId(lastIndex); - } - b.setAuthor(p.getAuthor() != null ? p.getAuthor() : committer); - b.setCommitter(committer); - b.setMessage(message(p)); - lastIndex = inserter.insert(b); - } - if (lastIndex.equals(acceptedOldIndex)) { - throw new NoOp(); - } - return lastIndex; - } - - private String message(Proposal p) { - StringBuilder m = new StringBuilder(); - String msg = p.getMessage(); - if (msg != null && !msg.isEmpty()) { - m.append(msg); - while (m.length() < 2 || m.charAt(m.length() - 2) != '\n' - || m.charAt(m.length() - 1) != '\n') { - m.append('\n'); - } - } - m.append(KetchConstants.TERM.getName()) - .append(": ") //$NON-NLS-1$ - .append(leader.getTerm()); - return m.toString(); - } - - void abort() { - for (Proposal p : todo) { - p.abort(); - } - } - - @Override - void success() { - for (Proposal p : todo) { - p.success(); - } - } - - private List makeStageList(Repository git, - ObjectInserter inserter) throws IOException { - // For each branch, collapse consecutive updates to only most recent, - // avoiding sending multiple objects in a rapid fast-forward chain, or - // rewritten content. - Map byRef = new HashMap<>(); - for (Proposal p : todo) { - for (Command c : p.getCommands()) { - Ref n = c.getNewRef(); - if (n != null && !n.isSymbolic()) { - byRef.put(n.getName(), n.getObjectId()); - } - } - } - if (byRef.isEmpty()) { - return Collections.emptyList(); - } - - Set newObjs = new HashSet<>(byRef.values()); - StageBuilder b = new StageBuilder( - leader.getSystem().getTxnStage(), - acceptedNewIndex); - return b.makeStageList(newObjs, git, inserter); - } - - private void blockUntil(ProposedTimestamp ts) - throws TimeIsUncertainException { - List times = todo.stream() - .flatMap(p -> p.getProposedTimestamps().stream()) - .collect(Collectors.toCollection(ArrayList::new)); - times.add(ts); - - try { - Duration maxWait = getSystem().getMaxWaitForMonotonicClock(); - ProposedTimestamp.blockUntil(times, maxWait); - } catch (InterruptedException | TimeoutException e) { - throw new TimeIsUncertainException(e); - } - } - - private static class NoOp extends Exception { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java deleted file mode 100644 index fac93c84b3..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/RemoteGitReplica.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; -import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NODELETE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.NotSupportedException; -import org.eclipse.jgit.errors.TransportException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.FetchConnection; -import org.eclipse.jgit.transport.PushConnection; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.RemoteConfig; -import org.eclipse.jgit.transport.RemoteRefUpdate; -import org.eclipse.jgit.transport.Transport; -import org.eclipse.jgit.transport.URIish; - -/** - * Representation of a Git repository on a remote replica system. - *

- * {@link org.eclipse.jgit.internal.ketch.KetchLeader} will contact the replica - * using the Git wire protocol. - *

- * The remote replica may be fully Ketch-aware, or a standard Git server. - */ -public class RemoteGitReplica extends KetchReplica { - private final URIish uri; - private final RemoteConfig remoteConfig; - - /** - * Configure a new remote. - * - * @param leader - * instance this replica follows. - * @param name - * unique-ish name identifying this remote for debugging. - * @param uri - * URI to connect to the follower's repository. - * @param cfg - * how Ketch should treat the remote system. - * @param rc - * optional remote configuration describing how to contact the - * peer repository. - */ - public RemoteGitReplica(KetchLeader leader, String name, URIish uri, - ReplicaConfig cfg, @Nullable RemoteConfig rc) { - super(leader, name, cfg); - this.uri = uri; - this.remoteConfig = rc; - } - - /** - * Get URI to contact the remote peer repository. - * - * @return URI to contact the remote peer repository. - */ - public URIish getURI() { - return uri; - } - - /** - * Get optional configuration describing how to contact the peer. - * - * @return optional configuration describing how to contact the peer. - */ - @Nullable - protected RemoteConfig getRemoteConfig() { - return remoteConfig; - } - - /** {@inheritDoc} */ - @Override - protected String describeForLog() { - return String.format("%s @ %s", getName(), getURI()); //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @Override - protected void startPush(ReplicaPushRequest req) { - getSystem().getExecutor().execute(() -> { - try (Repository git = getLeader().openRepository()) { - try { - push(git, req); - req.done(git); - } catch (Throwable err) { - req.setException(git, err); - } - } catch (IOException err) { - req.setException(null, err); - } - }); - } - - private void push(Repository repo, ReplicaPushRequest req) - throws NotSupportedException, TransportException, IOException { - Map adv; - List cmds = asUpdateList(req.getCommands()); - try (Transport transport = Transport.open(repo, uri)) { - RemoteConfig rc = getRemoteConfig(); - if (rc != null) { - transport.applyConfig(rc); - } - transport.setPushAtomic(true); - adv = push(repo, transport, cmds); - } - for (RemoteCommand c : cmds) { - c.copyStatusToResult(); - } - req.setRefs(adv); - } - - private Map push(Repository git, Transport transport, - List cmds) throws IOException { - Map updates = asUpdateMap(cmds); - try (PushConnection connection = transport.openPush()) { - Map adv = connection.getRefsMap(); - RemoteRefUpdate accepted = updates.get(getSystem().getTxnAccepted()); - if (accepted != null && !isExpectedValue(adv, accepted)) { - abort(cmds); - return adv; - } - - RemoteRefUpdate committed = updates.get(getSystem().getTxnCommitted()); - if (committed != null && !isExpectedValue(adv, committed)) { - abort(cmds); - return adv; - } - if (committed != null && getCommitMethod() == ALL_REFS) { - prepareCommit(git, cmds, updates, adv, - committed.getNewObjectId()); - } - - connection.push(NullProgressMonitor.INSTANCE, updates); - return adv; - } - } - - private static boolean isExpectedValue(Map adv, - RemoteRefUpdate u) { - Ref r = adv.get(u.getRemoteName()); - if (!AnyObjectId.isEqual(getId(r), u.getExpectedOldObjectId())) { - ((RemoteCommand) u).cmd.setResult(LOCK_FAILURE); - return false; - } - return true; - } - - private void prepareCommit(Repository git, List cmds, - Map updates, Map adv, - ObjectId committed) throws IOException { - for (ReceiveCommand cmd : prepareCommit(git, adv, committed)) { - RemoteCommand c = new RemoteCommand(cmd); - cmds.add(c); - updates.put(c.getRemoteName(), c); - } - } - - private static List asUpdateList( - Collection cmds) { - try { - List toPush = new ArrayList<>(cmds.size()); - for (ReceiveCommand cmd : cmds) { - toPush.add(new RemoteCommand(cmd)); - } - return toPush; - } catch (IOException e) { - // Cannot occur as no IO was required to build the command. - throw new IllegalStateException(e); - } - } - - private static Map asUpdateMap( - List cmds) { - Map m = new LinkedHashMap<>(); - for (RemoteCommand cmd : cmds) { - m.put(cmd.getRemoteName(), cmd); - } - return m; - } - - private static void abort(List cmds) { - List tmp = new ArrayList<>(cmds.size()); - for (RemoteCommand cmd : cmds) { - tmp.add(cmd.cmd); - } - ReceiveCommand.abort(tmp); - } - - /** {@inheritDoc} */ - @Override - protected void blockingFetch(Repository repo, ReplicaFetchRequest req) - throws NotSupportedException, TransportException { - try (Transport transport = Transport.open(repo, uri)) { - RemoteConfig rc = getRemoteConfig(); - if (rc != null) { - transport.applyConfig(rc); - } - fetch(transport, req); - } - } - - private void fetch(Transport transport, ReplicaFetchRequest req) - throws NotSupportedException, TransportException { - try (FetchConnection conn = transport.openFetch()) { - Map remoteRefs = conn.getRefsMap(); - req.setRefs(remoteRefs); - - List want = new ArrayList<>(); - for (String name : req.getWantRefs()) { - Ref ref = remoteRefs.get(name); - if (ref != null && ref.getObjectId() != null) { - want.add(ref); - } - } - for (ObjectId id : req.getWantObjects()) { - want.add(new ObjectIdRef.Unpeeled(NETWORK, id.name(), id)); - } - - conn.fetch(NullProgressMonitor.INSTANCE, want, - Collections. emptySet()); - } - } - - static class RemoteCommand extends RemoteRefUpdate { - final ReceiveCommand cmd; - - RemoteCommand(ReceiveCommand cmd) throws IOException { - super(null, null, - cmd.getNewId(), cmd.getRefName(), - true /* force update */, - null /* no local tracking ref */, - cmd.getOldId()); - this.cmd = cmd; - } - - void copyStatusToResult() { - if (cmd.getResult() == NOT_ATTEMPTED) { - switch (getStatus()) { - case OK: - case UP_TO_DATE: - case NON_EXISTING: - cmd.setResult(OK); - break; - - case REJECTED_NODELETE: - cmd.setResult(REJECTED_NODELETE); - break; - - case REJECTED_NONFASTFORWARD: - cmd.setResult(REJECTED_NONFASTFORWARD); - break; - - case REJECTED_OTHER_REASON: - cmd.setResult(REJECTED_OTHER_REASON, getMessage()); - break; - - default: - cmd.setResult(REJECTED_OTHER_REASON, getStatus().name()); - break; - } - } - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java deleted file mode 100644 index 1d323b8490..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaConfig.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static java.util.concurrent.TimeUnit.DAYS; -import static java.util.concurrent.TimeUnit.HOURS; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_COMMIT; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_SPEED; -import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; -import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod; -import org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed; -import org.eclipse.jgit.internal.ketch.KetchReplica.Participation; -import org.eclipse.jgit.lib.Config; - -/** - * Configures a {@link org.eclipse.jgit.internal.ketch.KetchReplica}. - */ -public class ReplicaConfig { - /** - * Read a configuration from a config block. - * - * @param cfg - * configuration to read. - * @param name - * of the replica being configured. - * @return replica configuration for {@code name}. - */ - public static ReplicaConfig newFromConfig(Config cfg, String name) { - return new ReplicaConfig().fromConfig(cfg, name); - } - - private Participation participation = Participation.FULL; - private CommitMethod commitMethod = CommitMethod.ALL_REFS; - private CommitSpeed commitSpeed = CommitSpeed.BATCHED; - private long minRetry = SECONDS.toMillis(5); - private long maxRetry = MINUTES.toMillis(1); - - /** - * Get participation of the replica in the system. - * - * @return participation of the replica in the system. - */ - public Participation getParticipation() { - return participation; - } - - /** - * Get how Ketch should apply committed changes. - * - * @return how Ketch should apply committed changes. - */ - public CommitMethod getCommitMethod() { - return commitMethod; - } - - /** - * Get how quickly should Ketch commit. - * - * @return how quickly should Ketch commit. - */ - public CommitSpeed getCommitSpeed() { - return commitSpeed; - } - - /** - * Returns the minimum wait delay before retrying a failure. - * - * @param unit - * to get retry delay in. - * @return minimum delay before retrying a failure. - */ - public long getMinRetry(TimeUnit unit) { - return unit.convert(minRetry, MILLISECONDS); - } - - /** - * Returns the maximum wait delay before retrying a failure. - * - * @param unit - * to get retry delay in. - * @return maximum delay before retrying a failure. - */ - public long getMaxRetry(TimeUnit unit) { - return unit.convert(maxRetry, MILLISECONDS); - } - - /** - * Update the configuration from a config block. - * - * @param cfg - * configuration to read. - * @param name - * of the replica being configured. - * @return {@code this} - */ - public ReplicaConfig fromConfig(Config cfg, String name) { - participation = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE, - participation); - commitMethod = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_COMMIT, - commitMethod); - commitSpeed = cfg.getEnum( - CONFIG_KEY_REMOTE, name, CONFIG_KEY_SPEED, - commitSpeed); - minRetry = getMillis(cfg, name, "ketch-minRetry", minRetry); //$NON-NLS-1$ - maxRetry = getMillis(cfg, name, "ketch-maxRetry", maxRetry); //$NON-NLS-1$ - return this; - } - - private static long getMillis(Config cfg, String name, String key, - long defaultValue) { - String valStr = cfg.getString(CONFIG_KEY_REMOTE, name, key); - if (valStr == null) { - return defaultValue; - } - - valStr = valStr.trim(); - if (valStr.isEmpty()) { - return defaultValue; - } - - Matcher m = UnitMap.PATTERN.matcher(valStr); - if (!m.matches()) { - return defaultValue; - } - - String digits = m.group(1); - String unitName = m.group(2).trim(); - TimeUnit unit = UnitMap.UNITS.get(unitName); - if (unit == null) { - return defaultValue; - } - - try { - if (digits.indexOf('.') == -1) { - return unit.toMillis(Long.parseLong(digits)); - } - - double val = Double.parseDouble(digits); - return (long) (val * unit.toMillis(1)); - } catch (NumberFormatException nfe) { - return defaultValue; - } - } - - static class UnitMap { - static final Pattern PATTERN = Pattern - .compile("^([1-9][0-9]*(?:\\.[0-9]*)?)\\s*(.*)$"); //$NON-NLS-1$ - - static final Map UNITS; - - static { - Map m = new HashMap<>(); - TimeUnit u = MILLISECONDS; - m.put("", u); //$NON-NLS-1$ - m.put("ms", u); //$NON-NLS-1$ - m.put("millis", u); //$NON-NLS-1$ - m.put("millisecond", u); //$NON-NLS-1$ - m.put("milliseconds", u); //$NON-NLS-1$ - - u = SECONDS; - m.put("s", u); //$NON-NLS-1$ - m.put("sec", u); //$NON-NLS-1$ - m.put("secs", u); //$NON-NLS-1$ - m.put("second", u); //$NON-NLS-1$ - m.put("seconds", u); //$NON-NLS-1$ - - u = MINUTES; - m.put("m", u); //$NON-NLS-1$ - m.put("min", u); //$NON-NLS-1$ - m.put("mins", u); //$NON-NLS-1$ - m.put("minute", u); //$NON-NLS-1$ - m.put("minutes", u); //$NON-NLS-1$ - - u = HOURS; - m.put("h", u); //$NON-NLS-1$ - m.put("hr", u); //$NON-NLS-1$ - m.put("hrs", u); //$NON-NLS-1$ - m.put("hour", u); //$NON-NLS-1$ - m.put("hours", u); //$NON-NLS-1$ - - u = DAYS; - m.put("d", u); //$NON-NLS-1$ - m.put("day", u); //$NON-NLS-1$ - m.put("days", u); //$NON-NLS-1$ - - UNITS = Collections.unmodifiableMap(m); - } - - private UnitMap() { - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java deleted file mode 100644 index f50ad62c80..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaFetchRequest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Map; -import java.util.Set; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.Ref; - -/** - * A fetch request to obtain objects from a replica, and its result. - */ -public class ReplicaFetchRequest { - private final Set wantRefs; - private final Set wantObjects; - private Map refs; - - /** - * Construct a new fetch request for a replica. - * - * @param wantRefs - * named references to be fetched. - * @param wantObjects - * specific objects to be fetched. - */ - public ReplicaFetchRequest(Set wantRefs, - Set wantObjects) { - this.wantRefs = wantRefs; - this.wantObjects = wantObjects; - } - - /** - * Get references to be fetched. - * - * @return references to be fetched. - */ - public Set getWantRefs() { - return wantRefs; - } - - /** - * Get objects to be fetched. - * - * @return objects to be fetched. - */ - public Set getWantObjects() { - return wantObjects; - } - - /** - * Get remote references, usually from the advertisement. - * - * @return remote references, usually from the advertisement. - */ - @Nullable - public Map getRefs() { - return refs; - } - - /** - * Set references observed from the replica. - * - * @param refs - * references observed from the replica. - */ - public void setRefs(Map refs) { - this.refs = refs; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java deleted file mode 100644 index 273760bc6e..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaPushRequest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Collection; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * A push request sending objects to a replica, and its result. - *

- * Implementors of {@link org.eclipse.jgit.internal.ketch.KetchReplica} must - * populate the command result fields, {@link #setRefs(Map)}, and call one of - * {@link #setException(Repository, Throwable)} or {@link #done(Repository)} to - * finish processing. - */ -public class ReplicaPushRequest { - private final KetchReplica replica; - private final Collection commands; - private Map refs; - private Throwable exception; - private boolean notified; - - /** - * Construct a new push request for a replica. - * - * @param replica - * the replica being pushed to. - * @param commands - * commands to be executed. - */ - public ReplicaPushRequest(KetchReplica replica, - Collection commands) { - this.replica = replica; - this.commands = commands; - } - - /** - * Get commands to be executed, and their results. - * - * @return commands to be executed, and their results. - */ - public Collection getCommands() { - return commands; - } - - /** - * Get remote references, usually from the advertisement. - * - * @return remote references, usually from the advertisement. - */ - @Nullable - public Map getRefs() { - return refs; - } - - /** - * Set references observed from the replica. - * - * @param refs - * references observed from the replica. - */ - public void setRefs(Map refs) { - this.refs = refs; - } - - /** - * Get exception thrown, if any. - * - * @return exception thrown, if any. - */ - @Nullable - public Throwable getException() { - return exception; - } - - /** - * Mark the request as crashing with a communication error. - *

- * This method may take significant time acquiring the leader lock and - * updating the Ketch state machine with the failure. - * - * @param repo - * local repository reference used by the push attempt. - * @param err - * exception thrown during communication. - */ - public void setException(@Nullable Repository repo, Throwable err) { - if (KetchReplica.log.isErrorEnabled()) { - KetchReplica.log.error(describe("failed"), err); //$NON-NLS-1$ - } - if (!notified) { - notified = true; - exception = err; - replica.afterPush(repo, this); - } - } - - /** - * Mark the request as completed without exception. - *

- * This method may take significant time acquiring the leader lock and - * updating the Ketch state machine with results from this replica. - * - * @param repo - * local repository reference used by the push attempt. - */ - public void done(Repository repo) { - if (KetchReplica.log.isDebugEnabled()) { - KetchReplica.log.debug(describe("completed")); //$NON-NLS-1$ - } - if (!notified) { - notified = true; - replica.afterPush(repo, this); - } - } - - private String describe(String heading) { - StringBuilder b = new StringBuilder(); - b.append("push to "); //$NON-NLS-1$ - b.append(replica.describeForLog()); - b.append(' ').append(heading).append(":\n"); //$NON-NLS-1$ - for (ReceiveCommand cmd : commands) { - b.append(String.format( - " %-12s %-12s %s %s", //$NON-NLS-1$ - LeaderSnapshot.str(cmd.getOldId()), - LeaderSnapshot.str(cmd.getNewId()), - cmd.getRefName(), - cmd.getResult())); - if (cmd.getMessage() != null) { - b.append(' ').append(cmd.getMessage()); - } - b.append('\n'); - } - return b.toString(); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java deleted file mode 100644 index 05e4ed693a..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/ReplicaSnapshot.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.util.Date; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.ObjectId; - -/** - * A snapshot of a replica. - * - * @see LeaderSnapshot - */ -public class ReplicaSnapshot { - final KetchReplica replica; - ObjectId accepted; - ObjectId committed; - KetchReplica.State state; - String error; - long retryAtMillis; - - ReplicaSnapshot(KetchReplica replica) { - this.replica = replica; - } - - /** - * Get the replica this snapshot describes the state of - * - * @return the replica this snapshot describes the state of - */ - public KetchReplica getReplica() { - return replica; - } - - /** - * Get current state of the replica - * - * @return current state of the replica - */ - public KetchReplica.State getState() { - return state; - } - - /** - * Get last known Git commit at {@code refs/txn/accepted} - * - * @return last known Git commit at {@code refs/txn/accepted} - */ - @Nullable - public ObjectId getAccepted() { - return accepted; - } - - /** - * Get last known Git commit at {@code refs/txn/committed} - * - * @return last known Git commit at {@code refs/txn/committed} - */ - @Nullable - public ObjectId getCommitted() { - return committed; - } - - /** - * Get error message - * - * @return if {@link #getState()} == - * {@link org.eclipse.jgit.internal.ketch.KetchReplica.State#OFFLINE} - * an optional human-readable message from the transport system - * explaining the failure. - */ - @Nullable - public String getErrorMessage() { - return error; - } - - /** - * Get when the leader will retry communication with the offline or lagging - * replica - * - * @return time (usually in the future) when the leader will retry - * communication with the offline or lagging replica; null if no - * retry is scheduled or necessary. - */ - @Nullable - public Date getRetryAt() { - return retryAtMillis > 0 ? new Date(retryAtMillis) : null; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java deleted file mode 100644 index 05da5be056..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/Round.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.io.IOException; -import java.util.List; - -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** - * One round-trip to all replicas proposing a log entry. - *

- * In Raft a log entry represents a state transition at a specific index in the - * replicated log. The leader can only append log entries to the log. - *

- * In Ketch a log entry is recorded under the {@code refs/txn} namespace. This - * occurs when: - *

    - *
  • a replica wants to establish itself as a new leader by proposing a new - * term (see {@link ElectionRound}) - *
  • an established leader wants to gain consensus on new {@link Proposal}s - * (see {@link ProposalRound}) - *
- */ -abstract class Round { - final KetchLeader leader; - final LogIndex acceptedOldIndex; - LogIndex acceptedNewIndex; - List stageCommands; - - Round(KetchLeader leader, LogIndex head) { - this.leader = leader; - this.acceptedOldIndex = head; - } - - KetchSystem getSystem() { - return leader.getSystem(); - } - - /** - * Creates a commit for {@code refs/txn/accepted} and calls - * {@link #runAsync(AnyObjectId)} to begin execution of the round across - * the system. - *

- * If references are being updated (such as in a {@link ProposalRound}) the - * RefTree may be modified. - *

- * Invoked without {@link KetchLeader#lock} to build objects. - * - * @throws IOException - * the round cannot build new objects within the leader's - * repository. The leader may be unable to execute. - */ - abstract void start() throws IOException; - - /** - * Asynchronously distribute the round's new value for - * {@code refs/txn/accepted} to all replicas. - *

- * Invoked by {@link #start()} after new commits have been created for the - * log. The method passes {@code newId} to {@link KetchLeader} to be - * distributed to all known replicas. - * - * @param newId - * new value for {@code refs/txn/accepted}. - */ - void runAsync(AnyObjectId newId) { - acceptedNewIndex = acceptedOldIndex.nextIndex(newId); - leader.runAsync(this); - } - - /** - * Notify the round it was accepted by a majority of the system. - *

- * Invoked by the leader with {@link KetchLeader#lock} held by the caller. - */ - abstract void success(); -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java deleted file mode 100644 index 40d86e1a85..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/StageBuilder.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.treewalk.EmptyTreeIterator; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.treewalk.filter.TreeFilter; - -/** - * Constructs a set of commands to stage content during a proposal. - */ -public class StageBuilder { - /** - * Acceptable number of references to send in a single stage transaction. - *

- * If the number of unique objects exceeds this amount the builder will - * attempt to decrease the reference count by chaining commits.. - */ - private static final int SMALL_BATCH_SIZE = 5; - - /** - * Acceptable number of commits to chain together using parent pointers. - *

- * When staging many unique commits the {@link StageBuilder} batches - * together unrelated commits as parents of a temporary commit. After the - * proposal completes the temporary commit is discarded and can be garbage - * collected by all replicas. - */ - private static final int TEMP_PARENT_BATCH_SIZE = 128; - - private static final byte[] PEEL = { ' ', '^' }; - - private final String txnStage; - private final String txnId; - - /** - * Construct a stage builder for a transaction. - * - * @param txnStageNamespace - * namespace for transaction references to build - * {@code "txnStageNamespace/txnId.n"} style names. - * @param txnId - * identifier used to name temporary staging refs. - */ - public StageBuilder(String txnStageNamespace, ObjectId txnId) { - this.txnStage = txnStageNamespace; - this.txnId = txnId.name(); - } - - /** - * Compare two RefTrees and return commands to stage new objects. - *

- * This method ignores the lineage between the two RefTrees and does a - * straight diff on the two trees. New objects will be staged. The diff - * strategy is useful to catch-up a lagging replica, without sending every - * intermediate step. This may mean the replica does not have the same - * object set as other replicas if there are rewinds or branch deletes. - * - * @param git - * source repository to read {@code oldTree} and {@code newTree} - * from. - * @param oldTree - * accepted RefTree on the replica ({@code refs/txn/accepted}). - * Use {@link org.eclipse.jgit.lib.ObjectId#zeroId()} if the - * remote does not have any ref tree, e.g. a new replica catching - * up. - * @param newTree - * RefTree being sent to the replica. The trees will be compared. - * @return list of commands to create {@code "refs/txn/stage/..."} - * references on replicas anchoring new objects into the repository - * while a transaction gains consensus. - * @throws java.io.IOException - * {@code git} cannot be accessed to compare {@code oldTree} and - * {@code newTree} to build the object set. - */ - public List makeStageList(Repository git, ObjectId oldTree, - ObjectId newTree) throws IOException { - try (RevWalk rw = new RevWalk(git); - TreeWalk tw = new TreeWalk(rw.getObjectReader()); - ObjectInserter ins = git.newObjectInserter()) { - if (AnyObjectId.isEqual(oldTree, ObjectId.zeroId())) { - tw.addTree(new EmptyTreeIterator()); - } else { - tw.addTree(rw.parseTree(oldTree)); - } - tw.addTree(rw.parseTree(newTree)); - tw.setFilter(TreeFilter.ANY_DIFF); - tw.setRecursive(true); - - Set newObjs = new HashSet<>(); - while (tw.next()) { - if (tw.getRawMode(1) == TYPE_GITLINK - && !tw.isPathSuffix(PEEL, 2)) { - newObjs.add(tw.getObjectId(1)); - } - } - - List cmds = makeStageList(newObjs, git, ins); - ins.flush(); - return cmds; - } - } - - /** - * Construct a set of commands to stage objects on a replica. - * - * @param newObjs - * objects to send to a replica. - * @param git - * local repository to read source objects from. Required to - * perform minification of {@code newObjs}. - * @param inserter - * inserter to write temporary commit objects during minification - * if many new branches are created by {@code newObjs}. - * @return list of commands to create {@code "refs/txn/stage/..."} - * references on replicas anchoring {@code newObjs} into the - * repository while a transaction gains consensus. - * @throws java.io.IOException - * {@code git} cannot be accessed to perform minification of - * {@code newObjs}. - */ - public List makeStageList(Set newObjs, - @Nullable Repository git, @Nullable ObjectInserter inserter) - throws IOException { - if (git == null || newObjs.size() <= SMALL_BATCH_SIZE) { - // Without a source repository can only construct unique set. - List cmds = new ArrayList<>(newObjs.size()); - for (ObjectId id : newObjs) { - stage(cmds, id); - } - return cmds; - } - - List cmds = new ArrayList<>(); - List commits = new ArrayList<>(); - reduceObjects(cmds, commits, git, newObjs); - - if (inserter == null || commits.size() <= 1 - || (cmds.size() + commits.size()) <= SMALL_BATCH_SIZE) { - // Without an inserter to aggregate commits, or for a small set of - // commits just send one stage ref per commit. - for (RevCommit c : commits) { - stage(cmds, c.copy()); - } - return cmds; - } - - // 'commits' is sorted most recent to least recent commit. - // Group batches of commits and build a chain. - // TODO(sop) Cluster by restricted graphs to support filtering. - ObjectId tip = null; - for (int end = commits.size(); end > 0;) { - int start = Math.max(0, end - TEMP_PARENT_BATCH_SIZE); - List batch = commits.subList(start, end); - List parents = new ArrayList<>(1 + batch.size()); - if (tip != null) { - parents.add(tip); - } - parents.addAll(batch); - - CommitBuilder b = new CommitBuilder(); - b.setTreeId(batch.get(0).getTree()); - b.setParentIds(parents); - b.setAuthor(tmpAuthor(batch)); - b.setCommitter(b.getAuthor()); - tip = inserter.insert(b); - end = start; - } - stage(cmds, tip); - return cmds; - } - - private static PersonIdent tmpAuthor(List commits) { - // Construct a predictable author using most recent commit time. - int t = 0; - for (int i = 0; i < commits.size();) { - t = Math.max(t, commits.get(i).getCommitTime()); - } - String name = "Ketch Stage"; //$NON-NLS-1$ - String email = "tmp@tmp"; //$NON-NLS-1$ - return new PersonIdent(name, email, t * 1000L, 0); - } - - private void reduceObjects(List cmds, - List commits, Repository git, - Set newObjs) throws IOException { - try (RevWalk rw = new RevWalk(git)) { - rw.setRetainBody(false); - - for (ObjectId id : newObjs) { - RevObject obj = rw.parseAny(id); - if (obj instanceof RevCommit) { - rw.markStart((RevCommit) obj); - } else { - stage(cmds, id); - } - } - - for (RevCommit c; (c = rw.next()) != null;) { - commits.add(c); - rw.markUninteresting(c); - } - } - } - - private void stage(List cmds, ObjectId id) { - int estLen = txnStage.length() + txnId.length() + 5; - StringBuilder n = new StringBuilder(estLen); - n.append(txnStage).append(txnId).append('.'); - n.append(Integer.toHexString(cmds.size())); - cmds.add(new ReceiveCommand(ObjectId.zeroId(), id, n.toString())); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java deleted file mode 100644 index f665e6a438..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/TimeIsUncertainException.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.ketch; - -import java.io.IOException; - -import org.eclipse.jgit.internal.JGitText; - -class TimeIsUncertainException extends IOException { - private static final long serialVersionUID = 1L; - - TimeIsUncertainException() { - super(JGitText.get().timeIsUncertain); - } - - TimeIsUncertainException(Exception e) { - super(JGitText.get().timeIsUncertain, e); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java deleted file mode 100644 index dfe03752ca..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/ketch/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Distributed consensus system built on Git. - */ -package org.eclipse.jgit.internal.ketch; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index 876cbec161..26d5b5b176 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -13,7 +13,6 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; -import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; @@ -45,7 +44,6 @@ import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; -import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -95,7 +93,6 @@ public class DfsGarbageCollector { private Set allHeadsAndTags; private Set allTags; private Set nonHeads; - private Set txnHeads; private Set tagTargets; /** @@ -318,7 +315,6 @@ public class DfsGarbageCollector { allHeadsAndTags = new HashSet<>(); allTags = new HashSet<>(); nonHeads = new HashSet<>(); - txnHeads = new HashSet<>(); tagTargets = new HashSet<>(); for (Ref ref : refsBefore) { if (ref.isSymbolic() || ref.getObjectId() == null) { @@ -328,8 +324,6 @@ public class DfsGarbageCollector { allHeads.add(ref.getObjectId()); } else if (isTag(ref)) { allTags.add(ref.getObjectId()); - } else if (RefTreeNames.isRefTree(refdb, ref.getName())) { - txnHeads.add(ref.getObjectId()); } else { nonHeads.add(ref.getObjectId()); } @@ -355,7 +349,6 @@ public class DfsGarbageCollector { try { packHeads(pm); packRest(pm); - packRefTreeGraph(pm); packGarbage(pm); objdb.commitPack(newPackDesc, toPrune()); rollback = false; @@ -559,19 +552,6 @@ public class DfsGarbageCollector { } } - private void packRefTreeGraph(ProgressMonitor pm) throws IOException { - if (txnHeads.isEmpty()) - return; - - try (PackWriter pw = newPackWriter()) { - for (ObjectIdSet packedObjs : newPackObj) - pw.excludeObjects(packedObjs); - pw.preparePack(pm, txnHeads, NONE); - if (0 < pw.getObjectCount()) - writePack(GC_TXN, pw, pm, 0 /* unknown pack size */); - } - } - private void packGarbage(ProgressMonitor pm) throws IOException { PackConfig cfg = new PackConfig(packConfig); cfg.setReuseDeltas(true); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 4dab3b20c5..46ec87df54 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -104,13 +104,6 @@ public abstract class DfsObjDatabase extends ObjectDatabase { /** Created from non-heads by {@link DfsGarbageCollector}. */ GC_REST, - /** - * RefTreeGraph pack was created by Git garbage collection. - * - * @see DfsGarbageCollector - */ - GC_TXN, - /** * Pack was created by Git garbage collection. *

@@ -133,7 +126,6 @@ public abstract class DfsObjDatabase extends ObjectDatabase { .add(COMPACT) .add(GC) .add(GC_REST) - .add(GC_TXN) .add(UNREACHABLE_GARBAGE) .build(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java index 3f113a3ee3..8e124e3c20 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjectRepresentation.java @@ -48,7 +48,6 @@ class DfsObjectRepresentation extends StoredObjectRepresentation { switch (pack.getPackDescription().getPackSource()) { case GC: case GC_REST: - case GC_TXN: return true; default: return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java index 0c8755fca3..4f418ab4db 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackDescription.java @@ -533,7 +533,6 @@ public class DfsPackDescription { switch (s) { case GC: case GC_REST: - case GC_TXN: return true; default: return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index fd052cec28..a2daef30d8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -40,7 +40,6 @@ import org.eclipse.jgit.events.IndexChangedEvent; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle; import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepository; -import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; import org.eclipse.jgit.lib.BaseRepositoryBuilder; import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.ConfigConstants; @@ -182,9 +181,6 @@ public class FileRepository extends Repository { if (StringUtils.equalsIgnoreCase(reftype, ConfigConstants.CONFIG_REF_STORAGE_REFTABLE)) { refs = new FileReftableDatabase(this); - } else if (StringUtils.equalsIgnoreCase(reftype, - ConfigConstants.CONFIG_REFSTORAGE_REFTREE)) { - refs = new RefTreeDatabase(this, new RefDirectory(this)); } else { throw new IOException(JGitText.get().unknownRepositoryFormat); } @@ -640,7 +636,7 @@ public class FileRepository extends Repository { refsHeadsFile.delete(); // RefDirectory wants to create the refs/ directory from scratch, so // remove that too. - refsFile.delete(); + refsFile.delete(); // remove HEAD so its previous invalid value doesn't cause issues. headFile.delete(); @@ -668,7 +664,7 @@ public class FileRepository extends Repository { for (ReflogEntry e : logs) { logWriter.log(r.getName(), e); } - } + } } try (RevWalk rw = new RevWalk(this)) { @@ -768,7 +764,7 @@ public class FileRepository extends Repository { FileUtils.delete(refsFile, FileUtils.RECURSIVE); for (String r : additional) { new File(getDirectory(), r).delete(); - } + } } FileUtils.mkdir(refsFile, true); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 1f2fe1057f..324075269a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -60,7 +60,6 @@ import org.eclipse.jgit.errors.NoWorkTreeException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; @@ -802,7 +801,6 @@ public class GC { Set txnHeads = new HashSet<>(); Set tagTargets = new HashSet<>(); Set indexObjects = listNonHEADIndexObjects(); - RefDatabase refdb = repo.getRefDatabase(); for (Ref ref : refsBefore) { checkCancelled(); @@ -814,8 +812,6 @@ public class GC { allHeads.add(ref.getObjectId()); } else if (isTag(ref)) { allTags.add(ref.getObjectId()); - } else if (RefTreeNames.isRefTree(refdb, ref.getName())) { - txnHeads.add(ref.getObjectId()); } else { nonHeads.add(ref.getObjectId()); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java deleted file mode 100644 index 5138636089..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/AlwaysFailUpdate.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import java.io.IOException; - -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; - -/** Update that always rejects with {@code LOCK_FAILURE}. */ -class AlwaysFailUpdate extends RefUpdate { - private final RefTreeDatabase refdb; - - AlwaysFailUpdate(RefTreeDatabase refdb, String name) { - super(new ObjectIdRef.Unpeeled(Ref.Storage.NEW, name, null)); - this.refdb = refdb; - setCheckConflicting(false); - } - - /** {@inheritDoc} */ - @Override - protected RefDatabase getRefDatabase() { - return refdb; - } - - /** {@inheritDoc} */ - @Override - protected Repository getRepository() { - return refdb.getRepository(); - } - - /** {@inheritDoc} */ - @Override - protected boolean tryLock(boolean deref) throws IOException { - return false; - } - - /** {@inheritDoc} */ - @Override - protected void unlock() { - // No locks are held here. - } - - /** {@inheritDoc} */ - @Override - protected Result doUpdate(Result desiredResult) { - return Result.LOCK_FAILURE; - } - - /** {@inheritDoc} */ - @Override - protected Result doDelete(Result desiredResult) { - return Result.LOCK_FAILURE; - } - - /** {@inheritDoc} */ - @Override - protected Result doLink(String target) { - return Result.LOCK_FAILURE; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java deleted file mode 100644 index bb06a9e0cf..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Command.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; -import org.eclipse.jgit.transport.ReceiveCommand.Result; - -/** - * Command to create, update or delete an entry inside a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - *

- * Unlike {@link org.eclipse.jgit.transport.ReceiveCommand} (which can only - * update a reference to an {@link org.eclipse.jgit.lib.ObjectId}), a RefTree - * Command can also create, modify or delete symbolic references to a target - * reference. - *

- * RefTree Commands may wrap a {@code ReceiveCommand} to allow callers to - * process an existing ReceiveCommand against a RefTree. - *

- * Commands should be passed into - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree#apply(java.util.Collection)} - * for processing. - */ -public class Command { - /** - * Set unprocessed commands as failed due to transaction aborted. - *

- * If a command is still - * {@link org.eclipse.jgit.transport.ReceiveCommand.Result#NOT_ATTEMPTED} it - * will be set to - * {@link org.eclipse.jgit.transport.ReceiveCommand.Result#REJECTED_OTHER_REASON}. - * If {@code why} is non-null its contents will be used as the message for - * the first command status. - * - * @param commands - * commands to mark as failed. - * @param why - * optional message to set on the first aborted command. - */ - public static void abort(Iterable commands, @Nullable String why) { - if (why == null || why.isEmpty()) { - why = JGitText.get().transactionAborted; - } - for (Command c : commands) { - if (c.getResult() == NOT_ATTEMPTED) { - c.setResult(REJECTED_OTHER_REASON, why); - why = JGitText.get().transactionAborted; - } - } - } - - private final Ref oldRef; - private final Ref newRef; - private final ReceiveCommand cmd; - private Result result; - - /** - * Create a command to create, update or delete a reference. - *

- * At least one of {@code oldRef} or {@code newRef} must be supplied. - * - * @param oldRef - * expected value. Null if the ref should not exist. - * @param newRef - * desired value, must be peeled if not null and not symbolic. - * Null to delete the ref. - */ - public Command(@Nullable Ref oldRef, @Nullable Ref newRef) { - this.oldRef = oldRef; - this.newRef = newRef; - this.cmd = null; - this.result = NOT_ATTEMPTED; - - if (oldRef == null && newRef == null) { - throw new IllegalArgumentException(); - } - if (newRef != null && !newRef.isPeeled() && !newRef.isSymbolic()) { - throw new IllegalArgumentException(); - } - if (oldRef != null && newRef != null - && !oldRef.getName().equals(newRef.getName())) { - throw new IllegalArgumentException(); - } - } - - /** - * Construct a RefTree command wrapped around a ReceiveCommand. - * - * @param rw - * walk instance to peel the {@code newId}. - * @param cmd - * command received from a push client. - * @throws org.eclipse.jgit.errors.MissingObjectException - * {@code oldId} or {@code newId} is missing. - * @throws java.io.IOException - * {@code oldId} or {@code newId} cannot be peeled. - */ - public Command(RevWalk rw, ReceiveCommand cmd) - throws MissingObjectException, IOException { - this.oldRef = toRef(rw, cmd.getOldId(), cmd.getOldSymref(), - cmd.getRefName(), false); - this.newRef = toRef(rw, cmd.getNewId(), cmd.getNewSymref(), - cmd.getRefName(), true); - this.cmd = cmd; - } - - static Ref toRef(RevWalk rw, ObjectId id, @Nullable String target, - String name, boolean mustExist) - throws MissingObjectException, IOException { - if (target != null) { - return new SymbolicRef(name, - new ObjectIdRef.Unpeeled(NETWORK, target, id)); - } else if (ObjectId.zeroId().equals(id)) { - return null; - } - - try { - RevObject o = rw.parseAny(id); - if (o instanceof RevTag) { - RevObject p = rw.peel(o); - return new ObjectIdRef.PeeledTag(NETWORK, name, id, p.copy()); - } - return new ObjectIdRef.PeeledNonTag(NETWORK, name, id); - } catch (MissingObjectException e) { - if (mustExist) { - throw e; - } - return new ObjectIdRef.Unpeeled(NETWORK, name, id); - } - } - - /** - * Get name of the reference affected by this command. - * - * @return name of the reference affected by this command. - */ - public String getRefName() { - if (cmd != null) { - return cmd.getRefName(); - } else if (newRef != null) { - return newRef.getName(); - } - return oldRef.getName(); - } - - /** - * Set the result of this command. - * - * @param result - * the command result. - */ - public void setResult(Result result) { - setResult(result, null); - } - - /** - * Set the result of this command. - * - * @param result - * the command result. - * @param why - * optional message explaining the result status. - */ - public void setResult(Result result, @Nullable String why) { - if (cmd != null) { - cmd.setResult(result, why); - } else { - this.result = result; - } - } - - /** - * Get result of executing this command. - * - * @return result of executing this command. - */ - public Result getResult() { - return cmd != null ? cmd.getResult() : result; - } - - /** - * Get optional message explaining command failure. - * - * @return optional message explaining command failure. - */ - @Nullable - public String getMessage() { - return cmd != null ? cmd.getMessage() : null; - } - - /** - * Old peeled reference. - * - * @return the old reference; null if the command is creating the reference. - */ - @Nullable - public Ref getOldRef() { - return oldRef; - } - - /** - * New peeled reference. - * - * @return the new reference; null if the command is deleting the reference. - */ - @Nullable - public Ref getNewRef() { - return newRef; - } - - /** {@inheritDoc} */ - @Override - public String toString() { - StringBuilder s = new StringBuilder(); - append(s, oldRef, "CREATE"); //$NON-NLS-1$ - s.append(' '); - append(s, newRef, "DELETE"); //$NON-NLS-1$ - s.append(' ').append(getRefName()); - s.append(' ').append(getResult()); - if (getMessage() != null) { - s.append(' ').append(getMessage()); - } - return s.toString(); - } - - private static void append(StringBuilder s, Ref r, String nullName) { - if (r == null) { - s.append(nullName); - } else if (r.isSymbolic()) { - s.append(r.getTarget().getName()); - } else { - ObjectId id = r.getObjectId(); - if (id != null) { - s.append(id.name()); - } - } - } - - /** - * Check the entry is consistent with either the old or the new ref. - * - * @param entry - * current entry; null if the entry does not exist. - * @return true if entry matches {@link #getOldRef()} or - * {@link #getNewRef()}; otherwise false. - */ - boolean checkRef(@Nullable DirCacheEntry entry) { - if (entry != null && entry.getRawMode() == 0) { - entry = null; - } - return check(entry, oldRef) || check(entry, newRef); - } - - private static boolean check(@Nullable DirCacheEntry cur, - @Nullable Ref exp) { - if (cur == null) { - // Does not exist, ok if oldRef does not exist. - return exp == null; - } else if (exp == null) { - // Expected to not exist, but currently exists, fail. - return false; - } - - if (exp.isSymbolic()) { - String dst = exp.getTarget().getName(); - return cur.getRawMode() == TYPE_SYMLINK - && cur.getObjectId().equals(symref(dst)); - } - - return cur.getRawMode() == TYPE_GITLINK - && cur.getObjectId().equals(exp.getObjectId()); - } - - static ObjectId symref(String s) { - @SuppressWarnings("resource") - ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); - return fmt.idFor(OBJ_BLOB, encode(s)); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java deleted file mode 100644 index 6f12e9ccb6..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTree.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.R_REFS; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.GITLINK; -import static org.eclipse.jgit.lib.FileMode.SYMLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.dircache.DirCache; -import org.eclipse.jgit.dircache.DirCacheBuilder; -import org.eclipse.jgit.dircache.DirCacheEditor; -import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; -import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; -import org.eclipse.jgit.dircache.DirCacheEntry; -import org.eclipse.jgit.errors.CorruptObjectException; -import org.eclipse.jgit.errors.DirCacheNameConflictException; -import org.eclipse.jgit.errors.IncorrectObjectTypeException; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.util.RawParseUtils; - -/** - * Tree of references in the reference graph. - *

- * The root corresponds to the {@code "refs/"} subdirectory, for example the - * default reference {@code "refs/heads/master"} is stored at path - * {@code "heads/master"} in a {@code RefTree}. - *

- * Normal references are stored as {@link org.eclipse.jgit.lib.FileMode#GITLINK} - * tree entries. The ObjectId in the tree entry is the ObjectId the reference - * refers to. - *

- * Symbolic references are stored as - * {@link org.eclipse.jgit.lib.FileMode#SYMLINK} entries, with the blob storing - * the name of the target reference. - *

- * Annotated tags also store the peeled object using a {@code GITLINK} entry - * with the suffix " ^" (space carrot), for example - * {@code "tags/v1.0"} stores the annotated tag object, while - * "tags/v1.0 ^" stores the commit the tag annotates. - *

- * {@code HEAD} is a special case and stored as {@code "..HEAD"}. - */ -public class RefTree { - /** Suffix applied to GITLINK to indicate its the peeled value of a tag. */ - public static final String PEELED_SUFFIX = " ^"; //$NON-NLS-1$ - static final String ROOT_DOTDOT = ".."; //$NON-NLS-1$ - - /** - * Create an empty reference tree. - * - * @return a new empty reference tree. - */ - public static RefTree newEmptyTree() { - return new RefTree(DirCache.newInCore()); - } - - /** - * Load a reference tree. - * - * @param reader - * reader to scan the reference tree with. - * @param tree - * the tree to read. - * @return the ref tree read from the commit. - * @throws java.io.IOException - * the repository cannot be accessed through the reader. - * @throws org.eclipse.jgit.errors.CorruptObjectException - * a tree object is corrupt and cannot be read. - * @throws org.eclipse.jgit.errors.IncorrectObjectTypeException - * a tree object wasn't actually a tree. - * @throws org.eclipse.jgit.errors.MissingObjectException - * a reference tree object doesn't exist. - */ - public static RefTree read(ObjectReader reader, RevTree tree) - throws MissingObjectException, IncorrectObjectTypeException, - CorruptObjectException, IOException { - return new RefTree(DirCache.read(reader, tree)); - } - - private DirCache contents; - private Map pendingBlobs; - - private RefTree(DirCache dc) { - this.contents = dc; - } - - /** - * Read one reference. - *

- * References are always returned peeled - * ({@link org.eclipse.jgit.lib.Ref#isPeeled()} is true). If the reference - * points to an annotated tag, the returned reference will be peeled and - * contain {@link org.eclipse.jgit.lib.Ref#getPeeledObjectId()}. - *

- * If the reference is a symbolic reference and the chain depth is less than - * {@link org.eclipse.jgit.lib.RefDatabase#MAX_SYMBOLIC_REF_DEPTH} the - * returned reference is resolved. If the chain depth is longer, the - * symbolic reference is returned without resolving. - * - * @param reader - * to access objects necessary to read the requested reference. - * @param name - * name of the reference to read. - * @return the reference; null if it does not exist. - * @throws java.io.IOException - * cannot read a symbolic reference target. - */ - @Nullable - public Ref exactRef(ObjectReader reader, String name) throws IOException { - Ref r = readRef(reader, name); - if (r == null) { - return null; - } else if (r.isSymbolic()) { - return resolve(reader, r, 0); - } - - DirCacheEntry p = contents.getEntry(peeledPath(name)); - if (p != null && p.getRawMode() == TYPE_GITLINK) { - return new ObjectIdRef.PeeledTag(PACKED, r.getName(), - r.getObjectId(), p.getObjectId()); - } - return r; - } - - private Ref readRef(ObjectReader reader, String name) throws IOException { - DirCacheEntry e = contents.getEntry(refPath(name)); - return e != null ? toRef(reader, e, name) : null; - } - - private Ref toRef(ObjectReader reader, DirCacheEntry e, String name) - throws IOException { - int mode = e.getRawMode(); - if (mode == TYPE_GITLINK) { - ObjectId id = e.getObjectId(); - return new ObjectIdRef.PeeledNonTag(PACKED, name, id); - } - - if (mode == TYPE_SYMLINK) { - ObjectId id = e.getObjectId(); - String n = pendingBlobs != null ? pendingBlobs.get(id) : null; - if (n == null) { - byte[] bin = reader.open(id, OBJ_BLOB).getCachedBytes(); - n = RawParseUtils.decode(bin); - } - Ref dst = new ObjectIdRef.Unpeeled(NEW, n, null); - return new SymbolicRef(name, dst); - } - - return null; // garbage file or something; not a reference. - } - - private Ref resolve(ObjectReader reader, Ref ref, int depth) - throws IOException { - if (ref.isSymbolic() && depth < MAX_SYMBOLIC_REF_DEPTH) { - Ref r = readRef(reader, ref.getTarget().getName()); - if (r == null) { - return ref; - } - Ref dst = resolve(reader, r, depth + 1); - return new SymbolicRef(ref.getName(), dst); - } - return ref; - } - - /** - * Attempt a batch of commands against this RefTree. - *

- * The batch is applied atomically, either all commands apply at once, or - * they all reject and the RefTree is left unmodified. - *

- * On success (when this method returns {@code true}) the command results - * are left as-is (probably {@code NOT_ATTEMPTED}). Result fields are set - * only when this method returns {@code false} to indicate failure. - * - * @param cmdList - * to apply. All commands should still have result NOT_ATTEMPTED. - * @return true if the commands applied; false if they were rejected. - */ - public boolean apply(Collection cmdList) { - try { - DirCacheEditor ed = contents.editor(); - for (Command cmd : cmdList) { - if (!isValidRef(cmd)) { - cmd.setResult(REJECTED_OTHER_REASON, - JGitText.get().funnyRefname); - Command.abort(cmdList, null); - return false; - } - apply(ed, cmd); - } - ed.finish(); - return true; - } catch (DirCacheNameConflictException e) { - String r1 = refName(e.getPath1()); - String r2 = refName(e.getPath2()); - for (Command cmd : cmdList) { - if (r1.equals(cmd.getRefName()) - || r2.equals(cmd.getRefName())) { - cmd.setResult(LOCK_FAILURE); - break; - } - } - Command.abort(cmdList, null); - return false; - } catch (LockFailureException e) { - Command.abort(cmdList, null); - return false; - } - } - - private static boolean isValidRef(Command cmd) { - String n = cmd.getRefName(); - return HEAD.equals(n) || Repository.isValidRefName(n); - } - - private void apply(DirCacheEditor ed, Command cmd) { - String path = refPath(cmd.getRefName()); - Ref oldRef = cmd.getOldRef(); - final Ref newRef = cmd.getNewRef(); - - if (newRef == null) { - checkRef(contents.getEntry(path), cmd); - ed.add(new DeletePath(path)); - cleanupPeeledRef(ed, oldRef); - return; - } - - if (newRef.isSymbolic()) { - final String dst = newRef.getTarget().getName(); - ed.add(new PathEdit(path) { - @Override - public void apply(DirCacheEntry ent) { - checkRef(ent, cmd); - ObjectId id = Command.symref(dst); - ent.setFileMode(SYMLINK); - ent.setObjectId(id); - if (pendingBlobs == null) { - pendingBlobs = new HashMap<>(4); - } - pendingBlobs.put(id, dst); - } - }.setReplace(false)); - cleanupPeeledRef(ed, oldRef); - return; - } - - ed.add(new PathEdit(path) { - @Override - public void apply(DirCacheEntry ent) { - checkRef(ent, cmd); - ent.setFileMode(GITLINK); - ent.setObjectId(newRef.getObjectId()); - } - }.setReplace(false)); - - if (newRef.getPeeledObjectId() != null) { - ed.add(new PathEdit(peeledPath(newRef.getName())) { - @Override - public void apply(DirCacheEntry ent) { - ent.setFileMode(GITLINK); - ent.setObjectId(newRef.getPeeledObjectId()); - } - }.setReplace(false)); - } else { - cleanupPeeledRef(ed, oldRef); - } - } - - private static void checkRef(@Nullable DirCacheEntry ent, Command cmd) { - if (!cmd.checkRef(ent)) { - cmd.setResult(LOCK_FAILURE); - throw new LockFailureException(); - } - } - - private static void cleanupPeeledRef(DirCacheEditor ed, Ref ref) { - if (ref != null && !ref.isSymbolic() - && (!ref.isPeeled() || ref.getPeeledObjectId() != null)) { - ed.add(new DeletePath(peeledPath(ref.getName()))); - } - } - - /** - * Convert a path name in a RefTree to the reference name known by Git. - * - * @param path - * name read from the RefTree structure, for example - * {@code "heads/master"}. - * @return reference name for the path, {@code "refs/heads/master"}. - */ - public static String refName(String path) { - if (path.startsWith(ROOT_DOTDOT)) { - return path.substring(2); - } - return R_REFS + path; - } - - static String refPath(String name) { - if (name.startsWith(R_REFS)) { - return name.substring(R_REFS.length()); - } - return ROOT_DOTDOT + name; - } - - private static String peeledPath(String name) { - return refPath(name) + PEELED_SUFFIX; - } - - /** - * Write this reference tree. - * - * @param inserter - * inserter to use when writing trees to the object database. - * Caller is responsible for flushing the inserter before trying - * to read the objects, or exposing them through a reference. - * @return the top level tree. - * @throws java.io.IOException - * a tree could not be written. - */ - public ObjectId writeTree(ObjectInserter inserter) throws IOException { - if (pendingBlobs != null) { - for (String s : pendingBlobs.values()) { - inserter.insert(OBJ_BLOB, encode(s)); - } - pendingBlobs = null; - } - return contents.writeTree(inserter); - } - - /** - * Create a deep copy of this RefTree. - * - * @return a deep copy of this RefTree. - */ - public RefTree copy() { - RefTree r = new RefTree(DirCache.newInCore()); - DirCacheBuilder b = r.contents.builder(); - for (int i = 0; i < contents.getEntryCount(); i++) { - b.add(new DirCacheEntry(contents.getEntry(i))); - } - b.finish(); - if (pendingBlobs != null) { - r.pendingBlobs = new HashMap<>(pendingBlobs); - } - return r; - } - - private static class LockFailureException extends RuntimeException { - private static final long serialVersionUID = 1L; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java deleted file mode 100644 index b154b95adc..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeBatch.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_TREE; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; -import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE; -import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.CommitBuilder; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** Batch update a {@link RefTreeDatabase}. */ -class RefTreeBatch extends BatchRefUpdate { - private final RefTreeDatabase refdb; - private Ref src; - private ObjectId parentCommitId; - private ObjectId parentTreeId; - private RefTree tree; - private PersonIdent author; - private ObjectId newCommitId; - - RefTreeBatch(RefTreeDatabase refdb) { - super(refdb); - this.refdb = refdb; - } - - /** {@inheritDoc} */ - @Override - public void execute(RevWalk rw, ProgressMonitor monitor) - throws IOException { - List todo = new ArrayList<>(getCommands().size()); - for (ReceiveCommand c : getCommands()) { - if (!isAllowNonFastForwards()) { - if (c.getType() == UPDATE) { - c.updateType(rw); - } - if (c.getType() == UPDATE_NONFASTFORWARD) { - c.setResult(REJECTED_NONFASTFORWARD); - if (isAtomic()) { - ReceiveCommand.abort(getCommands()); - return; - } - continue; - } - } - todo.add(new Command(rw, c)); - } - init(rw); - execute(rw, todo); - } - - void init(RevWalk rw) throws IOException { - src = refdb.getBootstrap().exactRef(refdb.getTxnCommitted()); - if (src != null && src.getObjectId() != null) { - RevCommit c = rw.parseCommit(src.getObjectId()); - parentCommitId = c; - parentTreeId = c.getTree(); - tree = RefTree.read(rw.getObjectReader(), c.getTree()); - } else { - parentCommitId = ObjectId.zeroId(); - try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { - parentTreeId = fmt.idFor(OBJ_TREE, new byte[] {}); - } - tree = RefTree.newEmptyTree(); - } - } - - @Nullable - Ref exactRef(ObjectReader reader, String name) throws IOException { - return tree.exactRef(reader, name); - } - - /** - * Execute an update from {@link RefTreeUpdate} or {@link RefTreeRename}. - * - * @param rw - * current RevWalk handling the update or rename. - * @param todo - * commands to execute. Must never be a bootstrap reference name. - * @throws IOException - * the storage system is unable to read or write data. - */ - void execute(RevWalk rw, List todo) throws IOException { - for (Command c : todo) { - if (c.getResult() != NOT_ATTEMPTED) { - Command.abort(todo, null); - return; - } - if (refdb.conflictsWithBootstrap(c.getRefName())) { - c.setResult(REJECTED_OTHER_REASON, MessageFormat - .format(JGitText.get().invalidRefName, c.getRefName())); - Command.abort(todo, null); - return; - } - } - - if (apply(todo) && newCommitId != null) { - commit(rw, todo); - } - } - - private boolean apply(List todo) throws IOException { - if (!tree.apply(todo)) { - // apply set rejection information on commands. - return false; - } - - Repository repo = refdb.getRepository(); - try (ObjectInserter ins = repo.newObjectInserter()) { - CommitBuilder b = new CommitBuilder(); - b.setTreeId(tree.writeTree(ins)); - if (parentTreeId.equals(b.getTreeId())) { - for (Command c : todo) { - c.setResult(OK); - } - return true; - } - if (!parentCommitId.equals(ObjectId.zeroId())) { - b.setParentId(parentCommitId); - } - - author = getRefLogIdent(); - if (author == null) { - author = new PersonIdent(repo); - } - b.setAuthor(author); - b.setCommitter(author); - b.setMessage(getRefLogMessage()); - newCommitId = ins.insert(b); - ins.flush(); - } - return true; - } - - private void commit(RevWalk rw, List todo) throws IOException { - ReceiveCommand commit = new ReceiveCommand( - parentCommitId, newCommitId, - refdb.getTxnCommitted()); - updateBootstrap(rw, commit); - - if (commit.getResult() == OK) { - for (Command c : todo) { - c.setResult(OK); - } - } else { - Command.abort(todo, commit.getResult().name()); - } - } - - private void updateBootstrap(RevWalk rw, ReceiveCommand commit) - throws IOException { - BatchRefUpdate u = refdb.getBootstrap().newBatchUpdate(); - u.setAllowNonFastForwards(true); - u.setPushCertificate(getPushCertificate()); - if (isRefLogDisabled()) { - u.disableRefLog(); - } else { - u.setRefLogIdent(author); - u.setRefLogMessage(getRefLogMessage(), false); - } - u.addCommand(commit); - u.execute(rw, NullProgressMonitor.INSTANCE); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java deleted file mode 100644 index 34b8f2cd36..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeDatabase.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.lib.BatchRefUpdate; -import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Ref.Storage; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefRename; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.util.RefList; -import org.eclipse.jgit.util.RefMap; - -/** - * Reference database backed by a - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - *

- * The storage for RefTreeDatabase has two parts. The main part is a native Git - * tree object stored under the {@code refs/txn} namespace. To avoid cycles, - * references to {@code refs/txn} are not stored in that tree object, but - * instead in a "bootstrap" layer, which is a separate - * {@link org.eclipse.jgit.lib.RefDatabase} such as - * {@link org.eclipse.jgit.internal.storage.file.RefDirectory} using local - * reference files inside of {@code $GIT_DIR/refs}. - */ -public class RefTreeDatabase extends RefDatabase { - private final Repository repo; - private final RefDatabase bootstrap; - private final String txnCommitted; - - @Nullable - private final String txnNamespace; - private volatile Scanner.Result refs; - - /** - * Create a RefTreeDb for a repository. - * - * @param repo - * the repository using references in this database. - * @param bootstrap - * bootstrap reference database storing the references that - * anchor the - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - */ - public RefTreeDatabase(Repository repo, RefDatabase bootstrap) { - Config cfg = repo.getConfig(); - String committed = cfg.getString("reftree", null, "committedRef"); //$NON-NLS-1$ //$NON-NLS-2$ - if (committed == null || committed.isEmpty()) { - committed = "refs/txn/committed"; //$NON-NLS-1$ - } - - this.repo = repo; - this.bootstrap = bootstrap; - this.txnNamespace = initNamespace(committed); - this.txnCommitted = committed; - } - - /** - * Create a RefTreeDb for a repository. - * - * @param repo - * the repository using references in this database. - * @param bootstrap - * bootstrap reference database storing the references that - * anchor the - * {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. - * @param txnCommitted - * name of the bootstrap reference holding the committed RefTree. - */ - public RefTreeDatabase(Repository repo, RefDatabase bootstrap, - String txnCommitted) { - this.repo = repo; - this.bootstrap = bootstrap; - this.txnNamespace = initNamespace(txnCommitted); - this.txnCommitted = txnCommitted; - } - - private static String initNamespace(String committed) { - int s = committed.lastIndexOf('/'); - if (s < 0) { - return null; - } - return committed.substring(0, s + 1); // Keep trailing '/'. - } - - Repository getRepository() { - return repo; - } - - /** - * Get the bootstrap reference database - * - * @return the bootstrap reference database, which must be used to access - * {@link #getTxnCommitted()}, {@link #getTxnNamespace()}. - */ - public RefDatabase getBootstrap() { - return bootstrap; - } - - /** - * Get name of bootstrap reference anchoring committed RefTree. - * - * @return name of bootstrap reference anchoring committed RefTree. - */ - public String getTxnCommitted() { - return txnCommitted; - } - - /** - * Get namespace used by bootstrap layer. - * - * @return namespace used by bootstrap layer, e.g. {@code refs/txn/}. Always - * ends in {@code '/'}. - */ - @Nullable - public String getTxnNamespace() { - return txnNamespace; - } - - /** {@inheritDoc} */ - @Override - public void create() throws IOException { - bootstrap.create(); - } - - /** {@inheritDoc} */ - @Override - public boolean performsAtomicTransactions() { - return true; - } - - /** {@inheritDoc} */ - @Override - public void refresh() { - bootstrap.refresh(); - } - - /** {@inheritDoc} */ - @Override - public void close() { - refs = null; - bootstrap.close(); - } - - /** {@inheritDoc} */ - @Override - public Ref exactRef(String name) throws IOException { - if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { - // Pass through names like MERGE_HEAD, ORIG_HEAD, FETCH_HEAD. - return bootstrap.exactRef(name); - } else if (conflictsWithBootstrap(name)) { - return null; - } - - boolean partial = false; - Ref src = bootstrap.exactRef(txnCommitted); - Scanner.Result c = refs; - if (c == null || !c.refTreeId.equals(idOf(src))) { - c = Scanner.scanRefTree(repo, src, prefixOf(name), false); - partial = true; - } - - Ref r = c.all.get(name); - if (r != null && r.isSymbolic()) { - r = c.sym.get(name); - if (partial && r.getObjectId() == null) { - // Attempting exactRef("HEAD") with partial scan will leave - // an unresolved symref as its target e.g. refs/heads/master - // was not read by the partial scan. Scan everything instead. - return getRefs(ALL).get(name); - } - } - return r; - } - - private static String prefixOf(String name) { - int s = name.lastIndexOf('/'); - if (s >= 0) { - return name.substring(0, s); - } - return ""; //$NON-NLS-1$ - } - - /** {@inheritDoc} */ - @Override - public Map getRefs(String prefix) throws IOException { - if (!prefix.isEmpty() && prefix.charAt(prefix.length() - 1) != '/') { - return new HashMap<>(0); - } - - Ref src = bootstrap.exactRef(txnCommitted); - Scanner.Result c = refs; - if (c == null || !c.refTreeId.equals(idOf(src))) { - c = Scanner.scanRefTree(repo, src, prefix, true); - if (prefix.isEmpty()) { - refs = c; - } - } - return new RefMap(prefix, RefList. emptyList(), c.all, c.sym); - } - - private static ObjectId idOf(@Nullable Ref src) { - return src != null && src.getObjectId() != null - ? src.getObjectId() - : ObjectId.zeroId(); - } - - /** {@inheritDoc} */ - @Override - public List getAdditionalRefs() throws IOException { - Collection txnRefs; - if (txnNamespace != null) { - txnRefs = bootstrap.getRefsByPrefix(txnNamespace); - } else { - Ref r = bootstrap.exactRef(txnCommitted); - if (r != null && r.getObjectId() != null) { - txnRefs = Collections.singleton(r); - } else { - txnRefs = Collections.emptyList(); - } - } - - List otherRefs = bootstrap.getAdditionalRefs(); - List all = new ArrayList<>(txnRefs.size() + otherRefs.size()); - all.addAll(txnRefs); - all.addAll(otherRefs); - return all; - } - - /** {@inheritDoc} */ - @Override - public Ref peel(Ref ref) throws IOException { - Ref i = ref.getLeaf(); - ObjectId id = i.getObjectId(); - if (i.isPeeled() || id == null) { - return ref; - } - try (RevWalk rw = new RevWalk(repo)) { - RevObject obj = rw.parseAny(id); - if (obj instanceof RevTag) { - ObjectId p = rw.peel(obj).copy(); - i = new ObjectIdRef.PeeledTag(PACKED, i.getName(), id, p); - } else { - i = new ObjectIdRef.PeeledNonTag(PACKED, i.getName(), id); - } - } - return recreate(ref, i); - } - - private static Ref recreate(Ref old, Ref leaf) { - if (old.isSymbolic()) { - Ref dst = recreate(old.getTarget(), leaf); - return new SymbolicRef(old.getName(), dst); - } - return leaf; - } - - /** {@inheritDoc} */ - @Override - public boolean isNameConflicting(String name) throws IOException { - return conflictsWithBootstrap(name) - || !getConflictingNames(name).isEmpty(); - } - - /** {@inheritDoc} */ - @Override - public BatchRefUpdate newBatchUpdate() { - return new RefTreeBatch(this); - } - - /** {@inheritDoc} */ - @Override - public RefUpdate newUpdate(String name, boolean detach) throws IOException { - if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { - return bootstrap.newUpdate(name, detach); - } - if (conflictsWithBootstrap(name)) { - return new AlwaysFailUpdate(this, name); - } - - Ref r = exactRef(name); - if (r == null) { - r = new ObjectIdRef.Unpeeled(Storage.NEW, name, null); - } - - boolean detaching = detach && r.isSymbolic(); - if (detaching) { - r = new ObjectIdRef.Unpeeled(LOOSE, name, r.getObjectId()); - } - - RefTreeUpdate u = new RefTreeUpdate(this, r); - if (detaching) { - u.setDetachingSymbolicRef(); - } - return u; - } - - /** {@inheritDoc} */ - @Override - public RefRename newRename(String fromName, String toName) - throws IOException { - RefUpdate from = newUpdate(fromName, true); - RefUpdate to = newUpdate(toName, true); - return new RefTreeRename(this, from, to); - } - - boolean conflictsWithBootstrap(String name) { - if (txnNamespace != null && name.startsWith(txnNamespace)) { - return true; - } else if (txnCommitted.equals(name)) { - return true; - } - - if (name.indexOf('/') < 0 && !HEAD.equals(name)) { - return true; - } - - if (name.length() > txnCommitted.length() - && name.charAt(txnCommitted.length()) == '/' - && name.startsWith(txnCommitted)) { - return true; - } - return false; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java deleted file mode 100644 index eec0da29e2..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeNames.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import org.eclipse.jgit.lib.RefDatabase; - -/** - * Magic reference name logic for RefTrees. - */ -public class RefTreeNames { - /** - * Suffix used on a {@link RefTreeDatabase#getTxnNamespace()} for user data. - *

- * A {@link RefTreeDatabase}'s namespace may include a subspace (e.g. - * {@code "refs/txn/stage/"}) containing commit objects from the usual user - * portion of the repository (e.g. {@code "refs/heads/"}). These should be - * packed by the garbage collector alongside other user content rather than - * with the RefTree. - */ - private static final String STAGE = "stage/"; //$NON-NLS-1$ - - /** - * Determine if the reference is likely to be a RefTree. - * - * @param refdb - * database instance. - * @param ref - * reference name. - * @return {@code true} if the reference is a RefTree. - */ - public static boolean isRefTree(RefDatabase refdb, String ref) { - if (refdb instanceof RefTreeDatabase) { - RefTreeDatabase b = (RefTreeDatabase) refdb; - if (ref.equals(b.getTxnCommitted())) { - return true; - } - - String namespace = b.getTxnNamespace(); - if (namespace != null - && ref.startsWith(namespace) - && !ref.startsWith(namespace + STAGE)) { - return true; - } - } - return false; - } - - private RefTreeNames() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java deleted file mode 100644 index ccf8f75e7d..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeRename.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.HEAD; -import static org.eclipse.jgit.lib.RefUpdate.Result.REJECTED; -import static org.eclipse.jgit.lib.RefUpdate.Result.RENAMED; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefRename; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.RefUpdate.Result; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevWalk; - -/** Single reference rename to {@link RefTreeDatabase}. */ -class RefTreeRename extends RefRename { - private final RefTreeDatabase refdb; - - RefTreeRename(RefTreeDatabase refdb, RefUpdate src, RefUpdate dst) { - super(src, dst); - this.refdb = refdb; - } - - /** {@inheritDoc} */ - @Override - protected Result doRename() throws IOException { - try (RevWalk rw = new RevWalk(refdb.getRepository())) { - RefTreeBatch batch = new RefTreeBatch(refdb); - batch.setRefLogIdent(getRefLogIdent()); - batch.setRefLogMessage(getRefLogMessage(), false); - batch.init(rw); - - Ref head = batch.exactRef(rw.getObjectReader(), HEAD); - Ref oldRef = batch.exactRef(rw.getObjectReader(), source.getName()); - if (oldRef == null) { - return REJECTED; - } - - Ref newRef = asNew(oldRef); - List mv = new ArrayList<>(3); - mv.add(new Command(oldRef, null)); - mv.add(new Command(null, newRef)); - if (head != null && head.isSymbolic() - && head.getTarget().getName().equals(oldRef.getName())) { - mv.add(new Command( - head, - new SymbolicRef(head.getName(), newRef))); - } - batch.execute(rw, mv); - return RefTreeUpdate.translate(mv.get(1).getResult(), RENAMED); - } - } - - private Ref asNew(Ref src) { - String name = destination.getName(); - if (src.isSymbolic()) { - return new SymbolicRef(name, src.getTarget()); - } - - ObjectId peeled = src.getPeeledObjectId(); - if (peeled != null) { - return new ObjectIdRef.PeeledTag( - src.getStorage(), - name, - src.getObjectId(), - peeled); - } - - return new ObjectIdRef.PeeledNonTag( - src.getStorage(), - name, - src.getObjectId()); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java deleted file mode 100644 index 6e6ccd9986..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/RefTreeUpdate.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; - -import java.io.IOException; -import java.util.Collections; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.RefDatabase; -import org.eclipse.jgit.lib.RefUpdate; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.transport.ReceiveCommand; - -/** Single reference update to {@link RefTreeDatabase}. */ -class RefTreeUpdate extends RefUpdate { - private final RefTreeDatabase refdb; - private RevWalk rw; - private RefTreeBatch batch; - private Ref oldRef; - - RefTreeUpdate(RefTreeDatabase refdb, Ref ref) { - super(ref); - this.refdb = refdb; - setCheckConflicting(false); // Done automatically by doUpdate. - } - - /** {@inheritDoc} */ - @Override - protected RefDatabase getRefDatabase() { - return refdb; - } - - /** {@inheritDoc} */ - @Override - protected Repository getRepository() { - return refdb.getRepository(); - } - - /** {@inheritDoc} */ - @Override - protected boolean tryLock(boolean deref) throws IOException { - rw = new RevWalk(getRepository()); - batch = new RefTreeBatch(refdb); - batch.init(rw); - oldRef = batch.exactRef(rw.getObjectReader(), getName()); - if (oldRef != null && oldRef.getObjectId() != null) { - setOldObjectId(oldRef.getObjectId()); - } else if (oldRef == null && getExpectedOldObjectId() != null) { - setOldObjectId(ObjectId.zeroId()); - } - return true; - } - - /** {@inheritDoc} */ - @Override - protected void unlock() { - batch = null; - if (rw != null) { - rw.close(); - rw = null; - } - } - - /** {@inheritDoc} */ - @Override - protected Result doUpdate(Result desiredResult) throws IOException { - return run(newRef(getName(), getNewObjectId()), desiredResult); - } - - private Ref newRef(String name, ObjectId id) - throws MissingObjectException, IOException { - RevObject o = rw.parseAny(id); - if (o instanceof RevTag) { - RevObject p = rw.peel(o); - return new ObjectIdRef.PeeledTag(LOOSE, name, id, p.copy()); - } - return new ObjectIdRef.PeeledNonTag(LOOSE, name, id); - } - - /** {@inheritDoc} */ - @Override - protected Result doDelete(Result desiredResult) throws IOException { - return run(null, desiredResult); - } - - /** {@inheritDoc} */ - @Override - protected Result doLink(String target) throws IOException { - Ref dst = new ObjectIdRef.Unpeeled(NEW, target, null); - SymbolicRef n = new SymbolicRef(getName(), dst); - Result desiredResult = getRef().getStorage() == NEW - ? Result.NEW - : Result.FORCED; - return run(n, desiredResult); - } - - private Result run(@Nullable Ref newRef, Result desiredResult) - throws IOException { - Command c = new Command(oldRef, newRef); - batch.setRefLogIdent(getRefLogIdent()); - batch.setRefLogMessage(getRefLogMessage(), isRefLogIncludingResult()); - batch.execute(rw, Collections.singletonList(c)); - return translate(c.getResult(), desiredResult); - } - - static Result translate(ReceiveCommand.Result r, Result desiredResult) { - switch (r) { - case OK: - return desiredResult; - - case LOCK_FAILURE: - return Result.LOCK_FAILURE; - - case NOT_ATTEMPTED: - return Result.NOT_ATTEMPTED; - - case REJECTED_MISSING_OBJECT: - return Result.IO_FAILURE; - - case REJECTED_CURRENT_BRANCH: - return Result.REJECTED_CURRENT_BRANCH; - - case REJECTED_OTHER_REASON: - case REJECTED_NOCREATE: - case REJECTED_NODELETE: - case REJECTED_NONFASTFORWARD: - default: - return Result.REJECTED; - } - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java deleted file mode 100644 index 3f5122911c..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftree/Scanner.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright (C) 2016, Google Inc. and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.reftree; - -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.eclipse.jgit.lib.Constants.R_REFS; -import static org.eclipse.jgit.lib.Constants.encode; -import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; -import static org.eclipse.jgit.lib.FileMode.TYPE_TREE; -import static org.eclipse.jgit.lib.Ref.Storage.NEW; -import static org.eclipse.jgit.lib.Ref.Storage.PACKED; -import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; - -import java.io.IOException; - -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.errors.IncorrectObjectTypeException; -import org.eclipse.jgit.lib.AnyObjectId; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdRef; -import org.eclipse.jgit.lib.ObjectReader; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.SymbolicRef; -import org.eclipse.jgit.revwalk.RevTree; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.treewalk.AbstractTreeIterator; -import org.eclipse.jgit.treewalk.CanonicalTreeParser; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.Paths; -import org.eclipse.jgit.util.RawParseUtils; -import org.eclipse.jgit.util.RefList; - -/** A tree parser that extracts references from a {@link RefTree}. */ -class Scanner { - private static final int MAX_SYMLINK_BYTES = 10 << 10; - private static final byte[] BINARY_R_REFS = encode(R_REFS); - private static final byte[] REFS_DOT_DOT = encode("refs/.."); //$NON-NLS-1$ - - static class Result { - final ObjectId refTreeId; - final RefList all; - final RefList sym; - - Result(ObjectId id, RefList all, RefList sym) { - this.refTreeId = id; - this.all = all; - this.sym = sym; - } - } - - /** - * Scan a {@link RefTree} and parse entries into {@link Ref} instances. - * - * @param repo - * source repository containing the commit and tree objects that - * make up the RefTree. - * @param src - * bootstrap reference such as {@code refs/txn/committed} to read - * the reference tree tip from. The current ObjectId will be - * included in {@link Result#refTreeId}. - * @param prefix - * if non-empty a reference prefix to scan only a subdirectory. - * For example {@code prefix = "refs/heads/"} will limit the scan - * to only the {@code "heads"} directory of the RefTree, avoiding - * other directories like {@code "tags"}. Empty string reads all - * entries in the RefTree. - * @param recursive - * if true recurse into subdirectories of the reference tree; - * false to read only one level. Callers may use false during an - * implementation of {@code exactRef(String)} where only one - * reference is needed out of a specific subtree. - * @return sorted list of references after parsing. - * @throws IOException - * tree cannot be accessed from the repository. - */ - static Result scanRefTree(Repository repo, @Nullable Ref src, String prefix, - boolean recursive) throws IOException { - RefList.Builder all = new RefList.Builder<>(); - RefList.Builder sym = new RefList.Builder<>(); - - ObjectId srcId; - if (src != null && src.getObjectId() != null) { - try (ObjectReader reader = repo.newObjectReader()) { - srcId = src.getObjectId(); - scan(reader, srcId, prefix, recursive, all, sym); - } - } else { - srcId = ObjectId.zeroId(); - } - - RefList aList = all.toRefList(); - for (int idx = 0; idx < sym.size();) { - Ref s = sym.get(idx); - Ref r = resolve(s, 0, aList); - if (r != null) { - sym.set(idx++, r); - } else { - // Remove broken symbolic reference, they don't exist. - sym.remove(idx); - int rm = aList.find(s.getName()); - if (0 <= rm) { - aList = aList.remove(rm); - } - } - } - return new Result(srcId, aList, sym.toRefList()); - } - - private static void scan(ObjectReader reader, AnyObjectId srcId, - String prefix, boolean recursive, - RefList.Builder all, RefList.Builder sym) - throws IncorrectObjectTypeException, IOException { - CanonicalTreeParser p = createParserAtPath(reader, srcId, prefix); - if (p == null) { - return; - } - - while (!p.eof()) { - int mode = p.getEntryRawMode(); - if (mode == TYPE_TREE) { - if (recursive) { - p = p.createSubtreeIterator(reader); - } else { - p = p.next(); - } - continue; - } - - if (!curElementHasPeelSuffix(p)) { - Ref r = toRef(reader, mode, p); - if (r != null) { - all.add(r); - if (r.isSymbolic()) { - sym.add(r); - } - } - } else if (mode == TYPE_GITLINK) { - peel(all, p); - } - p = p.next(); - } - } - - private static CanonicalTreeParser createParserAtPath(ObjectReader reader, - AnyObjectId srcId, String prefix) throws IOException { - ObjectId root = toTree(reader, srcId); - if (prefix.isEmpty()) { - return new CanonicalTreeParser(BINARY_R_REFS, reader, root); - } - - String dir = RefTree.refPath(Paths.stripTrailingSeparator(prefix)); - TreeWalk tw = TreeWalk.forPath(reader, dir, root); - if (tw == null || !tw.isSubtree()) { - return null; - } - - ObjectId id = tw.getObjectId(0); - return new CanonicalTreeParser(encode(prefix), reader, id); - } - - private static Ref resolve(Ref ref, int depth, RefList refs) - throws IOException { - if (!ref.isSymbolic()) { - return ref; - } else if (MAX_SYMBOLIC_REF_DEPTH <= depth) { - return null; - } - - Ref r = refs.get(ref.getTarget().getName()); - if (r == null) { - return ref; - } - - Ref dst = resolve(r, depth + 1, refs); - if (dst == null) { - return null; - } - return new SymbolicRef(ref.getName(), dst); - } - - private static RevTree toTree(ObjectReader reader, AnyObjectId id) - throws IOException { - try (RevWalk rw = new RevWalk(reader)) { - return rw.parseTree(id); - } - } - - private static boolean curElementHasPeelSuffix(AbstractTreeIterator itr) { - int n = itr.getEntryPathLength(); - byte[] c = itr.getEntryPathBuffer(); - return n > 2 && c[n - 2] == ' ' && c[n - 1] == '^'; - } - - private static void peel(RefList.Builder all, CanonicalTreeParser p) { - String name = refName(p, true); - for (int idx = all.size() - 1; 0 <= idx; idx--) { - Ref r = all.get(idx); - int cmp = r.getName().compareTo(name); - if (cmp == 0) { - all.set(idx, new ObjectIdRef.PeeledTag(PACKED, r.getName(), - r.getObjectId(), p.getEntryObjectId())); - break; - } else if (cmp < 0) { - // Stray peeled name without matching base name; skip entry. - break; - } - } - } - - private static Ref toRef(ObjectReader reader, int mode, - CanonicalTreeParser p) throws IOException { - if (mode == TYPE_GITLINK) { - String name = refName(p, false); - ObjectId id = p.getEntryObjectId(); - return new ObjectIdRef.PeeledNonTag(PACKED, name, id); - - } else if (mode == TYPE_SYMLINK) { - ObjectId id = p.getEntryObjectId(); - byte[] bin = reader.open(id, OBJ_BLOB) - .getCachedBytes(MAX_SYMLINK_BYTES); - String dst = RawParseUtils.decode(bin); - Ref trg = new ObjectIdRef.Unpeeled(NEW, dst, null); - String name = refName(p, false); - return new SymbolicRef(name, trg); - } - return null; - } - - private static String refName(CanonicalTreeParser p, boolean peel) { - byte[] buf = p.getEntryPathBuffer(); - int len = p.getEntryPathLength(); - if (peel) { - len -= 2; - } - int ptr = 0; - if (RawParseUtils.match(buf, ptr, REFS_DOT_DOT) > 0) { - ptr = 7; - } - return RawParseUtils.decode(buf, ptr, len); - } - - private Scanner() { - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 2587947c3f..954a75cb2c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -553,12 +553,6 @@ public final class ConfigConstants { */ public static final String CONFIG_REF_STORAGE_REFTABLE = "reftable"; - /** - * The "reftree" refStorage format - * @since 5.7 - */ - public static final String CONFIG_REFSTORAGE_REFTREE = "reftree"; - /** * The "jmx" section * @since 5.1.13