This was experimental code and never used in production. Change-Id: Ia3da7f2b82d9e365cec2ccf9397cbc47439cd150 Signed-off-by: Han-Wen Nienhuys <hanwen@google.com> Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>tags/v5.11.0.202102031030-m2
org.eclipse.jgit.dircache;version="[5.11.0,5.12.0)", | org.eclipse.jgit.dircache;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.errors;version="[5.11.0,5.12.0)", | org.eclipse.jgit.errors;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.gitrepo;version="[5.11.0,5.12.0)", | org.eclipse.jgit.gitrepo;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.ketch;version="[5.11.0,5.12.0)", | |||||
org.eclipse.jgit.internal.storage.file;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.storage.file;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.storage.io;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.storage.io;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.storage.pack;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.storage.pack;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.storage.reftable;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.storage.reftable;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.storage.reftree;version="[5.11.0,5.12.0)", | |||||
org.eclipse.jgit.lfs;version="[5.11.0,5.12.0)", | org.eclipse.jgit.lfs;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.lfs.server;version="[5.11.0,5.12.0)", | org.eclipse.jgit.lfs.server;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.lfs.server.fs;version="[5.11.0,5.12.0)", | org.eclipse.jgit.lfs.server.fs;version="[5.11.0,5.12.0)", |
org.eclipse.jgit.pgm.debug.ReadDirCache | org.eclipse.jgit.pgm.debug.ReadDirCache | ||||
org.eclipse.jgit.pgm.debug.ReadReftable | org.eclipse.jgit.pgm.debug.ReadReftable | ||||
org.eclipse.jgit.pgm.debug.RebuildCommitGraph | org.eclipse.jgit.pgm.debug.RebuildCommitGraph | ||||
org.eclipse.jgit.pgm.debug.RebuildRefTree | |||||
org.eclipse.jgit.pgm.debug.ShowCacheTree | org.eclipse.jgit.pgm.debug.ShowCacheTree | ||||
org.eclipse.jgit.pgm.debug.ShowCommands | org.eclipse.jgit.pgm.debug.ShowCommands | ||||
org.eclipse.jgit.pgm.debug.ShowDirCache | org.eclipse.jgit.pgm.debug.ShowDirCache |
metaVar_filepattern=filepattern | metaVar_filepattern=filepattern | ||||
metaVar_gitDir=GIT_DIR | metaVar_gitDir=GIT_DIR | ||||
metaVar_hostName=HOSTNAME | metaVar_hostName=HOSTNAME | ||||
metaVar_ketchServerType=SERVERTYPE | |||||
metaVar_lfsStorage=STORAGE | metaVar_lfsStorage=STORAGE | ||||
metaVar_linesOfContext=lines | metaVar_linesOfContext=lines | ||||
metaVar_message=message | metaVar_message=message | ||||
usage_Gc=Cleanup unnecessary files and optimize the local repository | usage_Gc=Cleanup unnecessary files and optimize the local repository | ||||
usage_Glog=View commit history as a graph | usage_Glog=View commit history as a graph | ||||
usage_IndexPack=Build pack index file for an existing packed archive | usage_IndexPack=Build pack index file for an existing packed archive | ||||
usage_ketchServerType=Ketch server type | |||||
usage_LFSDirectory=Directory to store large objects | usage_LFSDirectory=Directory to store large objects | ||||
usage_LFSPort=Server http port | usage_LFSPort=Server http port | ||||
usage_LFSRunStore=Store (fs | s3), store lfs objects in file system or Amazon S3 | usage_LFSRunStore=Store (fs | s3), store lfs objects in file system or Amazon S3 | ||||
usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files | usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files | ||||
usage_ReadDirCache= Read the DirCache 100 times | usage_ReadDirCache= Read the DirCache 100 times | ||||
usage_RebuildCommitGraph=Recreate a repository from another one's commit graph | usage_RebuildCommitGraph=Recreate a repository from another one's commit graph | ||||
usage_RebuildRefTree=Copy references into a RefTree | |||||
usage_RebuildRefTreeEnable=set extensions.refStorage = reftree | |||||
usage_Remote=Manage set of tracked repositories | usage_Remote=Manage set of tracked repositories | ||||
usage_RepositoryToReadFrom=Repository to read from | usage_RepositoryToReadFrom=Repository to read from | ||||
usage_RepositoryToReceiveInto=Repository to receive into | usage_RepositoryToReceiveInto=Repository to receive into |
import java.io.File; | import java.io.File; | ||||
import java.io.IOException; | import java.io.IOException; | ||||
import java.net.InetSocketAddress; | import java.net.InetSocketAddress; | ||||
import java.net.URISyntaxException; | |||||
import java.text.MessageFormat; | import java.text.MessageFormat; | ||||
import java.util.ArrayList; | import java.util.ArrayList; | ||||
import java.util.List; | import java.util.List; | ||||
import java.util.concurrent.Executors; | import java.util.concurrent.Executors; | ||||
import org.eclipse.jgit.errors.ConfigInvalidException; | import org.eclipse.jgit.errors.ConfigInvalidException; | ||||
import org.eclipse.jgit.internal.ketch.KetchLeader; | |||||
import org.eclipse.jgit.internal.ketch.KetchLeaderCache; | |||||
import org.eclipse.jgit.internal.ketch.KetchPreReceive; | |||||
import org.eclipse.jgit.internal.ketch.KetchSystem; | |||||
import org.eclipse.jgit.internal.ketch.KetchText; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.lib.StoredConfig; | import org.eclipse.jgit.lib.StoredConfig; | ||||
import org.eclipse.jgit.pgm.internal.CLIText; | import org.eclipse.jgit.pgm.internal.CLIText; | ||||
import org.eclipse.jgit.storage.file.FileBasedConfig; | import org.eclipse.jgit.storage.file.FileBasedConfig; | ||||
import org.eclipse.jgit.storage.pack.PackConfig; | import org.eclipse.jgit.storage.pack.PackConfig; | ||||
import org.eclipse.jgit.transport.DaemonClient; | import org.eclipse.jgit.transport.DaemonClient; | ||||
import org.eclipse.jgit.transport.DaemonService; | import org.eclipse.jgit.transport.DaemonService; | ||||
import org.eclipse.jgit.transport.ReceivePack; | |||||
import org.eclipse.jgit.transport.resolver.FileResolver; | import org.eclipse.jgit.transport.resolver.FileResolver; | ||||
import org.eclipse.jgit.transport.resolver.ReceivePackFactory; | |||||
import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; | |||||
import org.eclipse.jgit.util.FS; | import org.eclipse.jgit.util.FS; | ||||
import org.eclipse.jgit.util.SystemReader; | import org.eclipse.jgit.util.SystemReader; | ||||
import org.kohsuke.args4j.Argument; | import org.kohsuke.args4j.Argument; | ||||
@Option(name = "--export-all", usage = "usage_exportWithoutGitDaemonExportOk") | @Option(name = "--export-all", usage = "usage_exportWithoutGitDaemonExportOk") | ||||
boolean exportAll; | boolean exportAll; | ||||
@Option(name = "--ketch", metaVar = "metaVar_ketchServerType", usage = "usage_ketchServerType") | |||||
KetchServerType ketchServerType; | |||||
enum KetchServerType { | |||||
LEADER; | |||||
} | |||||
@Argument(required = true, metaVar = "metaVar_directory", usage = "usage_directoriesToExport") | @Argument(required = true, metaVar = "metaVar_directory", usage = "usage_directoriesToExport") | ||||
List<File> directory = new ArrayList<>(); | List<File> directory = new ArrayList<>(); | ||||
} | } | ||||
cfg = new FileBasedConfig(configFile, FS.DETECTED); | cfg = new FileBasedConfig(configFile, FS.DETECTED); | ||||
} | } | ||||
cfg.load(); | |||||
new WindowCacheConfig().fromConfig(cfg).install(); | |||||
packConfig.fromConfig(cfg); | |||||
cfg.load(); | |||||
new WindowCacheConfig().fromConfig(cfg).install(); | |||||
packConfig.fromConfig(cfg); | |||||
int threads = packConfig.getThreads(); | int threads = packConfig.getThreads(); | ||||
if (threads <= 0) | if (threads <= 0) | ||||
service(d, n).setOverridable(true); | service(d, n).setOverridable(true); | ||||
for (String n : forbidOverride) | for (String n : forbidOverride) | ||||
service(d, n).setOverridable(false); | service(d, n).setOverridable(false); | ||||
if (ketchServerType == KetchServerType.LEADER) { | |||||
startKetchLeader(d); | |||||
} | |||||
d.start(); | d.start(); | ||||
outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress())); | outw.println(MessageFormat.format(CLIText.get().listeningOn, d.getAddress())); | ||||
} | } | ||||
throw die(MessageFormat.format(CLIText.get().serviceNotSupported, n)); | throw die(MessageFormat.format(CLIText.get().serviceNotSupported, n)); | ||||
return svc; | return svc; | ||||
} | } | ||||
private void startKetchLeader(org.eclipse.jgit.transport.Daemon daemon) { | |||||
KetchSystem system = new KetchSystem(); | |||||
final KetchLeaderCache leaders = new KetchLeaderCache(system); | |||||
final ReceivePackFactory<DaemonClient> factory; | |||||
factory = daemon.getReceivePackFactory(); | |||||
daemon.setReceivePackFactory((DaemonClient req, Repository repo) -> { | |||||
ReceivePack rp = factory.create(req, repo); | |||||
KetchLeader leader; | |||||
try { | |||||
leader = leaders.get(repo); | |||||
} catch (URISyntaxException err) { | |||||
throw new ServiceNotEnabledException( | |||||
KetchText.get().invalidFollowerUri, err); | |||||
} | |||||
rp.setPreReceiveHook(new KetchPreReceive(leader)); | |||||
return rp; | |||||
}); | |||||
} | |||||
} | } |
/* | |||||
* Copyright (C) 2015, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.pgm.debug; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import java.io.IOException; | |||||
import java.util.ArrayList; | |||||
import java.util.List; | |||||
import org.eclipse.jgit.internal.storage.reftree.RefTree; | |||||
import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; | |||||
import org.eclipse.jgit.lib.CommitBuilder; | |||||
import org.eclipse.jgit.lib.ConfigConstants; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.ObjectReader; | |||||
import org.eclipse.jgit.lib.PersonIdent; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
import org.eclipse.jgit.lib.RefUpdate; | |||||
import org.eclipse.jgit.lib.StoredConfig; | |||||
import org.eclipse.jgit.pgm.Command; | |||||
import org.eclipse.jgit.pgm.TextBuiltin; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.kohsuke.args4j.Option; | |||||
@Command(usage = "usage_RebuildRefTree") | |||||
class RebuildRefTree extends TextBuiltin { | |||||
@Option(name = "--enable", usage = "usage_RebuildRefTreeEnable") | |||||
boolean enable; | |||||
private String txnNamespace; | |||||
private String txnCommitted; | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected void run() throws Exception { | |||||
try (ObjectReader reader = db.newObjectReader(); | |||||
RevWalk rw = new RevWalk(reader); | |||||
ObjectInserter inserter = db.newObjectInserter()) { | |||||
RefDatabase refDb = db.getRefDatabase(); | |||||
if (refDb instanceof RefTreeDatabase) { | |||||
RefTreeDatabase d = (RefTreeDatabase) refDb; | |||||
refDb = d.getBootstrap(); | |||||
txnNamespace = d.getTxnNamespace(); | |||||
txnCommitted = d.getTxnCommitted(); | |||||
} else { | |||||
RefTreeDatabase d = new RefTreeDatabase(db, refDb); | |||||
txnNamespace = d.getTxnNamespace(); | |||||
txnCommitted = d.getTxnCommitted(); | |||||
} | |||||
errw.format("Rebuilding %s from %s", //$NON-NLS-1$ | |||||
txnCommitted, refDb.getClass().getSimpleName()); | |||||
errw.println(); | |||||
errw.flush(); | |||||
CommitBuilder b = new CommitBuilder(); | |||||
Ref ref = refDb.exactRef(txnCommitted); | |||||
RefUpdate update = refDb.newUpdate(txnCommitted, true); | |||||
ObjectId oldTreeId; | |||||
if (ref != null && ref.getObjectId() != null) { | |||||
ObjectId oldId = ref.getObjectId(); | |||||
update.setExpectedOldObjectId(oldId); | |||||
b.setParentId(oldId); | |||||
oldTreeId = rw.parseCommit(oldId).getTree(); | |||||
} else { | |||||
update.setExpectedOldObjectId(ObjectId.zeroId()); | |||||
oldTreeId = ObjectId.zeroId(); | |||||
} | |||||
RefTree tree = rebuild(refDb); | |||||
b.setTreeId(tree.writeTree(inserter)); | |||||
b.setAuthor(new PersonIdent(db)); | |||||
b.setCommitter(b.getAuthor()); | |||||
if (b.getTreeId().equals(oldTreeId)) { | |||||
return; | |||||
} | |||||
update.setNewObjectId(inserter.insert(b)); | |||||
inserter.flush(); | |||||
RefUpdate.Result result = update.update(rw); | |||||
switch (result) { | |||||
case NEW: | |||||
case FAST_FORWARD: | |||||
break; | |||||
default: | |||||
throw die(String.format("%s: %s", update.getName(), result)); //$NON-NLS-1$ | |||||
} | |||||
if (enable && !(db.getRefDatabase() instanceof RefTreeDatabase)) { | |||||
StoredConfig cfg = db.getConfig(); | |||||
cfg.setInt(ConfigConstants.CONFIG_CORE_SECTION, null, | |||||
ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION, 1); | |||||
cfg.setString(ConfigConstants.CONFIG_EXTENSIONS_SECTION, null, | |||||
ConfigConstants.CONFIG_KEY_REFSTORAGE, | |||||
ConfigConstants.CONFIG_REFSTORAGE_REFTREE); | |||||
cfg.save(); | |||||
errw.println("Enabled reftree."); //$NON-NLS-1$ | |||||
errw.flush(); | |||||
} | |||||
} | |||||
} | |||||
private RefTree rebuild(RefDatabase refdb) throws IOException { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
List<org.eclipse.jgit.internal.storage.reftree.Command> cmds | |||||
= new ArrayList<>(); | |||||
Ref head = refdb.exactRef(HEAD); | |||||
if (head != null) { | |||||
cmds.add(new org.eclipse.jgit.internal.storage.reftree.Command( | |||||
null, | |||||
head)); | |||||
} | |||||
for (Ref r : refdb.getRefs()) { | |||||
if (r.getName().equals(txnCommitted) || r.getName().equals(HEAD) | |||||
|| r.getName().startsWith(txnNamespace)) { | |||||
continue; | |||||
} | |||||
cmds.add(new org.eclipse.jgit.internal.storage.reftree.Command( | |||||
null, | |||||
db.getRefDatabase().peel(r))); | |||||
} | |||||
tree.apply(cmds); | |||||
return tree; | |||||
} | |||||
} |
org.eclipse.jgit.internal.storage.io;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.storage.io;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.storage.pack;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.storage.pack;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.storage.reftable;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.storage.reftable;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.storage.reftree;version="[5.11.0,5.12.0)", | |||||
org.eclipse.jgit.internal.transport.connectivity;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.transport.connectivity;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.transport.http;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.transport.http;version="[5.11.0,5.12.0)", | ||||
org.eclipse.jgit.internal.transport.parser;version="[5.11.0,5.12.0)", | org.eclipse.jgit.internal.transport.parser;version="[5.11.0,5.12.0)", |
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; | |||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; | ||||
DfsPackDescription.objectLookupComparator( | DfsPackDescription.objectLookupComparator( | ||||
new PackSource.ComparatorBuilder() | new PackSource.ComparatorBuilder() | ||||
.add(GC) | .add(GC) | ||||
.add(INSERT, RECEIVE, GC_REST, GC_TXN, UNREACHABLE_GARBAGE) | |||||
.add(INSERT, RECEIVE, GC_REST, UNREACHABLE_GARBAGE) | |||||
.add(COMPACT) | .add(COMPACT) | ||||
.build()), | .build()), | ||||
a, b); | a, b); |
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.DEFAULT_COMPARATOR; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.DEFAULT_COMPARATOR; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; | |||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; | ||||
assertEquals(0, DEFAULT_COMPARATOR.compare(COMPACT, COMPACT)); | assertEquals(0, DEFAULT_COMPARATOR.compare(COMPACT, COMPACT)); | ||||
assertEquals(0, DEFAULT_COMPARATOR.compare(GC, GC)); | assertEquals(0, DEFAULT_COMPARATOR.compare(GC, GC)); | ||||
assertEquals(0, DEFAULT_COMPARATOR.compare(GC_REST, GC_REST)); | assertEquals(0, DEFAULT_COMPARATOR.compare(GC_REST, GC_REST)); | ||||
assertEquals(0, DEFAULT_COMPARATOR.compare(GC_TXN, GC_TXN)); | |||||
assertEquals(0, DEFAULT_COMPARATOR.compare(UNREACHABLE_GARBAGE, UNREACHABLE_GARBAGE)); | assertEquals(0, DEFAULT_COMPARATOR.compare(UNREACHABLE_GARBAGE, UNREACHABLE_GARBAGE)); | ||||
assertEquals(0, DEFAULT_COMPARATOR.compare(INSERT, RECEIVE)); | assertEquals(0, DEFAULT_COMPARATOR.compare(INSERT, RECEIVE)); | ||||
assertEquals(-1, DEFAULT_COMPARATOR.compare(GC, GC_REST)); | assertEquals(-1, DEFAULT_COMPARATOR.compare(GC, GC_REST)); | ||||
assertEquals(1, DEFAULT_COMPARATOR.compare(GC_REST, GC)); | assertEquals(1, DEFAULT_COMPARATOR.compare(GC_REST, GC)); | ||||
assertEquals(-1, DEFAULT_COMPARATOR.compare(GC_REST, GC_TXN)); | |||||
assertEquals(1, DEFAULT_COMPARATOR.compare(GC_TXN, GC_REST)); | |||||
assertEquals(-1, DEFAULT_COMPARATOR.compare(GC_TXN, UNREACHABLE_GARBAGE)); | |||||
assertEquals(1, DEFAULT_COMPARATOR.compare(UNREACHABLE_GARBAGE, GC_TXN)); | |||||
} | } | ||||
} | } |
/* | |||||
* Copyright (C) 2016 Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import static org.eclipse.jgit.lib.Constants.MASTER; | |||||
import static org.eclipse.jgit.lib.Constants.ORIG_HEAD; | |||||
import static org.eclipse.jgit.lib.Constants.R_HEADS; | |||||
import static org.eclipse.jgit.lib.RefDatabase.ALL; | |||||
import static org.junit.Assert.assertEquals; | |||||
import static org.junit.Assert.assertFalse; | |||||
import java.io.File; | |||||
import java.io.IOException; | |||||
import java.util.List; | |||||
import org.eclipse.jgit.internal.storage.file.FileRepository; | |||||
import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; | |||||
import org.eclipse.jgit.junit.TestRepository; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
import org.eclipse.jgit.lib.RefUpdate; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.storage.file.FileBasedConfig; | |||||
import org.eclipse.jgit.storage.file.FileRepositoryBuilder; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
public class LocalDiskRefTreeDatabaseTest extends LocalDiskRepositoryTestCase { | |||||
private FileRepository repo; | |||||
private RefTreeDatabase refdb; | |||||
private RefDatabase bootstrap; | |||||
private TestRepository<FileRepository> testRepo; | |||||
private RevCommit A; | |||||
private RevCommit B; | |||||
@Override | |||||
@Before | |||||
public void setUp() throws Exception { | |||||
super.setUp(); | |||||
FileRepository init = createWorkRepository(); | |||||
FileBasedConfig cfg = init.getConfig(); | |||||
cfg.setInt("core", null, "repositoryformatversion", 1); | |||||
cfg.setString("extensions", null, "refStorage", "reftree"); | |||||
cfg.save(); | |||||
repo = (FileRepository) new FileRepositoryBuilder() | |||||
.setGitDir(init.getDirectory()) | |||||
.build(); | |||||
refdb = (RefTreeDatabase) repo.getRefDatabase(); | |||||
bootstrap = refdb.getBootstrap(); | |||||
addRepoToClose(repo); | |||||
RefUpdate head = refdb.newUpdate(HEAD, true); | |||||
head.link(R_HEADS + MASTER); | |||||
testRepo = new TestRepository<>(init); | |||||
A = testRepo.commit().create(); | |||||
B = testRepo.commit(testRepo.getRevWalk().parseCommit(A)); | |||||
} | |||||
@Test | |||||
public void testHeadOrigHead() throws IOException { | |||||
RefUpdate master = refdb.newUpdate(HEAD, false); | |||||
master.setExpectedOldObjectId(ObjectId.zeroId()); | |||||
master.setNewObjectId(A); | |||||
assertEquals(RefUpdate.Result.NEW, master.update()); | |||||
assertEquals(A, refdb.exactRef(HEAD).getObjectId()); | |||||
RefUpdate orig = refdb.newUpdate(ORIG_HEAD, true); | |||||
orig.setNewObjectId(B); | |||||
assertEquals(RefUpdate.Result.NEW, orig.update()); | |||||
File origFile = new File(repo.getDirectory(), ORIG_HEAD); | |||||
assertEquals(B.name() + '\n', read(origFile)); | |||||
assertEquals(B, bootstrap.exactRef(ORIG_HEAD).getObjectId()); | |||||
assertEquals(B, refdb.exactRef(ORIG_HEAD).getObjectId()); | |||||
assertFalse(refdb.getRefs(ALL).containsKey(ORIG_HEAD)); | |||||
List<Ref> addl = refdb.getAdditionalRefs(); | |||||
assertEquals(2, addl.size()); | |||||
assertEquals(ORIG_HEAD, addl.get(1).getName()); | |||||
assertEquals(B, addl.get(1).getObjectId()); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2010, 2013, 2016 Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import static org.eclipse.jgit.lib.Constants.ORIG_HEAD; | |||||
import static org.eclipse.jgit.lib.Constants.R_HEADS; | |||||
import static org.eclipse.jgit.lib.Constants.R_TAGS; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.PACKED; | |||||
import static org.eclipse.jgit.lib.RefDatabase.ALL; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import static org.junit.Assert.assertEquals; | |||||
import static org.junit.Assert.assertFalse; | |||||
import static org.junit.Assert.assertNotEquals; | |||||
import static org.junit.Assert.assertNotNull; | |||||
import static org.junit.Assert.assertNull; | |||||
import static org.junit.Assert.assertSame; | |||||
import static org.junit.Assert.assertTrue; | |||||
import static org.junit.Assert.fail; | |||||
import java.io.IOException; | |||||
import java.text.MessageFormat; | |||||
import java.util.Arrays; | |||||
import java.util.Collections; | |||||
import java.util.List; | |||||
import java.util.Map; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; | |||||
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; | |||||
import org.eclipse.jgit.junit.TestRepository; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.BatchRefUpdate; | |||||
import org.eclipse.jgit.lib.CommitBuilder; | |||||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.ObjectReader; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
import org.eclipse.jgit.lib.RefUpdate; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevTag; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
public class RefTreeDatabaseTest { | |||||
private InMemRefTreeRepo repo; | |||||
private RefTreeDatabase refdb; | |||||
private RefDatabase bootstrap; | |||||
private TestRepository<InMemRefTreeRepo> testRepo; | |||||
private RevCommit A; | |||||
private RevCommit B; | |||||
private RevTag v1_0; | |||||
@Before | |||||
public void setUp() throws Exception { | |||||
repo = new InMemRefTreeRepo(new DfsRepositoryDescription("test")); | |||||
bootstrap = refdb.getBootstrap(); | |||||
testRepo = new TestRepository<>(repo); | |||||
A = testRepo.commit().create(); | |||||
B = testRepo.commit(testRepo.getRevWalk().parseCommit(A)); | |||||
v1_0 = testRepo.tag("v1_0", B); | |||||
testRepo.getRevWalk().parseBody(v1_0); | |||||
} | |||||
@Test | |||||
public void testSupportsAtomic() { | |||||
assertTrue(refdb.performsAtomicTransactions()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_EmptyDatabase() throws IOException { | |||||
assertTrue("no references", refdb.getRefs(ALL).isEmpty()); | |||||
assertTrue("no references", refdb.getRefs(R_HEADS).isEmpty()); | |||||
assertTrue("no references", refdb.getRefs(R_TAGS).isEmpty()); | |||||
assertTrue("no references", refdb.getAdditionalRefs().isEmpty()); | |||||
} | |||||
@Test | |||||
public void testGetAdditionalRefs() throws IOException { | |||||
update("refs/heads/master", A); | |||||
List<Ref> addl = refdb.getAdditionalRefs(); | |||||
assertEquals(1, addl.size()); | |||||
assertEquals("refs/txn/committed", addl.get(0).getName()); | |||||
assertEquals(getTxnCommitted(), addl.get(0).getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_HeadOnOneBranch() throws IOException { | |||||
symref(HEAD, "refs/heads/master"); | |||||
update("refs/heads/master", A); | |||||
Map<String, Ref> all = refdb.getRefs(ALL); | |||||
assertEquals(2, all.size()); | |||||
assertTrue("has HEAD", all.containsKey(HEAD)); | |||||
assertTrue("has master", all.containsKey("refs/heads/master")); | |||||
Ref head = all.get(HEAD); | |||||
Ref master = all.get("refs/heads/master"); | |||||
assertEquals(HEAD, head.getName()); | |||||
assertTrue(head.isSymbolic()); | |||||
assertSame(LOOSE, head.getStorage()); | |||||
assertSame("uses same ref as target", master, head.getTarget()); | |||||
assertEquals("refs/heads/master", master.getName()); | |||||
assertFalse(master.isSymbolic()); | |||||
assertSame(PACKED, master.getStorage()); | |||||
assertEquals(A, master.getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_DetachedHead() throws IOException { | |||||
update(HEAD, A); | |||||
Map<String, Ref> all = refdb.getRefs(ALL); | |||||
assertEquals(1, all.size()); | |||||
assertTrue("has HEAD", all.containsKey(HEAD)); | |||||
Ref head = all.get(HEAD); | |||||
assertEquals(HEAD, head.getName()); | |||||
assertFalse(head.isSymbolic()); | |||||
assertSame(PACKED, head.getStorage()); | |||||
assertEquals(A, head.getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_DeeplyNestedBranch() throws IOException { | |||||
String name = "refs/heads/a/b/c/d/e/f/g/h/i/j/k"; | |||||
update(name, A); | |||||
Map<String, Ref> all = refdb.getRefs(ALL); | |||||
assertEquals(1, all.size()); | |||||
Ref r = all.get(name); | |||||
assertEquals(name, r.getName()); | |||||
assertFalse(r.isSymbolic()); | |||||
assertSame(PACKED, r.getStorage()); | |||||
assertEquals(A, r.getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_HeadBranchNotBorn() throws IOException { | |||||
update("refs/heads/A", A); | |||||
update("refs/heads/B", B); | |||||
Map<String, Ref> all = refdb.getRefs(ALL); | |||||
assertEquals(2, all.size()); | |||||
assertFalse("no HEAD", all.containsKey(HEAD)); | |||||
Ref a = all.get("refs/heads/A"); | |||||
Ref b = all.get("refs/heads/B"); | |||||
assertEquals(A, a.getObjectId()); | |||||
assertEquals(B, b.getObjectId()); | |||||
assertEquals("refs/heads/A", a.getName()); | |||||
assertEquals("refs/heads/B", b.getName()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_HeadsOnly() throws IOException { | |||||
update("refs/heads/A", A); | |||||
update("refs/heads/B", B); | |||||
update("refs/tags/v1.0", v1_0); | |||||
Map<String, Ref> heads = refdb.getRefs(R_HEADS); | |||||
assertEquals(2, heads.size()); | |||||
Ref a = heads.get("A"); | |||||
Ref b = heads.get("B"); | |||||
assertEquals("refs/heads/A", a.getName()); | |||||
assertEquals("refs/heads/B", b.getName()); | |||||
assertEquals(A, a.getObjectId()); | |||||
assertEquals(B, b.getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_TagsOnly() throws IOException { | |||||
update("refs/heads/A", A); | |||||
update("refs/heads/B", B); | |||||
update("refs/tags/v1.0", v1_0); | |||||
Map<String, Ref> tags = refdb.getRefs(R_TAGS); | |||||
assertEquals(1, tags.size()); | |||||
Ref a = tags.get("v1.0"); | |||||
assertEquals("refs/tags/v1.0", a.getName()); | |||||
assertEquals(v1_0, a.getObjectId()); | |||||
assertTrue(a.isPeeled()); | |||||
assertEquals(v1_0.getObject(), a.getPeeledObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_HeadsSymref() throws IOException { | |||||
symref("refs/heads/other", "refs/heads/master"); | |||||
update("refs/heads/master", A); | |||||
Map<String, Ref> heads = refdb.getRefs(R_HEADS); | |||||
assertEquals(2, heads.size()); | |||||
Ref master = heads.get("master"); | |||||
Ref other = heads.get("other"); | |||||
assertEquals("refs/heads/master", master.getName()); | |||||
assertEquals(A, master.getObjectId()); | |||||
assertEquals("refs/heads/other", other.getName()); | |||||
assertEquals(A, other.getObjectId()); | |||||
assertSame(master, other.getTarget()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_InvalidPrefixes() throws IOException { | |||||
update("refs/heads/A", A); | |||||
assertTrue("empty refs/heads", refdb.getRefs("refs/heads").isEmpty()); | |||||
assertTrue("empty objects", refdb.getRefs("objects").isEmpty()); | |||||
assertTrue("empty objects/", refdb.getRefs("objects/").isEmpty()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_DiscoversNew() throws IOException { | |||||
update("refs/heads/master", A); | |||||
Map<String, Ref> orig = refdb.getRefs(ALL); | |||||
update("refs/heads/next", B); | |||||
Map<String, Ref> next = refdb.getRefs(ALL); | |||||
assertEquals(1, orig.size()); | |||||
assertEquals(2, next.size()); | |||||
assertFalse(orig.containsKey("refs/heads/next")); | |||||
assertTrue(next.containsKey("refs/heads/next")); | |||||
assertEquals(A, next.get("refs/heads/master").getObjectId()); | |||||
assertEquals(B, next.get("refs/heads/next").getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_DiscoversModified() throws IOException { | |||||
symref(HEAD, "refs/heads/master"); | |||||
update("refs/heads/master", A); | |||||
Map<String, Ref> all = refdb.getRefs(ALL); | |||||
assertEquals(A, all.get(HEAD).getObjectId()); | |||||
update("refs/heads/master", B); | |||||
all = refdb.getRefs(ALL); | |||||
assertEquals(B, all.get(HEAD).getObjectId()); | |||||
assertEquals(B, refdb.exactRef(HEAD).getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRefs_CycleInSymbolicRef() throws IOException { | |||||
symref("refs/1", "refs/2"); | |||||
symref("refs/2", "refs/3"); | |||||
symref("refs/3", "refs/4"); | |||||
symref("refs/4", "refs/5"); | |||||
symref("refs/5", "refs/end"); | |||||
update("refs/end", A); | |||||
Map<String, Ref> all = refdb.getRefs(ALL); | |||||
Ref r = all.get("refs/1"); | |||||
assertNotNull("has 1", r); | |||||
assertEquals("refs/1", r.getName()); | |||||
assertEquals(A, r.getObjectId()); | |||||
assertTrue(r.isSymbolic()); | |||||
r = r.getTarget(); | |||||
assertEquals("refs/2", r.getName()); | |||||
assertEquals(A, r.getObjectId()); | |||||
assertTrue(r.isSymbolic()); | |||||
r = r.getTarget(); | |||||
assertEquals("refs/3", r.getName()); | |||||
assertEquals(A, r.getObjectId()); | |||||
assertTrue(r.isSymbolic()); | |||||
r = r.getTarget(); | |||||
assertEquals("refs/4", r.getName()); | |||||
assertEquals(A, r.getObjectId()); | |||||
assertTrue(r.isSymbolic()); | |||||
r = r.getTarget(); | |||||
assertEquals("refs/5", r.getName()); | |||||
assertEquals(A, r.getObjectId()); | |||||
assertTrue(r.isSymbolic()); | |||||
r = r.getTarget(); | |||||
assertEquals("refs/end", r.getName()); | |||||
assertEquals(A, r.getObjectId()); | |||||
assertFalse(r.isSymbolic()); | |||||
symref("refs/5", "refs/6"); | |||||
symref("refs/6", "refs/end"); | |||||
all = refdb.getRefs(ALL); | |||||
assertNull("mising 1 due to cycle", all.get("refs/1")); | |||||
assertEquals(A, all.get("refs/2").getObjectId()); | |||||
assertEquals(A, all.get("refs/3").getObjectId()); | |||||
assertEquals(A, all.get("refs/4").getObjectId()); | |||||
assertEquals(A, all.get("refs/5").getObjectId()); | |||||
assertEquals(A, all.get("refs/6").getObjectId()); | |||||
assertEquals(A, all.get("refs/end").getObjectId()); | |||||
} | |||||
@Test | |||||
public void testGetRef_NonExistingBranchConfig() throws IOException { | |||||
assertNull("find branch config", refdb.findRef("config")); | |||||
assertNull("find branch config", refdb.findRef("refs/heads/config")); | |||||
} | |||||
@Test | |||||
public void testGetRef_FindBranchConfig() throws IOException { | |||||
update("refs/heads/config", A); | |||||
for (String t : new String[] { "config", "refs/heads/config" }) { | |||||
Ref r = refdb.findRef(t); | |||||
assertNotNull("find branch config (" + t + ")", r); | |||||
assertEquals("for " + t, "refs/heads/config", r.getName()); | |||||
assertEquals("for " + t, A, r.getObjectId()); | |||||
} | |||||
} | |||||
@Test | |||||
public void testFirstExactRef() throws IOException { | |||||
update("refs/heads/A", A); | |||||
update("refs/tags/v1.0", v1_0); | |||||
Ref a = refdb.firstExactRef("refs/heads/A", "refs/tags/v1.0"); | |||||
Ref one = refdb.firstExactRef("refs/tags/v1.0", "refs/heads/A"); | |||||
assertEquals("refs/heads/A", a.getName()); | |||||
assertEquals("refs/tags/v1.0", one.getName()); | |||||
assertEquals(A, a.getObjectId()); | |||||
assertEquals(v1_0, one.getObjectId()); | |||||
} | |||||
@Test | |||||
public void testExactRef_DiscoversModified() throws IOException { | |||||
symref(HEAD, "refs/heads/master"); | |||||
update("refs/heads/master", A); | |||||
assertEquals(A, refdb.exactRef(HEAD).getObjectId()); | |||||
update("refs/heads/master", B); | |||||
assertEquals(B, refdb.exactRef(HEAD).getObjectId()); | |||||
} | |||||
@Test | |||||
public void testIsNameConflicting() throws IOException { | |||||
update("refs/heads/a/b", A); | |||||
update("refs/heads/q", B); | |||||
// new references cannot replace an existing container | |||||
assertTrue(refdb.isNameConflicting("refs")); | |||||
assertTrue(refdb.isNameConflicting("refs/heads")); | |||||
assertTrue(refdb.isNameConflicting("refs/heads/a")); | |||||
// existing reference is not conflicting | |||||
assertFalse(refdb.isNameConflicting("refs/heads/a/b")); | |||||
// new references are not conflicting | |||||
assertFalse(refdb.isNameConflicting("refs/heads/a/d")); | |||||
assertFalse(refdb.isNameConflicting("refs/heads/master")); | |||||
// existing reference must not be used as a container | |||||
assertTrue(refdb.isNameConflicting("refs/heads/a/b/c")); | |||||
assertTrue(refdb.isNameConflicting("refs/heads/q/master")); | |||||
// refs/txn/ names always conflict. | |||||
assertTrue(refdb.isNameConflicting(refdb.getTxnCommitted())); | |||||
assertTrue(refdb.isNameConflicting("refs/txn/foo")); | |||||
} | |||||
@Test | |||||
public void testUpdate_RefusesRefsTxnNamespace() throws IOException { | |||||
ObjectId txnId = getTxnCommitted(); | |||||
RefUpdate u = refdb.newUpdate("refs/txn/tmp", false); | |||||
u.setNewObjectId(B); | |||||
assertEquals(RefUpdate.Result.LOCK_FAILURE, u.update()); | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
ReceiveCommand cmd = command(null, B, "refs/txn/tmp"); | |||||
BatchRefUpdate batch = refdb.newBatchUpdate(); | |||||
batch.addCommand(cmd); | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
batch.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); | |||||
assertEquals(MessageFormat.format(JGitText.get().invalidRefName, | |||||
"refs/txn/tmp"), cmd.getMessage()); | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
} | |||||
@Test | |||||
public void testUpdate_RefusesDotLockInRefName() throws IOException { | |||||
ObjectId txnId = getTxnCommitted(); | |||||
RefUpdate u = refdb.newUpdate("refs/heads/pu.lock", false); | |||||
u.setNewObjectId(B); | |||||
assertEquals(RefUpdate.Result.REJECTED, u.update()); | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
ReceiveCommand cmd = command(null, B, "refs/heads/pu.lock"); | |||||
BatchRefUpdate batch = refdb.newBatchUpdate(); | |||||
batch.addCommand(cmd); | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
batch.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); | |||||
assertEquals(JGitText.get().funnyRefname, cmd.getMessage()); | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
} | |||||
@Test | |||||
public void testUpdate_RefusesOrigHeadOnBare() throws IOException { | |||||
assertTrue(refdb.getRepository().isBare()); | |||||
ObjectId txnId = getTxnCommitted(); | |||||
RefUpdate orig = refdb.newUpdate(ORIG_HEAD, true); | |||||
orig.setNewObjectId(B); | |||||
assertEquals(RefUpdate.Result.LOCK_FAILURE, orig.update()); | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
ReceiveCommand cmd = command(null, B, ORIG_HEAD); | |||||
BatchRefUpdate batch = refdb.newBatchUpdate(); | |||||
batch.addCommand(cmd); | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
batch.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertEquals(REJECTED_OTHER_REASON, cmd.getResult()); | |||||
assertEquals( | |||||
MessageFormat.format(JGitText.get().invalidRefName, ORIG_HEAD), | |||||
cmd.getMessage()); | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
} | |||||
@Test | |||||
public void testBatchRefUpdate_NonFastForwardAborts() throws IOException { | |||||
update("refs/heads/master", A); | |||||
update("refs/heads/masters", B); | |||||
ObjectId txnId = getTxnCommitted(); | |||||
List<ReceiveCommand> commands = Arrays.asList( | |||||
command(A, B, "refs/heads/master"), | |||||
command(B, A, "refs/heads/masters")); | |||||
BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); | |||||
batchUpdate.addCommand(commands); | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
assertEquals(REJECTED_NONFASTFORWARD, | |||||
commands.get(1).getResult()); | |||||
assertEquals(REJECTED_OTHER_REASON, | |||||
commands.get(0).getResult()); | |||||
assertEquals(JGitText.get().transactionAborted, | |||||
commands.get(0).getMessage()); | |||||
} | |||||
@Test | |||||
public void testBatchRefUpdate_ForceUpdate() throws IOException { | |||||
update("refs/heads/master", A); | |||||
update("refs/heads/masters", B); | |||||
ObjectId txnId = getTxnCommitted(); | |||||
List<ReceiveCommand> commands = Arrays.asList( | |||||
command(A, B, "refs/heads/master"), | |||||
command(B, A, "refs/heads/masters")); | |||||
BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); | |||||
batchUpdate.setAllowNonFastForwards(true); | |||||
batchUpdate.addCommand(commands); | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertNotEquals(txnId, getTxnCommitted()); | |||||
Map<String, Ref> refs = refdb.getRefs(ALL); | |||||
assertEquals(OK, commands.get(0).getResult()); | |||||
assertEquals(OK, commands.get(1).getResult()); | |||||
assertEquals( | |||||
"[refs/heads/master, refs/heads/masters]", | |||||
refs.keySet().toString()); | |||||
assertEquals(B.getId(), refs.get("refs/heads/master").getObjectId()); | |||||
assertEquals(A.getId(), refs.get("refs/heads/masters").getObjectId()); | |||||
} | |||||
@Test | |||||
public void testBatchRefUpdate_NonFastForwardDoesNotDoExpensiveMergeCheck() | |||||
throws IOException { | |||||
update("refs/heads/master", B); | |||||
ObjectId txnId = getTxnCommitted(); | |||||
List<ReceiveCommand> commands = Arrays.asList( | |||||
command(B, A, "refs/heads/master")); | |||||
BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); | |||||
batchUpdate.setAllowNonFastForwards(true); | |||||
batchUpdate.addCommand(commands); | |||||
try (RevWalk rw = new RevWalk(repo) { | |||||
@Override | |||||
public boolean isMergedInto(RevCommit base, RevCommit tip) { | |||||
fail("isMergedInto() should not be called"); | |||||
return false; | |||||
} | |||||
}) { | |||||
batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertNotEquals(txnId, getTxnCommitted()); | |||||
Map<String, Ref> refs = refdb.getRefs(ALL); | |||||
assertEquals(OK, commands.get(0).getResult()); | |||||
assertEquals(A.getId(), refs.get("refs/heads/master").getObjectId()); | |||||
} | |||||
@Test | |||||
public void testBatchRefUpdate_ConflictCausesAbort() throws IOException { | |||||
update("refs/heads/master", A); | |||||
update("refs/heads/masters", B); | |||||
ObjectId txnId = getTxnCommitted(); | |||||
List<ReceiveCommand> commands = Arrays.asList( | |||||
command(A, B, "refs/heads/master"), | |||||
command(null, A, "refs/heads/master/x"), | |||||
command(null, A, "refs/heads")); | |||||
BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); | |||||
batchUpdate.setAllowNonFastForwards(true); | |||||
batchUpdate.addCommand(commands); | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertEquals(txnId, getTxnCommitted()); | |||||
assertEquals(LOCK_FAILURE, commands.get(0).getResult()); | |||||
assertEquals(REJECTED_OTHER_REASON, commands.get(1).getResult()); | |||||
assertEquals(JGitText.get().transactionAborted, | |||||
commands.get(1).getMessage()); | |||||
assertEquals(REJECTED_OTHER_REASON, commands.get(2).getResult()); | |||||
assertEquals(JGitText.get().transactionAborted, | |||||
commands.get(2).getMessage()); | |||||
} | |||||
@Test | |||||
public void testBatchRefUpdate_NoConflictIfDeleted() throws IOException { | |||||
update("refs/heads/master", A); | |||||
update("refs/heads/masters", B); | |||||
ObjectId txnId = getTxnCommitted(); | |||||
List<ReceiveCommand> commands = Arrays.asList( | |||||
command(A, B, "refs/heads/master"), | |||||
command(null, A, "refs/heads/masters/x"), | |||||
command(B, null, "refs/heads/masters")); | |||||
BatchRefUpdate batchUpdate = refdb.newBatchUpdate(); | |||||
batchUpdate.setAllowNonFastForwards(true); | |||||
batchUpdate.addCommand(commands); | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
batchUpdate.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
assertNotEquals(txnId, getTxnCommitted()); | |||||
assertEquals(OK, commands.get(0).getResult()); | |||||
assertEquals(OK, commands.get(1).getResult()); | |||||
assertEquals(OK, commands.get(2).getResult()); | |||||
Map<String, Ref> refs = refdb.getRefs(ALL); | |||||
assertEquals( | |||||
"[refs/heads/master, refs/heads/masters/x]", | |||||
refs.keySet().toString()); | |||||
assertEquals(A.getId(), refs.get("refs/heads/masters/x").getObjectId()); | |||||
} | |||||
private ObjectId getTxnCommitted() throws IOException { | |||||
Ref r = bootstrap.exactRef(refdb.getTxnCommitted()); | |||||
if (r != null && r.getObjectId() != null) { | |||||
return r.getObjectId(); | |||||
} | |||||
return ObjectId.zeroId(); | |||||
} | |||||
private static ReceiveCommand command(AnyObjectId a, AnyObjectId b, | |||||
String name) { | |||||
return new ReceiveCommand( | |||||
a != null ? a.copy() : ObjectId.zeroId(), | |||||
b != null ? b.copy() : ObjectId.zeroId(), | |||||
name); | |||||
} | |||||
private void symref(String name, String dst) | |||||
throws IOException { | |||||
commit((ObjectReader reader, RefTree tree) -> { | |||||
Ref old = tree.exactRef(reader, name); | |||||
Command n = new Command(old, new SymbolicRef(name, | |||||
new ObjectIdRef.Unpeeled(Ref.Storage.NEW, dst, null))); | |||||
return tree.apply(Collections.singleton(n)); | |||||
}); | |||||
} | |||||
private void update(String name, ObjectId id) | |||||
throws IOException { | |||||
commit((ObjectReader reader, RefTree tree) -> { | |||||
Ref old = tree.exactRef(reader, name); | |||||
Command n; | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
n = new Command(old, Command.toRef(rw, id, null, name, true)); | |||||
} | |||||
return tree.apply(Collections.singleton(n)); | |||||
}); | |||||
} | |||||
interface Function { | |||||
boolean apply(ObjectReader reader, RefTree tree) throws IOException; | |||||
} | |||||
private void commit(Function fun) throws IOException { | |||||
try (ObjectReader reader = repo.newObjectReader(); | |||||
ObjectInserter inserter = repo.newObjectInserter(); | |||||
RevWalk rw = new RevWalk(reader)) { | |||||
RefUpdate u = bootstrap.newUpdate(refdb.getTxnCommitted(), false); | |||||
CommitBuilder cb = new CommitBuilder(); | |||||
testRepo.setAuthorAndCommitter(cb); | |||||
Ref ref = bootstrap.exactRef(refdb.getTxnCommitted()); | |||||
RefTree tree; | |||||
if (ref != null && ref.getObjectId() != null) { | |||||
tree = RefTree.read(reader, rw.parseTree(ref.getObjectId())); | |||||
cb.setParentId(ref.getObjectId()); | |||||
u.setExpectedOldObjectId(ref.getObjectId()); | |||||
} else { | |||||
tree = RefTree.newEmptyTree(); | |||||
u.setExpectedOldObjectId(ObjectId.zeroId()); | |||||
} | |||||
assertTrue(fun.apply(reader, tree)); | |||||
cb.setTreeId(tree.writeTree(inserter)); | |||||
u.setNewObjectId(inserter.insert(cb)); | |||||
inserter.flush(); | |||||
switch (u.update(rw)) { | |||||
case NEW: | |||||
case FAST_FORWARD: | |||||
break; | |||||
default: | |||||
fail("Expected " + u.getName() + " to update"); | |||||
} | |||||
} | |||||
} | |||||
private class InMemRefTreeRepo extends InMemoryRepository { | |||||
private final RefTreeDatabase refs; | |||||
InMemRefTreeRepo(DfsRepositoryDescription repoDesc) { | |||||
super(repoDesc); | |||||
refs = new RefTreeDatabase(this, super.getRefDatabase(), | |||||
"refs/txn/committed"); | |||||
RefTreeDatabaseTest.this.refdb = refs; | |||||
} | |||||
@Override | |||||
public RefDatabase getRefDatabase() { | |||||
return refs; | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import static org.eclipse.jgit.lib.Constants.R_HEADS; | |||||
import static org.eclipse.jgit.lib.Constants.R_TAGS; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.NEW; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import static org.junit.Assert.assertEquals; | |||||
import static org.junit.Assert.assertFalse; | |||||
import static org.junit.Assert.assertNotNull; | |||||
import static org.junit.Assert.assertNull; | |||||
import static org.junit.Assert.assertSame; | |||||
import static org.junit.Assert.assertTrue; | |||||
import java.io.IOException; | |||||
import java.util.Arrays; | |||||
import java.util.Collections; | |||||
import org.eclipse.jgit.errors.MissingObjectException; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.internal.storage.dfs.DfsRepositoryDescription; | |||||
import org.eclipse.jgit.internal.storage.dfs.InMemoryRepository; | |||||
import org.eclipse.jgit.junit.TestRepository; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.ObjectReader; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevBlob; | |||||
import org.eclipse.jgit.revwalk.RevTag; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
public class RefTreeTest { | |||||
private static final String R_MASTER = R_HEADS + "master"; | |||||
private InMemoryRepository repo; | |||||
private TestRepository<InMemoryRepository> git; | |||||
@Before | |||||
public void setUp() throws IOException { | |||||
repo = new InMemoryRepository(new DfsRepositoryDescription("RefTree")); | |||||
git = new TestRepository<>(repo); | |||||
} | |||||
@Test | |||||
public void testEmptyTree() throws IOException { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
try (ObjectReader reader = repo.newObjectReader()) { | |||||
assertNull(HEAD, tree.exactRef(reader, HEAD)); | |||||
assertNull("master", tree.exactRef(reader, R_MASTER)); | |||||
} | |||||
} | |||||
@Test | |||||
public void testApplyThenReadMaster() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob id = git.blob("A"); | |||||
Command cmd = new Command(null, ref(R_MASTER, id)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
assertSame(NOT_ATTEMPTED, cmd.getResult()); | |||||
try (ObjectReader reader = repo.newObjectReader()) { | |||||
Ref m = tree.exactRef(reader, R_MASTER); | |||||
assertNotNull(R_MASTER, m); | |||||
assertEquals(R_MASTER, m.getName()); | |||||
assertEquals(id, m.getObjectId()); | |||||
assertTrue("peeled", m.isPeeled()); | |||||
} | |||||
} | |||||
@Test | |||||
public void testUpdateMaster() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob id1 = git.blob("A"); | |||||
Command cmd1 = new Command(null, ref(R_MASTER, id1)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd1))); | |||||
assertSame(NOT_ATTEMPTED, cmd1.getResult()); | |||||
RevBlob id2 = git.blob("B"); | |||||
Command cmd2 = new Command(ref(R_MASTER, id1), ref(R_MASTER, id2)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd2))); | |||||
assertSame(NOT_ATTEMPTED, cmd2.getResult()); | |||||
try (ObjectReader reader = repo.newObjectReader()) { | |||||
Ref m = tree.exactRef(reader, R_MASTER); | |||||
assertNotNull(R_MASTER, m); | |||||
assertEquals(R_MASTER, m.getName()); | |||||
assertEquals(id2, m.getObjectId()); | |||||
assertTrue("peeled", m.isPeeled()); | |||||
} | |||||
} | |||||
@Test | |||||
public void testHeadSymref() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob id = git.blob("A"); | |||||
Command cmd1 = new Command(null, ref(R_MASTER, id)); | |||||
Command cmd2 = new Command(null, symref(HEAD, R_MASTER)); | |||||
assertTrue(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); | |||||
assertSame(NOT_ATTEMPTED, cmd1.getResult()); | |||||
assertSame(NOT_ATTEMPTED, cmd2.getResult()); | |||||
try (ObjectReader reader = repo.newObjectReader()) { | |||||
Ref m = tree.exactRef(reader, HEAD); | |||||
assertNotNull(HEAD, m); | |||||
assertEquals(HEAD, m.getName()); | |||||
assertTrue("symbolic", m.isSymbolic()); | |||||
assertNotNull(m.getTarget()); | |||||
assertEquals(R_MASTER, m.getTarget().getName()); | |||||
assertEquals(id, m.getTarget().getObjectId()); | |||||
} | |||||
// Writing flushes some buffers, re-read from blob. | |||||
ObjectId newId = write(tree); | |||||
try (ObjectReader reader = repo.newObjectReader(); | |||||
RevWalk rw = new RevWalk(reader)) { | |||||
tree = RefTree.read(reader, rw.parseTree(newId)); | |||||
Ref m = tree.exactRef(reader, HEAD); | |||||
assertEquals(R_MASTER, m.getTarget().getName()); | |||||
} | |||||
} | |||||
@Test | |||||
public void testTagIsPeeled() throws Exception { | |||||
String name = "v1.0"; | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob id = git.blob("A"); | |||||
RevTag tag = git.tag(name, id); | |||||
String ref = R_TAGS + name; | |||||
Command cmd = create(ref, tag); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
assertSame(NOT_ATTEMPTED, cmd.getResult()); | |||||
try (ObjectReader reader = repo.newObjectReader()) { | |||||
Ref m = tree.exactRef(reader, ref); | |||||
assertNotNull(ref, m); | |||||
assertEquals(ref, m.getName()); | |||||
assertEquals(tag, m.getObjectId()); | |||||
assertTrue("peeled", m.isPeeled()); | |||||
assertEquals(id, m.getPeeledObjectId()); | |||||
} | |||||
} | |||||
@Test | |||||
public void testApplyAlreadyExists() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob a = git.blob("A"); | |||||
Command cmd = new Command(null, ref(R_MASTER, a)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
ObjectId treeId = write(tree); | |||||
RevBlob b = git.blob("B"); | |||||
Command cmd1 = create(R_MASTER, b); | |||||
Command cmd2 = create(R_MASTER, b); | |||||
assertFalse(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); | |||||
assertSame(LOCK_FAILURE, cmd1.getResult()); | |||||
assertSame(REJECTED_OTHER_REASON, cmd2.getResult()); | |||||
assertEquals(JGitText.get().transactionAborted, cmd2.getMessage()); | |||||
assertEquals(treeId, write(tree)); | |||||
} | |||||
@Test | |||||
public void testApplyWrongOldId() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob a = git.blob("A"); | |||||
Command cmd = new Command(null, ref(R_MASTER, a)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
ObjectId treeId = write(tree); | |||||
RevBlob b = git.blob("B"); | |||||
RevBlob c = git.blob("C"); | |||||
Command cmd1 = update(R_MASTER, b, c); | |||||
Command cmd2 = create(R_MASTER, b); | |||||
assertFalse(tree.apply(Arrays.asList(new Command[] { cmd1, cmd2 }))); | |||||
assertSame(LOCK_FAILURE, cmd1.getResult()); | |||||
assertSame(REJECTED_OTHER_REASON, cmd2.getResult()); | |||||
assertEquals(JGitText.get().transactionAborted, cmd2.getMessage()); | |||||
assertEquals(treeId, write(tree)); | |||||
} | |||||
@Test | |||||
public void testApplyWrongOldIdButAlreadyCurrentIsNoOp() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob a = git.blob("A"); | |||||
Command cmd = new Command(null, ref(R_MASTER, a)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
ObjectId treeId = write(tree); | |||||
RevBlob b = git.blob("B"); | |||||
cmd = update(R_MASTER, b, a); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
assertEquals(treeId, write(tree)); | |||||
} | |||||
@Test | |||||
public void testApplyCannotCreateSubdirectory() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob a = git.blob("A"); | |||||
Command cmd = new Command(null, ref(R_MASTER, a)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
ObjectId treeId = write(tree); | |||||
RevBlob b = git.blob("B"); | |||||
Command cmd1 = create(R_MASTER + "/fail", b); | |||||
assertFalse(tree.apply(Collections.singletonList(cmd1))); | |||||
assertSame(LOCK_FAILURE, cmd1.getResult()); | |||||
assertEquals(treeId, write(tree)); | |||||
} | |||||
@Test | |||||
public void testApplyCannotCreateParentRef() throws Exception { | |||||
RefTree tree = RefTree.newEmptyTree(); | |||||
RevBlob a = git.blob("A"); | |||||
Command cmd = new Command(null, ref(R_MASTER, a)); | |||||
assertTrue(tree.apply(Collections.singletonList(cmd))); | |||||
ObjectId treeId = write(tree); | |||||
RevBlob b = git.blob("B"); | |||||
Command cmd1 = create("refs/heads", b); | |||||
assertFalse(tree.apply(Collections.singletonList(cmd1))); | |||||
assertSame(LOCK_FAILURE, cmd1.getResult()); | |||||
assertEquals(treeId, write(tree)); | |||||
} | |||||
private static Ref ref(String name, ObjectId id) { | |||||
return new ObjectIdRef.PeeledNonTag(LOOSE, name, id); | |||||
} | |||||
private static Ref symref(String name, String dest) { | |||||
Ref d = new ObjectIdRef.PeeledNonTag(NEW, dest, null); | |||||
return new SymbolicRef(name, d); | |||||
} | |||||
private Command create(String name, ObjectId id) | |||||
throws MissingObjectException, IOException { | |||||
return update(name, ObjectId.zeroId(), id); | |||||
} | |||||
private Command update(String name, ObjectId oldId, ObjectId newId) | |||||
throws MissingObjectException, IOException { | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
return new Command(rw, new ReceiveCommand(oldId, newId, name)); | |||||
} | |||||
} | |||||
private ObjectId write(RefTree tree) throws IOException { | |||||
try (ObjectInserter ins = repo.newObjectInserter()) { | |||||
ObjectId id = tree.writeTree(ins); | |||||
ins.flush(); | |||||
return id; | |||||
} | |||||
} | |||||
} |
<?xml version="1.0" encoding="UTF-8" standalone="no"?> | <?xml version="1.0" encoding="UTF-8" standalone="no"?> | ||||
<component id="org.eclipse.jgit" version="2"> | <component id="org.eclipse.jgit" version="2"> | ||||
<resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants"> | <resource path="src/org/eclipse/jgit/lib/ConfigConstants.java" type="org.eclipse.jgit.lib.ConfigConstants"> | ||||
<filter id="338755678"> | |||||
<message_arguments> | |||||
<message_argument value="org.eclipse.jgit.lib.ConfigConstants"/> | |||||
<message_argument value="CONFIG_REFSTORAGE_REFTREE"/> | |||||
</message_arguments> | |||||
</filter> | |||||
<filter id="1141899266"> | <filter id="1141899266"> | ||||
<message_arguments> | <message_arguments> | ||||
<message_argument value="5.9"/> | <message_argument value="5.9"/> |
org.eclipse.jgit.http.test", | org.eclipse.jgit.http.test", | ||||
org.eclipse.jgit.internal.fsck;version="5.11.0"; | org.eclipse.jgit.internal.fsck;version="5.11.0"; | ||||
x-friends:="org.eclipse.jgit.test", | x-friends:="org.eclipse.jgit.test", | ||||
org.eclipse.jgit.internal.ketch;version="5.11.0"; | |||||
x-friends:="org.eclipse.jgit.junit, | |||||
org.eclipse.jgit.test, | |||||
org.eclipse.jgit.pgm", | |||||
org.eclipse.jgit.internal.revwalk;version="5.11.0";x-internal:=true, | org.eclipse.jgit.internal.revwalk;version="5.11.0";x-internal:=true, | ||||
org.eclipse.jgit.internal.storage.dfs;version="5.11.0"; | org.eclipse.jgit.internal.storage.dfs;version="5.11.0"; | ||||
x-friends:="org.eclipse.jgit.test, | x-friends:="org.eclipse.jgit.test, | ||||
org.eclipse.jgit.junit, | org.eclipse.jgit.junit, | ||||
org.eclipse.jgit.test, | org.eclipse.jgit.test, | ||||
org.eclipse.jgit.pgm", | org.eclipse.jgit.pgm", | ||||
org.eclipse.jgit.internal.storage.reftree;version="5.11.0"; | |||||
x-friends:="org.eclipse.jgit.junit, | |||||
org.eclipse.jgit.test, | |||||
org.eclipse.jgit.pgm", | |||||
org.eclipse.jgit.internal.submodule;version="5.11.0";x-internal:=true, | org.eclipse.jgit.internal.submodule;version="5.11.0";x-internal:=true, | ||||
org.eclipse.jgit.internal.transport.connectivity;version="5.11.0"; | org.eclipse.jgit.internal.transport.connectivity;version="5.11.0"; | ||||
x-friends:="org.eclipse.jgit.test", | x-friends:="org.eclipse.jgit.test", |
accepted=accepted. | |||||
cannotFetchFromLocalReplica=cannot fetch from LocalReplica | |||||
failed=failed! | |||||
invalidFollowerUri=invalid follower URI | |||||
leaderFailedToStore=leader failed to store | |||||
localReplicaRequired=LocalReplica instance is required | |||||
mismatchedTxnNamespace=mismatched txnNamespace; expected {0} found {1} | |||||
outsideTxnNamespace=ref {0} is outside of txnNamespace {1} | |||||
proposingUpdates=Proposing updates | |||||
queuedProposalFailedToApply=queued proposal failed to apply | |||||
starting=starting! | |||||
unsupportedVoterCount=unsupported voter count {0}, expected one of {1} | |||||
waitingForQueue=Waiting for queue |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static java.util.concurrent.TimeUnit.SECONDS; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.TERM; | |||||
import java.io.IOException; | |||||
import java.util.List; | |||||
import java.util.concurrent.TimeoutException; | |||||
import org.eclipse.jgit.lib.CommitBuilder; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.lib.TreeFormatter; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.util.time.ProposedTimestamp; | |||||
import org.slf4j.Logger; | |||||
import org.slf4j.LoggerFactory; | |||||
/** | |||||
* The initial {@link Round} for a leaderless repository, used to establish a | |||||
* leader. | |||||
*/ | |||||
class ElectionRound extends Round { | |||||
private static final Logger log = LoggerFactory.getLogger(ElectionRound.class); | |||||
private long term; | |||||
ElectionRound(KetchLeader leader, LogIndex head) { | |||||
super(leader, head); | |||||
} | |||||
@Override | |||||
void start() throws IOException { | |||||
ObjectId id; | |||||
try (Repository git = leader.openRepository(); | |||||
ProposedTimestamp ts = getSystem().getClock().propose(); | |||||
ObjectInserter inserter = git.newObjectInserter()) { | |||||
id = bumpTerm(git, ts, inserter); | |||||
inserter.flush(); | |||||
blockUntil(ts); | |||||
} | |||||
runAsync(id); | |||||
} | |||||
@Override | |||||
void success() { | |||||
// Do nothing upon election, KetchLeader will copy the term. | |||||
} | |||||
long getTerm() { | |||||
return term; | |||||
} | |||||
private ObjectId bumpTerm(Repository git, ProposedTimestamp ts, | |||||
ObjectInserter inserter) throws IOException { | |||||
CommitBuilder b = new CommitBuilder(); | |||||
if (!ObjectId.zeroId().equals(acceptedOldIndex)) { | |||||
try (RevWalk rw = new RevWalk(git)) { | |||||
RevCommit c = rw.parseCommit(acceptedOldIndex); | |||||
if (getSystem().requireMonotonicLeaderElections()) { | |||||
if (ts.read(SECONDS) < c.getCommitTime()) { | |||||
throw new TimeIsUncertainException(); | |||||
} | |||||
} | |||||
b.setTreeId(c.getTree()); | |||||
b.setParentId(acceptedOldIndex); | |||||
term = parseTerm(c.getFooterLines(TERM)) + 1; | |||||
} | |||||
} else { | |||||
term = 1; | |||||
b.setTreeId(inserter.insert(new TreeFormatter())); | |||||
} | |||||
StringBuilder msg = new StringBuilder(); | |||||
msg.append(KetchConstants.TERM.getName()) | |||||
.append(": ") //$NON-NLS-1$ | |||||
.append(term); | |||||
String tag = leader.getSystem().newLeaderTag(); | |||||
if (tag != null && !tag.isEmpty()) { | |||||
msg.append(' ').append(tag); | |||||
} | |||||
b.setAuthor(leader.getSystem().newCommitter(ts)); | |||||
b.setCommitter(b.getAuthor()); | |||||
b.setMessage(msg.toString()); | |||||
if (log.isDebugEnabled()) { | |||||
log.debug("Trying to elect myself " + b.getMessage()); //$NON-NLS-1$ | |||||
} | |||||
return inserter.insert(b); | |||||
} | |||||
private static long parseTerm(List<String> footer) { | |||||
if (footer.isEmpty()) { | |||||
return 0; | |||||
} | |||||
String s = footer.get(0); | |||||
int p = s.indexOf(' '); | |||||
if (p > 0) { | |||||
s = s.substring(0, p); | |||||
} | |||||
return Long.parseLong(s, 10); | |||||
} | |||||
private void blockUntil(ProposedTimestamp ts) throws IOException { | |||||
try { | |||||
ts.blockUntil(getSystem().getMaxWaitForMonotonicClock()); | |||||
} catch (InterruptedException | TimeoutException e) { | |||||
throw new TimeIsUncertainException(e); | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import org.eclipse.jgit.revwalk.FooterKey; | |||||
/** | |||||
* Frequently used constants in a Ketch system. | |||||
*/ | |||||
public class KetchConstants { | |||||
/** | |||||
* Default reference namespace holding {@link #ACCEPTED} and | |||||
* {@link #COMMITTED} references and the {@link #STAGE} sub-namespace. | |||||
*/ | |||||
public static final String DEFAULT_TXN_NAMESPACE = "refs/txn/"; //$NON-NLS-1$ | |||||
/** Reference name holding the RefTree accepted by a follower. */ | |||||
public static final String ACCEPTED = "accepted"; //$NON-NLS-1$ | |||||
/** Reference name holding the RefTree known to be committed. */ | |||||
public static final String COMMITTED = "committed"; //$NON-NLS-1$ | |||||
/** Reference subdirectory holding proposed heads. */ | |||||
public static final String STAGE = "stage/"; //$NON-NLS-1$ | |||||
/** Footer containing the current term. */ | |||||
public static final FooterKey TERM = new FooterKey("Term"); //$NON-NLS-1$ | |||||
/** Section for Ketch configuration ({@code ketch}). */ | |||||
public static final String CONFIG_SECTION_KETCH = "ketch"; //$NON-NLS-1$ | |||||
/** Behavior for a replica ({@code remote.$name.ketch-type}) */ | |||||
public static final String CONFIG_KEY_TYPE = "ketch-type"; //$NON-NLS-1$ | |||||
/** Behavior for a replica ({@code remote.$name.ketch-commit}) */ | |||||
public static final String CONFIG_KEY_COMMIT = "ketch-commit"; //$NON-NLS-1$ | |||||
/** Behavior for a replica ({@code remote.$name.ketch-speed}) */ | |||||
public static final String CONFIG_KEY_SPEED = "ketch-speed"; //$NON-NLS-1$ | |||||
private KetchConstants() { | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.KetchLeader.State.CANDIDATE; | |||||
import static org.eclipse.jgit.internal.ketch.KetchLeader.State.LEADER; | |||||
import static org.eclipse.jgit.internal.ketch.KetchLeader.State.SHUTDOWN; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.Participation.FOLLOWER_ONLY; | |||||
import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; | |||||
import java.io.IOException; | |||||
import java.text.MessageFormat; | |||||
import java.util.ArrayList; | |||||
import java.util.Arrays; | |||||
import java.util.Collection; | |||||
import java.util.List; | |||||
import java.util.concurrent.locks.Lock; | |||||
import java.util.concurrent.locks.ReentrantLock; | |||||
import org.eclipse.jgit.internal.storage.reftree.RefTree; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.slf4j.Logger; | |||||
import org.slf4j.LoggerFactory; | |||||
/** | |||||
* A leader managing consensus across remote followers. | |||||
* <p> | |||||
* A leader instance starts up in | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#CANDIDATE} and tries | |||||
* to begin a new term by sending an | |||||
* {@link org.eclipse.jgit.internal.ketch.ElectionRound} to all replicas. Its | |||||
* term starts if a majority of replicas have accepted this leader instance for | |||||
* the term. | |||||
* <p> | |||||
* Once elected by a majority the instance enters | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER} and runs | |||||
* proposals offered to {@link #queueProposal(Proposal)}. This continues until | |||||
* the leader is timed out for inactivity, or is deposed by a competing leader | |||||
* gaining its own majority. | |||||
* <p> | |||||
* Once timed out or deposed this {@code KetchLeader} instance should be | |||||
* discarded, and a new instance takes over. | |||||
* <p> | |||||
* Each leader instance coordinates a group of | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchReplica}s. Replica instances are | |||||
* owned by the leader instance and must be discarded when the leader is | |||||
* discarded. | |||||
* <p> | |||||
* In Ketch all push requests are issued through the leader. The steps are as | |||||
* follows (see {@link org.eclipse.jgit.internal.ketch.KetchPreReceive} for an | |||||
* example): | |||||
* <ul> | |||||
* <li>Create a {@link org.eclipse.jgit.internal.ketch.Proposal} with the | |||||
* {@link org.eclipse.jgit.transport.ReceiveCommand}s that represent the push. | |||||
* <li>Invoke {@link #queueProposal(Proposal)} on the leader instance. | |||||
* <li>Wait for consensus with | |||||
* {@link org.eclipse.jgit.internal.ketch.Proposal#await()}. | |||||
* <li>To examine the status of the push, check | |||||
* {@link org.eclipse.jgit.internal.ketch.Proposal#getCommands()}, looking at | |||||
* {@link org.eclipse.jgit.internal.storage.reftree.Command#getResult()}. | |||||
* </ul> | |||||
* <p> | |||||
* The leader gains consensus by first pushing the needed objects and a | |||||
* {@link org.eclipse.jgit.internal.storage.reftree.RefTree} representing the | |||||
* desired target repository state to the {@code refs/txn/accepted} branch on | |||||
* each of the replicas. Once a majority has succeeded, the leader commits the | |||||
* state by either pushing the {@code refs/txn/accepted} value to | |||||
* {@code refs/txn/committed} (for Ketch-aware replicas) or by pushing updates | |||||
* to {@code refs/heads/master}, etc. for stock Git replicas. | |||||
* <p> | |||||
* Internally, the actual transport to replicas is performed on background | |||||
* threads via the {@link org.eclipse.jgit.internal.ketch.KetchSystem}'s | |||||
* executor service. For performance, the | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader}, | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchReplica} and | |||||
* {@link org.eclipse.jgit.internal.ketch.Proposal} objects share some state, | |||||
* and may invoke each other's methods on different threads. This access is | |||||
* protected by the leader's {@link #lock} object. Care must be taken to prevent | |||||
* concurrent access by correctly obtaining the leader's lock. | |||||
*/ | |||||
public abstract class KetchLeader { | |||||
private static final Logger log = LoggerFactory.getLogger(KetchLeader.class); | |||||
/** Current state of the leader instance. */ | |||||
public enum State { | |||||
/** Newly created instance trying to elect itself leader. */ | |||||
CANDIDATE, | |||||
/** Leader instance elected by a majority. */ | |||||
LEADER, | |||||
/** Instance has been deposed by another with a more recent term. */ | |||||
DEPOSED, | |||||
/** Leader has been gracefully shutdown, e.g. due to inactivity. */ | |||||
SHUTDOWN; | |||||
} | |||||
private final KetchSystem system; | |||||
/** Leader's knowledge of replicas for this repository. */ | |||||
private KetchReplica[] voters; | |||||
private KetchReplica[] followers; | |||||
private LocalReplica self; | |||||
/** | |||||
* Lock protecting all data within this leader instance. | |||||
* <p> | |||||
* This lock extends into the {@link KetchReplica} instances used by the | |||||
* leader. They share the same lock instance to simplify concurrency. | |||||
*/ | |||||
final Lock lock; | |||||
private State state = CANDIDATE; | |||||
/** Term of this leader, once elected. */ | |||||
private long term; | |||||
/** | |||||
* Pending proposals accepted into the queue in FIFO order. | |||||
* <p> | |||||
* These proposals were preflighted and do not contain any conflicts with | |||||
* each other and their expectations matched the leader's local view of the | |||||
* agreed upon {@code refs/txn/accepted} tree. | |||||
*/ | |||||
private final List<Proposal> queued; | |||||
/** | |||||
* State of the repository's RefTree after applying all entries in | |||||
* {@link #queued}. New proposals must be consistent with this tree to be | |||||
* appended to the end of {@link #queued}. | |||||
* <p> | |||||
* Must be deep-copied with {@link RefTree#copy()} if | |||||
* {@link #roundHoldsReferenceToRefTree} is {@code true}. | |||||
*/ | |||||
private RefTree refTree; | |||||
/** | |||||
* If {@code true} {@link #refTree} must be duplicated before queuing the | |||||
* next proposal. The {@link #refTree} was passed into the constructor of a | |||||
* {@link ProposalRound}, and that external reference to the {@link RefTree} | |||||
* object is held by the proposal until it materializes the tree object in | |||||
* the object store. This field is set {@code true} when the proposal begins | |||||
* execution and set {@code false} once tree objects are persisted in the | |||||
* local repository's object store or {@link #refTree} is replaced with a | |||||
* copy to isolate it from any running rounds. | |||||
* <p> | |||||
* If proposals arrive less frequently than the {@code RefTree} is written | |||||
* out to the repository the {@link #roundHoldsReferenceToRefTree} behavior | |||||
* avoids duplicating {@link #refTree}, reducing both time and memory used. | |||||
* However if proposals arrive more frequently {@link #refTree} must be | |||||
* duplicated to prevent newly queued proposals from corrupting the | |||||
* {@link #runningRound}. | |||||
*/ | |||||
volatile boolean roundHoldsReferenceToRefTree; | |||||
/** End of the leader's log. */ | |||||
private LogIndex headIndex; | |||||
/** Leader knows this (and all prior) states are committed. */ | |||||
private LogIndex committedIndex; | |||||
/** | |||||
* Is the leader idle with no work pending? If {@code true} there is no work | |||||
* for the leader (normal state). This field is {@code false} when the | |||||
* leader thread is scheduled for execution, or while {@link #runningRound} | |||||
* defines a round in progress. | |||||
*/ | |||||
private boolean idle; | |||||
/** Current round the leader is preparing and waiting for a vote on. */ | |||||
private Round runningRound; | |||||
/** | |||||
* Construct a leader for a Ketch instance. | |||||
* | |||||
* @param system | |||||
* Ketch system configuration the leader must adhere to. | |||||
*/ | |||||
protected KetchLeader(KetchSystem system) { | |||||
this.system = system; | |||||
this.lock = new ReentrantLock(true /* fair */); | |||||
this.queued = new ArrayList<>(4); | |||||
this.idle = true; | |||||
} | |||||
/** @return system configuration. */ | |||||
KetchSystem getSystem() { | |||||
return system; | |||||
} | |||||
/** | |||||
* Configure the replicas used by this Ketch instance. | |||||
* <p> | |||||
* Replicas should be configured once at creation before any proposals are | |||||
* executed. Once elections happen, <b>reconfiguration is a complicated | |||||
* concept that is not currently supported</b>. | |||||
* | |||||
* @param replicas | |||||
* members participating with the same repository. | |||||
*/ | |||||
public void setReplicas(Collection<KetchReplica> replicas) { | |||||
List<KetchReplica> v = new ArrayList<>(5); | |||||
List<KetchReplica> f = new ArrayList<>(5); | |||||
for (KetchReplica r : replicas) { | |||||
switch (r.getParticipation()) { | |||||
case FULL: | |||||
v.add(r); | |||||
break; | |||||
case FOLLOWER_ONLY: | |||||
f.add(r); | |||||
break; | |||||
} | |||||
} | |||||
Collection<Integer> validVoters = validVoterCounts(); | |||||
if (!validVoters.contains(Integer.valueOf(v.size()))) { | |||||
throw new IllegalArgumentException(MessageFormat.format( | |||||
KetchText.get().unsupportedVoterCount, | |||||
Integer.valueOf(v.size()), | |||||
validVoters)); | |||||
} | |||||
LocalReplica me = findLocal(v); | |||||
if (me == null) { | |||||
throw new IllegalArgumentException( | |||||
KetchText.get().localReplicaRequired); | |||||
} | |||||
lock.lock(); | |||||
try { | |||||
voters = v.toArray(new KetchReplica[0]); | |||||
followers = f.toArray(new KetchReplica[0]); | |||||
self = me; | |||||
} finally { | |||||
lock.unlock(); | |||||
} | |||||
} | |||||
private static Collection<Integer> validVoterCounts() { | |||||
@SuppressWarnings("boxing") | |||||
Integer[] valid = { | |||||
// An odd number of voting replicas is required. | |||||
1, 3, 5, 7, 9 }; | |||||
return Arrays.asList(valid); | |||||
} | |||||
private static LocalReplica findLocal(Collection<KetchReplica> voters) { | |||||
for (KetchReplica r : voters) { | |||||
if (r instanceof LocalReplica) { | |||||
return (LocalReplica) r; | |||||
} | |||||
} | |||||
return null; | |||||
} | |||||
/** | |||||
* Get an instance of the repository for use by a leader thread. | |||||
* <p> | |||||
* The caller will close the repository. | |||||
* | |||||
* @return opened repository for use by the leader thread. | |||||
* @throws java.io.IOException | |||||
* cannot reopen the repository for the leader. | |||||
*/ | |||||
protected abstract Repository openRepository() throws IOException; | |||||
/** | |||||
* Queue a reference update proposal for consensus. | |||||
* <p> | |||||
* This method does not wait for consensus to be reached. The proposal is | |||||
* checked to look for risks of conflicts, and then submitted into the queue | |||||
* for distribution as soon as possible. | |||||
* <p> | |||||
* Callers must use {@link org.eclipse.jgit.internal.ketch.Proposal#await()} | |||||
* to see if the proposal is done. | |||||
* | |||||
* @param proposal | |||||
* the proposed reference updates to queue for consideration. | |||||
* Once execution is complete the individual reference result | |||||
* fields will be populated with the outcome. | |||||
* @throws java.lang.InterruptedException | |||||
* current thread was interrupted. The proposal may have been | |||||
* aborted if it was not yet queued for execution. | |||||
* @throws java.io.IOException | |||||
* unrecoverable error preventing proposals from being attempted | |||||
* by this leader. | |||||
*/ | |||||
public void queueProposal(Proposal proposal) | |||||
throws InterruptedException, IOException { | |||||
try { | |||||
lock.lockInterruptibly(); | |||||
} catch (InterruptedException e) { | |||||
proposal.abort(); | |||||
throw e; | |||||
} | |||||
try { | |||||
if (refTree == null) { | |||||
initialize(); | |||||
for (Proposal p : queued) { | |||||
refTree.apply(p.getCommands()); | |||||
} | |||||
} else if (roundHoldsReferenceToRefTree) { | |||||
refTree = refTree.copy(); | |||||
roundHoldsReferenceToRefTree = false; | |||||
} | |||||
if (!refTree.apply(proposal.getCommands())) { | |||||
// A conflict exists so abort the proposal. | |||||
proposal.abort(); | |||||
return; | |||||
} | |||||
queued.add(proposal); | |||||
proposal.notifyState(QUEUED); | |||||
if (idle) { | |||||
scheduleLeader(); | |||||
} | |||||
} finally { | |||||
lock.unlock(); | |||||
} | |||||
} | |||||
private void initialize() throws IOException { | |||||
try (Repository git = openRepository(); RevWalk rw = new RevWalk(git)) { | |||||
self.initialize(git); | |||||
ObjectId accepted = self.getTxnAccepted(); | |||||
if (!ObjectId.zeroId().equals(accepted)) { | |||||
RevCommit c = rw.parseCommit(accepted); | |||||
headIndex = LogIndex.unknown(accepted); | |||||
refTree = RefTree.read(rw.getObjectReader(), c.getTree()); | |||||
} else { | |||||
headIndex = LogIndex.unknown(ObjectId.zeroId()); | |||||
refTree = RefTree.newEmptyTree(); | |||||
} | |||||
} | |||||
} | |||||
private void scheduleLeader() { | |||||
idle = false; | |||||
system.getExecutor().execute(this::runLeader); | |||||
} | |||||
private void runLeader() { | |||||
Round round; | |||||
lock.lock(); | |||||
try { | |||||
switch (state) { | |||||
case CANDIDATE: | |||||
round = new ElectionRound(this, headIndex); | |||||
break; | |||||
case LEADER: | |||||
round = newProposalRound(); | |||||
break; | |||||
case DEPOSED: | |||||
case SHUTDOWN: | |||||
default: | |||||
log.warn("Leader cannot run {}", state); //$NON-NLS-1$ | |||||
// TODO(sop): Redirect proposals. | |||||
return; | |||||
} | |||||
} finally { | |||||
lock.unlock(); | |||||
} | |||||
try { | |||||
round.start(); | |||||
} catch (IOException e) { | |||||
// TODO(sop) Depose leader if it cannot use its repository. | |||||
log.error(KetchText.get().leaderFailedToStore, e); | |||||
lock.lock(); | |||||
try { | |||||
nextRound(); | |||||
} finally { | |||||
lock.unlock(); | |||||
} | |||||
} | |||||
} | |||||
private ProposalRound newProposalRound() { | |||||
List<Proposal> todo = new ArrayList<>(queued); | |||||
queued.clear(); | |||||
roundHoldsReferenceToRefTree = true; | |||||
return new ProposalRound(this, headIndex, todo, refTree); | |||||
} | |||||
/** @return term of this leader's reign. */ | |||||
long getTerm() { | |||||
return term; | |||||
} | |||||
/** @return end of the leader's log. */ | |||||
LogIndex getHead() { | |||||
return headIndex; | |||||
} | |||||
/** | |||||
* @return state leader knows it has committed across a quorum of replicas. | |||||
*/ | |||||
LogIndex getCommitted() { | |||||
return committedIndex; | |||||
} | |||||
boolean isIdle() { | |||||
return idle; | |||||
} | |||||
void runAsync(Round round) { | |||||
lock.lock(); | |||||
try { | |||||
// End of the log is this round. Once transport begins it is | |||||
// reasonable to assume at least one replica will eventually get | |||||
// this, and there is reasonable probability it commits. | |||||
headIndex = round.acceptedNewIndex; | |||||
runningRound = round; | |||||
for (KetchReplica replica : voters) { | |||||
replica.pushTxnAcceptedAsync(round); | |||||
} | |||||
for (KetchReplica replica : followers) { | |||||
replica.pushTxnAcceptedAsync(round); | |||||
} | |||||
} finally { | |||||
lock.unlock(); | |||||
} | |||||
} | |||||
/** | |||||
* Asynchronous signal from a replica after completion. | |||||
* <p> | |||||
* Must be called while {@link #lock} is held by the replica. | |||||
* | |||||
* @param replica | |||||
* replica posting a completion event. | |||||
*/ | |||||
void onReplicaUpdate(KetchReplica replica) { | |||||
if (log.isDebugEnabled()) { | |||||
log.debug("Replica {} finished:\n{}", //$NON-NLS-1$ | |||||
replica.describeForLog(), snapshot()); | |||||
} | |||||
if (replica.getParticipation() == FOLLOWER_ONLY) { | |||||
// Followers cannot vote, so votes haven't changed. | |||||
return; | |||||
} else if (runningRound == null) { | |||||
// No round running, no need to tally votes. | |||||
return; | |||||
} | |||||
assert headIndex.equals(runningRound.acceptedNewIndex); | |||||
int matching = 0; | |||||
for (KetchReplica r : voters) { | |||||
if (r.hasAccepted(headIndex)) { | |||||
matching++; | |||||
} | |||||
} | |||||
int quorum = voters.length / 2 + 1; | |||||
boolean success = matching >= quorum; | |||||
if (!success) { | |||||
return; | |||||
} | |||||
switch (state) { | |||||
case CANDIDATE: | |||||
term = ((ElectionRound) runningRound).getTerm(); | |||||
state = LEADER; | |||||
if (log.isDebugEnabled()) { | |||||
log.debug("Won election, running term " + term); //$NON-NLS-1$ | |||||
} | |||||
//$FALL-THROUGH$ | |||||
case LEADER: | |||||
committedIndex = headIndex; | |||||
if (log.isDebugEnabled()) { | |||||
log.debug("Committed {} in term {}", //$NON-NLS-1$ | |||||
committedIndex.describeForLog(), | |||||
Long.valueOf(term)); | |||||
} | |||||
nextRound(); | |||||
commitAsync(replica); | |||||
notifySuccess(runningRound); | |||||
if (log.isDebugEnabled()) { | |||||
log.debug("Leader state:\n{}", snapshot()); //$NON-NLS-1$ | |||||
} | |||||
break; | |||||
default: | |||||
log.debug("Leader ignoring replica while in {}", state); //$NON-NLS-1$ | |||||
break; | |||||
} | |||||
} | |||||
private void notifySuccess(Round round) { | |||||
// Drop the leader lock while notifying Proposal listeners. | |||||
lock.unlock(); | |||||
try { | |||||
round.success(); | |||||
} finally { | |||||
lock.lock(); | |||||
} | |||||
} | |||||
private void commitAsync(KetchReplica caller) { | |||||
for (KetchReplica r : voters) { | |||||
if (r == caller) { | |||||
continue; | |||||
} | |||||
if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { | |||||
r.pushCommitAsync(committedIndex); | |||||
} | |||||
} | |||||
for (KetchReplica r : followers) { | |||||
if (r == caller) { | |||||
continue; | |||||
} | |||||
if (r.shouldPushUnbatchedCommit(committedIndex, isIdle())) { | |||||
r.pushCommitAsync(committedIndex); | |||||
} | |||||
} | |||||
} | |||||
/** Schedule the next round; invoked while {@link #lock} is held. */ | |||||
void nextRound() { | |||||
runningRound = null; | |||||
if (queued.isEmpty()) { | |||||
idle = true; | |||||
} else { | |||||
// Caller holds lock. Reschedule leader on a new thread so | |||||
// the call stack can unwind and lock is not held unexpectedly | |||||
// during prepare for the next round. | |||||
scheduleLeader(); | |||||
} | |||||
} | |||||
/** | |||||
* Snapshot this leader | |||||
* | |||||
* @return snapshot of this leader | |||||
*/ | |||||
public LeaderSnapshot snapshot() { | |||||
lock.lock(); | |||||
try { | |||||
LeaderSnapshot s = new LeaderSnapshot(); | |||||
s.state = state; | |||||
s.term = term; | |||||
s.headIndex = headIndex; | |||||
s.committedIndex = committedIndex; | |||||
s.idle = isIdle(); | |||||
for (KetchReplica r : voters) { | |||||
s.replicas.add(r.snapshot()); | |||||
} | |||||
for (KetchReplica r : followers) { | |||||
s.replicas.add(r.snapshot()); | |||||
} | |||||
return s; | |||||
} finally { | |||||
lock.unlock(); | |||||
} | |||||
} | |||||
/** | |||||
* Gracefully shutdown this leader and cancel outstanding operations. | |||||
*/ | |||||
public void shutdown() { | |||||
lock.lock(); | |||||
try { | |||||
if (state != SHUTDOWN) { | |||||
state = SHUTDOWN; | |||||
for (KetchReplica r : voters) { | |||||
r.shutdown(); | |||||
} | |||||
for (KetchReplica r : followers) { | |||||
r.shutdown(); | |||||
} | |||||
} | |||||
} finally { | |||||
lock.unlock(); | |||||
} | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public String toString() { | |||||
return snapshot().toString(); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import java.net.URISyntaxException; | |||||
import java.util.concurrent.ConcurrentHashMap; | |||||
import java.util.concurrent.ConcurrentMap; | |||||
import java.util.concurrent.locks.Lock; | |||||
import java.util.concurrent.locks.ReentrantLock; | |||||
import org.eclipse.jgit.internal.storage.dfs.DfsRepository; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
/** | |||||
* A cache of live leader instances, keyed by repository. | |||||
* <p> | |||||
* Ketch only assigns a leader to a repository when needed. If | |||||
* {@link #get(Repository)} is called for a repository that does not have a | |||||
* leader, the leader is created and added to the cache. | |||||
*/ | |||||
public class KetchLeaderCache { | |||||
private final KetchSystem system; | |||||
private final ConcurrentMap<String, KetchLeader> leaders; | |||||
private final Lock startLock; | |||||
/** | |||||
* Initialize a new leader cache. | |||||
* | |||||
* @param system | |||||
* system configuration for the leaders | |||||
*/ | |||||
public KetchLeaderCache(KetchSystem system) { | |||||
this.system = system; | |||||
leaders = new ConcurrentHashMap<>(); | |||||
startLock = new ReentrantLock(true /* fair */); | |||||
} | |||||
/** | |||||
* Lookup the leader instance for a given repository. | |||||
* | |||||
* @param repo | |||||
* repository to get the leader for. | |||||
* @return the leader instance for the repository. | |||||
* @throws java.net.URISyntaxException | |||||
* remote configuration contains an invalid URL. | |||||
*/ | |||||
public KetchLeader get(Repository repo) | |||||
throws URISyntaxException { | |||||
String key = computeKey(repo); | |||||
KetchLeader leader = leaders.get(key); | |||||
if (leader != null) { | |||||
return leader; | |||||
} | |||||
return startLeader(key, repo); | |||||
} | |||||
private KetchLeader startLeader(String key, Repository repo) | |||||
throws URISyntaxException { | |||||
startLock.lock(); | |||||
try { | |||||
KetchLeader leader = leaders.get(key); | |||||
if (leader != null) { | |||||
return leader; | |||||
} | |||||
leader = system.createLeader(repo); | |||||
leaders.put(key, leader); | |||||
return leader; | |||||
} finally { | |||||
startLock.unlock(); | |||||
} | |||||
} | |||||
private static String computeKey(Repository repo) { | |||||
if (repo instanceof DfsRepository) { | |||||
DfsRepository dfs = (DfsRepository) repo; | |||||
return dfs.getDescription().getRepositoryName(); | |||||
} | |||||
if (repo.getDirectory() != null) { | |||||
return repo.getDirectory().toURI().toString(); | |||||
} | |||||
throw new IllegalArgumentException(); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static java.util.concurrent.TimeUnit.MILLISECONDS; | |||||
import static java.util.concurrent.TimeUnit.SECONDS; | |||||
import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; | |||||
import static org.eclipse.jgit.internal.ketch.Proposal.State.QUEUED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import java.io.IOException; | |||||
import java.util.Collection; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.transport.PreReceiveHook; | |||||
import org.eclipse.jgit.transport.ProgressSpinner; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.transport.ReceivePack; | |||||
import org.slf4j.Logger; | |||||
import org.slf4j.LoggerFactory; | |||||
/** | |||||
* PreReceiveHook for handling push traffic in a Ketch system. | |||||
* <p> | |||||
* Install an instance on {@link org.eclipse.jgit.transport.ReceivePack} to | |||||
* capture the commands and other connection state and relay them through the | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader}, allowing the leader to | |||||
* gain consensus about the new reference state. | |||||
*/ | |||||
public class KetchPreReceive implements PreReceiveHook { | |||||
private static final Logger log = LoggerFactory.getLogger(KetchPreReceive.class); | |||||
private final KetchLeader leader; | |||||
/** | |||||
* Construct a hook executing updates through a | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader}. | |||||
* | |||||
* @param leader | |||||
* leader for this repository. | |||||
*/ | |||||
public KetchPreReceive(KetchLeader leader) { | |||||
this.leader = leader; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public void onPreReceive(ReceivePack rp, Collection<ReceiveCommand> cmds) { | |||||
cmds = ReceiveCommand.filter(cmds, NOT_ATTEMPTED); | |||||
if (cmds.isEmpty()) { | |||||
return; | |||||
} | |||||
try { | |||||
Proposal proposal = new Proposal(rp.getRevWalk(), cmds) | |||||
.setPushCertificate(rp.getPushCertificate()) | |||||
.setAuthor(rp.getRefLogIdent()) | |||||
.setMessage("push"); //$NON-NLS-1$ | |||||
leader.queueProposal(proposal); | |||||
if (proposal.isDone()) { | |||||
// This failed fast, e.g. conflict or bad precondition. | |||||
return; | |||||
} | |||||
ProgressSpinner spinner = new ProgressSpinner( | |||||
rp.getMessageOutputStream()); | |||||
if (proposal.getState() == QUEUED) { | |||||
waitForQueue(proposal, spinner); | |||||
} | |||||
if (!proposal.isDone()) { | |||||
waitForPropose(proposal, spinner); | |||||
} | |||||
} catch (IOException | InterruptedException e) { | |||||
String msg = JGitText.get().transactionAborted; | |||||
for (ReceiveCommand cmd : cmds) { | |||||
if (cmd.getResult() == NOT_ATTEMPTED) { | |||||
cmd.setResult(REJECTED_OTHER_REASON, msg); | |||||
} | |||||
} | |||||
log.error(msg, e); | |||||
} | |||||
} | |||||
private void waitForQueue(Proposal proposal, ProgressSpinner spinner) | |||||
throws InterruptedException { | |||||
spinner.beginTask(KetchText.get().waitingForQueue, 1, SECONDS); | |||||
while (!proposal.awaitStateChange(QUEUED, 250, MILLISECONDS)) { | |||||
spinner.update(); | |||||
} | |||||
switch (proposal.getState()) { | |||||
case RUNNING: | |||||
default: | |||||
spinner.endTask(KetchText.get().starting); | |||||
break; | |||||
case EXECUTED: | |||||
spinner.endTask(KetchText.get().accepted); | |||||
break; | |||||
case ABORTED: | |||||
spinner.endTask(KetchText.get().failed); | |||||
break; | |||||
} | |||||
} | |||||
private void waitForPropose(Proposal proposal, ProgressSpinner spinner) | |||||
throws InterruptedException { | |||||
spinner.beginTask(KetchText.get().proposingUpdates, 2, SECONDS); | |||||
while (!proposal.await(250, MILLISECONDS)) { | |||||
spinner.update(); | |||||
} | |||||
spinner.endTask(proposal.getState() == EXECUTED | |||||
? KetchText.get().accepted | |||||
: KetchText.get().failed); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static java.util.concurrent.TimeUnit.MILLISECONDS; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.BATCHED; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed.FAST; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.CURRENT; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Type.CREATE; | |||||
import java.io.IOException; | |||||
import java.lang.ref.WeakReference; | |||||
import java.util.ArrayList; | |||||
import java.util.Collection; | |||||
import java.util.HashMap; | |||||
import java.util.Iterator; | |||||
import java.util.List; | |||||
import java.util.Map; | |||||
import java.util.concurrent.Callable; | |||||
import java.util.concurrent.Future; | |||||
import org.eclipse.jgit.annotations.NonNull; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.internal.storage.reftree.RefTree; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.treewalk.TreeWalk; | |||||
import org.eclipse.jgit.util.FileUtils; | |||||
import org.eclipse.jgit.util.SystemReader; | |||||
import org.slf4j.Logger; | |||||
import org.slf4j.LoggerFactory; | |||||
/** | |||||
* A Ketch replica, either {@link org.eclipse.jgit.internal.ketch.LocalReplica} | |||||
* or {@link org.eclipse.jgit.internal.ketch.RemoteGitReplica}. | |||||
* <p> | |||||
* Replicas can be either a stock Git replica, or a Ketch-aware replica. | |||||
* <p> | |||||
* A stock Git replica has no special knowledge of Ketch and simply stores | |||||
* objects and references. Ketch communicates with the stock Git replica using | |||||
* the Git push wire protocol. The | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader} commits an agreed upon | |||||
* state by pushing all references to the Git replica, for example | |||||
* {@code "refs/heads/master"} is pushed during commit. Stock Git replicas use | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS} to | |||||
* record the final state. | |||||
* <p> | |||||
* Ketch-aware replicas understand the {@code RefTree} sent during the proposal | |||||
* and during commit are able to update their own reference space to match the | |||||
* state represented by the {@code RefTree}. Ketch-aware replicas typically use | |||||
* a {@link org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase} and | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#TXN_COMMITTED} | |||||
* to record the final state. | |||||
* <p> | |||||
* KetchReplica instances are tightly coupled with a single | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader}. Some state may be | |||||
* accessed by the leader thread and uses the leader's own | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} to protect shared | |||||
* data. | |||||
*/ | |||||
public abstract class KetchReplica { | |||||
static final Logger log = LoggerFactory.getLogger(KetchReplica.class); | |||||
private static final byte[] PEEL = { ' ', '^' }; | |||||
/** Participation of a replica in establishing consensus. */ | |||||
public enum Participation { | |||||
/** Replica can vote. */ | |||||
FULL, | |||||
/** Replica does not vote, but tracks leader. */ | |||||
FOLLOWER_ONLY; | |||||
} | |||||
/** How this replica wants to receive Ketch commit operations. */ | |||||
public enum CommitMethod { | |||||
/** All references are pushed to the peer as standard Git. */ | |||||
ALL_REFS, | |||||
/** Only {@code refs/txn/committed} is written/updated. */ | |||||
TXN_COMMITTED; | |||||
} | |||||
/** Delay before committing to a replica. */ | |||||
public enum CommitSpeed { | |||||
/** | |||||
* Send the commit immediately, even if it could be batched with the | |||||
* next proposal. | |||||
*/ | |||||
FAST, | |||||
/** | |||||
* If the next proposal is available, batch the commit with it, | |||||
* otherwise just send the commit. This generates less network use, but | |||||
* may provide slower consistency on the replica. | |||||
*/ | |||||
BATCHED; | |||||
} | |||||
/** Current state of a replica. */ | |||||
public enum State { | |||||
/** Leader has not yet contacted the replica. */ | |||||
UNKNOWN, | |||||
/** Replica is behind the consensus. */ | |||||
LAGGING, | |||||
/** Replica matches the consensus. */ | |||||
CURRENT, | |||||
/** Replica has a different (or unknown) history. */ | |||||
DIVERGENT, | |||||
/** Replica's history contains the leader's history. */ | |||||
AHEAD, | |||||
/** Replica can not be contacted. */ | |||||
OFFLINE; | |||||
} | |||||
private final KetchLeader leader; | |||||
private final String replicaName; | |||||
private final Participation participation; | |||||
private final CommitMethod commitMethod; | |||||
private final CommitSpeed commitSpeed; | |||||
private final long minRetryMillis; | |||||
private final long maxRetryMillis; | |||||
private final Map<ObjectId, List<ReceiveCommand>> staged; | |||||
private final Map<String, ReceiveCommand> running; | |||||
private final Map<String, ReceiveCommand> waiting; | |||||
private final List<ReplicaPushRequest> queued; | |||||
/** | |||||
* Value known for {@code "refs/txn/accepted"}. | |||||
* <p> | |||||
* Raft literature refers to this as {@code matchIndex}. | |||||
*/ | |||||
private ObjectId txnAccepted; | |||||
/** | |||||
* Value known for {@code "refs/txn/committed"}. | |||||
* <p> | |||||
* Raft literature refers to this as {@code commitIndex}. In traditional | |||||
* Raft this is a state variable inside the follower implementation, but | |||||
* Ketch keeps it in the leader. | |||||
*/ | |||||
private ObjectId txnCommitted; | |||||
/** What is happening with this replica. */ | |||||
private State state = UNKNOWN; | |||||
private String error; | |||||
/** Scheduled retry due to communication failure. */ | |||||
private Future<?> retryFuture; | |||||
private long lastRetryMillis; | |||||
private long retryAtMillis; | |||||
/** | |||||
* Configure a replica representation. | |||||
* | |||||
* @param leader | |||||
* instance this replica follows. | |||||
* @param name | |||||
* unique-ish name identifying this replica for debugging. | |||||
* @param cfg | |||||
* how Ketch should treat the replica. | |||||
*/ | |||||
protected KetchReplica(KetchLeader leader, String name, ReplicaConfig cfg) { | |||||
this.leader = leader; | |||||
this.replicaName = name; | |||||
this.participation = cfg.getParticipation(); | |||||
this.commitMethod = cfg.getCommitMethod(); | |||||
this.commitSpeed = cfg.getCommitSpeed(); | |||||
this.minRetryMillis = cfg.getMinRetry(MILLISECONDS); | |||||
this.maxRetryMillis = cfg.getMaxRetry(MILLISECONDS); | |||||
this.staged = new HashMap<>(); | |||||
this.running = new HashMap<>(); | |||||
this.waiting = new HashMap<>(); | |||||
this.queued = new ArrayList<>(4); | |||||
} | |||||
/** | |||||
* Get system configuration. | |||||
* | |||||
* @return system configuration. | |||||
*/ | |||||
public KetchSystem getSystem() { | |||||
return getLeader().getSystem(); | |||||
} | |||||
/** | |||||
* Get leader instance this replica follows. | |||||
* | |||||
* @return leader instance this replica follows. | |||||
*/ | |||||
public KetchLeader getLeader() { | |||||
return leader; | |||||
} | |||||
/** | |||||
* Get unique-ish name for debugging. | |||||
* | |||||
* @return unique-ish name for debugging. | |||||
*/ | |||||
public String getName() { | |||||
return replicaName; | |||||
} | |||||
/** | |||||
* Get description of this replica for error/debug logging purposes. | |||||
* | |||||
* @return description of this replica for error/debug logging purposes. | |||||
*/ | |||||
protected String describeForLog() { | |||||
return getName(); | |||||
} | |||||
/** | |||||
* Get how the replica participates in this Ketch system. | |||||
* | |||||
* @return how the replica participates in this Ketch system. | |||||
*/ | |||||
public Participation getParticipation() { | |||||
return participation; | |||||
} | |||||
/** | |||||
* Get how Ketch will commit to the repository. | |||||
* | |||||
* @return how Ketch will commit to the repository. | |||||
*/ | |||||
public CommitMethod getCommitMethod() { | |||||
return commitMethod; | |||||
} | |||||
/** | |||||
* Get when Ketch will commit to the repository. | |||||
* | |||||
* @return when Ketch will commit to the repository. | |||||
*/ | |||||
public CommitSpeed getCommitSpeed() { | |||||
return commitSpeed; | |||||
} | |||||
/** | |||||
* Called by leader to perform graceful shutdown. | |||||
* <p> | |||||
* Default implementation cancels any scheduled retry. Subclasses may add | |||||
* additional logic before or after calling {@code super.shutdown()}. | |||||
* <p> | |||||
* Called with {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held | |||||
* by caller. | |||||
*/ | |||||
protected void shutdown() { | |||||
Future<?> f = retryFuture; | |||||
if (f != null) { | |||||
retryFuture = null; | |||||
f.cancel(true); | |||||
} | |||||
} | |||||
ReplicaSnapshot snapshot() { | |||||
ReplicaSnapshot s = new ReplicaSnapshot(this); | |||||
s.accepted = txnAccepted; | |||||
s.committed = txnCommitted; | |||||
s.state = state; | |||||
s.error = error; | |||||
s.retryAtMillis = waitingForRetry() ? retryAtMillis : 0; | |||||
return s; | |||||
} | |||||
/** | |||||
* Update the leader's view of the replica after a poll. | |||||
* <p> | |||||
* Called with {@link KetchLeader#lock} held by caller. | |||||
* | |||||
* @param refs | |||||
* map of refs from the replica. | |||||
*/ | |||||
void initialize(Map<String, Ref> refs) { | |||||
if (txnAccepted == null) { | |||||
txnAccepted = getId(refs.get(getSystem().getTxnAccepted())); | |||||
} | |||||
if (txnCommitted == null) { | |||||
txnCommitted = getId(refs.get(getSystem().getTxnCommitted())); | |||||
} | |||||
} | |||||
ObjectId getTxnAccepted() { | |||||
return txnAccepted; | |||||
} | |||||
boolean hasAccepted(LogIndex id) { | |||||
return equals(txnAccepted, id); | |||||
} | |||||
private static boolean equals(@Nullable ObjectId a, LogIndex b) { | |||||
return a != null && b != null && AnyObjectId.isEqual(a, b); | |||||
} | |||||
/** | |||||
* Schedule a proposal round with the replica. | |||||
* <p> | |||||
* Called with {@link KetchLeader#lock} held by caller. | |||||
* | |||||
* @param round | |||||
* current round being run by the leader. | |||||
*/ | |||||
void pushTxnAcceptedAsync(Round round) { | |||||
List<ReceiveCommand> cmds = new ArrayList<>(); | |||||
if (commitSpeed == BATCHED) { | |||||
LogIndex committedIndex = leader.getCommitted(); | |||||
if (equals(txnAccepted, committedIndex) | |||||
&& !equals(txnCommitted, committedIndex)) { | |||||
prepareTxnCommitted(cmds, committedIndex); | |||||
} | |||||
} | |||||
// TODO(sop) Lagging replicas should build accept on the fly. | |||||
if (round.stageCommands != null) { | |||||
for (ReceiveCommand cmd : round.stageCommands) { | |||||
// TODO(sop): Do not send certain object graphs to replica. | |||||
cmds.add(copy(cmd)); | |||||
} | |||||
} | |||||
cmds.add(new ReceiveCommand( | |||||
round.acceptedOldIndex, round.acceptedNewIndex, | |||||
getSystem().getTxnAccepted())); | |||||
pushAsync(new ReplicaPushRequest(this, cmds)); | |||||
} | |||||
private static ReceiveCommand copy(ReceiveCommand c) { | |||||
return new ReceiveCommand(c.getOldId(), c.getNewId(), c.getRefName()); | |||||
} | |||||
boolean shouldPushUnbatchedCommit(LogIndex committed, boolean leaderIdle) { | |||||
return (leaderIdle || commitSpeed == FAST) && hasAccepted(committed); | |||||
} | |||||
void pushCommitAsync(LogIndex committed) { | |||||
List<ReceiveCommand> cmds = new ArrayList<>(); | |||||
prepareTxnCommitted(cmds, committed); | |||||
pushAsync(new ReplicaPushRequest(this, cmds)); | |||||
} | |||||
private void prepareTxnCommitted(List<ReceiveCommand> cmds, | |||||
ObjectId committed) { | |||||
removeStaged(cmds, committed); | |||||
cmds.add(new ReceiveCommand( | |||||
txnCommitted, committed, | |||||
getSystem().getTxnCommitted())); | |||||
} | |||||
private void removeStaged(List<ReceiveCommand> cmds, ObjectId committed) { | |||||
List<ReceiveCommand> a = staged.remove(committed); | |||||
if (a != null) { | |||||
delete(cmds, a); | |||||
} | |||||
if (staged.isEmpty() || !(committed instanceof LogIndex)) { | |||||
return; | |||||
} | |||||
LogIndex committedIndex = (LogIndex) committed; | |||||
Iterator<Map.Entry<ObjectId, List<ReceiveCommand>>> itr = staged | |||||
.entrySet().iterator(); | |||||
while (itr.hasNext()) { | |||||
Map.Entry<ObjectId, List<ReceiveCommand>> e = itr.next(); | |||||
if (e.getKey() instanceof LogIndex) { | |||||
LogIndex stagedIndex = (LogIndex) e.getKey(); | |||||
if (stagedIndex.isBefore(committedIndex)) { | |||||
delete(cmds, e.getValue()); | |||||
itr.remove(); | |||||
} | |||||
} | |||||
} | |||||
} | |||||
private static void delete(List<ReceiveCommand> cmds, | |||||
List<ReceiveCommand> createCmds) { | |||||
for (ReceiveCommand cmd : createCmds) { | |||||
ObjectId id = cmd.getNewId(); | |||||
String name = cmd.getRefName(); | |||||
cmds.add(new ReceiveCommand(id, ObjectId.zeroId(), name)); | |||||
} | |||||
} | |||||
/** | |||||
* Determine the next push for this replica (if any) and start it. | |||||
* <p> | |||||
* If the replica has successfully accepted the committed state of the | |||||
* leader, this method will push all references to the replica using the | |||||
* configured {@link CommitMethod}. | |||||
* <p> | |||||
* If the replica is {@link State#LAGGING} this method will begin catch up | |||||
* by sending a more recent {@code refs/txn/accepted}. | |||||
* <p> | |||||
* Must be invoked with {@link KetchLeader#lock} held by caller. | |||||
*/ | |||||
private void runNextPushRequest() { | |||||
LogIndex committed = leader.getCommitted(); | |||||
if (!equals(txnCommitted, committed) | |||||
&& shouldPushUnbatchedCommit(committed, leader.isIdle())) { | |||||
pushCommitAsync(committed); | |||||
} | |||||
if (queued.isEmpty() || !running.isEmpty() || waitingForRetry()) { | |||||
return; | |||||
} | |||||
// Collapse all queued requests into a single request. | |||||
Map<String, ReceiveCommand> cmdMap = new HashMap<>(); | |||||
for (ReplicaPushRequest req : queued) { | |||||
for (ReceiveCommand cmd : req.getCommands()) { | |||||
String name = cmd.getRefName(); | |||||
ReceiveCommand old = cmdMap.remove(name); | |||||
if (old != null) { | |||||
cmd = new ReceiveCommand( | |||||
old.getOldId(), cmd.getNewId(), | |||||
name); | |||||
} | |||||
cmdMap.put(name, cmd); | |||||
} | |||||
} | |||||
queued.clear(); | |||||
waiting.clear(); | |||||
List<ReceiveCommand> next = new ArrayList<>(cmdMap.values()); | |||||
for (ReceiveCommand cmd : next) { | |||||
running.put(cmd.getRefName(), cmd); | |||||
} | |||||
startPush(new ReplicaPushRequest(this, next)); | |||||
} | |||||
private void pushAsync(ReplicaPushRequest req) { | |||||
if (defer(req)) { | |||||
// TODO(sop) Collapse during long retry outage. | |||||
for (ReceiveCommand cmd : req.getCommands()) { | |||||
waiting.put(cmd.getRefName(), cmd); | |||||
} | |||||
queued.add(req); | |||||
} else { | |||||
for (ReceiveCommand cmd : req.getCommands()) { | |||||
running.put(cmd.getRefName(), cmd); | |||||
} | |||||
startPush(req); | |||||
} | |||||
} | |||||
private boolean defer(ReplicaPushRequest req) { | |||||
if (waitingForRetry()) { | |||||
// Prior communication failure; everything is deferred. | |||||
return true; | |||||
} | |||||
for (ReceiveCommand nextCmd : req.getCommands()) { | |||||
ReceiveCommand priorCmd = waiting.get(nextCmd.getRefName()); | |||||
if (priorCmd == null) { | |||||
priorCmd = running.get(nextCmd.getRefName()); | |||||
} | |||||
if (priorCmd != null) { | |||||
// Another request pending on same ref; that must go first. | |||||
// Verify priorCmd.newId == nextCmd.oldId? | |||||
return true; | |||||
} | |||||
} | |||||
return false; | |||||
} | |||||
private boolean waitingForRetry() { | |||||
Future<?> f = retryFuture; | |||||
return f != null && !f.isDone(); | |||||
} | |||||
private void retryLater(ReplicaPushRequest req) { | |||||
Collection<ReceiveCommand> cmds = req.getCommands(); | |||||
for (ReceiveCommand cmd : cmds) { | |||||
cmd.setResult(NOT_ATTEMPTED, null); | |||||
if (!waiting.containsKey(cmd.getRefName())) { | |||||
waiting.put(cmd.getRefName(), cmd); | |||||
} | |||||
} | |||||
queued.add(0, new ReplicaPushRequest(this, cmds)); | |||||
if (!waitingForRetry()) { | |||||
long delay = FileUtils | |||||
.delay(lastRetryMillis, minRetryMillis, maxRetryMillis); | |||||
if (log.isDebugEnabled()) { | |||||
log.debug("Retrying {} after {} ms", //$NON-NLS-1$ | |||||
describeForLog(), Long.valueOf(delay)); | |||||
} | |||||
lastRetryMillis = delay; | |||||
retryAtMillis = SystemReader.getInstance().getCurrentTime() + delay; | |||||
retryFuture = getSystem().getExecutor() | |||||
.schedule(new WeakRetryPush(this), delay, MILLISECONDS); | |||||
} | |||||
} | |||||
/** Weakly holds a retrying replica, allowing it to garbage collect. */ | |||||
static class WeakRetryPush extends WeakReference<KetchReplica> | |||||
implements Callable<Void> { | |||||
WeakRetryPush(KetchReplica r) { | |||||
super(r); | |||||
} | |||||
@Override | |||||
public Void call() throws Exception { | |||||
KetchReplica r = get(); | |||||
if (r != null) { | |||||
r.doRetryPush(); | |||||
} | |||||
return null; | |||||
} | |||||
} | |||||
private void doRetryPush() { | |||||
leader.lock.lock(); | |||||
try { | |||||
retryFuture = null; | |||||
runNextPushRequest(); | |||||
} finally { | |||||
leader.lock.unlock(); | |||||
} | |||||
} | |||||
/** | |||||
* Begin executing a single push. | |||||
* <p> | |||||
* This method must move processing onto another thread. Called with | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock} held by caller. | |||||
* | |||||
* @param req | |||||
* the request to send to the replica. | |||||
*/ | |||||
protected abstract void startPush(ReplicaPushRequest req); | |||||
/** | |||||
* Callback from {@link ReplicaPushRequest} upon success or failure. | |||||
* <p> | |||||
* Acquires the {@link KetchLeader#lock} and updates the leader's internal | |||||
* knowledge about this replica to reflect what has been learned during a | |||||
* push to the replica. In some cases of divergence this method may take | |||||
* some time to determine how the replica has diverged; to reduce contention | |||||
* this is evaluated before acquiring the leader lock. | |||||
* | |||||
* @param repo | |||||
* local repository instance used by the push thread. | |||||
* @param req | |||||
* push request just attempted. | |||||
*/ | |||||
void afterPush(@Nullable Repository repo, ReplicaPushRequest req) { | |||||
ReceiveCommand acceptCmd = null; | |||||
ReceiveCommand commitCmd = null; | |||||
List<ReceiveCommand> stages = null; | |||||
for (ReceiveCommand cmd : req.getCommands()) { | |||||
String name = cmd.getRefName(); | |||||
if (name.equals(getSystem().getTxnAccepted())) { | |||||
acceptCmd = cmd; | |||||
} else if (name.equals(getSystem().getTxnCommitted())) { | |||||
commitCmd = cmd; | |||||
} else if (cmd.getResult() == OK && cmd.getType() == CREATE | |||||
&& name.startsWith(getSystem().getTxnStage())) { | |||||
if (stages == null) { | |||||
stages = new ArrayList<>(); | |||||
} | |||||
stages.add(cmd); | |||||
} | |||||
} | |||||
State newState = null; | |||||
ObjectId acceptId = readId(req, acceptCmd); | |||||
if (repo != null && acceptCmd != null && acceptCmd.getResult() != OK | |||||
&& req.getException() == null) { | |||||
try (LagCheck lag = new LagCheck(this, repo)) { | |||||
newState = lag.check(acceptId, acceptCmd); | |||||
acceptId = lag.getRemoteId(); | |||||
} | |||||
} | |||||
leader.lock.lock(); | |||||
try { | |||||
for (ReceiveCommand cmd : req.getCommands()) { | |||||
running.remove(cmd.getRefName()); | |||||
} | |||||
Throwable err = req.getException(); | |||||
if (err != null) { | |||||
state = OFFLINE; | |||||
error = err.toString(); | |||||
retryLater(req); | |||||
leader.onReplicaUpdate(this); | |||||
return; | |||||
} | |||||
lastRetryMillis = 0; | |||||
error = null; | |||||
updateView(req, acceptId, commitCmd); | |||||
if (acceptCmd != null && acceptCmd.getResult() == OK) { | |||||
state = hasAccepted(leader.getHead()) ? CURRENT : LAGGING; | |||||
if (stages != null) { | |||||
staged.put(acceptCmd.getNewId(), stages); | |||||
} | |||||
} else if (newState != null) { | |||||
state = newState; | |||||
} | |||||
leader.onReplicaUpdate(this); | |||||
runNextPushRequest(); | |||||
} finally { | |||||
leader.lock.unlock(); | |||||
} | |||||
} | |||||
private void updateView(ReplicaPushRequest req, @Nullable ObjectId acceptId, | |||||
ReceiveCommand commitCmd) { | |||||
if (acceptId != null) { | |||||
txnAccepted = acceptId; | |||||
} | |||||
ObjectId committed = readId(req, commitCmd); | |||||
if (committed != null) { | |||||
txnCommitted = committed; | |||||
} else if (acceptId != null && txnCommitted == null) { | |||||
// Initialize during first conversation. | |||||
Map<String, Ref> adv = req.getRefs(); | |||||
if (adv != null) { | |||||
Ref refs = adv.get(getSystem().getTxnCommitted()); | |||||
txnCommitted = getId(refs); | |||||
} | |||||
} | |||||
} | |||||
@Nullable | |||||
private static ObjectId readId(ReplicaPushRequest req, | |||||
@Nullable ReceiveCommand cmd) { | |||||
if (cmd == null) { | |||||
// Ref was not in the command list, do not trust advertisement. | |||||
return null; | |||||
} else if (cmd.getResult() == OK) { | |||||
// Currently at newId. | |||||
return cmd.getNewId(); | |||||
} | |||||
Map<String, Ref> refs = req.getRefs(); | |||||
return refs != null ? getId(refs.get(cmd.getRefName())) : null; | |||||
} | |||||
/** | |||||
* Fetch objects from the remote using the calling thread. | |||||
* <p> | |||||
* Called without {@link org.eclipse.jgit.internal.ketch.KetchLeader#lock}. | |||||
* | |||||
* @param repo | |||||
* local repository to fetch objects into. | |||||
* @param req | |||||
* the request to fetch from a replica. | |||||
* @throws java.io.IOException | |||||
* communication with the replica was not possible. | |||||
*/ | |||||
protected abstract void blockingFetch(Repository repo, | |||||
ReplicaFetchRequest req) throws IOException; | |||||
/** | |||||
* Build a list of commands to commit | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod#ALL_REFS}. | |||||
* | |||||
* @param git | |||||
* local leader repository to read committed state from. | |||||
* @param current | |||||
* all known references in the replica's repository. Typically | |||||
* this comes from a push advertisement. | |||||
* @param committed | |||||
* state being pushed to {@code refs/txn/committed}. | |||||
* @return commands to update during commit. | |||||
* @throws java.io.IOException | |||||
* cannot read the committed state. | |||||
*/ | |||||
protected Collection<ReceiveCommand> prepareCommit(Repository git, | |||||
Map<String, Ref> current, ObjectId committed) throws IOException { | |||||
List<ReceiveCommand> delta = new ArrayList<>(); | |||||
Map<String, Ref> remote = new HashMap<>(current); | |||||
try (RevWalk rw = new RevWalk(git); | |||||
TreeWalk tw = new TreeWalk(rw.getObjectReader())) { | |||||
tw.setRecursive(true); | |||||
tw.addTree(rw.parseCommit(committed).getTree()); | |||||
while (tw.next()) { | |||||
if (tw.getRawMode(0) != TYPE_GITLINK | |||||
|| tw.isPathSuffix(PEEL, 2)) { | |||||
// Symbolic references cannot be pushed. | |||||
// Caching peeled values is handled remotely. | |||||
continue; | |||||
} | |||||
// TODO(sop) Do not send certain ref names to replica. | |||||
String name = RefTree.refName(tw.getPathString()); | |||||
Ref oldRef = remote.remove(name); | |||||
ObjectId oldId = getId(oldRef); | |||||
ObjectId newId = tw.getObjectId(0); | |||||
if (!AnyObjectId.isEqual(oldId, newId)) { | |||||
delta.add(new ReceiveCommand(oldId, newId, name)); | |||||
} | |||||
} | |||||
} | |||||
// Delete any extra references not in the committed state. | |||||
for (Ref ref : remote.values()) { | |||||
if (canDelete(ref)) { | |||||
delta.add(new ReceiveCommand( | |||||
ref.getObjectId(), ObjectId.zeroId(), | |||||
ref.getName())); | |||||
} | |||||
} | |||||
return delta; | |||||
} | |||||
boolean canDelete(Ref ref) { | |||||
String name = ref.getName(); | |||||
if (HEAD.equals(name)) { | |||||
return false; | |||||
} | |||||
if (name.startsWith(getSystem().getTxnNamespace())) { | |||||
return false; | |||||
} | |||||
// TODO(sop) Do not delete precious names from replica. | |||||
return true; | |||||
} | |||||
@NonNull | |||||
static ObjectId getId(@Nullable Ref ref) { | |||||
if (ref != null) { | |||||
ObjectId id = ref.getObjectId(); | |||||
if (id != null) { | |||||
return id; | |||||
} | |||||
} | |||||
return ObjectId.zeroId(); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.ACCEPTED; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.COMMITTED; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_SECTION_KETCH; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.DEFAULT_TXN_NAMESPACE; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.STAGE; | |||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_NAME; | |||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; | |||||
import java.net.URISyntaxException; | |||||
import java.time.Duration; | |||||
import java.util.ArrayList; | |||||
import java.util.List; | |||||
import java.util.Random; | |||||
import java.util.concurrent.Executors; | |||||
import java.util.concurrent.ScheduledExecutorService; | |||||
import java.util.concurrent.ThreadFactory; | |||||
import java.util.concurrent.atomic.AtomicInteger; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.lib.Config; | |||||
import org.eclipse.jgit.lib.PersonIdent; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.transport.RemoteConfig; | |||||
import org.eclipse.jgit.transport.URIish; | |||||
import org.eclipse.jgit.util.time.MonotonicClock; | |||||
import org.eclipse.jgit.util.time.MonotonicSystemClock; | |||||
import org.eclipse.jgit.util.time.ProposedTimestamp; | |||||
import org.slf4j.Logger; | |||||
import org.slf4j.LoggerFactory; | |||||
/** | |||||
* Ketch system-wide configuration. | |||||
* <p> | |||||
* This class provides useful defaults for testing and small proof of concepts. | |||||
* Full scale installations are expected to subclass and override methods to | |||||
* provide consistent configuration across all managed repositories. | |||||
* <p> | |||||
* Servers should configure their own | |||||
* {@link java.util.concurrent.ScheduledExecutorService}. | |||||
*/ | |||||
public class KetchSystem { | |||||
private static final Random RNG = new Random(); | |||||
/** | |||||
* Get default executor, one thread per available processor. | |||||
* | |||||
* @return default executor, one thread per available processor. | |||||
*/ | |||||
public static ScheduledExecutorService defaultExecutor() { | |||||
return DefaultExecutorHolder.I; | |||||
} | |||||
private final ScheduledExecutorService executor; | |||||
private final MonotonicClock clock; | |||||
private final String txnNamespace; | |||||
private final String txnAccepted; | |||||
private final String txnCommitted; | |||||
private final String txnStage; | |||||
/** | |||||
* Create a default system with a thread pool of 1 thread per CPU. | |||||
*/ | |||||
public KetchSystem() { | |||||
this(defaultExecutor(), new MonotonicSystemClock(), DEFAULT_TXN_NAMESPACE); | |||||
} | |||||
/** | |||||
* Create a Ketch system with the provided executor service. | |||||
* | |||||
* @param executor | |||||
* thread pool to run background operations. | |||||
* @param clock | |||||
* clock to create timestamps. | |||||
* @param txnNamespace | |||||
* reference namespace for the RefTree graph and associated | |||||
* transaction state. Must begin with {@code "refs/"} and end | |||||
* with {@code '/'}, for example {@code "refs/txn/"}. | |||||
*/ | |||||
public KetchSystem(ScheduledExecutorService executor, MonotonicClock clock, | |||||
String txnNamespace) { | |||||
this.executor = executor; | |||||
this.clock = clock; | |||||
this.txnNamespace = txnNamespace; | |||||
this.txnAccepted = txnNamespace + ACCEPTED; | |||||
this.txnCommitted = txnNamespace + COMMITTED; | |||||
this.txnStage = txnNamespace + STAGE; | |||||
} | |||||
/** | |||||
* Get executor to perform background operations. | |||||
* | |||||
* @return executor to perform background operations. | |||||
*/ | |||||
public ScheduledExecutorService getExecutor() { | |||||
return executor; | |||||
} | |||||
/** | |||||
* Get clock to obtain timestamps from. | |||||
* | |||||
* @return clock to obtain timestamps from. | |||||
*/ | |||||
public MonotonicClock getClock() { | |||||
return clock; | |||||
} | |||||
/** | |||||
* Get how long the leader will wait for the {@link #getClock()}'s | |||||
* {@code ProposedTimestamp} used in commits proposed to the RefTree graph | |||||
* ({@link #getTxnAccepted()}) | |||||
* | |||||
* @return how long the leader will wait for the {@link #getClock()}'s | |||||
* {@code ProposedTimestamp} used in commits proposed to the RefTree | |||||
* graph ({@link #getTxnAccepted()}). Defaults to 5 seconds. | |||||
*/ | |||||
public Duration getMaxWaitForMonotonicClock() { | |||||
return Duration.ofSeconds(5); | |||||
} | |||||
/** | |||||
* Whether elections should require monotonically increasing commit | |||||
* timestamps | |||||
* | |||||
* @return {@code true} if elections should require monotonically increasing | |||||
* commit timestamps. This requires a very good | |||||
* {@link org.eclipse.jgit.util.time.MonotonicClock}. | |||||
*/ | |||||
public boolean requireMonotonicLeaderElections() { | |||||
return false; | |||||
} | |||||
/** | |||||
* Get the namespace used for the RefTree graph and transaction management. | |||||
* | |||||
* @return reference namespace such as {@code "refs/txn/"}. | |||||
*/ | |||||
public String getTxnNamespace() { | |||||
return txnNamespace; | |||||
} | |||||
/** | |||||
* Get name of the accepted RefTree graph. | |||||
* | |||||
* @return name of the accepted RefTree graph. | |||||
*/ | |||||
public String getTxnAccepted() { | |||||
return txnAccepted; | |||||
} | |||||
/** | |||||
* Get name of the committed RefTree graph. | |||||
* | |||||
* @return name of the committed RefTree graph. | |||||
*/ | |||||
public String getTxnCommitted() { | |||||
return txnCommitted; | |||||
} | |||||
/** | |||||
* Get prefix for staged objects, e.g. {@code "refs/txn/stage/"}. | |||||
* | |||||
* @return prefix for staged objects, e.g. {@code "refs/txn/stage/"}. | |||||
*/ | |||||
public String getTxnStage() { | |||||
return txnStage; | |||||
} | |||||
/** | |||||
* Create new committer {@code PersonIdent} for ketch system | |||||
* | |||||
* @param time | |||||
* timestamp for the committer. | |||||
* @return identity line for the committer header of a RefTreeGraph. | |||||
*/ | |||||
public PersonIdent newCommitter(ProposedTimestamp time) { | |||||
String name = "ketch"; //$NON-NLS-1$ | |||||
String email = "ketch@system"; //$NON-NLS-1$ | |||||
return new PersonIdent(name, email, time); | |||||
} | |||||
/** | |||||
* Construct a random tag to identify a candidate during leader election. | |||||
* <p> | |||||
* Multiple processes trying to elect themselves leaders at exactly the same | |||||
* time (rounded to seconds) using the same | |||||
* {@link #newCommitter(ProposedTimestamp)} identity strings, for the same | |||||
* term, may generate the same ObjectId for the election commit and falsely | |||||
* assume they have both won. | |||||
* <p> | |||||
* Candidates add this tag to their election ballot commit to disambiguate | |||||
* the election. The tag only needs to be unique for a given triplet of | |||||
* {@link #newCommitter(ProposedTimestamp)}, system time (rounded to | |||||
* seconds), and term. If every replica in the system uses a unique | |||||
* {@code newCommitter} (such as including the host name after the | |||||
* {@code "@"} in the email address) the tag could be the empty string. | |||||
* <p> | |||||
* The default implementation generates a few bytes of random data. | |||||
* | |||||
* @return unique tag; null or empty string if {@code newCommitter()} is | |||||
* sufficiently unique to identify the leader. | |||||
*/ | |||||
@Nullable | |||||
public String newLeaderTag() { | |||||
int n = RNG.nextInt(1 << (6 * 4)); | |||||
return String.format("%06x", Integer.valueOf(n)); //$NON-NLS-1$ | |||||
} | |||||
/** | |||||
* Construct the KetchLeader instance of a repository. | |||||
* | |||||
* @param repo | |||||
* local repository stored by the leader. | |||||
* @return leader instance. | |||||
* @throws java.net.URISyntaxException | |||||
* a follower configuration contains an unsupported URI. | |||||
*/ | |||||
public KetchLeader createLeader(Repository repo) | |||||
throws URISyntaxException { | |||||
KetchLeader leader = new KetchLeader(this) { | |||||
@Override | |||||
protected Repository openRepository() { | |||||
repo.incrementOpen(); | |||||
return repo; | |||||
} | |||||
}; | |||||
leader.setReplicas(createReplicas(leader, repo)); | |||||
return leader; | |||||
} | |||||
/** | |||||
* Get the collection of replicas for a repository. | |||||
* <p> | |||||
* The collection of replicas must include the local repository. | |||||
* | |||||
* @param leader | |||||
* the leader driving these replicas. | |||||
* @param repo | |||||
* repository to get the replicas of. | |||||
* @return collection of replicas for the specified repository. | |||||
* @throws java.net.URISyntaxException | |||||
* a configured URI is invalid. | |||||
*/ | |||||
protected List<KetchReplica> createReplicas(KetchLeader leader, | |||||
Repository repo) throws URISyntaxException { | |||||
List<KetchReplica> replicas = new ArrayList<>(); | |||||
Config cfg = repo.getConfig(); | |||||
String localName = getLocalName(cfg); | |||||
for (String name : cfg.getSubsections(CONFIG_KEY_REMOTE)) { | |||||
if (!hasParticipation(cfg, name)) { | |||||
continue; | |||||
} | |||||
ReplicaConfig kc = ReplicaConfig.newFromConfig(cfg, name); | |||||
if (name.equals(localName)) { | |||||
replicas.add(new LocalReplica(leader, name, kc)); | |||||
continue; | |||||
} | |||||
RemoteConfig rc = new RemoteConfig(cfg, name); | |||||
List<URIish> uris = rc.getPushURIs(); | |||||
if (uris.isEmpty()) { | |||||
uris = rc.getURIs(); | |||||
} | |||||
for (URIish uri : uris) { | |||||
String n = uris.size() == 1 ? name : uri.getHost(); | |||||
replicas.add(new RemoteGitReplica(leader, n, uri, kc, rc)); | |||||
} | |||||
} | |||||
return replicas; | |||||
} | |||||
private static boolean hasParticipation(Config cfg, String name) { | |||||
return cfg.getString(CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE) != null; | |||||
} | |||||
private static String getLocalName(Config cfg) { | |||||
return cfg.getString(CONFIG_SECTION_KETCH, null, CONFIG_KEY_NAME); | |||||
} | |||||
static class DefaultExecutorHolder { | |||||
private static final Logger log = LoggerFactory.getLogger(KetchSystem.class); | |||||
static final ScheduledExecutorService I = create(); | |||||
private static ScheduledExecutorService create() { | |||||
int cores = Runtime.getRuntime().availableProcessors(); | |||||
int threads = Math.max(5, cores); | |||||
log.info("Using {} threads", Integer.valueOf(threads)); //$NON-NLS-1$ | |||||
return Executors.newScheduledThreadPool( | |||||
threads, | |||||
new ThreadFactory() { | |||||
private final AtomicInteger threadCnt = new AtomicInteger(); | |||||
@Override | |||||
public Thread newThread(Runnable r) { | |||||
int id = threadCnt.incrementAndGet(); | |||||
Thread thr = new Thread(r); | |||||
thr.setName("KetchExecutor-" + id); //$NON-NLS-1$ | |||||
return thr; | |||||
} | |||||
}); | |||||
} | |||||
private DefaultExecutorHolder() { | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import org.eclipse.jgit.nls.NLS; | |||||
import org.eclipse.jgit.nls.TranslationBundle; | |||||
/** | |||||
* Translation bundle for the Ketch implementation. | |||||
*/ | |||||
public class KetchText extends TranslationBundle { | |||||
/** | |||||
* Get an instance of this translation bundle. | |||||
* | |||||
* @return instance of this translation bundle. | |||||
*/ | |||||
public static KetchText get() { | |||||
return NLS.getBundleFor(KetchText.class); | |||||
} | |||||
// @formatter:off | |||||
/***/ public String accepted; | |||||
/***/ public String cannotFetchFromLocalReplica; | |||||
/***/ public String failed; | |||||
/***/ public String invalidFollowerUri; | |||||
/***/ public String leaderFailedToStore; | |||||
/***/ public String localReplicaRequired; | |||||
/***/ public String mismatchedTxnNamespace; | |||||
/***/ public String outsideTxnNamespace; | |||||
/***/ public String proposingUpdates; | |||||
/***/ public String queuedProposalFailedToApply; | |||||
/***/ public String starting; | |||||
/***/ public String unsupportedVoterCount; | |||||
/***/ public String waitingForQueue; | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.AHEAD; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.DIVERGENT; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.LAGGING; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.UNKNOWN; | |||||
import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; | |||||
import java.io.IOException; | |||||
import java.util.Collections; | |||||
import java.util.Map; | |||||
import org.eclipse.jgit.errors.MissingObjectException; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
/** | |||||
* A helper to check if a {@link KetchReplica} is ahead or behind the leader. | |||||
*/ | |||||
class LagCheck implements AutoCloseable { | |||||
private final KetchReplica replica; | |||||
private final Repository repo; | |||||
private RevWalk rw; | |||||
private ObjectId remoteId; | |||||
LagCheck(KetchReplica replica, Repository repo) { | |||||
this.replica = replica; | |||||
this.repo = repo; | |||||
initRevWalk(); | |||||
} | |||||
private void initRevWalk() { | |||||
if (rw != null) { | |||||
rw.close(); | |||||
} | |||||
rw = new RevWalk(repo); | |||||
rw.setRetainBody(false); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public void close() { | |||||
if (rw != null) { | |||||
rw.close(); | |||||
rw = null; | |||||
} | |||||
} | |||||
ObjectId getRemoteId() { | |||||
return remoteId; | |||||
} | |||||
KetchReplica.State check(ObjectId acceptId, ReceiveCommand acceptCmd) { | |||||
remoteId = acceptId; | |||||
if (remoteId == null) { | |||||
// Nothing advertised by the replica, value is unknown. | |||||
return UNKNOWN; | |||||
} | |||||
if (AnyObjectId.isEqual(remoteId, ObjectId.zeroId())) { | |||||
// Replica does not have the txnAccepted reference. | |||||
return LAGGING; | |||||
} | |||||
try { | |||||
RevCommit remote; | |||||
try { | |||||
remote = parseRemoteCommit(acceptCmd.getRefName()); | |||||
} catch (RefGoneException gone) { | |||||
// Replica does not have the txnAccepted reference. | |||||
return LAGGING; | |||||
} catch (MissingObjectException notFound) { | |||||
// Local repository does not know this commit so it cannot | |||||
// be including the replica's log. | |||||
return DIVERGENT; | |||||
} | |||||
RevCommit head = rw.parseCommit(acceptCmd.getNewId()); | |||||
if (rw.isMergedInto(remote, head)) { | |||||
return LAGGING; | |||||
} | |||||
// TODO(sop) Check term to see if my leader was deposed. | |||||
if (rw.isMergedInto(head, remote)) { | |||||
return AHEAD; | |||||
} | |||||
return DIVERGENT; | |||||
} catch (IOException err) { | |||||
KetchReplica.log.error(String.format( | |||||
"Cannot compare %s", //$NON-NLS-1$ | |||||
acceptCmd.getRefName()), err); | |||||
return UNKNOWN; | |||||
} | |||||
} | |||||
private RevCommit parseRemoteCommit(String refName) | |||||
throws IOException, MissingObjectException, RefGoneException { | |||||
try { | |||||
return rw.parseCommit(remoteId); | |||||
} catch (MissingObjectException notLocal) { | |||||
// Fall through and try to acquire the object by fetching it. | |||||
} | |||||
ReplicaFetchRequest fetch = new ReplicaFetchRequest( | |||||
Collections.singleton(refName), | |||||
Collections.<ObjectId> emptySet()); | |||||
try { | |||||
replica.blockingFetch(repo, fetch); | |||||
} catch (IOException fetchErr) { | |||||
KetchReplica.log.error(String.format( | |||||
"Cannot fetch %s (%s) from %s", //$NON-NLS-1$ | |||||
remoteId.abbreviate(8).name(), refName, | |||||
replica.describeForLog()), fetchErr); | |||||
throw new MissingObjectException(remoteId, OBJ_COMMIT); | |||||
} | |||||
Map<String, Ref> adv = fetch.getRefs(); | |||||
if (adv == null) { | |||||
throw new MissingObjectException(remoteId, OBJ_COMMIT); | |||||
} | |||||
Ref ref = adv.get(refName); | |||||
if (ref == null || ref.getObjectId() == null) { | |||||
throw new RefGoneException(); | |||||
} | |||||
initRevWalk(); | |||||
remoteId = ref.getObjectId(); | |||||
return rw.parseCommit(remoteId); | |||||
} | |||||
private static class RefGoneException extends Exception { | |||||
private static final long serialVersionUID = 1L; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.State.OFFLINE; | |||||
import java.util.ArrayList; | |||||
import java.util.Collection; | |||||
import java.util.Collections; | |||||
import java.util.List; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
/** | |||||
* A snapshot of a leader and its view of the world. | |||||
*/ | |||||
public class LeaderSnapshot { | |||||
final List<ReplicaSnapshot> replicas = new ArrayList<>(); | |||||
KetchLeader.State state; | |||||
long term; | |||||
LogIndex headIndex; | |||||
LogIndex committedIndex; | |||||
boolean idle; | |||||
LeaderSnapshot() { | |||||
} | |||||
/** | |||||
* Get unmodifiable view of configured replicas. | |||||
* | |||||
* @return unmodifiable view of configured replicas. | |||||
*/ | |||||
public Collection<ReplicaSnapshot> getReplicas() { | |||||
return Collections.unmodifiableList(replicas); | |||||
} | |||||
/** | |||||
* Get current state of the leader. | |||||
* | |||||
* @return current state of the leader. | |||||
*/ | |||||
public KetchLeader.State getState() { | |||||
return state; | |||||
} | |||||
/** | |||||
* Whether the leader is not running a round to reach consensus, and has no | |||||
* rounds queued. | |||||
* | |||||
* @return {@code true} if the leader is not running a round to reach | |||||
* consensus, and has no rounds queued. | |||||
*/ | |||||
public boolean isIdle() { | |||||
return idle; | |||||
} | |||||
/** | |||||
* Get term of this leader | |||||
* | |||||
* @return term of this leader. Valid only if {@link #getState()} is | |||||
* currently | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader.State#LEADER}. | |||||
*/ | |||||
public long getTerm() { | |||||
return term; | |||||
} | |||||
/** | |||||
* Get end of the leader's log | |||||
* | |||||
* @return end of the leader's log; null if leader hasn't started up enough | |||||
* to begin its own election. | |||||
*/ | |||||
@Nullable | |||||
public LogIndex getHead() { | |||||
return headIndex; | |||||
} | |||||
/** | |||||
* Get state the leader knows is committed on a majority of participant | |||||
* replicas | |||||
* | |||||
* @return state the leader knows is committed on a majority of participant | |||||
* replicas. Null until the leader instance has committed a log | |||||
* index within its own term. | |||||
*/ | |||||
@Nullable | |||||
public LogIndex getCommitted() { | |||||
return committedIndex; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public String toString() { | |||||
StringBuilder s = new StringBuilder(); | |||||
s.append(isIdle() ? "IDLE" : "RUNNING"); //$NON-NLS-1$ //$NON-NLS-2$ | |||||
s.append(" state ").append(getState()); //$NON-NLS-1$ | |||||
if (getTerm() > 0) { | |||||
s.append(" term ").append(getTerm()); //$NON-NLS-1$ | |||||
} | |||||
s.append('\n'); | |||||
s.append(String.format( | |||||
"%-10s %12s %12s\n", //$NON-NLS-1$ | |||||
"Replica", "Accepted", "Committed")); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ | |||||
s.append("------------------------------------\n"); //$NON-NLS-1$ | |||||
debug(s, "(leader)", getHead(), getCommitted()); //$NON-NLS-1$ | |||||
s.append('\n'); | |||||
for (ReplicaSnapshot r : getReplicas()) { | |||||
debug(s, r); | |||||
s.append('\n'); | |||||
} | |||||
s.append('\n'); | |||||
return s.toString(); | |||||
} | |||||
private static void debug(StringBuilder b, ReplicaSnapshot s) { | |||||
KetchReplica replica = s.getReplica(); | |||||
debug(b, replica.getName(), s.getAccepted(), s.getCommitted()); | |||||
b.append(String.format(" %-8s %s", //$NON-NLS-1$ | |||||
replica.getParticipation(), s.getState())); | |||||
if (s.getState() == OFFLINE) { | |||||
String err = s.getErrorMessage(); | |||||
if (err != null) { | |||||
b.append(" (").append(err).append(')'); //$NON-NLS-1$ | |||||
} | |||||
} | |||||
} | |||||
private static void debug(StringBuilder s, String name, | |||||
ObjectId accepted, ObjectId committed) { | |||||
s.append(String.format( | |||||
"%-10s %-12s %-12s", //$NON-NLS-1$ | |||||
name, str(accepted), str(committed))); | |||||
} | |||||
static String str(ObjectId c) { | |||||
if (c instanceof LogIndex) { | |||||
return ((LogIndex) c).describeForLog(); | |||||
} else if (c != null) { | |||||
return c.abbreviate(8).name(); | |||||
} | |||||
return "-"; //$NON-NLS-1$ | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.TXN_COMMITTED; | |||||
import static org.eclipse.jgit.lib.RefDatabase.ALL; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import java.io.IOException; | |||||
import java.text.MessageFormat; | |||||
import java.util.ArrayList; | |||||
import java.util.Collection; | |||||
import java.util.List; | |||||
import java.util.Map; | |||||
import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; | |||||
import org.eclipse.jgit.lib.BatchRefUpdate; | |||||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.util.time.MonotonicClock; | |||||
import org.eclipse.jgit.util.time.ProposedTimestamp; | |||||
/** | |||||
* Ketch replica running on the same system as the | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader}. | |||||
*/ | |||||
public class LocalReplica extends KetchReplica { | |||||
/** | |||||
* Configure a local replica. | |||||
* | |||||
* @param leader | |||||
* instance this replica follows. | |||||
* @param name | |||||
* unique-ish name identifying this replica for debugging. | |||||
* @param cfg | |||||
* how Ketch should treat the local system. | |||||
*/ | |||||
public LocalReplica(KetchLeader leader, String name, ReplicaConfig cfg) { | |||||
super(leader, name, cfg); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected String describeForLog() { | |||||
return String.format("%s (leader)", getName()); //$NON-NLS-1$ | |||||
} | |||||
/** | |||||
* Initializes local replica by reading accepted and committed references. | |||||
* <p> | |||||
* Loads accepted and committed references from the reference database of | |||||
* the local replica and stores their current ObjectIds in memory. | |||||
* | |||||
* @param repo | |||||
* repository to initialize state from. | |||||
* @throws IOException | |||||
* cannot read repository state. | |||||
*/ | |||||
void initialize(Repository repo) throws IOException { | |||||
RefDatabase refdb = repo.getRefDatabase(); | |||||
if (refdb instanceof RefTreeDatabase) { | |||||
RefTreeDatabase treeDb = (RefTreeDatabase) refdb; | |||||
String txnNamespace = getSystem().getTxnNamespace(); | |||||
if (!txnNamespace.equals(treeDb.getTxnNamespace())) { | |||||
throw new IOException(MessageFormat.format( | |||||
KetchText.get().mismatchedTxnNamespace, | |||||
txnNamespace, treeDb.getTxnNamespace())); | |||||
} | |||||
refdb = treeDb.getBootstrap(); | |||||
} | |||||
initialize(refdb.exactRef( | |||||
getSystem().getTxnAccepted(), | |||||
getSystem().getTxnCommitted())); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected void startPush(ReplicaPushRequest req) { | |||||
getSystem().getExecutor().execute(() -> { | |||||
MonotonicClock clk = getSystem().getClock(); | |||||
try (Repository git = getLeader().openRepository(); | |||||
ProposedTimestamp ts = clk.propose()) { | |||||
try { | |||||
update(git, req, ts); | |||||
req.done(git); | |||||
} catch (Throwable err) { | |||||
req.setException(git, err); | |||||
} | |||||
} catch (IOException err) { | |||||
req.setException(null, err); | |||||
} | |||||
}); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected void blockingFetch(Repository repo, ReplicaFetchRequest req) | |||||
throws IOException { | |||||
throw new IOException(KetchText.get().cannotFetchFromLocalReplica); | |||||
} | |||||
private void update(Repository git, ReplicaPushRequest req, | |||||
ProposedTimestamp ts) throws IOException { | |||||
RefDatabase refdb = git.getRefDatabase(); | |||||
CommitMethod method = getCommitMethod(); | |||||
// Local replica probably uses RefTreeDatabase, the request should | |||||
// be only for the txnNamespace, so drop to the bootstrap layer. | |||||
if (refdb instanceof RefTreeDatabase) { | |||||
if (!isOnlyTxnNamespace(req.getCommands())) { | |||||
return; | |||||
} | |||||
refdb = ((RefTreeDatabase) refdb).getBootstrap(); | |||||
method = TXN_COMMITTED; | |||||
} | |||||
BatchRefUpdate batch = refdb.newBatchUpdate(); | |||||
batch.addProposedTimestamp(ts); | |||||
batch.setRefLogIdent(getSystem().newCommitter(ts)); | |||||
batch.setRefLogMessage("ketch", false); //$NON-NLS-1$ | |||||
batch.setAllowNonFastForwards(true); | |||||
// RefDirectory updates multiple references sequentially. | |||||
// Run everything else first, then accepted (if present), | |||||
// then committed (if present). This ensures an earlier | |||||
// failure will not update these critical references. | |||||
ReceiveCommand accepted = null; | |||||
ReceiveCommand committed = null; | |||||
for (ReceiveCommand cmd : req.getCommands()) { | |||||
String name = cmd.getRefName(); | |||||
if (name.equals(getSystem().getTxnAccepted())) { | |||||
accepted = cmd; | |||||
} else if (name.equals(getSystem().getTxnCommitted())) { | |||||
committed = cmd; | |||||
} else { | |||||
batch.addCommand(cmd); | |||||
} | |||||
} | |||||
if (committed != null && method == ALL_REFS) { | |||||
Map<String, Ref> refs = refdb.getRefs(ALL); | |||||
batch.addCommand(prepareCommit(git, refs, committed.getNewId())); | |||||
} | |||||
if (accepted != null) { | |||||
batch.addCommand(accepted); | |||||
} | |||||
if (committed != null) { | |||||
batch.addCommand(committed); | |||||
} | |||||
try (RevWalk rw = new RevWalk(git)) { | |||||
batch.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
// KetchReplica only cares about accepted and committed in | |||||
// advertisement. If they failed, store the current values | |||||
// back in the ReplicaPushRequest. | |||||
List<String> failed = new ArrayList<>(2); | |||||
checkFailed(failed, accepted); | |||||
checkFailed(failed, committed); | |||||
if (!failed.isEmpty()) { | |||||
String[] arr = failed.toArray(new String[0]); | |||||
req.setRefs(refdb.exactRef(arr)); | |||||
} | |||||
} | |||||
private static void checkFailed(List<String> failed, ReceiveCommand cmd) { | |||||
if (cmd != null && cmd.getResult() != OK) { | |||||
failed.add(cmd.getRefName()); | |||||
} | |||||
} | |||||
private boolean isOnlyTxnNamespace(Collection<ReceiveCommand> cmdList) { | |||||
// Be paranoid and reject non txnNamespace names, this | |||||
// is a programming error in Ketch that should not occur. | |||||
String txnNamespace = getSystem().getTxnNamespace(); | |||||
for (ReceiveCommand cmd : cmdList) { | |||||
if (!cmd.getRefName().startsWith(txnNamespace)) { | |||||
cmd.setResult(REJECTED_OTHER_REASON, | |||||
MessageFormat.format( | |||||
KetchText.get().outsideTxnNamespace, | |||||
cmd.getRefName(), txnNamespace)); | |||||
ReceiveCommand.abort(cmdList); | |||||
return false; | |||||
} | |||||
} | |||||
return true; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
/** | |||||
* An ObjectId for a commit extended with incrementing log index. | |||||
* <p> | |||||
* For any two LogIndex instances, {@code A} is an ancestor of {@code C} | |||||
* reachable through parent edges in the graph if {@code A.index < C.index}. | |||||
* LogIndex provides a performance optimization for Ketch, the same information | |||||
* can be obtained from {@link org.eclipse.jgit.revwalk.RevWalk}. | |||||
* <p> | |||||
* Index values are only valid within a single | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader} instance after it has won | |||||
* an election. By restricting scope to a single leader new leaders do not need | |||||
* to traverse the entire history to determine the next {@code index} for new | |||||
* proposals. This differs from Raft, where leader election uses the log index | |||||
* and the term number to determine which replica holds a sufficiently | |||||
* up-to-date log. Since Ketch uses Git objects for storage of its replicated | |||||
* log, it keeps the term number as Raft does but uses standard Git operations | |||||
* to imply the log index. | |||||
* <p> | |||||
* {@link org.eclipse.jgit.internal.ketch.Round#runAsync(AnyObjectId)} bumps the | |||||
* index as each new round is constructed. | |||||
*/ | |||||
public class LogIndex extends ObjectId { | |||||
static LogIndex unknown(AnyObjectId id) { | |||||
return new LogIndex(id, 0); | |||||
} | |||||
private final long index; | |||||
private LogIndex(AnyObjectId id, long index) { | |||||
super(id); | |||||
this.index = index; | |||||
} | |||||
LogIndex nextIndex(AnyObjectId id) { | |||||
return new LogIndex(id, index + 1); | |||||
} | |||||
/** | |||||
* Get index provided by the current leader instance. | |||||
* | |||||
* @return index provided by the current leader instance. | |||||
*/ | |||||
public long getIndex() { | |||||
return index; | |||||
} | |||||
/** | |||||
* Check if this log position committed before another log position. | |||||
* <p> | |||||
* Only valid for log positions in memory for the current leader. | |||||
* | |||||
* @param c | |||||
* other (more recent) log position. | |||||
* @return true if this log position was before {@code c} or equal to c and | |||||
* therefore any agreement of {@code c} implies agreement on this | |||||
* log position. | |||||
*/ | |||||
boolean isBefore(LogIndex c) { | |||||
return index <= c.index; | |||||
} | |||||
/** | |||||
* Create string suitable for debug logging containing the log index and | |||||
* abbreviated ObjectId. | |||||
* | |||||
* @return string suitable for debug logging containing the log index and | |||||
* abbreviated ObjectId. | |||||
*/ | |||||
@SuppressWarnings("boxing") | |||||
public String describeForLog() { | |||||
return String.format("%5d/%s", index, abbreviate(6).name()); //$NON-NLS-1$ | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@SuppressWarnings("boxing") | |||||
@Override | |||||
public String toString() { | |||||
return String.format("LogId[%5d/%s]", index, name()); //$NON-NLS-1$ | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.Proposal.State.ABORTED; | |||||
import static org.eclipse.jgit.internal.ketch.Proposal.State.EXECUTED; | |||||
import static org.eclipse.jgit.internal.ketch.Proposal.State.NEW; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; | |||||
import java.io.IOException; | |||||
import java.util.ArrayList; | |||||
import java.util.Collection; | |||||
import java.util.Collections; | |||||
import java.util.List; | |||||
import java.util.concurrent.CopyOnWriteArrayList; | |||||
import java.util.concurrent.TimeUnit; | |||||
import java.util.concurrent.atomic.AtomicReference; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.errors.MissingObjectException; | |||||
import org.eclipse.jgit.internal.storage.reftree.Command; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.PersonIdent; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.PushCertificate; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.util.time.ProposedTimestamp; | |||||
/** | |||||
* A proposal to be applied in a Ketch system. | |||||
* <p> | |||||
* Pushing to a Ketch leader results in the leader making a proposal. The | |||||
* proposal includes the list of reference updates. The leader attempts to send | |||||
* the proposal to a quorum of replicas by pushing the proposal to a "staging" | |||||
* area under the {@code refs/txn/stage/} namespace. If the proposal succeeds | |||||
* then the changes are durable and the leader can commit the proposal. | |||||
* <p> | |||||
* Proposals are executed by | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader#queueProposal(Proposal)}, | |||||
* which runs them asynchronously in the background. Proposals are thread-safe | |||||
* futures allowing callers to {@link #await()} for results or be notified by | |||||
* callback using {@link #addListener(Runnable)}. | |||||
*/ | |||||
public class Proposal { | |||||
/** Current state of the proposal. */ | |||||
public enum State { | |||||
/** Proposal has not yet been given to a {@link KetchLeader}. */ | |||||
NEW(false), | |||||
/** | |||||
* Proposal was validated and has entered the queue, but a round | |||||
* containing this proposal has not started yet. | |||||
*/ | |||||
QUEUED(false), | |||||
/** Round containing the proposal has begun and is in progress. */ | |||||
RUNNING(false), | |||||
/** | |||||
* Proposal was executed through a round. Individual results from | |||||
* {@link Proposal#getCommands()}, {@link Command#getResult()} explain | |||||
* the success or failure outcome. | |||||
*/ | |||||
EXECUTED(true), | |||||
/** Proposal was aborted and did not reach consensus. */ | |||||
ABORTED(true); | |||||
private final boolean done; | |||||
private State(boolean done) { | |||||
this.done = done; | |||||
} | |||||
/** @return true if this is a terminal state. */ | |||||
public boolean isDone() { | |||||
return done; | |||||
} | |||||
} | |||||
private final List<Command> commands; | |||||
private PersonIdent author; | |||||
private String message; | |||||
private PushCertificate pushCert; | |||||
private List<ProposedTimestamp> timestamps; | |||||
private final List<Runnable> listeners = new CopyOnWriteArrayList<>(); | |||||
private final AtomicReference<State> state = new AtomicReference<>(NEW); | |||||
/** | |||||
* Create a proposal from a list of Ketch commands. | |||||
* | |||||
* @param cmds | |||||
* prepared list of commands. | |||||
*/ | |||||
public Proposal(List<Command> cmds) { | |||||
commands = Collections.unmodifiableList(new ArrayList<>(cmds)); | |||||
} | |||||
/** | |||||
* Create a proposal from a collection of received commands. | |||||
* | |||||
* @param rw | |||||
* walker to assist in preparing commands. | |||||
* @param cmds | |||||
* list of pending commands. | |||||
* @throws org.eclipse.jgit.errors.MissingObjectException | |||||
* newId of a command is not found locally. | |||||
* @throws java.io.IOException | |||||
* local objects cannot be accessed. | |||||
*/ | |||||
public Proposal(RevWalk rw, Collection<ReceiveCommand> cmds) | |||||
throws MissingObjectException, IOException { | |||||
commands = asCommandList(rw, cmds); | |||||
} | |||||
private static List<Command> asCommandList(RevWalk rw, | |||||
Collection<ReceiveCommand> cmds) | |||||
throws MissingObjectException, IOException { | |||||
List<Command> commands = new ArrayList<>(cmds.size()); | |||||
for (ReceiveCommand cmd : cmds) { | |||||
commands.add(new Command(rw, cmd)); | |||||
} | |||||
return Collections.unmodifiableList(commands); | |||||
} | |||||
/** | |||||
* Get commands from this proposal. | |||||
* | |||||
* @return commands from this proposal. | |||||
*/ | |||||
public Collection<Command> getCommands() { | |||||
return commands; | |||||
} | |||||
/** | |||||
* Get optional author of the proposal. | |||||
* | |||||
* @return optional author of the proposal. | |||||
*/ | |||||
@Nullable | |||||
public PersonIdent getAuthor() { | |||||
return author; | |||||
} | |||||
/** | |||||
* Set the author for the proposal. | |||||
* | |||||
* @param who | |||||
* optional identity of the author of the proposal. | |||||
* @return {@code this} | |||||
*/ | |||||
public Proposal setAuthor(@Nullable PersonIdent who) { | |||||
author = who; | |||||
return this; | |||||
} | |||||
/** | |||||
* Get optional message for the commit log of the RefTree. | |||||
* | |||||
* @return optional message for the commit log of the RefTree. | |||||
*/ | |||||
@Nullable | |||||
public String getMessage() { | |||||
return message; | |||||
} | |||||
/** | |||||
* Set the message to appear in the commit log of the RefTree. | |||||
* | |||||
* @param msg | |||||
* message text for the commit. | |||||
* @return {@code this} | |||||
*/ | |||||
public Proposal setMessage(@Nullable String msg) { | |||||
message = msg != null && !msg.isEmpty() ? msg : null; | |||||
return this; | |||||
} | |||||
/** | |||||
* Get optional certificate signing the references. | |||||
* | |||||
* @return optional certificate signing the references. | |||||
*/ | |||||
@Nullable | |||||
public PushCertificate getPushCertificate() { | |||||
return pushCert; | |||||
} | |||||
/** | |||||
* Set the push certificate signing the references. | |||||
* | |||||
* @param cert | |||||
* certificate, may be null. | |||||
* @return {@code this} | |||||
*/ | |||||
public Proposal setPushCertificate(@Nullable PushCertificate cert) { | |||||
pushCert = cert; | |||||
return this; | |||||
} | |||||
/** | |||||
* Get timestamps that Ketch must block for. | |||||
* | |||||
* @return timestamps that Ketch must block for. These may have been used as | |||||
* commit times inside the objects involved in the proposal. | |||||
*/ | |||||
public List<ProposedTimestamp> getProposedTimestamps() { | |||||
if (timestamps != null) { | |||||
return timestamps; | |||||
} | |||||
return Collections.emptyList(); | |||||
} | |||||
/** | |||||
* Request the proposal to wait for the affected timestamps to resolve. | |||||
* | |||||
* @param ts | |||||
* a {@link org.eclipse.jgit.util.time.ProposedTimestamp} object. | |||||
* @return {@code this}. | |||||
*/ | |||||
public Proposal addProposedTimestamp(ProposedTimestamp ts) { | |||||
if (timestamps == null) { | |||||
timestamps = new ArrayList<>(4); | |||||
} | |||||
timestamps.add(ts); | |||||
return this; | |||||
} | |||||
/** | |||||
* Add a callback to be invoked when the proposal is done. | |||||
* <p> | |||||
* A proposal is done when it has entered either | |||||
* {@link org.eclipse.jgit.internal.ketch.Proposal.State#EXECUTED} or | |||||
* {@link org.eclipse.jgit.internal.ketch.Proposal.State#ABORTED} state. If | |||||
* the proposal is already done {@code callback.run()} is immediately | |||||
* invoked on the caller's thread. | |||||
* | |||||
* @param callback | |||||
* method to run after the proposal is done. The callback may be | |||||
* run on a Ketch system thread and should be completed quickly. | |||||
*/ | |||||
public void addListener(Runnable callback) { | |||||
boolean runNow = false; | |||||
synchronized (state) { | |||||
if (state.get().isDone()) { | |||||
runNow = true; | |||||
} else { | |||||
listeners.add(callback); | |||||
} | |||||
} | |||||
if (runNow) { | |||||
callback.run(); | |||||
} | |||||
} | |||||
/** Set command result as OK. */ | |||||
void success() { | |||||
for (Command c : commands) { | |||||
if (c.getResult() == NOT_ATTEMPTED) { | |||||
c.setResult(OK); | |||||
} | |||||
} | |||||
notifyState(EXECUTED); | |||||
} | |||||
/** Mark commands as "transaction aborted". */ | |||||
void abort() { | |||||
Command.abort(commands, null); | |||||
notifyState(ABORTED); | |||||
} | |||||
/** | |||||
* Read the current state of the proposal. | |||||
* | |||||
* @return read the current state of the proposal. | |||||
*/ | |||||
public State getState() { | |||||
return state.get(); | |||||
} | |||||
/** | |||||
* Whether the proposal was attempted | |||||
* | |||||
* @return {@code true} if the proposal was attempted. A true value does not | |||||
* mean consensus was reached, only that the proposal was considered | |||||
* and will not be making any more progress beyond its current | |||||
* state. | |||||
*/ | |||||
public boolean isDone() { | |||||
return state.get().isDone(); | |||||
} | |||||
/** | |||||
* Wait for the proposal to be attempted and {@link #isDone()} to be true. | |||||
* | |||||
* @throws java.lang.InterruptedException | |||||
* caller was interrupted before proposal executed. | |||||
*/ | |||||
public void await() throws InterruptedException { | |||||
synchronized (state) { | |||||
while (!state.get().isDone()) { | |||||
state.wait(); | |||||
} | |||||
} | |||||
} | |||||
/** | |||||
* Wait for the proposal to be attempted and {@link #isDone()} to be true. | |||||
* | |||||
* @param wait | |||||
* how long to wait. | |||||
* @param unit | |||||
* unit describing the wait time. | |||||
* @return true if the proposal is done; false if the method timed out. | |||||
* @throws java.lang.InterruptedException | |||||
* caller was interrupted before proposal executed. | |||||
*/ | |||||
public boolean await(long wait, TimeUnit unit) throws InterruptedException { | |||||
synchronized (state) { | |||||
if (state.get().isDone()) { | |||||
return true; | |||||
} | |||||
state.wait(unit.toMillis(wait)); | |||||
return state.get().isDone(); | |||||
} | |||||
} | |||||
/** | |||||
* Wait for the proposal to exit a state. | |||||
* | |||||
* @param notIn | |||||
* state the proposal should not be in to return. | |||||
* @param wait | |||||
* how long to wait. | |||||
* @param unit | |||||
* unit describing the wait time. | |||||
* @return true if the proposal exited the state; false on time out. | |||||
* @throws java.lang.InterruptedException | |||||
* caller was interrupted before proposal executed. | |||||
*/ | |||||
public boolean awaitStateChange(State notIn, long wait, TimeUnit unit) | |||||
throws InterruptedException { | |||||
synchronized (state) { | |||||
if (state.get() != notIn) { | |||||
return true; | |||||
} | |||||
state.wait(unit.toMillis(wait)); | |||||
return state.get() != notIn; | |||||
} | |||||
} | |||||
void notifyState(State s) { | |||||
synchronized (state) { | |||||
state.set(s); | |||||
state.notifyAll(); | |||||
} | |||||
if (s.isDone()) { | |||||
for (Runnable callback : listeners) { | |||||
callback.run(); | |||||
} | |||||
listeners.clear(); | |||||
} | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public String toString() { | |||||
StringBuilder s = new StringBuilder(); | |||||
s.append("Ketch Proposal {\n"); //$NON-NLS-1$ | |||||
s.append(" ").append(state.get()).append('\n'); //$NON-NLS-1$ | |||||
if (author != null) { | |||||
s.append(" author ").append(author).append('\n'); //$NON-NLS-1$ | |||||
} | |||||
if (message != null) { | |||||
s.append(" message ").append(message).append('\n'); //$NON-NLS-1$ | |||||
} | |||||
for (Command c : commands) { | |||||
s.append(" "); //$NON-NLS-1$ | |||||
format(s, c.getOldRef(), "CREATE"); //$NON-NLS-1$ | |||||
s.append(' '); | |||||
format(s, c.getNewRef(), "DELETE"); //$NON-NLS-1$ | |||||
s.append(' ').append(c.getRefName()); | |||||
if (c.getResult() != ReceiveCommand.Result.NOT_ATTEMPTED) { | |||||
s.append(' ').append(c.getResult()); // $NON-NLS-1$ | |||||
} | |||||
s.append('\n'); | |||||
} | |||||
s.append('}'); | |||||
return s.toString(); | |||||
} | |||||
private static void format(StringBuilder s, @Nullable Ref r, String n) { | |||||
if (r == null) { | |||||
s.append(n); | |||||
} else if (r.isSymbolic()) { | |||||
s.append(r.getTarget().getName()); | |||||
} else { | |||||
ObjectId id = r.getObjectId(); | |||||
if (id != null) { | |||||
s.append(id.abbreviate(8).name()); | |||||
} | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.Proposal.State.RUNNING; | |||||
import java.io.IOException; | |||||
import java.time.Duration; | |||||
import java.util.ArrayList; | |||||
import java.util.Collections; | |||||
import java.util.HashMap; | |||||
import java.util.HashSet; | |||||
import java.util.List; | |||||
import java.util.Map; | |||||
import java.util.Set; | |||||
import java.util.concurrent.TimeoutException; | |||||
import java.util.stream.Collectors; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.internal.storage.reftree.Command; | |||||
import org.eclipse.jgit.internal.storage.reftree.RefTree; | |||||
import org.eclipse.jgit.lib.CommitBuilder; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.PersonIdent; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.util.time.ProposedTimestamp; | |||||
/** A {@link Round} that aggregates and sends user {@link Proposal}s. */ | |||||
class ProposalRound extends Round { | |||||
private final List<Proposal> todo; | |||||
private RefTree queuedTree; | |||||
ProposalRound(KetchLeader leader, LogIndex head, List<Proposal> todo, | |||||
@Nullable RefTree tree) { | |||||
super(leader, head); | |||||
this.todo = todo; | |||||
if (tree != null && canCombine(todo)) { | |||||
this.queuedTree = tree; | |||||
} else { | |||||
leader.roundHoldsReferenceToRefTree = false; | |||||
} | |||||
} | |||||
private static boolean canCombine(List<Proposal> todo) { | |||||
Proposal first = todo.get(0); | |||||
for (int i = 1; i < todo.size(); i++) { | |||||
if (!canCombine(first, todo.get(i))) { | |||||
return false; | |||||
} | |||||
} | |||||
return true; | |||||
} | |||||
private static boolean canCombine(Proposal a, Proposal b) { | |||||
String aMsg = nullToEmpty(a.getMessage()); | |||||
String bMsg = nullToEmpty(b.getMessage()); | |||||
return aMsg.equals(bMsg) && canCombine(a.getAuthor(), b.getAuthor()); | |||||
} | |||||
private static String nullToEmpty(@Nullable String str) { | |||||
return str != null ? str : ""; //$NON-NLS-1$ | |||||
} | |||||
private static boolean canCombine(@Nullable PersonIdent a, | |||||
@Nullable PersonIdent b) { | |||||
if (a != null && b != null) { | |||||
// Same name and email address. Combine timestamp as the two | |||||
// proposals are running concurrently and appear together or | |||||
// not at all from the point of view of an outside reader. | |||||
return a.getName().equals(b.getName()) | |||||
&& a.getEmailAddress().equals(b.getEmailAddress()); | |||||
} | |||||
// If a and b are null, both will be the system identity. | |||||
return a == null && b == null; | |||||
} | |||||
@Override | |||||
void start() throws IOException { | |||||
for (Proposal p : todo) { | |||||
p.notifyState(RUNNING); | |||||
} | |||||
try { | |||||
ObjectId id; | |||||
try (Repository git = leader.openRepository(); | |||||
ProposedTimestamp ts = getSystem().getClock().propose()) { | |||||
id = insertProposals(git, ts); | |||||
blockUntil(ts); | |||||
} | |||||
runAsync(id); | |||||
} catch (NoOp e) { | |||||
for (Proposal p : todo) { | |||||
p.success(); | |||||
} | |||||
leader.lock.lock(); | |||||
try { | |||||
leader.nextRound(); | |||||
} finally { | |||||
leader.lock.unlock(); | |||||
} | |||||
} catch (IOException e) { | |||||
abort(); | |||||
throw e; | |||||
} | |||||
} | |||||
private ObjectId insertProposals(Repository git, ProposedTimestamp ts) | |||||
throws IOException, NoOp { | |||||
ObjectId id; | |||||
try (ObjectInserter inserter = git.newObjectInserter()) { | |||||
// TODO(sop) Process signed push certificates. | |||||
if (queuedTree != null) { | |||||
id = insertSingleProposal(git, ts, inserter); | |||||
} else { | |||||
id = insertMultiProposal(git, ts, inserter); | |||||
} | |||||
stageCommands = makeStageList(git, inserter); | |||||
inserter.flush(); | |||||
} | |||||
return id; | |||||
} | |||||
private ObjectId insertSingleProposal(Repository git, ProposedTimestamp ts, | |||||
ObjectInserter inserter) throws IOException, NoOp { | |||||
// Fast path: tree is passed in with all proposals applied. | |||||
ObjectId treeId = queuedTree.writeTree(inserter); | |||||
queuedTree = null; | |||||
leader.roundHoldsReferenceToRefTree = false; | |||||
if (!ObjectId.zeroId().equals(acceptedOldIndex)) { | |||||
try (RevWalk rw = new RevWalk(git)) { | |||||
RevCommit c = rw.parseCommit(acceptedOldIndex); | |||||
if (treeId.equals(c.getTree())) { | |||||
throw new NoOp(); | |||||
} | |||||
} | |||||
} | |||||
Proposal p = todo.get(0); | |||||
CommitBuilder b = new CommitBuilder(); | |||||
b.setTreeId(treeId); | |||||
if (!ObjectId.zeroId().equals(acceptedOldIndex)) { | |||||
b.setParentId(acceptedOldIndex); | |||||
} | |||||
b.setCommitter(leader.getSystem().newCommitter(ts)); | |||||
b.setAuthor(p.getAuthor() != null ? p.getAuthor() : b.getCommitter()); | |||||
b.setMessage(message(p)); | |||||
return inserter.insert(b); | |||||
} | |||||
private ObjectId insertMultiProposal(Repository git, ProposedTimestamp ts, | |||||
ObjectInserter inserter) throws IOException, NoOp { | |||||
// The tree was not passed in, or there are multiple proposals | |||||
// each needing their own commit. Reset the tree and replay each | |||||
// proposal in order as individual commits. | |||||
ObjectId lastIndex = acceptedOldIndex; | |||||
ObjectId oldTreeId; | |||||
RefTree tree; | |||||
if (ObjectId.zeroId().equals(lastIndex)) { | |||||
oldTreeId = ObjectId.zeroId(); | |||||
tree = RefTree.newEmptyTree(); | |||||
} else { | |||||
try (RevWalk rw = new RevWalk(git)) { | |||||
RevCommit c = rw.parseCommit(lastIndex); | |||||
oldTreeId = c.getTree(); | |||||
tree = RefTree.read(rw.getObjectReader(), c.getTree()); | |||||
} | |||||
} | |||||
PersonIdent committer = leader.getSystem().newCommitter(ts); | |||||
for (Proposal p : todo) { | |||||
if (!tree.apply(p.getCommands())) { | |||||
// This should not occur, previously during queuing the | |||||
// commands were successfully applied to the pending tree. | |||||
// Abort the entire round. | |||||
throw new IOException( | |||||
KetchText.get().queuedProposalFailedToApply); | |||||
} | |||||
ObjectId treeId = tree.writeTree(inserter); | |||||
if (treeId.equals(oldTreeId)) { | |||||
continue; | |||||
} | |||||
CommitBuilder b = new CommitBuilder(); | |||||
b.setTreeId(treeId); | |||||
if (!ObjectId.zeroId().equals(lastIndex)) { | |||||
b.setParentId(lastIndex); | |||||
} | |||||
b.setAuthor(p.getAuthor() != null ? p.getAuthor() : committer); | |||||
b.setCommitter(committer); | |||||
b.setMessage(message(p)); | |||||
lastIndex = inserter.insert(b); | |||||
} | |||||
if (lastIndex.equals(acceptedOldIndex)) { | |||||
throw new NoOp(); | |||||
} | |||||
return lastIndex; | |||||
} | |||||
private String message(Proposal p) { | |||||
StringBuilder m = new StringBuilder(); | |||||
String msg = p.getMessage(); | |||||
if (msg != null && !msg.isEmpty()) { | |||||
m.append(msg); | |||||
while (m.length() < 2 || m.charAt(m.length() - 2) != '\n' | |||||
|| m.charAt(m.length() - 1) != '\n') { | |||||
m.append('\n'); | |||||
} | |||||
} | |||||
m.append(KetchConstants.TERM.getName()) | |||||
.append(": ") //$NON-NLS-1$ | |||||
.append(leader.getTerm()); | |||||
return m.toString(); | |||||
} | |||||
void abort() { | |||||
for (Proposal p : todo) { | |||||
p.abort(); | |||||
} | |||||
} | |||||
@Override | |||||
void success() { | |||||
for (Proposal p : todo) { | |||||
p.success(); | |||||
} | |||||
} | |||||
private List<ReceiveCommand> makeStageList(Repository git, | |||||
ObjectInserter inserter) throws IOException { | |||||
// For each branch, collapse consecutive updates to only most recent, | |||||
// avoiding sending multiple objects in a rapid fast-forward chain, or | |||||
// rewritten content. | |||||
Map<String, ObjectId> byRef = new HashMap<>(); | |||||
for (Proposal p : todo) { | |||||
for (Command c : p.getCommands()) { | |||||
Ref n = c.getNewRef(); | |||||
if (n != null && !n.isSymbolic()) { | |||||
byRef.put(n.getName(), n.getObjectId()); | |||||
} | |||||
} | |||||
} | |||||
if (byRef.isEmpty()) { | |||||
return Collections.emptyList(); | |||||
} | |||||
Set<ObjectId> newObjs = new HashSet<>(byRef.values()); | |||||
StageBuilder b = new StageBuilder( | |||||
leader.getSystem().getTxnStage(), | |||||
acceptedNewIndex); | |||||
return b.makeStageList(newObjs, git, inserter); | |||||
} | |||||
private void blockUntil(ProposedTimestamp ts) | |||||
throws TimeIsUncertainException { | |||||
List<ProposedTimestamp> times = todo.stream() | |||||
.flatMap(p -> p.getProposedTimestamps().stream()) | |||||
.collect(Collectors.toCollection(ArrayList::new)); | |||||
times.add(ts); | |||||
try { | |||||
Duration maxWait = getSystem().getMaxWaitForMonotonicClock(); | |||||
ProposedTimestamp.blockUntil(times, maxWait); | |||||
} catch (InterruptedException | TimeoutException e) { | |||||
throw new TimeIsUncertainException(e); | |||||
} | |||||
} | |||||
private static class NoOp extends Exception { | |||||
private static final long serialVersionUID = 1L; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod.ALL_REFS; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NODELETE; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import java.io.IOException; | |||||
import java.util.ArrayList; | |||||
import java.util.Collection; | |||||
import java.util.Collections; | |||||
import java.util.LinkedHashMap; | |||||
import java.util.List; | |||||
import java.util.Map; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.errors.NotSupportedException; | |||||
import org.eclipse.jgit.errors.TransportException; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.transport.FetchConnection; | |||||
import org.eclipse.jgit.transport.PushConnection; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.transport.RemoteConfig; | |||||
import org.eclipse.jgit.transport.RemoteRefUpdate; | |||||
import org.eclipse.jgit.transport.Transport; | |||||
import org.eclipse.jgit.transport.URIish; | |||||
/** | |||||
* Representation of a Git repository on a remote replica system. | |||||
* <p> | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchLeader} will contact the replica | |||||
* using the Git wire protocol. | |||||
* <p> | |||||
* The remote replica may be fully Ketch-aware, or a standard Git server. | |||||
*/ | |||||
public class RemoteGitReplica extends KetchReplica { | |||||
private final URIish uri; | |||||
private final RemoteConfig remoteConfig; | |||||
/** | |||||
* Configure a new remote. | |||||
* | |||||
* @param leader | |||||
* instance this replica follows. | |||||
* @param name | |||||
* unique-ish name identifying this remote for debugging. | |||||
* @param uri | |||||
* URI to connect to the follower's repository. | |||||
* @param cfg | |||||
* how Ketch should treat the remote system. | |||||
* @param rc | |||||
* optional remote configuration describing how to contact the | |||||
* peer repository. | |||||
*/ | |||||
public RemoteGitReplica(KetchLeader leader, String name, URIish uri, | |||||
ReplicaConfig cfg, @Nullable RemoteConfig rc) { | |||||
super(leader, name, cfg); | |||||
this.uri = uri; | |||||
this.remoteConfig = rc; | |||||
} | |||||
/** | |||||
* Get URI to contact the remote peer repository. | |||||
* | |||||
* @return URI to contact the remote peer repository. | |||||
*/ | |||||
public URIish getURI() { | |||||
return uri; | |||||
} | |||||
/** | |||||
* Get optional configuration describing how to contact the peer. | |||||
* | |||||
* @return optional configuration describing how to contact the peer. | |||||
*/ | |||||
@Nullable | |||||
protected RemoteConfig getRemoteConfig() { | |||||
return remoteConfig; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected String describeForLog() { | |||||
return String.format("%s @ %s", getName(), getURI()); //$NON-NLS-1$ | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected void startPush(ReplicaPushRequest req) { | |||||
getSystem().getExecutor().execute(() -> { | |||||
try (Repository git = getLeader().openRepository()) { | |||||
try { | |||||
push(git, req); | |||||
req.done(git); | |||||
} catch (Throwable err) { | |||||
req.setException(git, err); | |||||
} | |||||
} catch (IOException err) { | |||||
req.setException(null, err); | |||||
} | |||||
}); | |||||
} | |||||
private void push(Repository repo, ReplicaPushRequest req) | |||||
throws NotSupportedException, TransportException, IOException { | |||||
Map<String, Ref> adv; | |||||
List<RemoteCommand> cmds = asUpdateList(req.getCommands()); | |||||
try (Transport transport = Transport.open(repo, uri)) { | |||||
RemoteConfig rc = getRemoteConfig(); | |||||
if (rc != null) { | |||||
transport.applyConfig(rc); | |||||
} | |||||
transport.setPushAtomic(true); | |||||
adv = push(repo, transport, cmds); | |||||
} | |||||
for (RemoteCommand c : cmds) { | |||||
c.copyStatusToResult(); | |||||
} | |||||
req.setRefs(adv); | |||||
} | |||||
private Map<String, Ref> push(Repository git, Transport transport, | |||||
List<RemoteCommand> cmds) throws IOException { | |||||
Map<String, RemoteRefUpdate> updates = asUpdateMap(cmds); | |||||
try (PushConnection connection = transport.openPush()) { | |||||
Map<String, Ref> adv = connection.getRefsMap(); | |||||
RemoteRefUpdate accepted = updates.get(getSystem().getTxnAccepted()); | |||||
if (accepted != null && !isExpectedValue(adv, accepted)) { | |||||
abort(cmds); | |||||
return adv; | |||||
} | |||||
RemoteRefUpdate committed = updates.get(getSystem().getTxnCommitted()); | |||||
if (committed != null && !isExpectedValue(adv, committed)) { | |||||
abort(cmds); | |||||
return adv; | |||||
} | |||||
if (committed != null && getCommitMethod() == ALL_REFS) { | |||||
prepareCommit(git, cmds, updates, adv, | |||||
committed.getNewObjectId()); | |||||
} | |||||
connection.push(NullProgressMonitor.INSTANCE, updates); | |||||
return adv; | |||||
} | |||||
} | |||||
private static boolean isExpectedValue(Map<String, Ref> adv, | |||||
RemoteRefUpdate u) { | |||||
Ref r = adv.get(u.getRemoteName()); | |||||
if (!AnyObjectId.isEqual(getId(r), u.getExpectedOldObjectId())) { | |||||
((RemoteCommand) u).cmd.setResult(LOCK_FAILURE); | |||||
return false; | |||||
} | |||||
return true; | |||||
} | |||||
private void prepareCommit(Repository git, List<RemoteCommand> cmds, | |||||
Map<String, RemoteRefUpdate> updates, Map<String, Ref> adv, | |||||
ObjectId committed) throws IOException { | |||||
for (ReceiveCommand cmd : prepareCommit(git, adv, committed)) { | |||||
RemoteCommand c = new RemoteCommand(cmd); | |||||
cmds.add(c); | |||||
updates.put(c.getRemoteName(), c); | |||||
} | |||||
} | |||||
private static List<RemoteCommand> asUpdateList( | |||||
Collection<ReceiveCommand> cmds) { | |||||
try { | |||||
List<RemoteCommand> toPush = new ArrayList<>(cmds.size()); | |||||
for (ReceiveCommand cmd : cmds) { | |||||
toPush.add(new RemoteCommand(cmd)); | |||||
} | |||||
return toPush; | |||||
} catch (IOException e) { | |||||
// Cannot occur as no IO was required to build the command. | |||||
throw new IllegalStateException(e); | |||||
} | |||||
} | |||||
private static Map<String, RemoteRefUpdate> asUpdateMap( | |||||
List<RemoteCommand> cmds) { | |||||
Map<String, RemoteRefUpdate> m = new LinkedHashMap<>(); | |||||
for (RemoteCommand cmd : cmds) { | |||||
m.put(cmd.getRemoteName(), cmd); | |||||
} | |||||
return m; | |||||
} | |||||
private static void abort(List<RemoteCommand> cmds) { | |||||
List<ReceiveCommand> tmp = new ArrayList<>(cmds.size()); | |||||
for (RemoteCommand cmd : cmds) { | |||||
tmp.add(cmd.cmd); | |||||
} | |||||
ReceiveCommand.abort(tmp); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected void blockingFetch(Repository repo, ReplicaFetchRequest req) | |||||
throws NotSupportedException, TransportException { | |||||
try (Transport transport = Transport.open(repo, uri)) { | |||||
RemoteConfig rc = getRemoteConfig(); | |||||
if (rc != null) { | |||||
transport.applyConfig(rc); | |||||
} | |||||
fetch(transport, req); | |||||
} | |||||
} | |||||
private void fetch(Transport transport, ReplicaFetchRequest req) | |||||
throws NotSupportedException, TransportException { | |||||
try (FetchConnection conn = transport.openFetch()) { | |||||
Map<String, Ref> remoteRefs = conn.getRefsMap(); | |||||
req.setRefs(remoteRefs); | |||||
List<Ref> want = new ArrayList<>(); | |||||
for (String name : req.getWantRefs()) { | |||||
Ref ref = remoteRefs.get(name); | |||||
if (ref != null && ref.getObjectId() != null) { | |||||
want.add(ref); | |||||
} | |||||
} | |||||
for (ObjectId id : req.getWantObjects()) { | |||||
want.add(new ObjectIdRef.Unpeeled(NETWORK, id.name(), id)); | |||||
} | |||||
conn.fetch(NullProgressMonitor.INSTANCE, want, | |||||
Collections.<ObjectId> emptySet()); | |||||
} | |||||
} | |||||
static class RemoteCommand extends RemoteRefUpdate { | |||||
final ReceiveCommand cmd; | |||||
RemoteCommand(ReceiveCommand cmd) throws IOException { | |||||
super(null, null, | |||||
cmd.getNewId(), cmd.getRefName(), | |||||
true /* force update */, | |||||
null /* no local tracking ref */, | |||||
cmd.getOldId()); | |||||
this.cmd = cmd; | |||||
} | |||||
void copyStatusToResult() { | |||||
if (cmd.getResult() == NOT_ATTEMPTED) { | |||||
switch (getStatus()) { | |||||
case OK: | |||||
case UP_TO_DATE: | |||||
case NON_EXISTING: | |||||
cmd.setResult(OK); | |||||
break; | |||||
case REJECTED_NODELETE: | |||||
cmd.setResult(REJECTED_NODELETE); | |||||
break; | |||||
case REJECTED_NONFASTFORWARD: | |||||
cmd.setResult(REJECTED_NONFASTFORWARD); | |||||
break; | |||||
case REJECTED_OTHER_REASON: | |||||
cmd.setResult(REJECTED_OTHER_REASON, getMessage()); | |||||
break; | |||||
default: | |||||
cmd.setResult(REJECTED_OTHER_REASON, getStatus().name()); | |||||
break; | |||||
} | |||||
} | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static java.util.concurrent.TimeUnit.DAYS; | |||||
import static java.util.concurrent.TimeUnit.HOURS; | |||||
import static java.util.concurrent.TimeUnit.MILLISECONDS; | |||||
import static java.util.concurrent.TimeUnit.MINUTES; | |||||
import static java.util.concurrent.TimeUnit.SECONDS; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_COMMIT; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_SPEED; | |||||
import static org.eclipse.jgit.internal.ketch.KetchConstants.CONFIG_KEY_TYPE; | |||||
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_REMOTE; | |||||
import java.util.Collections; | |||||
import java.util.HashMap; | |||||
import java.util.Map; | |||||
import java.util.concurrent.TimeUnit; | |||||
import java.util.regex.Matcher; | |||||
import java.util.regex.Pattern; | |||||
import org.eclipse.jgit.internal.ketch.KetchReplica.CommitMethod; | |||||
import org.eclipse.jgit.internal.ketch.KetchReplica.CommitSpeed; | |||||
import org.eclipse.jgit.internal.ketch.KetchReplica.Participation; | |||||
import org.eclipse.jgit.lib.Config; | |||||
/** | |||||
* Configures a {@link org.eclipse.jgit.internal.ketch.KetchReplica}. | |||||
*/ | |||||
public class ReplicaConfig { | |||||
/** | |||||
* Read a configuration from a config block. | |||||
* | |||||
* @param cfg | |||||
* configuration to read. | |||||
* @param name | |||||
* of the replica being configured. | |||||
* @return replica configuration for {@code name}. | |||||
*/ | |||||
public static ReplicaConfig newFromConfig(Config cfg, String name) { | |||||
return new ReplicaConfig().fromConfig(cfg, name); | |||||
} | |||||
private Participation participation = Participation.FULL; | |||||
private CommitMethod commitMethod = CommitMethod.ALL_REFS; | |||||
private CommitSpeed commitSpeed = CommitSpeed.BATCHED; | |||||
private long minRetry = SECONDS.toMillis(5); | |||||
private long maxRetry = MINUTES.toMillis(1); | |||||
/** | |||||
* Get participation of the replica in the system. | |||||
* | |||||
* @return participation of the replica in the system. | |||||
*/ | |||||
public Participation getParticipation() { | |||||
return participation; | |||||
} | |||||
/** | |||||
* Get how Ketch should apply committed changes. | |||||
* | |||||
* @return how Ketch should apply committed changes. | |||||
*/ | |||||
public CommitMethod getCommitMethod() { | |||||
return commitMethod; | |||||
} | |||||
/** | |||||
* Get how quickly should Ketch commit. | |||||
* | |||||
* @return how quickly should Ketch commit. | |||||
*/ | |||||
public CommitSpeed getCommitSpeed() { | |||||
return commitSpeed; | |||||
} | |||||
/** | |||||
* Returns the minimum wait delay before retrying a failure. | |||||
* | |||||
* @param unit | |||||
* to get retry delay in. | |||||
* @return minimum delay before retrying a failure. | |||||
*/ | |||||
public long getMinRetry(TimeUnit unit) { | |||||
return unit.convert(minRetry, MILLISECONDS); | |||||
} | |||||
/** | |||||
* Returns the maximum wait delay before retrying a failure. | |||||
* | |||||
* @param unit | |||||
* to get retry delay in. | |||||
* @return maximum delay before retrying a failure. | |||||
*/ | |||||
public long getMaxRetry(TimeUnit unit) { | |||||
return unit.convert(maxRetry, MILLISECONDS); | |||||
} | |||||
/** | |||||
* Update the configuration from a config block. | |||||
* | |||||
* @param cfg | |||||
* configuration to read. | |||||
* @param name | |||||
* of the replica being configured. | |||||
* @return {@code this} | |||||
*/ | |||||
public ReplicaConfig fromConfig(Config cfg, String name) { | |||||
participation = cfg.getEnum( | |||||
CONFIG_KEY_REMOTE, name, CONFIG_KEY_TYPE, | |||||
participation); | |||||
commitMethod = cfg.getEnum( | |||||
CONFIG_KEY_REMOTE, name, CONFIG_KEY_COMMIT, | |||||
commitMethod); | |||||
commitSpeed = cfg.getEnum( | |||||
CONFIG_KEY_REMOTE, name, CONFIG_KEY_SPEED, | |||||
commitSpeed); | |||||
minRetry = getMillis(cfg, name, "ketch-minRetry", minRetry); //$NON-NLS-1$ | |||||
maxRetry = getMillis(cfg, name, "ketch-maxRetry", maxRetry); //$NON-NLS-1$ | |||||
return this; | |||||
} | |||||
private static long getMillis(Config cfg, String name, String key, | |||||
long defaultValue) { | |||||
String valStr = cfg.getString(CONFIG_KEY_REMOTE, name, key); | |||||
if (valStr == null) { | |||||
return defaultValue; | |||||
} | |||||
valStr = valStr.trim(); | |||||
if (valStr.isEmpty()) { | |||||
return defaultValue; | |||||
} | |||||
Matcher m = UnitMap.PATTERN.matcher(valStr); | |||||
if (!m.matches()) { | |||||
return defaultValue; | |||||
} | |||||
String digits = m.group(1); | |||||
String unitName = m.group(2).trim(); | |||||
TimeUnit unit = UnitMap.UNITS.get(unitName); | |||||
if (unit == null) { | |||||
return defaultValue; | |||||
} | |||||
try { | |||||
if (digits.indexOf('.') == -1) { | |||||
return unit.toMillis(Long.parseLong(digits)); | |||||
} | |||||
double val = Double.parseDouble(digits); | |||||
return (long) (val * unit.toMillis(1)); | |||||
} catch (NumberFormatException nfe) { | |||||
return defaultValue; | |||||
} | |||||
} | |||||
static class UnitMap { | |||||
static final Pattern PATTERN = Pattern | |||||
.compile("^([1-9][0-9]*(?:\\.[0-9]*)?)\\s*(.*)$"); //$NON-NLS-1$ | |||||
static final Map<String, TimeUnit> UNITS; | |||||
static { | |||||
Map<String, TimeUnit> m = new HashMap<>(); | |||||
TimeUnit u = MILLISECONDS; | |||||
m.put("", u); //$NON-NLS-1$ | |||||
m.put("ms", u); //$NON-NLS-1$ | |||||
m.put("millis", u); //$NON-NLS-1$ | |||||
m.put("millisecond", u); //$NON-NLS-1$ | |||||
m.put("milliseconds", u); //$NON-NLS-1$ | |||||
u = SECONDS; | |||||
m.put("s", u); //$NON-NLS-1$ | |||||
m.put("sec", u); //$NON-NLS-1$ | |||||
m.put("secs", u); //$NON-NLS-1$ | |||||
m.put("second", u); //$NON-NLS-1$ | |||||
m.put("seconds", u); //$NON-NLS-1$ | |||||
u = MINUTES; | |||||
m.put("m", u); //$NON-NLS-1$ | |||||
m.put("min", u); //$NON-NLS-1$ | |||||
m.put("mins", u); //$NON-NLS-1$ | |||||
m.put("minute", u); //$NON-NLS-1$ | |||||
m.put("minutes", u); //$NON-NLS-1$ | |||||
u = HOURS; | |||||
m.put("h", u); //$NON-NLS-1$ | |||||
m.put("hr", u); //$NON-NLS-1$ | |||||
m.put("hrs", u); //$NON-NLS-1$ | |||||
m.put("hour", u); //$NON-NLS-1$ | |||||
m.put("hours", u); //$NON-NLS-1$ | |||||
u = DAYS; | |||||
m.put("d", u); //$NON-NLS-1$ | |||||
m.put("day", u); //$NON-NLS-1$ | |||||
m.put("days", u); //$NON-NLS-1$ | |||||
UNITS = Collections.unmodifiableMap(m); | |||||
} | |||||
private UnitMap() { | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import java.util.Map; | |||||
import java.util.Set; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
/** | |||||
* A fetch request to obtain objects from a replica, and its result. | |||||
*/ | |||||
public class ReplicaFetchRequest { | |||||
private final Set<String> wantRefs; | |||||
private final Set<ObjectId> wantObjects; | |||||
private Map<String, Ref> refs; | |||||
/** | |||||
* Construct a new fetch request for a replica. | |||||
* | |||||
* @param wantRefs | |||||
* named references to be fetched. | |||||
* @param wantObjects | |||||
* specific objects to be fetched. | |||||
*/ | |||||
public ReplicaFetchRequest(Set<String> wantRefs, | |||||
Set<ObjectId> wantObjects) { | |||||
this.wantRefs = wantRefs; | |||||
this.wantObjects = wantObjects; | |||||
} | |||||
/** | |||||
* Get references to be fetched. | |||||
* | |||||
* @return references to be fetched. | |||||
*/ | |||||
public Set<String> getWantRefs() { | |||||
return wantRefs; | |||||
} | |||||
/** | |||||
* Get objects to be fetched. | |||||
* | |||||
* @return objects to be fetched. | |||||
*/ | |||||
public Set<ObjectId> getWantObjects() { | |||||
return wantObjects; | |||||
} | |||||
/** | |||||
* Get remote references, usually from the advertisement. | |||||
* | |||||
* @return remote references, usually from the advertisement. | |||||
*/ | |||||
@Nullable | |||||
public Map<String, Ref> getRefs() { | |||||
return refs; | |||||
} | |||||
/** | |||||
* Set references observed from the replica. | |||||
* | |||||
* @param refs | |||||
* references observed from the replica. | |||||
*/ | |||||
public void setRefs(Map<String, Ref> refs) { | |||||
this.refs = refs; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import java.util.Collection; | |||||
import java.util.Map; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
/** | |||||
* A push request sending objects to a replica, and its result. | |||||
* <p> | |||||
* Implementors of {@link org.eclipse.jgit.internal.ketch.KetchReplica} must | |||||
* populate the command result fields, {@link #setRefs(Map)}, and call one of | |||||
* {@link #setException(Repository, Throwable)} or {@link #done(Repository)} to | |||||
* finish processing. | |||||
*/ | |||||
public class ReplicaPushRequest { | |||||
private final KetchReplica replica; | |||||
private final Collection<ReceiveCommand> commands; | |||||
private Map<String, Ref> refs; | |||||
private Throwable exception; | |||||
private boolean notified; | |||||
/** | |||||
* Construct a new push request for a replica. | |||||
* | |||||
* @param replica | |||||
* the replica being pushed to. | |||||
* @param commands | |||||
* commands to be executed. | |||||
*/ | |||||
public ReplicaPushRequest(KetchReplica replica, | |||||
Collection<ReceiveCommand> commands) { | |||||
this.replica = replica; | |||||
this.commands = commands; | |||||
} | |||||
/** | |||||
* Get commands to be executed, and their results. | |||||
* | |||||
* @return commands to be executed, and their results. | |||||
*/ | |||||
public Collection<ReceiveCommand> getCommands() { | |||||
return commands; | |||||
} | |||||
/** | |||||
* Get remote references, usually from the advertisement. | |||||
* | |||||
* @return remote references, usually from the advertisement. | |||||
*/ | |||||
@Nullable | |||||
public Map<String, Ref> getRefs() { | |||||
return refs; | |||||
} | |||||
/** | |||||
* Set references observed from the replica. | |||||
* | |||||
* @param refs | |||||
* references observed from the replica. | |||||
*/ | |||||
public void setRefs(Map<String, Ref> refs) { | |||||
this.refs = refs; | |||||
} | |||||
/** | |||||
* Get exception thrown, if any. | |||||
* | |||||
* @return exception thrown, if any. | |||||
*/ | |||||
@Nullable | |||||
public Throwable getException() { | |||||
return exception; | |||||
} | |||||
/** | |||||
* Mark the request as crashing with a communication error. | |||||
* <p> | |||||
* This method may take significant time acquiring the leader lock and | |||||
* updating the Ketch state machine with the failure. | |||||
* | |||||
* @param repo | |||||
* local repository reference used by the push attempt. | |||||
* @param err | |||||
* exception thrown during communication. | |||||
*/ | |||||
public void setException(@Nullable Repository repo, Throwable err) { | |||||
if (KetchReplica.log.isErrorEnabled()) { | |||||
KetchReplica.log.error(describe("failed"), err); //$NON-NLS-1$ | |||||
} | |||||
if (!notified) { | |||||
notified = true; | |||||
exception = err; | |||||
replica.afterPush(repo, this); | |||||
} | |||||
} | |||||
/** | |||||
* Mark the request as completed without exception. | |||||
* <p> | |||||
* This method may take significant time acquiring the leader lock and | |||||
* updating the Ketch state machine with results from this replica. | |||||
* | |||||
* @param repo | |||||
* local repository reference used by the push attempt. | |||||
*/ | |||||
public void done(Repository repo) { | |||||
if (KetchReplica.log.isDebugEnabled()) { | |||||
KetchReplica.log.debug(describe("completed")); //$NON-NLS-1$ | |||||
} | |||||
if (!notified) { | |||||
notified = true; | |||||
replica.afterPush(repo, this); | |||||
} | |||||
} | |||||
private String describe(String heading) { | |||||
StringBuilder b = new StringBuilder(); | |||||
b.append("push to "); //$NON-NLS-1$ | |||||
b.append(replica.describeForLog()); | |||||
b.append(' ').append(heading).append(":\n"); //$NON-NLS-1$ | |||||
for (ReceiveCommand cmd : commands) { | |||||
b.append(String.format( | |||||
" %-12s %-12s %s %s", //$NON-NLS-1$ | |||||
LeaderSnapshot.str(cmd.getOldId()), | |||||
LeaderSnapshot.str(cmd.getNewId()), | |||||
cmd.getRefName(), | |||||
cmd.getResult())); | |||||
if (cmd.getMessage() != null) { | |||||
b.append(' ').append(cmd.getMessage()); | |||||
} | |||||
b.append('\n'); | |||||
} | |||||
return b.toString(); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import java.util.Date; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
/** | |||||
* A snapshot of a replica. | |||||
* | |||||
* @see LeaderSnapshot | |||||
*/ | |||||
public class ReplicaSnapshot { | |||||
final KetchReplica replica; | |||||
ObjectId accepted; | |||||
ObjectId committed; | |||||
KetchReplica.State state; | |||||
String error; | |||||
long retryAtMillis; | |||||
ReplicaSnapshot(KetchReplica replica) { | |||||
this.replica = replica; | |||||
} | |||||
/** | |||||
* Get the replica this snapshot describes the state of | |||||
* | |||||
* @return the replica this snapshot describes the state of | |||||
*/ | |||||
public KetchReplica getReplica() { | |||||
return replica; | |||||
} | |||||
/** | |||||
* Get current state of the replica | |||||
* | |||||
* @return current state of the replica | |||||
*/ | |||||
public KetchReplica.State getState() { | |||||
return state; | |||||
} | |||||
/** | |||||
* Get last known Git commit at {@code refs/txn/accepted} | |||||
* | |||||
* @return last known Git commit at {@code refs/txn/accepted} | |||||
*/ | |||||
@Nullable | |||||
public ObjectId getAccepted() { | |||||
return accepted; | |||||
} | |||||
/** | |||||
* Get last known Git commit at {@code refs/txn/committed} | |||||
* | |||||
* @return last known Git commit at {@code refs/txn/committed} | |||||
*/ | |||||
@Nullable | |||||
public ObjectId getCommitted() { | |||||
return committed; | |||||
} | |||||
/** | |||||
* Get error message | |||||
* | |||||
* @return if {@link #getState()} == | |||||
* {@link org.eclipse.jgit.internal.ketch.KetchReplica.State#OFFLINE} | |||||
* an optional human-readable message from the transport system | |||||
* explaining the failure. | |||||
*/ | |||||
@Nullable | |||||
public String getErrorMessage() { | |||||
return error; | |||||
} | |||||
/** | |||||
* Get when the leader will retry communication with the offline or lagging | |||||
* replica | |||||
* | |||||
* @return time (usually in the future) when the leader will retry | |||||
* communication with the offline or lagging replica; null if no | |||||
* retry is scheduled or necessary. | |||||
*/ | |||||
@Nullable | |||||
public Date getRetryAt() { | |||||
return retryAtMillis > 0 ? new Date(retryAtMillis) : null; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import java.io.IOException; | |||||
import java.util.List; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
/** | |||||
* One round-trip to all replicas proposing a log entry. | |||||
* <p> | |||||
* In Raft a log entry represents a state transition at a specific index in the | |||||
* replicated log. The leader can only append log entries to the log. | |||||
* <p> | |||||
* In Ketch a log entry is recorded under the {@code refs/txn} namespace. This | |||||
* occurs when: | |||||
* <ul> | |||||
* <li>a replica wants to establish itself as a new leader by proposing a new | |||||
* term (see {@link ElectionRound}) | |||||
* <li>an established leader wants to gain consensus on new {@link Proposal}s | |||||
* (see {@link ProposalRound}) | |||||
* </ul> | |||||
*/ | |||||
abstract class Round { | |||||
final KetchLeader leader; | |||||
final LogIndex acceptedOldIndex; | |||||
LogIndex acceptedNewIndex; | |||||
List<ReceiveCommand> stageCommands; | |||||
Round(KetchLeader leader, LogIndex head) { | |||||
this.leader = leader; | |||||
this.acceptedOldIndex = head; | |||||
} | |||||
KetchSystem getSystem() { | |||||
return leader.getSystem(); | |||||
} | |||||
/** | |||||
* Creates a commit for {@code refs/txn/accepted} and calls | |||||
* {@link #runAsync(AnyObjectId)} to begin execution of the round across | |||||
* the system. | |||||
* <p> | |||||
* If references are being updated (such as in a {@link ProposalRound}) the | |||||
* RefTree may be modified. | |||||
* <p> | |||||
* Invoked without {@link KetchLeader#lock} to build objects. | |||||
* | |||||
* @throws IOException | |||||
* the round cannot build new objects within the leader's | |||||
* repository. The leader may be unable to execute. | |||||
*/ | |||||
abstract void start() throws IOException; | |||||
/** | |||||
* Asynchronously distribute the round's new value for | |||||
* {@code refs/txn/accepted} to all replicas. | |||||
* <p> | |||||
* Invoked by {@link #start()} after new commits have been created for the | |||||
* log. The method passes {@code newId} to {@link KetchLeader} to be | |||||
* distributed to all known replicas. | |||||
* | |||||
* @param newId | |||||
* new value for {@code refs/txn/accepted}. | |||||
*/ | |||||
void runAsync(AnyObjectId newId) { | |||||
acceptedNewIndex = acceptedOldIndex.nextIndex(newId); | |||||
leader.runAsync(this); | |||||
} | |||||
/** | |||||
* Notify the round it was accepted by a majority of the system. | |||||
* <p> | |||||
* Invoked by the leader with {@link KetchLeader#lock} held by the caller. | |||||
*/ | |||||
abstract void success(); | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; | |||||
import java.io.IOException; | |||||
import java.util.ArrayList; | |||||
import java.util.HashSet; | |||||
import java.util.List; | |||||
import java.util.Set; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.CommitBuilder; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.PersonIdent; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevObject; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.treewalk.EmptyTreeIterator; | |||||
import org.eclipse.jgit.treewalk.TreeWalk; | |||||
import org.eclipse.jgit.treewalk.filter.TreeFilter; | |||||
/** | |||||
* Constructs a set of commands to stage content during a proposal. | |||||
*/ | |||||
public class StageBuilder { | |||||
/** | |||||
* Acceptable number of references to send in a single stage transaction. | |||||
* <p> | |||||
* If the number of unique objects exceeds this amount the builder will | |||||
* attempt to decrease the reference count by chaining commits.. | |||||
*/ | |||||
private static final int SMALL_BATCH_SIZE = 5; | |||||
/** | |||||
* Acceptable number of commits to chain together using parent pointers. | |||||
* <p> | |||||
* When staging many unique commits the {@link StageBuilder} batches | |||||
* together unrelated commits as parents of a temporary commit. After the | |||||
* proposal completes the temporary commit is discarded and can be garbage | |||||
* collected by all replicas. | |||||
*/ | |||||
private static final int TEMP_PARENT_BATCH_SIZE = 128; | |||||
private static final byte[] PEEL = { ' ', '^' }; | |||||
private final String txnStage; | |||||
private final String txnId; | |||||
/** | |||||
* Construct a stage builder for a transaction. | |||||
* | |||||
* @param txnStageNamespace | |||||
* namespace for transaction references to build | |||||
* {@code "txnStageNamespace/txnId.n"} style names. | |||||
* @param txnId | |||||
* identifier used to name temporary staging refs. | |||||
*/ | |||||
public StageBuilder(String txnStageNamespace, ObjectId txnId) { | |||||
this.txnStage = txnStageNamespace; | |||||
this.txnId = txnId.name(); | |||||
} | |||||
/** | |||||
* Compare two RefTrees and return commands to stage new objects. | |||||
* <p> | |||||
* This method ignores the lineage between the two RefTrees and does a | |||||
* straight diff on the two trees. New objects will be staged. The diff | |||||
* strategy is useful to catch-up a lagging replica, without sending every | |||||
* intermediate step. This may mean the replica does not have the same | |||||
* object set as other replicas if there are rewinds or branch deletes. | |||||
* | |||||
* @param git | |||||
* source repository to read {@code oldTree} and {@code newTree} | |||||
* from. | |||||
* @param oldTree | |||||
* accepted RefTree on the replica ({@code refs/txn/accepted}). | |||||
* Use {@link org.eclipse.jgit.lib.ObjectId#zeroId()} if the | |||||
* remote does not have any ref tree, e.g. a new replica catching | |||||
* up. | |||||
* @param newTree | |||||
* RefTree being sent to the replica. The trees will be compared. | |||||
* @return list of commands to create {@code "refs/txn/stage/..."} | |||||
* references on replicas anchoring new objects into the repository | |||||
* while a transaction gains consensus. | |||||
* @throws java.io.IOException | |||||
* {@code git} cannot be accessed to compare {@code oldTree} and | |||||
* {@code newTree} to build the object set. | |||||
*/ | |||||
public List<ReceiveCommand> makeStageList(Repository git, ObjectId oldTree, | |||||
ObjectId newTree) throws IOException { | |||||
try (RevWalk rw = new RevWalk(git); | |||||
TreeWalk tw = new TreeWalk(rw.getObjectReader()); | |||||
ObjectInserter ins = git.newObjectInserter()) { | |||||
if (AnyObjectId.isEqual(oldTree, ObjectId.zeroId())) { | |||||
tw.addTree(new EmptyTreeIterator()); | |||||
} else { | |||||
tw.addTree(rw.parseTree(oldTree)); | |||||
} | |||||
tw.addTree(rw.parseTree(newTree)); | |||||
tw.setFilter(TreeFilter.ANY_DIFF); | |||||
tw.setRecursive(true); | |||||
Set<ObjectId> newObjs = new HashSet<>(); | |||||
while (tw.next()) { | |||||
if (tw.getRawMode(1) == TYPE_GITLINK | |||||
&& !tw.isPathSuffix(PEEL, 2)) { | |||||
newObjs.add(tw.getObjectId(1)); | |||||
} | |||||
} | |||||
List<ReceiveCommand> cmds = makeStageList(newObjs, git, ins); | |||||
ins.flush(); | |||||
return cmds; | |||||
} | |||||
} | |||||
/** | |||||
* Construct a set of commands to stage objects on a replica. | |||||
* | |||||
* @param newObjs | |||||
* objects to send to a replica. | |||||
* @param git | |||||
* local repository to read source objects from. Required to | |||||
* perform minification of {@code newObjs}. | |||||
* @param inserter | |||||
* inserter to write temporary commit objects during minification | |||||
* if many new branches are created by {@code newObjs}. | |||||
* @return list of commands to create {@code "refs/txn/stage/..."} | |||||
* references on replicas anchoring {@code newObjs} into the | |||||
* repository while a transaction gains consensus. | |||||
* @throws java.io.IOException | |||||
* {@code git} cannot be accessed to perform minification of | |||||
* {@code newObjs}. | |||||
*/ | |||||
public List<ReceiveCommand> makeStageList(Set<ObjectId> newObjs, | |||||
@Nullable Repository git, @Nullable ObjectInserter inserter) | |||||
throws IOException { | |||||
if (git == null || newObjs.size() <= SMALL_BATCH_SIZE) { | |||||
// Without a source repository can only construct unique set. | |||||
List<ReceiveCommand> cmds = new ArrayList<>(newObjs.size()); | |||||
for (ObjectId id : newObjs) { | |||||
stage(cmds, id); | |||||
} | |||||
return cmds; | |||||
} | |||||
List<ReceiveCommand> cmds = new ArrayList<>(); | |||||
List<RevCommit> commits = new ArrayList<>(); | |||||
reduceObjects(cmds, commits, git, newObjs); | |||||
if (inserter == null || commits.size() <= 1 | |||||
|| (cmds.size() + commits.size()) <= SMALL_BATCH_SIZE) { | |||||
// Without an inserter to aggregate commits, or for a small set of | |||||
// commits just send one stage ref per commit. | |||||
for (RevCommit c : commits) { | |||||
stage(cmds, c.copy()); | |||||
} | |||||
return cmds; | |||||
} | |||||
// 'commits' is sorted most recent to least recent commit. | |||||
// Group batches of commits and build a chain. | |||||
// TODO(sop) Cluster by restricted graphs to support filtering. | |||||
ObjectId tip = null; | |||||
for (int end = commits.size(); end > 0;) { | |||||
int start = Math.max(0, end - TEMP_PARENT_BATCH_SIZE); | |||||
List<RevCommit> batch = commits.subList(start, end); | |||||
List<ObjectId> parents = new ArrayList<>(1 + batch.size()); | |||||
if (tip != null) { | |||||
parents.add(tip); | |||||
} | |||||
parents.addAll(batch); | |||||
CommitBuilder b = new CommitBuilder(); | |||||
b.setTreeId(batch.get(0).getTree()); | |||||
b.setParentIds(parents); | |||||
b.setAuthor(tmpAuthor(batch)); | |||||
b.setCommitter(b.getAuthor()); | |||||
tip = inserter.insert(b); | |||||
end = start; | |||||
} | |||||
stage(cmds, tip); | |||||
return cmds; | |||||
} | |||||
private static PersonIdent tmpAuthor(List<RevCommit> commits) { | |||||
// Construct a predictable author using most recent commit time. | |||||
int t = 0; | |||||
for (int i = 0; i < commits.size();) { | |||||
t = Math.max(t, commits.get(i).getCommitTime()); | |||||
} | |||||
String name = "Ketch Stage"; //$NON-NLS-1$ | |||||
String email = "tmp@tmp"; //$NON-NLS-1$ | |||||
return new PersonIdent(name, email, t * 1000L, 0); | |||||
} | |||||
private void reduceObjects(List<ReceiveCommand> cmds, | |||||
List<RevCommit> commits, Repository git, | |||||
Set<ObjectId> newObjs) throws IOException { | |||||
try (RevWalk rw = new RevWalk(git)) { | |||||
rw.setRetainBody(false); | |||||
for (ObjectId id : newObjs) { | |||||
RevObject obj = rw.parseAny(id); | |||||
if (obj instanceof RevCommit) { | |||||
rw.markStart((RevCommit) obj); | |||||
} else { | |||||
stage(cmds, id); | |||||
} | |||||
} | |||||
for (RevCommit c; (c = rw.next()) != null;) { | |||||
commits.add(c); | |||||
rw.markUninteresting(c); | |||||
} | |||||
} | |||||
} | |||||
private void stage(List<ReceiveCommand> cmds, ObjectId id) { | |||||
int estLen = txnStage.length() + txnId.length() + 5; | |||||
StringBuilder n = new StringBuilder(estLen); | |||||
n.append(txnStage).append(txnId).append('.'); | |||||
n.append(Integer.toHexString(cmds.size())); | |||||
cmds.add(new ReceiveCommand(ObjectId.zeroId(), id, n.toString())); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; | |||||
import java.io.IOException; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
class TimeIsUncertainException extends IOException { | |||||
private static final long serialVersionUID = 1L; | |||||
TimeIsUncertainException() { | |||||
super(JGitText.get().timeIsUncertain); | |||||
} | |||||
TimeIsUncertainException(Exception e) { | |||||
super(JGitText.get().timeIsUncertain, e); | |||||
} | |||||
} |
/** | |||||
* Distributed consensus system built on Git. | |||||
*/ | |||||
package org.eclipse.jgit.internal.ketch; |
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN; | |||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE; | ||||
import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; | import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; | ||||
import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor; | import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor; | ||||
import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; | import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; | ||||
import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; | import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; | ||||
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | import org.eclipse.jgit.lib.AnyObjectId; | ||||
import org.eclipse.jgit.lib.Constants; | import org.eclipse.jgit.lib.Constants; | ||||
import org.eclipse.jgit.lib.NullProgressMonitor; | import org.eclipse.jgit.lib.NullProgressMonitor; | ||||
private Set<ObjectId> allHeadsAndTags; | private Set<ObjectId> allHeadsAndTags; | ||||
private Set<ObjectId> allTags; | private Set<ObjectId> allTags; | ||||
private Set<ObjectId> nonHeads; | private Set<ObjectId> nonHeads; | ||||
private Set<ObjectId> txnHeads; | |||||
private Set<ObjectId> tagTargets; | private Set<ObjectId> tagTargets; | ||||
/** | /** | ||||
allHeadsAndTags = new HashSet<>(); | allHeadsAndTags = new HashSet<>(); | ||||
allTags = new HashSet<>(); | allTags = new HashSet<>(); | ||||
nonHeads = new HashSet<>(); | nonHeads = new HashSet<>(); | ||||
txnHeads = new HashSet<>(); | |||||
tagTargets = new HashSet<>(); | tagTargets = new HashSet<>(); | ||||
for (Ref ref : refsBefore) { | for (Ref ref : refsBefore) { | ||||
if (ref.isSymbolic() || ref.getObjectId() == null) { | if (ref.isSymbolic() || ref.getObjectId() == null) { | ||||
allHeads.add(ref.getObjectId()); | allHeads.add(ref.getObjectId()); | ||||
} else if (isTag(ref)) { | } else if (isTag(ref)) { | ||||
allTags.add(ref.getObjectId()); | allTags.add(ref.getObjectId()); | ||||
} else if (RefTreeNames.isRefTree(refdb, ref.getName())) { | |||||
txnHeads.add(ref.getObjectId()); | |||||
} else { | } else { | ||||
nonHeads.add(ref.getObjectId()); | nonHeads.add(ref.getObjectId()); | ||||
} | } | ||||
try { | try { | ||||
packHeads(pm); | packHeads(pm); | ||||
packRest(pm); | packRest(pm); | ||||
packRefTreeGraph(pm); | |||||
packGarbage(pm); | packGarbage(pm); | ||||
objdb.commitPack(newPackDesc, toPrune()); | objdb.commitPack(newPackDesc, toPrune()); | ||||
rollback = false; | rollback = false; | ||||
} | } | ||||
} | } | ||||
private void packRefTreeGraph(ProgressMonitor pm) throws IOException { | |||||
if (txnHeads.isEmpty()) | |||||
return; | |||||
try (PackWriter pw = newPackWriter()) { | |||||
for (ObjectIdSet packedObjs : newPackObj) | |||||
pw.excludeObjects(packedObjs); | |||||
pw.preparePack(pm, txnHeads, NONE); | |||||
if (0 < pw.getObjectCount()) | |||||
writePack(GC_TXN, pw, pm, 0 /* unknown pack size */); | |||||
} | |||||
} | |||||
private void packGarbage(ProgressMonitor pm) throws IOException { | private void packGarbage(ProgressMonitor pm) throws IOException { | ||||
PackConfig cfg = new PackConfig(packConfig); | PackConfig cfg = new PackConfig(packConfig); | ||||
cfg.setReuseDeltas(true); | cfg.setReuseDeltas(true); |
/** Created from non-heads by {@link DfsGarbageCollector}. */ | /** Created from non-heads by {@link DfsGarbageCollector}. */ | ||||
GC_REST, | GC_REST, | ||||
/** | |||||
* RefTreeGraph pack was created by Git garbage collection. | |||||
* | |||||
* @see DfsGarbageCollector | |||||
*/ | |||||
GC_TXN, | |||||
/** | /** | ||||
* Pack was created by Git garbage collection. | * Pack was created by Git garbage collection. | ||||
* <p> | * <p> | ||||
.add(COMPACT) | .add(COMPACT) | ||||
.add(GC) | .add(GC) | ||||
.add(GC_REST) | .add(GC_REST) | ||||
.add(GC_TXN) | |||||
.add(UNREACHABLE_GARBAGE) | .add(UNREACHABLE_GARBAGE) | ||||
.build(); | .build(); | ||||
switch (pack.getPackDescription().getPackSource()) { | switch (pack.getPackDescription().getPackSource()) { | ||||
case GC: | case GC: | ||||
case GC_REST: | case GC_REST: | ||||
case GC_TXN: | |||||
return true; | return true; | ||||
default: | default: | ||||
return false; | return false; |
switch (s) { | switch (s) { | ||||
case GC: | case GC: | ||||
case GC_REST: | case GC_REST: | ||||
case GC_TXN: | |||||
return true; | return true; | ||||
default: | default: | ||||
return false; | return false; |
import org.eclipse.jgit.internal.JGitText; | import org.eclipse.jgit.internal.JGitText; | ||||
import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle; | import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateHandle; | ||||
import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepository; | import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepository; | ||||
import org.eclipse.jgit.internal.storage.reftree.RefTreeDatabase; | |||||
import org.eclipse.jgit.lib.BaseRepositoryBuilder; | import org.eclipse.jgit.lib.BaseRepositoryBuilder; | ||||
import org.eclipse.jgit.lib.BatchRefUpdate; | import org.eclipse.jgit.lib.BatchRefUpdate; | ||||
import org.eclipse.jgit.lib.ConfigConstants; | import org.eclipse.jgit.lib.ConfigConstants; | ||||
if (StringUtils.equalsIgnoreCase(reftype, | if (StringUtils.equalsIgnoreCase(reftype, | ||||
ConfigConstants.CONFIG_REF_STORAGE_REFTABLE)) { | ConfigConstants.CONFIG_REF_STORAGE_REFTABLE)) { | ||||
refs = new FileReftableDatabase(this); | refs = new FileReftableDatabase(this); | ||||
} else if (StringUtils.equalsIgnoreCase(reftype, | |||||
ConfigConstants.CONFIG_REFSTORAGE_REFTREE)) { | |||||
refs = new RefTreeDatabase(this, new RefDirectory(this)); | |||||
} else { | } else { | ||||
throw new IOException(JGitText.get().unknownRepositoryFormat); | throw new IOException(JGitText.get().unknownRepositoryFormat); | ||||
} | } | ||||
refsHeadsFile.delete(); | refsHeadsFile.delete(); | ||||
// RefDirectory wants to create the refs/ directory from scratch, so | // RefDirectory wants to create the refs/ directory from scratch, so | ||||
// remove that too. | // remove that too. | ||||
refsFile.delete(); | |||||
refsFile.delete(); | |||||
// remove HEAD so its previous invalid value doesn't cause issues. | // remove HEAD so its previous invalid value doesn't cause issues. | ||||
headFile.delete(); | headFile.delete(); | ||||
for (ReflogEntry e : logs) { | for (ReflogEntry e : logs) { | ||||
logWriter.log(r.getName(), e); | logWriter.log(r.getName(), e); | ||||
} | } | ||||
} | |||||
} | |||||
} | } | ||||
try (RevWalk rw = new RevWalk(this)) { | try (RevWalk rw = new RevWalk(this)) { | ||||
FileUtils.delete(refsFile, FileUtils.RECURSIVE); | FileUtils.delete(refsFile, FileUtils.RECURSIVE); | ||||
for (String r : additional) { | for (String r : additional) { | ||||
new File(getDirectory(), r).delete(); | new File(getDirectory(), r).delete(); | ||||
} | |||||
} | |||||
} | } | ||||
FileUtils.mkdir(refsFile, true); | FileUtils.mkdir(refsFile, true); |
import org.eclipse.jgit.internal.JGitText; | import org.eclipse.jgit.internal.JGitText; | ||||
import org.eclipse.jgit.internal.storage.pack.PackExt; | import org.eclipse.jgit.internal.storage.pack.PackExt; | ||||
import org.eclipse.jgit.internal.storage.pack.PackWriter; | import org.eclipse.jgit.internal.storage.pack.PackWriter; | ||||
import org.eclipse.jgit.internal.storage.reftree.RefTreeNames; | |||||
import org.eclipse.jgit.lib.ConfigConstants; | import org.eclipse.jgit.lib.ConfigConstants; | ||||
import org.eclipse.jgit.lib.Constants; | import org.eclipse.jgit.lib.Constants; | ||||
import org.eclipse.jgit.lib.FileMode; | import org.eclipse.jgit.lib.FileMode; | ||||
Set<ObjectId> txnHeads = new HashSet<>(); | Set<ObjectId> txnHeads = new HashSet<>(); | ||||
Set<ObjectId> tagTargets = new HashSet<>(); | Set<ObjectId> tagTargets = new HashSet<>(); | ||||
Set<ObjectId> indexObjects = listNonHEADIndexObjects(); | Set<ObjectId> indexObjects = listNonHEADIndexObjects(); | ||||
RefDatabase refdb = repo.getRefDatabase(); | |||||
for (Ref ref : refsBefore) { | for (Ref ref : refsBefore) { | ||||
checkCancelled(); | checkCancelled(); | ||||
allHeads.add(ref.getObjectId()); | allHeads.add(ref.getObjectId()); | ||||
} else if (isTag(ref)) { | } else if (isTag(ref)) { | ||||
allTags.add(ref.getObjectId()); | allTags.add(ref.getObjectId()); | ||||
} else if (RefTreeNames.isRefTree(refdb, ref.getName())) { | |||||
txnHeads.add(ref.getObjectId()); | |||||
} else { | } else { | ||||
nonHeads.add(ref.getObjectId()); | nonHeads.add(ref.getObjectId()); | ||||
} | } |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import java.io.IOException; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
import org.eclipse.jgit.lib.RefUpdate; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
/** Update that always rejects with {@code LOCK_FAILURE}. */ | |||||
class AlwaysFailUpdate extends RefUpdate { | |||||
private final RefTreeDatabase refdb; | |||||
AlwaysFailUpdate(RefTreeDatabase refdb, String name) { | |||||
super(new ObjectIdRef.Unpeeled(Ref.Storage.NEW, name, null)); | |||||
this.refdb = refdb; | |||||
setCheckConflicting(false); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected RefDatabase getRefDatabase() { | |||||
return refdb; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Repository getRepository() { | |||||
return refdb.getRepository(); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected boolean tryLock(boolean deref) throws IOException { | |||||
return false; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected void unlock() { | |||||
// No locks are held here. | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Result doUpdate(Result desiredResult) { | |||||
return Result.LOCK_FAILURE; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Result doDelete(Result desiredResult) { | |||||
return Result.LOCK_FAILURE; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Result doLink(String target) { | |||||
return Result.LOCK_FAILURE; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; | |||||
import static org.eclipse.jgit.lib.Constants.encode; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.NETWORK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import java.io.IOException; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.dircache.DirCacheEntry; | |||||
import org.eclipse.jgit.errors.MissingObjectException; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevObject; | |||||
import org.eclipse.jgit.revwalk.RevTag; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
import org.eclipse.jgit.transport.ReceiveCommand.Result; | |||||
/** | |||||
* Command to create, update or delete an entry inside a | |||||
* {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. | |||||
* <p> | |||||
* Unlike {@link org.eclipse.jgit.transport.ReceiveCommand} (which can only | |||||
* update a reference to an {@link org.eclipse.jgit.lib.ObjectId}), a RefTree | |||||
* Command can also create, modify or delete symbolic references to a target | |||||
* reference. | |||||
* <p> | |||||
* RefTree Commands may wrap a {@code ReceiveCommand} to allow callers to | |||||
* process an existing ReceiveCommand against a RefTree. | |||||
* <p> | |||||
* Commands should be passed into | |||||
* {@link org.eclipse.jgit.internal.storage.reftree.RefTree#apply(java.util.Collection)} | |||||
* for processing. | |||||
*/ | |||||
public class Command { | |||||
/** | |||||
* Set unprocessed commands as failed due to transaction aborted. | |||||
* <p> | |||||
* If a command is still | |||||
* {@link org.eclipse.jgit.transport.ReceiveCommand.Result#NOT_ATTEMPTED} it | |||||
* will be set to | |||||
* {@link org.eclipse.jgit.transport.ReceiveCommand.Result#REJECTED_OTHER_REASON}. | |||||
* If {@code why} is non-null its contents will be used as the message for | |||||
* the first command status. | |||||
* | |||||
* @param commands | |||||
* commands to mark as failed. | |||||
* @param why | |||||
* optional message to set on the first aborted command. | |||||
*/ | |||||
public static void abort(Iterable<Command> commands, @Nullable String why) { | |||||
if (why == null || why.isEmpty()) { | |||||
why = JGitText.get().transactionAborted; | |||||
} | |||||
for (Command c : commands) { | |||||
if (c.getResult() == NOT_ATTEMPTED) { | |||||
c.setResult(REJECTED_OTHER_REASON, why); | |||||
why = JGitText.get().transactionAborted; | |||||
} | |||||
} | |||||
} | |||||
private final Ref oldRef; | |||||
private final Ref newRef; | |||||
private final ReceiveCommand cmd; | |||||
private Result result; | |||||
/** | |||||
* Create a command to create, update or delete a reference. | |||||
* <p> | |||||
* At least one of {@code oldRef} or {@code newRef} must be supplied. | |||||
* | |||||
* @param oldRef | |||||
* expected value. Null if the ref should not exist. | |||||
* @param newRef | |||||
* desired value, must be peeled if not null and not symbolic. | |||||
* Null to delete the ref. | |||||
*/ | |||||
public Command(@Nullable Ref oldRef, @Nullable Ref newRef) { | |||||
this.oldRef = oldRef; | |||||
this.newRef = newRef; | |||||
this.cmd = null; | |||||
this.result = NOT_ATTEMPTED; | |||||
if (oldRef == null && newRef == null) { | |||||
throw new IllegalArgumentException(); | |||||
} | |||||
if (newRef != null && !newRef.isPeeled() && !newRef.isSymbolic()) { | |||||
throw new IllegalArgumentException(); | |||||
} | |||||
if (oldRef != null && newRef != null | |||||
&& !oldRef.getName().equals(newRef.getName())) { | |||||
throw new IllegalArgumentException(); | |||||
} | |||||
} | |||||
/** | |||||
* Construct a RefTree command wrapped around a ReceiveCommand. | |||||
* | |||||
* @param rw | |||||
* walk instance to peel the {@code newId}. | |||||
* @param cmd | |||||
* command received from a push client. | |||||
* @throws org.eclipse.jgit.errors.MissingObjectException | |||||
* {@code oldId} or {@code newId} is missing. | |||||
* @throws java.io.IOException | |||||
* {@code oldId} or {@code newId} cannot be peeled. | |||||
*/ | |||||
public Command(RevWalk rw, ReceiveCommand cmd) | |||||
throws MissingObjectException, IOException { | |||||
this.oldRef = toRef(rw, cmd.getOldId(), cmd.getOldSymref(), | |||||
cmd.getRefName(), false); | |||||
this.newRef = toRef(rw, cmd.getNewId(), cmd.getNewSymref(), | |||||
cmd.getRefName(), true); | |||||
this.cmd = cmd; | |||||
} | |||||
static Ref toRef(RevWalk rw, ObjectId id, @Nullable String target, | |||||
String name, boolean mustExist) | |||||
throws MissingObjectException, IOException { | |||||
if (target != null) { | |||||
return new SymbolicRef(name, | |||||
new ObjectIdRef.Unpeeled(NETWORK, target, id)); | |||||
} else if (ObjectId.zeroId().equals(id)) { | |||||
return null; | |||||
} | |||||
try { | |||||
RevObject o = rw.parseAny(id); | |||||
if (o instanceof RevTag) { | |||||
RevObject p = rw.peel(o); | |||||
return new ObjectIdRef.PeeledTag(NETWORK, name, id, p.copy()); | |||||
} | |||||
return new ObjectIdRef.PeeledNonTag(NETWORK, name, id); | |||||
} catch (MissingObjectException e) { | |||||
if (mustExist) { | |||||
throw e; | |||||
} | |||||
return new ObjectIdRef.Unpeeled(NETWORK, name, id); | |||||
} | |||||
} | |||||
/** | |||||
* Get name of the reference affected by this command. | |||||
* | |||||
* @return name of the reference affected by this command. | |||||
*/ | |||||
public String getRefName() { | |||||
if (cmd != null) { | |||||
return cmd.getRefName(); | |||||
} else if (newRef != null) { | |||||
return newRef.getName(); | |||||
} | |||||
return oldRef.getName(); | |||||
} | |||||
/** | |||||
* Set the result of this command. | |||||
* | |||||
* @param result | |||||
* the command result. | |||||
*/ | |||||
public void setResult(Result result) { | |||||
setResult(result, null); | |||||
} | |||||
/** | |||||
* Set the result of this command. | |||||
* | |||||
* @param result | |||||
* the command result. | |||||
* @param why | |||||
* optional message explaining the result status. | |||||
*/ | |||||
public void setResult(Result result, @Nullable String why) { | |||||
if (cmd != null) { | |||||
cmd.setResult(result, why); | |||||
} else { | |||||
this.result = result; | |||||
} | |||||
} | |||||
/** | |||||
* Get result of executing this command. | |||||
* | |||||
* @return result of executing this command. | |||||
*/ | |||||
public Result getResult() { | |||||
return cmd != null ? cmd.getResult() : result; | |||||
} | |||||
/** | |||||
* Get optional message explaining command failure. | |||||
* | |||||
* @return optional message explaining command failure. | |||||
*/ | |||||
@Nullable | |||||
public String getMessage() { | |||||
return cmd != null ? cmd.getMessage() : null; | |||||
} | |||||
/** | |||||
* Old peeled reference. | |||||
* | |||||
* @return the old reference; null if the command is creating the reference. | |||||
*/ | |||||
@Nullable | |||||
public Ref getOldRef() { | |||||
return oldRef; | |||||
} | |||||
/** | |||||
* New peeled reference. | |||||
* | |||||
* @return the new reference; null if the command is deleting the reference. | |||||
*/ | |||||
@Nullable | |||||
public Ref getNewRef() { | |||||
return newRef; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public String toString() { | |||||
StringBuilder s = new StringBuilder(); | |||||
append(s, oldRef, "CREATE"); //$NON-NLS-1$ | |||||
s.append(' '); | |||||
append(s, newRef, "DELETE"); //$NON-NLS-1$ | |||||
s.append(' ').append(getRefName()); | |||||
s.append(' ').append(getResult()); | |||||
if (getMessage() != null) { | |||||
s.append(' ').append(getMessage()); | |||||
} | |||||
return s.toString(); | |||||
} | |||||
private static void append(StringBuilder s, Ref r, String nullName) { | |||||
if (r == null) { | |||||
s.append(nullName); | |||||
} else if (r.isSymbolic()) { | |||||
s.append(r.getTarget().getName()); | |||||
} else { | |||||
ObjectId id = r.getObjectId(); | |||||
if (id != null) { | |||||
s.append(id.name()); | |||||
} | |||||
} | |||||
} | |||||
/** | |||||
* Check the entry is consistent with either the old or the new ref. | |||||
* | |||||
* @param entry | |||||
* current entry; null if the entry does not exist. | |||||
* @return true if entry matches {@link #getOldRef()} or | |||||
* {@link #getNewRef()}; otherwise false. | |||||
*/ | |||||
boolean checkRef(@Nullable DirCacheEntry entry) { | |||||
if (entry != null && entry.getRawMode() == 0) { | |||||
entry = null; | |||||
} | |||||
return check(entry, oldRef) || check(entry, newRef); | |||||
} | |||||
private static boolean check(@Nullable DirCacheEntry cur, | |||||
@Nullable Ref exp) { | |||||
if (cur == null) { | |||||
// Does not exist, ok if oldRef does not exist. | |||||
return exp == null; | |||||
} else if (exp == null) { | |||||
// Expected to not exist, but currently exists, fail. | |||||
return false; | |||||
} | |||||
if (exp.isSymbolic()) { | |||||
String dst = exp.getTarget().getName(); | |||||
return cur.getRawMode() == TYPE_SYMLINK | |||||
&& cur.getObjectId().equals(symref(dst)); | |||||
} | |||||
return cur.getRawMode() == TYPE_GITLINK | |||||
&& cur.getObjectId().equals(exp.getObjectId()); | |||||
} | |||||
static ObjectId symref(String s) { | |||||
@SuppressWarnings("resource") | |||||
ObjectInserter.Formatter fmt = new ObjectInserter.Formatter(); | |||||
return fmt.idFor(OBJ_BLOB, encode(s)); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; | |||||
import static org.eclipse.jgit.lib.Constants.R_REFS; | |||||
import static org.eclipse.jgit.lib.Constants.encode; | |||||
import static org.eclipse.jgit.lib.FileMode.GITLINK; | |||||
import static org.eclipse.jgit.lib.FileMode.SYMLINK; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.NEW; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.PACKED; | |||||
import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import java.io.IOException; | |||||
import java.util.Collection; | |||||
import java.util.HashMap; | |||||
import java.util.Map; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.dircache.DirCache; | |||||
import org.eclipse.jgit.dircache.DirCacheBuilder; | |||||
import org.eclipse.jgit.dircache.DirCacheEditor; | |||||
import org.eclipse.jgit.dircache.DirCacheEditor.DeletePath; | |||||
import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; | |||||
import org.eclipse.jgit.dircache.DirCacheEntry; | |||||
import org.eclipse.jgit.errors.CorruptObjectException; | |||||
import org.eclipse.jgit.errors.DirCacheNameConflictException; | |||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException; | |||||
import org.eclipse.jgit.errors.MissingObjectException; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.ObjectReader; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevTree; | |||||
import org.eclipse.jgit.util.RawParseUtils; | |||||
/** | |||||
* Tree of references in the reference graph. | |||||
* <p> | |||||
* The root corresponds to the {@code "refs/"} subdirectory, for example the | |||||
* default reference {@code "refs/heads/master"} is stored at path | |||||
* {@code "heads/master"} in a {@code RefTree}. | |||||
* <p> | |||||
* Normal references are stored as {@link org.eclipse.jgit.lib.FileMode#GITLINK} | |||||
* tree entries. The ObjectId in the tree entry is the ObjectId the reference | |||||
* refers to. | |||||
* <p> | |||||
* Symbolic references are stored as | |||||
* {@link org.eclipse.jgit.lib.FileMode#SYMLINK} entries, with the blob storing | |||||
* the name of the target reference. | |||||
* <p> | |||||
* Annotated tags also store the peeled object using a {@code GITLINK} entry | |||||
* with the suffix <code>" ^"</code> (space carrot), for example | |||||
* {@code "tags/v1.0"} stores the annotated tag object, while | |||||
* <code>"tags/v1.0 ^"</code> stores the commit the tag annotates. | |||||
* <p> | |||||
* {@code HEAD} is a special case and stored as {@code "..HEAD"}. | |||||
*/ | |||||
public class RefTree { | |||||
/** Suffix applied to GITLINK to indicate its the peeled value of a tag. */ | |||||
public static final String PEELED_SUFFIX = " ^"; //$NON-NLS-1$ | |||||
static final String ROOT_DOTDOT = ".."; //$NON-NLS-1$ | |||||
/** | |||||
* Create an empty reference tree. | |||||
* | |||||
* @return a new empty reference tree. | |||||
*/ | |||||
public static RefTree newEmptyTree() { | |||||
return new RefTree(DirCache.newInCore()); | |||||
} | |||||
/** | |||||
* Load a reference tree. | |||||
* | |||||
* @param reader | |||||
* reader to scan the reference tree with. | |||||
* @param tree | |||||
* the tree to read. | |||||
* @return the ref tree read from the commit. | |||||
* @throws java.io.IOException | |||||
* the repository cannot be accessed through the reader. | |||||
* @throws org.eclipse.jgit.errors.CorruptObjectException | |||||
* a tree object is corrupt and cannot be read. | |||||
* @throws org.eclipse.jgit.errors.IncorrectObjectTypeException | |||||
* a tree object wasn't actually a tree. | |||||
* @throws org.eclipse.jgit.errors.MissingObjectException | |||||
* a reference tree object doesn't exist. | |||||
*/ | |||||
public static RefTree read(ObjectReader reader, RevTree tree) | |||||
throws MissingObjectException, IncorrectObjectTypeException, | |||||
CorruptObjectException, IOException { | |||||
return new RefTree(DirCache.read(reader, tree)); | |||||
} | |||||
private DirCache contents; | |||||
private Map<ObjectId, String> pendingBlobs; | |||||
private RefTree(DirCache dc) { | |||||
this.contents = dc; | |||||
} | |||||
/** | |||||
* Read one reference. | |||||
* <p> | |||||
* References are always returned peeled | |||||
* ({@link org.eclipse.jgit.lib.Ref#isPeeled()} is true). If the reference | |||||
* points to an annotated tag, the returned reference will be peeled and | |||||
* contain {@link org.eclipse.jgit.lib.Ref#getPeeledObjectId()}. | |||||
* <p> | |||||
* If the reference is a symbolic reference and the chain depth is less than | |||||
* {@link org.eclipse.jgit.lib.RefDatabase#MAX_SYMBOLIC_REF_DEPTH} the | |||||
* returned reference is resolved. If the chain depth is longer, the | |||||
* symbolic reference is returned without resolving. | |||||
* | |||||
* @param reader | |||||
* to access objects necessary to read the requested reference. | |||||
* @param name | |||||
* name of the reference to read. | |||||
* @return the reference; null if it does not exist. | |||||
* @throws java.io.IOException | |||||
* cannot read a symbolic reference target. | |||||
*/ | |||||
@Nullable | |||||
public Ref exactRef(ObjectReader reader, String name) throws IOException { | |||||
Ref r = readRef(reader, name); | |||||
if (r == null) { | |||||
return null; | |||||
} else if (r.isSymbolic()) { | |||||
return resolve(reader, r, 0); | |||||
} | |||||
DirCacheEntry p = contents.getEntry(peeledPath(name)); | |||||
if (p != null && p.getRawMode() == TYPE_GITLINK) { | |||||
return new ObjectIdRef.PeeledTag(PACKED, r.getName(), | |||||
r.getObjectId(), p.getObjectId()); | |||||
} | |||||
return r; | |||||
} | |||||
private Ref readRef(ObjectReader reader, String name) throws IOException { | |||||
DirCacheEntry e = contents.getEntry(refPath(name)); | |||||
return e != null ? toRef(reader, e, name) : null; | |||||
} | |||||
private Ref toRef(ObjectReader reader, DirCacheEntry e, String name) | |||||
throws IOException { | |||||
int mode = e.getRawMode(); | |||||
if (mode == TYPE_GITLINK) { | |||||
ObjectId id = e.getObjectId(); | |||||
return new ObjectIdRef.PeeledNonTag(PACKED, name, id); | |||||
} | |||||
if (mode == TYPE_SYMLINK) { | |||||
ObjectId id = e.getObjectId(); | |||||
String n = pendingBlobs != null ? pendingBlobs.get(id) : null; | |||||
if (n == null) { | |||||
byte[] bin = reader.open(id, OBJ_BLOB).getCachedBytes(); | |||||
n = RawParseUtils.decode(bin); | |||||
} | |||||
Ref dst = new ObjectIdRef.Unpeeled(NEW, n, null); | |||||
return new SymbolicRef(name, dst); | |||||
} | |||||
return null; // garbage file or something; not a reference. | |||||
} | |||||
private Ref resolve(ObjectReader reader, Ref ref, int depth) | |||||
throws IOException { | |||||
if (ref.isSymbolic() && depth < MAX_SYMBOLIC_REF_DEPTH) { | |||||
Ref r = readRef(reader, ref.getTarget().getName()); | |||||
if (r == null) { | |||||
return ref; | |||||
} | |||||
Ref dst = resolve(reader, r, depth + 1); | |||||
return new SymbolicRef(ref.getName(), dst); | |||||
} | |||||
return ref; | |||||
} | |||||
/** | |||||
* Attempt a batch of commands against this RefTree. | |||||
* <p> | |||||
* The batch is applied atomically, either all commands apply at once, or | |||||
* they all reject and the RefTree is left unmodified. | |||||
* <p> | |||||
* On success (when this method returns {@code true}) the command results | |||||
* are left as-is (probably {@code NOT_ATTEMPTED}). Result fields are set | |||||
* only when this method returns {@code false} to indicate failure. | |||||
* | |||||
* @param cmdList | |||||
* to apply. All commands should still have result NOT_ATTEMPTED. | |||||
* @return true if the commands applied; false if they were rejected. | |||||
*/ | |||||
public boolean apply(Collection<Command> cmdList) { | |||||
try { | |||||
DirCacheEditor ed = contents.editor(); | |||||
for (Command cmd : cmdList) { | |||||
if (!isValidRef(cmd)) { | |||||
cmd.setResult(REJECTED_OTHER_REASON, | |||||
JGitText.get().funnyRefname); | |||||
Command.abort(cmdList, null); | |||||
return false; | |||||
} | |||||
apply(ed, cmd); | |||||
} | |||||
ed.finish(); | |||||
return true; | |||||
} catch (DirCacheNameConflictException e) { | |||||
String r1 = refName(e.getPath1()); | |||||
String r2 = refName(e.getPath2()); | |||||
for (Command cmd : cmdList) { | |||||
if (r1.equals(cmd.getRefName()) | |||||
|| r2.equals(cmd.getRefName())) { | |||||
cmd.setResult(LOCK_FAILURE); | |||||
break; | |||||
} | |||||
} | |||||
Command.abort(cmdList, null); | |||||
return false; | |||||
} catch (LockFailureException e) { | |||||
Command.abort(cmdList, null); | |||||
return false; | |||||
} | |||||
} | |||||
private static boolean isValidRef(Command cmd) { | |||||
String n = cmd.getRefName(); | |||||
return HEAD.equals(n) || Repository.isValidRefName(n); | |||||
} | |||||
private void apply(DirCacheEditor ed, Command cmd) { | |||||
String path = refPath(cmd.getRefName()); | |||||
Ref oldRef = cmd.getOldRef(); | |||||
final Ref newRef = cmd.getNewRef(); | |||||
if (newRef == null) { | |||||
checkRef(contents.getEntry(path), cmd); | |||||
ed.add(new DeletePath(path)); | |||||
cleanupPeeledRef(ed, oldRef); | |||||
return; | |||||
} | |||||
if (newRef.isSymbolic()) { | |||||
final String dst = newRef.getTarget().getName(); | |||||
ed.add(new PathEdit(path) { | |||||
@Override | |||||
public void apply(DirCacheEntry ent) { | |||||
checkRef(ent, cmd); | |||||
ObjectId id = Command.symref(dst); | |||||
ent.setFileMode(SYMLINK); | |||||
ent.setObjectId(id); | |||||
if (pendingBlobs == null) { | |||||
pendingBlobs = new HashMap<>(4); | |||||
} | |||||
pendingBlobs.put(id, dst); | |||||
} | |||||
}.setReplace(false)); | |||||
cleanupPeeledRef(ed, oldRef); | |||||
return; | |||||
} | |||||
ed.add(new PathEdit(path) { | |||||
@Override | |||||
public void apply(DirCacheEntry ent) { | |||||
checkRef(ent, cmd); | |||||
ent.setFileMode(GITLINK); | |||||
ent.setObjectId(newRef.getObjectId()); | |||||
} | |||||
}.setReplace(false)); | |||||
if (newRef.getPeeledObjectId() != null) { | |||||
ed.add(new PathEdit(peeledPath(newRef.getName())) { | |||||
@Override | |||||
public void apply(DirCacheEntry ent) { | |||||
ent.setFileMode(GITLINK); | |||||
ent.setObjectId(newRef.getPeeledObjectId()); | |||||
} | |||||
}.setReplace(false)); | |||||
} else { | |||||
cleanupPeeledRef(ed, oldRef); | |||||
} | |||||
} | |||||
private static void checkRef(@Nullable DirCacheEntry ent, Command cmd) { | |||||
if (!cmd.checkRef(ent)) { | |||||
cmd.setResult(LOCK_FAILURE); | |||||
throw new LockFailureException(); | |||||
} | |||||
} | |||||
private static void cleanupPeeledRef(DirCacheEditor ed, Ref ref) { | |||||
if (ref != null && !ref.isSymbolic() | |||||
&& (!ref.isPeeled() || ref.getPeeledObjectId() != null)) { | |||||
ed.add(new DeletePath(peeledPath(ref.getName()))); | |||||
} | |||||
} | |||||
/** | |||||
* Convert a path name in a RefTree to the reference name known by Git. | |||||
* | |||||
* @param path | |||||
* name read from the RefTree structure, for example | |||||
* {@code "heads/master"}. | |||||
* @return reference name for the path, {@code "refs/heads/master"}. | |||||
*/ | |||||
public static String refName(String path) { | |||||
if (path.startsWith(ROOT_DOTDOT)) { | |||||
return path.substring(2); | |||||
} | |||||
return R_REFS + path; | |||||
} | |||||
static String refPath(String name) { | |||||
if (name.startsWith(R_REFS)) { | |||||
return name.substring(R_REFS.length()); | |||||
} | |||||
return ROOT_DOTDOT + name; | |||||
} | |||||
private static String peeledPath(String name) { | |||||
return refPath(name) + PEELED_SUFFIX; | |||||
} | |||||
/** | |||||
* Write this reference tree. | |||||
* | |||||
* @param inserter | |||||
* inserter to use when writing trees to the object database. | |||||
* Caller is responsible for flushing the inserter before trying | |||||
* to read the objects, or exposing them through a reference. | |||||
* @return the top level tree. | |||||
* @throws java.io.IOException | |||||
* a tree could not be written. | |||||
*/ | |||||
public ObjectId writeTree(ObjectInserter inserter) throws IOException { | |||||
if (pendingBlobs != null) { | |||||
for (String s : pendingBlobs.values()) { | |||||
inserter.insert(OBJ_BLOB, encode(s)); | |||||
} | |||||
pendingBlobs = null; | |||||
} | |||||
return contents.writeTree(inserter); | |||||
} | |||||
/** | |||||
* Create a deep copy of this RefTree. | |||||
* | |||||
* @return a deep copy of this RefTree. | |||||
*/ | |||||
public RefTree copy() { | |||||
RefTree r = new RefTree(DirCache.newInCore()); | |||||
DirCacheBuilder b = r.contents.builder(); | |||||
for (int i = 0; i < contents.getEntryCount(); i++) { | |||||
b.add(new DirCacheEntry(contents.getEntry(i))); | |||||
} | |||||
b.finish(); | |||||
if (pendingBlobs != null) { | |||||
r.pendingBlobs = new HashMap<>(pendingBlobs); | |||||
} | |||||
return r; | |||||
} | |||||
private static class LockFailureException extends RuntimeException { | |||||
private static final long serialVersionUID = 1L; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.OBJ_TREE; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_OTHER_REASON; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE; | |||||
import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD; | |||||
import java.io.IOException; | |||||
import java.text.MessageFormat; | |||||
import java.util.ArrayList; | |||||
import java.util.List; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.internal.JGitText; | |||||
import org.eclipse.jgit.lib.BatchRefUpdate; | |||||
import org.eclipse.jgit.lib.CommitBuilder; | |||||
import org.eclipse.jgit.lib.NullProgressMonitor; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectInserter; | |||||
import org.eclipse.jgit.lib.ObjectReader; | |||||
import org.eclipse.jgit.lib.PersonIdent; | |||||
import org.eclipse.jgit.lib.ProgressMonitor; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.revwalk.RevCommit; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
/** Batch update a {@link RefTreeDatabase}. */ | |||||
class RefTreeBatch extends BatchRefUpdate { | |||||
private final RefTreeDatabase refdb; | |||||
private Ref src; | |||||
private ObjectId parentCommitId; | |||||
private ObjectId parentTreeId; | |||||
private RefTree tree; | |||||
private PersonIdent author; | |||||
private ObjectId newCommitId; | |||||
RefTreeBatch(RefTreeDatabase refdb) { | |||||
super(refdb); | |||||
this.refdb = refdb; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public void execute(RevWalk rw, ProgressMonitor monitor) | |||||
throws IOException { | |||||
List<Command> todo = new ArrayList<>(getCommands().size()); | |||||
for (ReceiveCommand c : getCommands()) { | |||||
if (!isAllowNonFastForwards()) { | |||||
if (c.getType() == UPDATE) { | |||||
c.updateType(rw); | |||||
} | |||||
if (c.getType() == UPDATE_NONFASTFORWARD) { | |||||
c.setResult(REJECTED_NONFASTFORWARD); | |||||
if (isAtomic()) { | |||||
ReceiveCommand.abort(getCommands()); | |||||
return; | |||||
} | |||||
continue; | |||||
} | |||||
} | |||||
todo.add(new Command(rw, c)); | |||||
} | |||||
init(rw); | |||||
execute(rw, todo); | |||||
} | |||||
void init(RevWalk rw) throws IOException { | |||||
src = refdb.getBootstrap().exactRef(refdb.getTxnCommitted()); | |||||
if (src != null && src.getObjectId() != null) { | |||||
RevCommit c = rw.parseCommit(src.getObjectId()); | |||||
parentCommitId = c; | |||||
parentTreeId = c.getTree(); | |||||
tree = RefTree.read(rw.getObjectReader(), c.getTree()); | |||||
} else { | |||||
parentCommitId = ObjectId.zeroId(); | |||||
try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) { | |||||
parentTreeId = fmt.idFor(OBJ_TREE, new byte[] {}); | |||||
} | |||||
tree = RefTree.newEmptyTree(); | |||||
} | |||||
} | |||||
@Nullable | |||||
Ref exactRef(ObjectReader reader, String name) throws IOException { | |||||
return tree.exactRef(reader, name); | |||||
} | |||||
/** | |||||
* Execute an update from {@link RefTreeUpdate} or {@link RefTreeRename}. | |||||
* | |||||
* @param rw | |||||
* current RevWalk handling the update or rename. | |||||
* @param todo | |||||
* commands to execute. Must never be a bootstrap reference name. | |||||
* @throws IOException | |||||
* the storage system is unable to read or write data. | |||||
*/ | |||||
void execute(RevWalk rw, List<Command> todo) throws IOException { | |||||
for (Command c : todo) { | |||||
if (c.getResult() != NOT_ATTEMPTED) { | |||||
Command.abort(todo, null); | |||||
return; | |||||
} | |||||
if (refdb.conflictsWithBootstrap(c.getRefName())) { | |||||
c.setResult(REJECTED_OTHER_REASON, MessageFormat | |||||
.format(JGitText.get().invalidRefName, c.getRefName())); | |||||
Command.abort(todo, null); | |||||
return; | |||||
} | |||||
} | |||||
if (apply(todo) && newCommitId != null) { | |||||
commit(rw, todo); | |||||
} | |||||
} | |||||
private boolean apply(List<Command> todo) throws IOException { | |||||
if (!tree.apply(todo)) { | |||||
// apply set rejection information on commands. | |||||
return false; | |||||
} | |||||
Repository repo = refdb.getRepository(); | |||||
try (ObjectInserter ins = repo.newObjectInserter()) { | |||||
CommitBuilder b = new CommitBuilder(); | |||||
b.setTreeId(tree.writeTree(ins)); | |||||
if (parentTreeId.equals(b.getTreeId())) { | |||||
for (Command c : todo) { | |||||
c.setResult(OK); | |||||
} | |||||
return true; | |||||
} | |||||
if (!parentCommitId.equals(ObjectId.zeroId())) { | |||||
b.setParentId(parentCommitId); | |||||
} | |||||
author = getRefLogIdent(); | |||||
if (author == null) { | |||||
author = new PersonIdent(repo); | |||||
} | |||||
b.setAuthor(author); | |||||
b.setCommitter(author); | |||||
b.setMessage(getRefLogMessage()); | |||||
newCommitId = ins.insert(b); | |||||
ins.flush(); | |||||
} | |||||
return true; | |||||
} | |||||
private void commit(RevWalk rw, List<Command> todo) throws IOException { | |||||
ReceiveCommand commit = new ReceiveCommand( | |||||
parentCommitId, newCommitId, | |||||
refdb.getTxnCommitted()); | |||||
updateBootstrap(rw, commit); | |||||
if (commit.getResult() == OK) { | |||||
for (Command c : todo) { | |||||
c.setResult(OK); | |||||
} | |||||
} else { | |||||
Command.abort(todo, commit.getResult().name()); | |||||
} | |||||
} | |||||
private void updateBootstrap(RevWalk rw, ReceiveCommand commit) | |||||
throws IOException { | |||||
BatchRefUpdate u = refdb.getBootstrap().newBatchUpdate(); | |||||
u.setAllowNonFastForwards(true); | |||||
u.setPushCertificate(getPushCertificate()); | |||||
if (isRefLogDisabled()) { | |||||
u.disableRefLog(); | |||||
} else { | |||||
u.setRefLogIdent(author); | |||||
u.setRefLogMessage(getRefLogMessage(), false); | |||||
} | |||||
u.addCommand(commit); | |||||
u.execute(rw, NullProgressMonitor.INSTANCE); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.PACKED; | |||||
import java.io.IOException; | |||||
import java.util.ArrayList; | |||||
import java.util.Collection; | |||||
import java.util.Collections; | |||||
import java.util.HashMap; | |||||
import java.util.List; | |||||
import java.util.Map; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.lib.BatchRefUpdate; | |||||
import org.eclipse.jgit.lib.Config; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Ref.Storage; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
import org.eclipse.jgit.lib.RefRename; | |||||
import org.eclipse.jgit.lib.RefUpdate; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevObject; | |||||
import org.eclipse.jgit.revwalk.RevTag; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.util.RefList; | |||||
import org.eclipse.jgit.util.RefMap; | |||||
/** | |||||
* Reference database backed by a | |||||
* {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. | |||||
* <p> | |||||
* The storage for RefTreeDatabase has two parts. The main part is a native Git | |||||
* tree object stored under the {@code refs/txn} namespace. To avoid cycles, | |||||
* references to {@code refs/txn} are not stored in that tree object, but | |||||
* instead in a "bootstrap" layer, which is a separate | |||||
* {@link org.eclipse.jgit.lib.RefDatabase} such as | |||||
* {@link org.eclipse.jgit.internal.storage.file.RefDirectory} using local | |||||
* reference files inside of {@code $GIT_DIR/refs}. | |||||
*/ | |||||
public class RefTreeDatabase extends RefDatabase { | |||||
private final Repository repo; | |||||
private final RefDatabase bootstrap; | |||||
private final String txnCommitted; | |||||
@Nullable | |||||
private final String txnNamespace; | |||||
private volatile Scanner.Result refs; | |||||
/** | |||||
* Create a RefTreeDb for a repository. | |||||
* | |||||
* @param repo | |||||
* the repository using references in this database. | |||||
* @param bootstrap | |||||
* bootstrap reference database storing the references that | |||||
* anchor the | |||||
* {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. | |||||
*/ | |||||
public RefTreeDatabase(Repository repo, RefDatabase bootstrap) { | |||||
Config cfg = repo.getConfig(); | |||||
String committed = cfg.getString("reftree", null, "committedRef"); //$NON-NLS-1$ //$NON-NLS-2$ | |||||
if (committed == null || committed.isEmpty()) { | |||||
committed = "refs/txn/committed"; //$NON-NLS-1$ | |||||
} | |||||
this.repo = repo; | |||||
this.bootstrap = bootstrap; | |||||
this.txnNamespace = initNamespace(committed); | |||||
this.txnCommitted = committed; | |||||
} | |||||
/** | |||||
* Create a RefTreeDb for a repository. | |||||
* | |||||
* @param repo | |||||
* the repository using references in this database. | |||||
* @param bootstrap | |||||
* bootstrap reference database storing the references that | |||||
* anchor the | |||||
* {@link org.eclipse.jgit.internal.storage.reftree.RefTree}. | |||||
* @param txnCommitted | |||||
* name of the bootstrap reference holding the committed RefTree. | |||||
*/ | |||||
public RefTreeDatabase(Repository repo, RefDatabase bootstrap, | |||||
String txnCommitted) { | |||||
this.repo = repo; | |||||
this.bootstrap = bootstrap; | |||||
this.txnNamespace = initNamespace(txnCommitted); | |||||
this.txnCommitted = txnCommitted; | |||||
} | |||||
private static String initNamespace(String committed) { | |||||
int s = committed.lastIndexOf('/'); | |||||
if (s < 0) { | |||||
return null; | |||||
} | |||||
return committed.substring(0, s + 1); // Keep trailing '/'. | |||||
} | |||||
Repository getRepository() { | |||||
return repo; | |||||
} | |||||
/** | |||||
* Get the bootstrap reference database | |||||
* | |||||
* @return the bootstrap reference database, which must be used to access | |||||
* {@link #getTxnCommitted()}, {@link #getTxnNamespace()}. | |||||
*/ | |||||
public RefDatabase getBootstrap() { | |||||
return bootstrap; | |||||
} | |||||
/** | |||||
* Get name of bootstrap reference anchoring committed RefTree. | |||||
* | |||||
* @return name of bootstrap reference anchoring committed RefTree. | |||||
*/ | |||||
public String getTxnCommitted() { | |||||
return txnCommitted; | |||||
} | |||||
/** | |||||
* Get namespace used by bootstrap layer. | |||||
* | |||||
* @return namespace used by bootstrap layer, e.g. {@code refs/txn/}. Always | |||||
* ends in {@code '/'}. | |||||
*/ | |||||
@Nullable | |||||
public String getTxnNamespace() { | |||||
return txnNamespace; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public void create() throws IOException { | |||||
bootstrap.create(); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public boolean performsAtomicTransactions() { | |||||
return true; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public void refresh() { | |||||
bootstrap.refresh(); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public void close() { | |||||
refs = null; | |||||
bootstrap.close(); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public Ref exactRef(String name) throws IOException { | |||||
if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { | |||||
// Pass through names like MERGE_HEAD, ORIG_HEAD, FETCH_HEAD. | |||||
return bootstrap.exactRef(name); | |||||
} else if (conflictsWithBootstrap(name)) { | |||||
return null; | |||||
} | |||||
boolean partial = false; | |||||
Ref src = bootstrap.exactRef(txnCommitted); | |||||
Scanner.Result c = refs; | |||||
if (c == null || !c.refTreeId.equals(idOf(src))) { | |||||
c = Scanner.scanRefTree(repo, src, prefixOf(name), false); | |||||
partial = true; | |||||
} | |||||
Ref r = c.all.get(name); | |||||
if (r != null && r.isSymbolic()) { | |||||
r = c.sym.get(name); | |||||
if (partial && r.getObjectId() == null) { | |||||
// Attempting exactRef("HEAD") with partial scan will leave | |||||
// an unresolved symref as its target e.g. refs/heads/master | |||||
// was not read by the partial scan. Scan everything instead. | |||||
return getRefs(ALL).get(name); | |||||
} | |||||
} | |||||
return r; | |||||
} | |||||
private static String prefixOf(String name) { | |||||
int s = name.lastIndexOf('/'); | |||||
if (s >= 0) { | |||||
return name.substring(0, s); | |||||
} | |||||
return ""; //$NON-NLS-1$ | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public Map<String, Ref> getRefs(String prefix) throws IOException { | |||||
if (!prefix.isEmpty() && prefix.charAt(prefix.length() - 1) != '/') { | |||||
return new HashMap<>(0); | |||||
} | |||||
Ref src = bootstrap.exactRef(txnCommitted); | |||||
Scanner.Result c = refs; | |||||
if (c == null || !c.refTreeId.equals(idOf(src))) { | |||||
c = Scanner.scanRefTree(repo, src, prefix, true); | |||||
if (prefix.isEmpty()) { | |||||
refs = c; | |||||
} | |||||
} | |||||
return new RefMap(prefix, RefList.<Ref> emptyList(), c.all, c.sym); | |||||
} | |||||
private static ObjectId idOf(@Nullable Ref src) { | |||||
return src != null && src.getObjectId() != null | |||||
? src.getObjectId() | |||||
: ObjectId.zeroId(); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public List<Ref> getAdditionalRefs() throws IOException { | |||||
Collection<Ref> txnRefs; | |||||
if (txnNamespace != null) { | |||||
txnRefs = bootstrap.getRefsByPrefix(txnNamespace); | |||||
} else { | |||||
Ref r = bootstrap.exactRef(txnCommitted); | |||||
if (r != null && r.getObjectId() != null) { | |||||
txnRefs = Collections.singleton(r); | |||||
} else { | |||||
txnRefs = Collections.emptyList(); | |||||
} | |||||
} | |||||
List<Ref> otherRefs = bootstrap.getAdditionalRefs(); | |||||
List<Ref> all = new ArrayList<>(txnRefs.size() + otherRefs.size()); | |||||
all.addAll(txnRefs); | |||||
all.addAll(otherRefs); | |||||
return all; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public Ref peel(Ref ref) throws IOException { | |||||
Ref i = ref.getLeaf(); | |||||
ObjectId id = i.getObjectId(); | |||||
if (i.isPeeled() || id == null) { | |||||
return ref; | |||||
} | |||||
try (RevWalk rw = new RevWalk(repo)) { | |||||
RevObject obj = rw.parseAny(id); | |||||
if (obj instanceof RevTag) { | |||||
ObjectId p = rw.peel(obj).copy(); | |||||
i = new ObjectIdRef.PeeledTag(PACKED, i.getName(), id, p); | |||||
} else { | |||||
i = new ObjectIdRef.PeeledNonTag(PACKED, i.getName(), id); | |||||
} | |||||
} | |||||
return recreate(ref, i); | |||||
} | |||||
private static Ref recreate(Ref old, Ref leaf) { | |||||
if (old.isSymbolic()) { | |||||
Ref dst = recreate(old.getTarget(), leaf); | |||||
return new SymbolicRef(old.getName(), dst); | |||||
} | |||||
return leaf; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public boolean isNameConflicting(String name) throws IOException { | |||||
return conflictsWithBootstrap(name) | |||||
|| !getConflictingNames(name).isEmpty(); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public BatchRefUpdate newBatchUpdate() { | |||||
return new RefTreeBatch(this); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public RefUpdate newUpdate(String name, boolean detach) throws IOException { | |||||
if (!repo.isBare() && name.indexOf('/') < 0 && !HEAD.equals(name)) { | |||||
return bootstrap.newUpdate(name, detach); | |||||
} | |||||
if (conflictsWithBootstrap(name)) { | |||||
return new AlwaysFailUpdate(this, name); | |||||
} | |||||
Ref r = exactRef(name); | |||||
if (r == null) { | |||||
r = new ObjectIdRef.Unpeeled(Storage.NEW, name, null); | |||||
} | |||||
boolean detaching = detach && r.isSymbolic(); | |||||
if (detaching) { | |||||
r = new ObjectIdRef.Unpeeled(LOOSE, name, r.getObjectId()); | |||||
} | |||||
RefTreeUpdate u = new RefTreeUpdate(this, r); | |||||
if (detaching) { | |||||
u.setDetachingSymbolicRef(); | |||||
} | |||||
return u; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
public RefRename newRename(String fromName, String toName) | |||||
throws IOException { | |||||
RefUpdate from = newUpdate(fromName, true); | |||||
RefUpdate to = newUpdate(toName, true); | |||||
return new RefTreeRename(this, from, to); | |||||
} | |||||
boolean conflictsWithBootstrap(String name) { | |||||
if (txnNamespace != null && name.startsWith(txnNamespace)) { | |||||
return true; | |||||
} else if (txnCommitted.equals(name)) { | |||||
return true; | |||||
} | |||||
if (name.indexOf('/') < 0 && !HEAD.equals(name)) { | |||||
return true; | |||||
} | |||||
if (name.length() > txnCommitted.length() | |||||
&& name.charAt(txnCommitted.length()) == '/' | |||||
&& name.startsWith(txnCommitted)) { | |||||
return true; | |||||
} | |||||
return false; | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
/** | |||||
* Magic reference name logic for RefTrees. | |||||
*/ | |||||
public class RefTreeNames { | |||||
/** | |||||
* Suffix used on a {@link RefTreeDatabase#getTxnNamespace()} for user data. | |||||
* <p> | |||||
* A {@link RefTreeDatabase}'s namespace may include a subspace (e.g. | |||||
* {@code "refs/txn/stage/"}) containing commit objects from the usual user | |||||
* portion of the repository (e.g. {@code "refs/heads/"}). These should be | |||||
* packed by the garbage collector alongside other user content rather than | |||||
* with the RefTree. | |||||
*/ | |||||
private static final String STAGE = "stage/"; //$NON-NLS-1$ | |||||
/** | |||||
* Determine if the reference is likely to be a RefTree. | |||||
* | |||||
* @param refdb | |||||
* database instance. | |||||
* @param ref | |||||
* reference name. | |||||
* @return {@code true} if the reference is a RefTree. | |||||
*/ | |||||
public static boolean isRefTree(RefDatabase refdb, String ref) { | |||||
if (refdb instanceof RefTreeDatabase) { | |||||
RefTreeDatabase b = (RefTreeDatabase) refdb; | |||||
if (ref.equals(b.getTxnCommitted())) { | |||||
return true; | |||||
} | |||||
String namespace = b.getTxnNamespace(); | |||||
if (namespace != null | |||||
&& ref.startsWith(namespace) | |||||
&& !ref.startsWith(namespace + STAGE)) { | |||||
return true; | |||||
} | |||||
} | |||||
return false; | |||||
} | |||||
private RefTreeNames() { | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.HEAD; | |||||
import static org.eclipse.jgit.lib.RefUpdate.Result.REJECTED; | |||||
import static org.eclipse.jgit.lib.RefUpdate.Result.RENAMED; | |||||
import java.io.IOException; | |||||
import java.util.ArrayList; | |||||
import java.util.List; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.RefRename; | |||||
import org.eclipse.jgit.lib.RefUpdate; | |||||
import org.eclipse.jgit.lib.RefUpdate.Result; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
/** Single reference rename to {@link RefTreeDatabase}. */ | |||||
class RefTreeRename extends RefRename { | |||||
private final RefTreeDatabase refdb; | |||||
RefTreeRename(RefTreeDatabase refdb, RefUpdate src, RefUpdate dst) { | |||||
super(src, dst); | |||||
this.refdb = refdb; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Result doRename() throws IOException { | |||||
try (RevWalk rw = new RevWalk(refdb.getRepository())) { | |||||
RefTreeBatch batch = new RefTreeBatch(refdb); | |||||
batch.setRefLogIdent(getRefLogIdent()); | |||||
batch.setRefLogMessage(getRefLogMessage(), false); | |||||
batch.init(rw); | |||||
Ref head = batch.exactRef(rw.getObjectReader(), HEAD); | |||||
Ref oldRef = batch.exactRef(rw.getObjectReader(), source.getName()); | |||||
if (oldRef == null) { | |||||
return REJECTED; | |||||
} | |||||
Ref newRef = asNew(oldRef); | |||||
List<Command> mv = new ArrayList<>(3); | |||||
mv.add(new Command(oldRef, null)); | |||||
mv.add(new Command(null, newRef)); | |||||
if (head != null && head.isSymbolic() | |||||
&& head.getTarget().getName().equals(oldRef.getName())) { | |||||
mv.add(new Command( | |||||
head, | |||||
new SymbolicRef(head.getName(), newRef))); | |||||
} | |||||
batch.execute(rw, mv); | |||||
return RefTreeUpdate.translate(mv.get(1).getResult(), RENAMED); | |||||
} | |||||
} | |||||
private Ref asNew(Ref src) { | |||||
String name = destination.getName(); | |||||
if (src.isSymbolic()) { | |||||
return new SymbolicRef(name, src.getTarget()); | |||||
} | |||||
ObjectId peeled = src.getPeeledObjectId(); | |||||
if (peeled != null) { | |||||
return new ObjectIdRef.PeeledTag( | |||||
src.getStorage(), | |||||
name, | |||||
src.getObjectId(), | |||||
peeled); | |||||
} | |||||
return new ObjectIdRef.PeeledNonTag( | |||||
src.getStorage(), | |||||
name, | |||||
src.getObjectId()); | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.LOOSE; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.NEW; | |||||
import java.io.IOException; | |||||
import java.util.Collections; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.errors.MissingObjectException; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.RefDatabase; | |||||
import org.eclipse.jgit.lib.RefUpdate; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevObject; | |||||
import org.eclipse.jgit.revwalk.RevTag; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.transport.ReceiveCommand; | |||||
/** Single reference update to {@link RefTreeDatabase}. */ | |||||
class RefTreeUpdate extends RefUpdate { | |||||
private final RefTreeDatabase refdb; | |||||
private RevWalk rw; | |||||
private RefTreeBatch batch; | |||||
private Ref oldRef; | |||||
RefTreeUpdate(RefTreeDatabase refdb, Ref ref) { | |||||
super(ref); | |||||
this.refdb = refdb; | |||||
setCheckConflicting(false); // Done automatically by doUpdate. | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected RefDatabase getRefDatabase() { | |||||
return refdb; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Repository getRepository() { | |||||
return refdb.getRepository(); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected boolean tryLock(boolean deref) throws IOException { | |||||
rw = new RevWalk(getRepository()); | |||||
batch = new RefTreeBatch(refdb); | |||||
batch.init(rw); | |||||
oldRef = batch.exactRef(rw.getObjectReader(), getName()); | |||||
if (oldRef != null && oldRef.getObjectId() != null) { | |||||
setOldObjectId(oldRef.getObjectId()); | |||||
} else if (oldRef == null && getExpectedOldObjectId() != null) { | |||||
setOldObjectId(ObjectId.zeroId()); | |||||
} | |||||
return true; | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected void unlock() { | |||||
batch = null; | |||||
if (rw != null) { | |||||
rw.close(); | |||||
rw = null; | |||||
} | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Result doUpdate(Result desiredResult) throws IOException { | |||||
return run(newRef(getName(), getNewObjectId()), desiredResult); | |||||
} | |||||
private Ref newRef(String name, ObjectId id) | |||||
throws MissingObjectException, IOException { | |||||
RevObject o = rw.parseAny(id); | |||||
if (o instanceof RevTag) { | |||||
RevObject p = rw.peel(o); | |||||
return new ObjectIdRef.PeeledTag(LOOSE, name, id, p.copy()); | |||||
} | |||||
return new ObjectIdRef.PeeledNonTag(LOOSE, name, id); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Result doDelete(Result desiredResult) throws IOException { | |||||
return run(null, desiredResult); | |||||
} | |||||
/** {@inheritDoc} */ | |||||
@Override | |||||
protected Result doLink(String target) throws IOException { | |||||
Ref dst = new ObjectIdRef.Unpeeled(NEW, target, null); | |||||
SymbolicRef n = new SymbolicRef(getName(), dst); | |||||
Result desiredResult = getRef().getStorage() == NEW | |||||
? Result.NEW | |||||
: Result.FORCED; | |||||
return run(n, desiredResult); | |||||
} | |||||
private Result run(@Nullable Ref newRef, Result desiredResult) | |||||
throws IOException { | |||||
Command c = new Command(oldRef, newRef); | |||||
batch.setRefLogIdent(getRefLogIdent()); | |||||
batch.setRefLogMessage(getRefLogMessage(), isRefLogIncludingResult()); | |||||
batch.execute(rw, Collections.singletonList(c)); | |||||
return translate(c.getResult(), desiredResult); | |||||
} | |||||
static Result translate(ReceiveCommand.Result r, Result desiredResult) { | |||||
switch (r) { | |||||
case OK: | |||||
return desiredResult; | |||||
case LOCK_FAILURE: | |||||
return Result.LOCK_FAILURE; | |||||
case NOT_ATTEMPTED: | |||||
return Result.NOT_ATTEMPTED; | |||||
case REJECTED_MISSING_OBJECT: | |||||
return Result.IO_FAILURE; | |||||
case REJECTED_CURRENT_BRANCH: | |||||
return Result.REJECTED_CURRENT_BRANCH; | |||||
case REJECTED_OTHER_REASON: | |||||
case REJECTED_NOCREATE: | |||||
case REJECTED_NODELETE: | |||||
case REJECTED_NONFASTFORWARD: | |||||
default: | |||||
return Result.REJECTED; | |||||
} | |||||
} | |||||
} |
/* | |||||
* Copyright (C) 2016, Google Inc. and others | |||||
* | |||||
* This program and the accompanying materials are made available under the | |||||
* terms of the Eclipse Distribution License v. 1.0 which is available at | |||||
* https://www.eclipse.org/org/documents/edl-v10.php. | |||||
* | |||||
* SPDX-License-Identifier: BSD-3-Clause | |||||
*/ | |||||
package org.eclipse.jgit.internal.storage.reftree; | |||||
import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; | |||||
import static org.eclipse.jgit.lib.Constants.R_REFS; | |||||
import static org.eclipse.jgit.lib.Constants.encode; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_GITLINK; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_SYMLINK; | |||||
import static org.eclipse.jgit.lib.FileMode.TYPE_TREE; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.NEW; | |||||
import static org.eclipse.jgit.lib.Ref.Storage.PACKED; | |||||
import static org.eclipse.jgit.lib.RefDatabase.MAX_SYMBOLIC_REF_DEPTH; | |||||
import java.io.IOException; | |||||
import org.eclipse.jgit.annotations.Nullable; | |||||
import org.eclipse.jgit.errors.IncorrectObjectTypeException; | |||||
import org.eclipse.jgit.lib.AnyObjectId; | |||||
import org.eclipse.jgit.lib.ObjectId; | |||||
import org.eclipse.jgit.lib.ObjectIdRef; | |||||
import org.eclipse.jgit.lib.ObjectReader; | |||||
import org.eclipse.jgit.lib.Ref; | |||||
import org.eclipse.jgit.lib.Repository; | |||||
import org.eclipse.jgit.lib.SymbolicRef; | |||||
import org.eclipse.jgit.revwalk.RevTree; | |||||
import org.eclipse.jgit.revwalk.RevWalk; | |||||
import org.eclipse.jgit.treewalk.AbstractTreeIterator; | |||||
import org.eclipse.jgit.treewalk.CanonicalTreeParser; | |||||
import org.eclipse.jgit.treewalk.TreeWalk; | |||||
import org.eclipse.jgit.util.Paths; | |||||
import org.eclipse.jgit.util.RawParseUtils; | |||||
import org.eclipse.jgit.util.RefList; | |||||
/** A tree parser that extracts references from a {@link RefTree}. */ | |||||
class Scanner { | |||||
private static final int MAX_SYMLINK_BYTES = 10 << 10; | |||||
private static final byte[] BINARY_R_REFS = encode(R_REFS); | |||||
private static final byte[] REFS_DOT_DOT = encode("refs/.."); //$NON-NLS-1$ | |||||
static class Result { | |||||
final ObjectId refTreeId; | |||||
final RefList<Ref> all; | |||||
final RefList<Ref> sym; | |||||
Result(ObjectId id, RefList<Ref> all, RefList<Ref> sym) { | |||||
this.refTreeId = id; | |||||
this.all = all; | |||||
this.sym = sym; | |||||
} | |||||
} | |||||
/** | |||||
* Scan a {@link RefTree} and parse entries into {@link Ref} instances. | |||||
* | |||||
* @param repo | |||||
* source repository containing the commit and tree objects that | |||||
* make up the RefTree. | |||||
* @param src | |||||
* bootstrap reference such as {@code refs/txn/committed} to read | |||||
* the reference tree tip from. The current ObjectId will be | |||||
* included in {@link Result#refTreeId}. | |||||
* @param prefix | |||||
* if non-empty a reference prefix to scan only a subdirectory. | |||||
* For example {@code prefix = "refs/heads/"} will limit the scan | |||||
* to only the {@code "heads"} directory of the RefTree, avoiding | |||||
* other directories like {@code "tags"}. Empty string reads all | |||||
* entries in the RefTree. | |||||
* @param recursive | |||||
* if true recurse into subdirectories of the reference tree; | |||||
* false to read only one level. Callers may use false during an | |||||
* implementation of {@code exactRef(String)} where only one | |||||
* reference is needed out of a specific subtree. | |||||
* @return sorted list of references after parsing. | |||||
* @throws IOException | |||||
* tree cannot be accessed from the repository. | |||||
*/ | |||||
static Result scanRefTree(Repository repo, @Nullable Ref src, String prefix, | |||||
boolean recursive) throws IOException { | |||||
RefList.Builder<Ref> all = new RefList.Builder<>(); | |||||
RefList.Builder<Ref> sym = new RefList.Builder<>(); | |||||
ObjectId srcId; | |||||
if (src != null && src.getObjectId() != null) { | |||||
try (ObjectReader reader = repo.newObjectReader()) { | |||||
srcId = src.getObjectId(); | |||||
scan(reader, srcId, prefix, recursive, all, sym); | |||||
} | |||||
} else { | |||||
srcId = ObjectId.zeroId(); | |||||
} | |||||
RefList<Ref> aList = all.toRefList(); | |||||
for (int idx = 0; idx < sym.size();) { | |||||
Ref s = sym.get(idx); | |||||
Ref r = resolve(s, 0, aList); | |||||
if (r != null) { | |||||
sym.set(idx++, r); | |||||
} else { | |||||
// Remove broken symbolic reference, they don't exist. | |||||
sym.remove(idx); | |||||
int rm = aList.find(s.getName()); | |||||
if (0 <= rm) { | |||||
aList = aList.remove(rm); | |||||
} | |||||
} | |||||
} | |||||
return new Result(srcId, aList, sym.toRefList()); | |||||
} | |||||
private static void scan(ObjectReader reader, AnyObjectId srcId, | |||||
String prefix, boolean recursive, | |||||
RefList.Builder<Ref> all, RefList.Builder<Ref> sym) | |||||
throws IncorrectObjectTypeException, IOException { | |||||
CanonicalTreeParser p = createParserAtPath(reader, srcId, prefix); | |||||
if (p == null) { | |||||
return; | |||||
} | |||||
while (!p.eof()) { | |||||
int mode = p.getEntryRawMode(); | |||||
if (mode == TYPE_TREE) { | |||||
if (recursive) { | |||||
p = p.createSubtreeIterator(reader); | |||||
} else { | |||||
p = p.next(); | |||||
} | |||||
continue; | |||||
} | |||||
if (!curElementHasPeelSuffix(p)) { | |||||
Ref r = toRef(reader, mode, p); | |||||
if (r != null) { | |||||
all.add(r); | |||||
if (r.isSymbolic()) { | |||||
sym.add(r); | |||||
} | |||||
} | |||||
} else if (mode == TYPE_GITLINK) { | |||||
peel(all, p); | |||||
} | |||||
p = p.next(); | |||||
} | |||||
} | |||||
private static CanonicalTreeParser createParserAtPath(ObjectReader reader, | |||||
AnyObjectId srcId, String prefix) throws IOException { | |||||
ObjectId root = toTree(reader, srcId); | |||||
if (prefix.isEmpty()) { | |||||
return new CanonicalTreeParser(BINARY_R_REFS, reader, root); | |||||
} | |||||
String dir = RefTree.refPath(Paths.stripTrailingSeparator(prefix)); | |||||
TreeWalk tw = TreeWalk.forPath(reader, dir, root); | |||||
if (tw == null || !tw.isSubtree()) { | |||||
return null; | |||||
} | |||||
ObjectId id = tw.getObjectId(0); | |||||
return new CanonicalTreeParser(encode(prefix), reader, id); | |||||
} | |||||
private static Ref resolve(Ref ref, int depth, RefList<Ref> refs) | |||||
throws IOException { | |||||
if (!ref.isSymbolic()) { | |||||
return ref; | |||||
} else if (MAX_SYMBOLIC_REF_DEPTH <= depth) { | |||||
return null; | |||||
} | |||||
Ref r = refs.get(ref.getTarget().getName()); | |||||
if (r == null) { | |||||
return ref; | |||||
} | |||||
Ref dst = resolve(r, depth + 1, refs); | |||||
if (dst == null) { | |||||
return null; | |||||
} | |||||
return new SymbolicRef(ref.getName(), dst); | |||||
} | |||||
private static RevTree toTree(ObjectReader reader, AnyObjectId id) | |||||
throws IOException { | |||||
try (RevWalk rw = new RevWalk(reader)) { | |||||
return rw.parseTree(id); | |||||
} | |||||
} | |||||
private static boolean curElementHasPeelSuffix(AbstractTreeIterator itr) { | |||||
int n = itr.getEntryPathLength(); | |||||
byte[] c = itr.getEntryPathBuffer(); | |||||
return n > 2 && c[n - 2] == ' ' && c[n - 1] == '^'; | |||||
} | |||||
private static void peel(RefList.Builder<Ref> all, CanonicalTreeParser p) { | |||||
String name = refName(p, true); | |||||
for (int idx = all.size() - 1; 0 <= idx; idx--) { | |||||
Ref r = all.get(idx); | |||||
int cmp = r.getName().compareTo(name); | |||||
if (cmp == 0) { | |||||
all.set(idx, new ObjectIdRef.PeeledTag(PACKED, r.getName(), | |||||
r.getObjectId(), p.getEntryObjectId())); | |||||
break; | |||||
} else if (cmp < 0) { | |||||
// Stray peeled name without matching base name; skip entry. | |||||
break; | |||||
} | |||||
} | |||||
} | |||||
private static Ref toRef(ObjectReader reader, int mode, | |||||
CanonicalTreeParser p) throws IOException { | |||||
if (mode == TYPE_GITLINK) { | |||||
String name = refName(p, false); | |||||
ObjectId id = p.getEntryObjectId(); | |||||
return new ObjectIdRef.PeeledNonTag(PACKED, name, id); | |||||
} else if (mode == TYPE_SYMLINK) { | |||||
ObjectId id = p.getEntryObjectId(); | |||||
byte[] bin = reader.open(id, OBJ_BLOB) | |||||
.getCachedBytes(MAX_SYMLINK_BYTES); | |||||
String dst = RawParseUtils.decode(bin); | |||||
Ref trg = new ObjectIdRef.Unpeeled(NEW, dst, null); | |||||
String name = refName(p, false); | |||||
return new SymbolicRef(name, trg); | |||||
} | |||||
return null; | |||||
} | |||||
private static String refName(CanonicalTreeParser p, boolean peel) { | |||||
byte[] buf = p.getEntryPathBuffer(); | |||||
int len = p.getEntryPathLength(); | |||||
if (peel) { | |||||
len -= 2; | |||||
} | |||||
int ptr = 0; | |||||
if (RawParseUtils.match(buf, ptr, REFS_DOT_DOT) > 0) { | |||||
ptr = 7; | |||||
} | |||||
return RawParseUtils.decode(buf, ptr, len); | |||||
} | |||||
private Scanner() { | |||||
} | |||||
} |
*/ | */ | ||||
public static final String CONFIG_REF_STORAGE_REFTABLE = "reftable"; | public static final String CONFIG_REF_STORAGE_REFTABLE = "reftable"; | ||||
/** | |||||
* The "reftree" refStorage format | |||||
* @since 5.7 | |||||
*/ | |||||
public static final String CONFIG_REFSTORAGE_REFTREE = "reftree"; | |||||
/** | /** | ||||
* The "jmx" section | * The "jmx" section | ||||
* @since 5.1.13 | * @since 5.1.13 |