From c44dd099a432094a12131cf60dfc8a19f5aa8101 Mon Sep 17 00:00:00 2001 From: James Moger Date: Wed, 13 Nov 2013 17:56:50 -0500 Subject: Implement mirror executor (issue-5) The mirror executor will fetch ref updates for repository mirrors. This feature is disabled by default and can be enabled by setting git.enableMirroring=true. The period between update checks is configurable, but it is global. An individual rpeository may not set it's own update schedule. Requirements: 1. you must manually clone the repository using native git git clone --mirror git://somewhere.com/myrepo.git 2. the "origin" remote must be the mirror source 3. the "origin" repository must be accessible without authentication OR the credentials must be embedded in the origin url (not recommended) Notes: 1. "origin" SSH urls are untested and not likely to work 2. mirrors cloned while Gitblit is running are likely to require clearing the gitblit cache (link on the repositories page of an administrator account) 3. Gitblit will automatically repair any invalid fetch refspecs with a "//" sequence. Change-Id: I4bbe3fb2df106366ae4c2313596d0fab0dfcac46 --- src/main/java/com/gitblit/GitBlit.java | 21 ++- src/main/java/com/gitblit/MirrorExecutor.java | 169 +++++++++++++++++++++ .../com/gitblit/client/IndicatorsRenderer.java | 8 + .../java/com/gitblit/git/GitblitReceivePack.java | 8 + .../java/com/gitblit/models/RepositoryModel.java | 1 + src/main/java/com/gitblit/models/TeamModel.java | 2 +- src/main/java/com/gitblit/models/UserModel.java | 2 +- src/main/java/com/gitblit/utils/JGitUtils.java | 50 ++++++ .../com/gitblit/wicket/GitBlitWebApp.properties | 5 +- .../com/gitblit/wicket/pages/RepositoryPage.html | 4 + .../com/gitblit/wicket/pages/RepositoryPage.java | 9 +- .../wicket/panels/ProjectRepositoryPanel.html | 1 + .../wicket/panels/ProjectRepositoryPanel.java | 6 + .../gitblit/wicket/panels/RepositoriesPanel.html | 2 +- .../gitblit/wicket/panels/RepositoriesPanel.java | 7 + 15 files changed, 289 insertions(+), 6 deletions(-) create mode 100644 src/main/java/com/gitblit/MirrorExecutor.java (limited to 'src/main/java') diff --git a/src/main/java/com/gitblit/GitBlit.java b/src/main/java/com/gitblit/GitBlit.java index f313b6e3..a0e8b0a3 100644 --- a/src/main/java/com/gitblit/GitBlit.java +++ b/src/main/java/com/gitblit/GitBlit.java @@ -164,7 +164,7 @@ public class GitBlit implements ServletContextListener { private final Logger logger = LoggerFactory.getLogger(GitBlit.class); - private final ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(5); + private final ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(10); private final List federationRegistrations = Collections .synchronizedList(new ArrayList()); @@ -207,6 +207,8 @@ public class GitBlit implements ServletContextListener { private GCExecutor gcExecutor; + private MirrorExecutor mirrorExecutor; + private TimeZone timezone; private FileBasedConfig projectConfigs; @@ -2035,6 +2037,7 @@ public class GitBlit implements ServletContextListener { model.origin = config.getString("remote", "origin", "url"); if (model.origin != null) { model.origin = model.origin.replace('\\', '/'); + model.isMirror = config.getBoolean("remote", "origin", "mirror", false); } model.preReceiveScripts = new ArrayList(Arrays.asList(config.getStringList( Constants.CONFIG_GITBLIT, null, "preReceiveScript"))); @@ -3505,6 +3508,7 @@ public class GitBlit implements ServletContextListener { mailExecutor = new MailExecutor(settings); luceneExecutor = new LuceneExecutor(settings, repositoriesFolder); gcExecutor = new GCExecutor(settings); + mirrorExecutor = new MirrorExecutor(settings); // initialize utilities String prefix = settings.getString(Keys.git.userRepositoryPrefix, "~"); @@ -3544,6 +3548,7 @@ public class GitBlit implements ServletContextListener { configureMailExecutor(); configureLuceneIndexing(); configureGarbageCollector(); + configureMirrorExecutor(); if (startFederation) { configureFederation(); } @@ -3595,6 +3600,19 @@ public class GitBlit implements ServletContextListener { } } + protected void configureMirrorExecutor() { + if (mirrorExecutor.isReady()) { + int mins = TimeUtils.convertFrequencyToMinutes(settings.getString(Keys.git.mirrorPeriod, "30 mins")); + if (mins < 5) { + mins = 5; + } + int delay = 1; + scheduledExecutor.scheduleAtFixedRate(mirrorExecutor, delay, mins, TimeUnit.MINUTES); + logger.info("Mirror executor is scheduled to fetch updates every {} minutes.", mins); + logger.info("Next scheduled mirror fetch is in {} minutes", delay); + } + } + protected void configureJGit() { // Configure JGit WindowCacheConfig cfg = new WindowCacheConfig(); @@ -3864,6 +3882,7 @@ public class GitBlit implements ServletContextListener { scheduledExecutor.shutdownNow(); luceneExecutor.close(); gcExecutor.close(); + mirrorExecutor.close(); if (fanoutService != null) { fanoutService.stop(); } diff --git a/src/main/java/com/gitblit/MirrorExecutor.java b/src/main/java/com/gitblit/MirrorExecutor.java new file mode 100644 index 00000000..21c194f6 --- /dev/null +++ b/src/main/java/com/gitblit/MirrorExecutor.java @@ -0,0 +1,169 @@ +/* + * Copyright 2013 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit; + +import java.text.MessageFormat; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.lib.RefUpdate.Result; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.transport.FetchResult; +import org.eclipse.jgit.transport.RemoteConfig; +import org.eclipse.jgit.transport.TrackingRefUpdate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.gitblit.models.RepositoryModel; +import com.gitblit.models.UserModel; +import com.gitblit.utils.JGitUtils; + +/** + * The Mirror executor handles periodic fetching of mirrored repositories. + * + * @author James Moger + * + */ +public class MirrorExecutor implements Runnable { + + private final Logger logger = LoggerFactory.getLogger(MirrorExecutor.class); + + private final Set repairAttempted = Collections.synchronizedSet(new HashSet()); + + private final IStoredSettings settings; + + private AtomicBoolean running = new AtomicBoolean(false); + + private AtomicBoolean forceClose = new AtomicBoolean(false); + + private final UserModel gitblitUser; + + public MirrorExecutor(IStoredSettings settings) { + this.settings = settings; + this.gitblitUser = new UserModel("gitblit"); + this.gitblitUser.displayName = "Gitblit"; + } + + public boolean isReady() { + return settings.getBoolean(Keys.git.enableMirroring, false); + } + + public boolean isRunning() { + return running.get(); + } + + public void close() { + forceClose.set(true); + } + + @Override + public void run() { + if (!isReady()) { + return; + } + + running.set(true); + + for (String repositoryName : GitBlit.self().getRepositoryList()) { + if (forceClose.get()) { + break; + } + if (GitBlit.self().isCollectingGarbage(repositoryName)) { + logger.debug("mirror is skipping {} garbagecollection", repositoryName); + continue; + } + RepositoryModel model = null; + Repository repository = null; + try { + model = GitBlit.self().getRepositoryModel(repositoryName); + if (!model.isMirror && !model.isBare) { + // repository must be a valid bare git mirror + logger.debug("mirror is skipping {} !mirror !bare", repositoryName); + continue; + } + + repository = GitBlit.self().getRepository(repositoryName); + if (repository == null) { + logger.warn(MessageFormat.format("MirrorExecutor is missing repository {0}?!?", repositoryName)); + continue; + } + + // automatically repair (some) invalid fetch ref specs + if (!repairAttempted.contains(repositoryName)) { + repairAttempted.add(repositoryName); + JGitUtils.repairFetchSpecs(repository); + } + + // find the first mirror remote - there should only be one + StoredConfig rc = repository.getConfig(); + RemoteConfig mirror = null; + List configs = RemoteConfig.getAllRemoteConfigs(rc); + for (RemoteConfig config : configs) { + if (config.isMirror()) { + mirror = config; + break; + } + } + + if (mirror == null) { + // repository does not have a mirror remote + logger.debug("mirror is skipping {} no mirror remote found", repositoryName); + continue; + } + + logger.debug("checking {} remote {} for ref updates", repositoryName, mirror.getName()); + final boolean testing = false; + Git git = new Git(repository); + FetchResult result = git.fetch().setRemote(mirror.getName()).setDryRun(testing).call(); + Collection refUpdates = result.getTrackingRefUpdates(); + if (refUpdates.size() > 0) { + for (TrackingRefUpdate ru : refUpdates) { + StringBuilder sb = new StringBuilder(); + sb.append("updated mirror "); + sb.append(repositoryName); + sb.append(" "); + sb.append(ru.getRemoteName()); + sb.append(" -> "); + sb.append(ru.getLocalName()); + if (ru.getResult() == Result.FORCED) { + sb.append(" (forced)"); + } + sb.append(" "); + sb.append(ru.getOldObjectId() == null ? "" : ru.getOldObjectId().abbreviate(7).name()); + sb.append(".."); + sb.append(ru.getNewObjectId() == null ? "" : ru.getNewObjectId().abbreviate(7).name()); + logger.info(sb.toString()); + } + } + } catch (Exception e) { + logger.error("Error updating mirror " + repositoryName, e); + } finally { + // cleanup + if (repository != null) { + repository.close(); + } + } + } + + running.set(false); + } +} diff --git a/src/main/java/com/gitblit/client/IndicatorsRenderer.java b/src/main/java/com/gitblit/client/IndicatorsRenderer.java index 5b61df66..5883ab04 100644 --- a/src/main/java/com/gitblit/client/IndicatorsRenderer.java +++ b/src/main/java/com/gitblit/client/IndicatorsRenderer.java @@ -56,6 +56,8 @@ public class IndicatorsRenderer extends JPanel implements TableCellRenderer, Ser private final ImageIcon sparkleshareIcon; + private final ImageIcon mirrorIcon; + public IndicatorsRenderer() { super(new FlowLayout(FlowLayout.RIGHT, 1, 0)); blankIcon = new ImageIcon(getClass().getResource("/blank.png")); @@ -67,6 +69,7 @@ public class IndicatorsRenderer extends JPanel implements TableCellRenderer, Ser federatedIcon = new ImageIcon(getClass().getResource("/federated_16x16.png")); forkIcon = new ImageIcon(getClass().getResource("/commit_divide_16x16.png")); sparkleshareIcon = new ImageIcon(getClass().getResource("/star_16x16.png")); + mirrorIcon = new ImageIcon(getClass().getResource("/mirror_16x16.png")); } @Override @@ -85,6 +88,11 @@ public class IndicatorsRenderer extends JPanel implements TableCellRenderer, Ser tooltip.append(Translation.get("gb.isSparkleshared")).append("
"); add(icon); } + if (model.isMirror) { + JLabel icon = new JLabel(mirrorIcon); + tooltip.append(Translation.get("gb.isMirror")).append("
"); + add(icon); + } if (model.isFork()) { JLabel icon = new JLabel(forkIcon); tooltip.append(Translation.get("gb.isFork")).append("
"); diff --git a/src/main/java/com/gitblit/git/GitblitReceivePack.java b/src/main/java/com/gitblit/git/GitblitReceivePack.java index e6ff5721..ba200b25 100644 --- a/src/main/java/com/gitblit/git/GitblitReceivePack.java +++ b/src/main/java/com/gitblit/git/GitblitReceivePack.java @@ -120,6 +120,14 @@ public class GitblitReceivePack extends ReceivePack implements PreReceiveHook, P @Override public void onPreReceive(ReceivePack rp, Collection commands) { + if (repository.isMirror) { + // repository is a mirror + for (ReceiveCommand cmd : commands) { + sendRejection(cmd, "Gitblit does not allow pushes to \"{0}\" because it is a mirror!", repository.name); + } + return; + } + if (repository.isFrozen) { // repository is frozen/readonly for (ReceiveCommand cmd : commands) { diff --git a/src/main/java/com/gitblit/models/RepositoryModel.java b/src/main/java/com/gitblit/models/RepositoryModel.java index 88854652..40a5acd2 100644 --- a/src/main/java/com/gitblit/models/RepositoryModel.java +++ b/src/main/java/com/gitblit/models/RepositoryModel.java @@ -65,6 +65,7 @@ public class RepositoryModel implements Serializable, Comparable availableRefs; diff --git a/src/main/java/com/gitblit/models/TeamModel.java b/src/main/java/com/gitblit/models/TeamModel.java index 54f194b1..a1928283 100644 --- a/src/main/java/com/gitblit/models/TeamModel.java +++ b/src/main/java/com/gitblit/models/TeamModel.java @@ -206,7 +206,7 @@ public class TeamModel implements Serializable, Comparable { // determine maximum permission for the repository final AccessPermission maxPermission = - (repository.isFrozen || !repository.isBare) ? + (repository.isFrozen || !repository.isBare || repository.isMirror) ? AccessPermission.CLONE : AccessPermission.REWIND; if (AccessRestrictionType.NONE.equals(repository.accessRestriction)) { diff --git a/src/main/java/com/gitblit/models/UserModel.java b/src/main/java/com/gitblit/models/UserModel.java index b4fdb66f..446db3ab 100644 --- a/src/main/java/com/gitblit/models/UserModel.java +++ b/src/main/java/com/gitblit/models/UserModel.java @@ -292,7 +292,7 @@ public class UserModel implements Principal, Serializable, Comparable // determine maximum permission for the repository final AccessPermission maxPermission = - (repository.isFrozen || !repository.isBare) ? + (repository.isFrozen || !repository.isBare || repository.isMirror) ? AccessPermission.CLONE : AccessPermission.REWIND; if (AccessRestrictionType.NONE.equals(repository.accessRestriction)) { diff --git a/src/main/java/com/gitblit/utils/JGitUtils.java b/src/main/java/com/gitblit/utils/JGitUtils.java index be2860de..5584fb5c 100644 --- a/src/main/java/com/gitblit/utils/JGitUtils.java +++ b/src/main/java/com/gitblit/utils/JGitUtils.java @@ -2096,4 +2096,54 @@ public class JGitUtils { } return StringUtils.decodeString(content); } + + /** + * Automatic repair of (some) invalid refspecs. These are the result of a + * bug in JGit cloning where a double forward-slash was injected. :( + * + * @param repository + * @return true, if the refspecs were repaired + */ + public static boolean repairFetchSpecs(Repository repository) { + StoredConfig rc = repository.getConfig(); + + // auto-repair broken fetch ref specs + for (String name : rc.getSubsections("remote")) { + int invalidSpecs = 0; + int repairedSpecs = 0; + List specs = new ArrayList(); + for (String spec : rc.getStringList("remote", name, "fetch")) { + try { + RefSpec rs = new RefSpec(spec); + // valid spec + specs.add(spec); + } catch (IllegalArgumentException e) { + // invalid spec + invalidSpecs++; + if (spec.contains("//")) { + // auto-repair this known spec bug + spec = spec.replace("//", "/"); + specs.add(spec); + repairedSpecs++; + } + } + } + + if (invalidSpecs == repairedSpecs && repairedSpecs > 0) { + // the fetch specs were automatically repaired + rc.setStringList("remote", name, "fetch", specs); + try { + rc.save(); + rc.load(); + LOGGER.debug("repaired {} invalid fetch refspecs for {}", repairedSpecs, repository.getDirectory()); + return true; + } catch (Exception e) { + LOGGER.error(null, e); + } + } else if (invalidSpecs > 0) { + LOGGER.error("mirror executor found {} invalid fetch refspecs for {}", invalidSpecs, repository.getDirectory()); + } + } + return false; + } } diff --git a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties index 526093ab..feaa9c6e 100644 --- a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties +++ b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties @@ -504,4 +504,7 @@ gb.noActivityToday = there has been no activity today gb.anonymousUser= anonymous gb.commitMessageRenderer = commit message renderer gb.diffStat = {0} insertions & {1} deletions -gb.home = home \ No newline at end of file +gb.home = home +gb.isMirror = this repository is a mirror +gb.mirrorOf = mirror of {0} +gb.mirrorWarning = this repository is a mirror and can not receive pushes \ No newline at end of file diff --git a/src/main/java/com/gitblit/wicket/pages/RepositoryPage.html b/src/main/java/com/gitblit/wicket/pages/RepositoryPage.html index 1af9127e..0acc6dbc 100644 --- a/src/main/java/com/gitblit/wicket/pages/RepositoryPage.html +++ b/src/main/java/com/gitblit/wicket/pages/RepositoryPage.html @@ -62,6 +62,10 @@

[forked from] [origin repository]

+ + +

[origin repository]

+
diff --git a/src/main/java/com/gitblit/wicket/pages/RepositoryPage.java b/src/main/java/com/gitblit/wicket/pages/RepositoryPage.java index d0d801ef..70a2b9eb 100644 --- a/src/main/java/com/gitblit/wicket/pages/RepositoryPage.java +++ b/src/main/java/com/gitblit/wicket/pages/RepositoryPage.java @@ -260,7 +260,14 @@ public abstract class RepositoryPage extends RootPage { // indicate origin repository RepositoryModel model = getRepositoryModel(); if (StringUtils.isEmpty(model.originRepository)) { - add(new Label("originRepository").setVisible(false)); + if (model.isMirror) { + Fragment mirrorFrag = new Fragment("originRepository", "mirrorFragment", this); + Label lbl = new Label("originRepository", MessageFormat.format(getString("gb.mirrorOf"), "" + model.origin + "")); + mirrorFrag.add(lbl.setEscapeModelStrings(false)); + add(mirrorFrag); + } else { + add(new Label("originRepository").setVisible(false)); + } } else { RepositoryModel origin = GitBlit.self().getRepositoryModel(model.originRepository); if (origin == null) { diff --git a/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.html b/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.html index 54c1e92b..d5a87d3f 100644 --- a/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.html +++ b/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.html @@ -39,6 +39,7 @@
+ diff --git a/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.java b/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.java index f6c80bd4..ed5780f5 100644 --- a/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.java +++ b/src/main/java/com/gitblit/wicket/panels/ProjectRepositoryPanel.java @@ -87,6 +87,12 @@ public class ProjectRepositoryPanel extends BasePanel { add(WicketUtils.newClearPixel("sparkleshareIcon").setVisible(false)); } + if (entry.isMirror) { + add(WicketUtils.newImage("mirrorIcon", "mirror_16x16.png", localizer.getString("gb.isMirror", parent))); + } else { + add(WicketUtils.newClearPixel("mirrorIcon").setVisible(false)); + } + if (entry.isFrozen) { add(WicketUtils.newImage("frozenIcon", "cold_16x16.png", localizer.getString("gb.isFrozen", parent))); } else { diff --git a/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.html b/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.html index e080103b..0cf3ef29 100644 --- a/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.html +++ b/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.html @@ -89,7 +89,7 @@ [repository name] [repository description] [repository owner] - + [last change] [repository size] diff --git a/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.java b/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.java index cb271504..9de387a8 100644 --- a/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.java +++ b/src/main/java/com/gitblit/wicket/panels/RepositoriesPanel.java @@ -243,6 +243,13 @@ public class RepositoriesPanel extends BasePanel { row.add(WicketUtils.newClearPixel("sparkleshareIcon").setVisible(false)); } + if (entry.isMirror) { + row.add(WicketUtils.newImage("mirrorIcon", "mirror_16x16.png", + getString("gb.isMirror"))); + } else { + row.add(WicketUtils.newClearPixel("mirrorIcon").setVisible(false)); + } + if (entry.isFork()) { row.add(WicketUtils.newImage("forkIcon", "commit_divide_16x16.png", getString("gb.isFork"))); -- cgit v1.2.3