From: James Moger Date: Mon, 26 Sep 2011 19:33:19 +0000 (-0400) Subject: Implemented a Federation Client. Bare clone tweaks. Documentation. X-Git-Tag: v0.6.0~8 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=f6740d55ff80bc6e16da5c3df0ee1ba2235d6629;p=gitblit.git Implemented a Federation Client. Bare clone tweaks. Documentation. --- diff --git a/.gitignore b/.gitignore index 5236c647..389f6635 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ /war /*.war /proposals +/*.jar +/federation.properties \ No newline at end of file diff --git a/NOTICE b/NOTICE index 8c8a2a90..41b61f12 100644 --- a/NOTICE +++ b/NOTICE @@ -142,6 +142,14 @@ ant-googlecode New BSD License http://code.google.com/p/ant-googlecode + +--------------------------------------------------------------------------- +GenJar +--------------------------------------------------------------------------- + GenJar, released under the + Apache Software License, Version 1.1. + + http://genjar.sourceforge.net --------------------------------------------------------------------------- Fancybox image viewer diff --git a/build.xml b/build.xml index 10259434..da0dee09 100644 --- a/build.xml +++ b/build.xml @@ -5,6 +5,9 @@ + + + @@ -81,6 +84,7 @@ + @@ -104,7 +108,7 @@ - + @@ -116,8 +120,9 @@ + - + @@ -144,6 +149,7 @@ + @@ -255,6 +261,9 @@ + + + @@ -362,6 +371,45 @@ + + + Building Gitblit Federation Client ${gb.version} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -539,6 +590,16 @@ targetfilename="gitblit-${gb.version}.war" summary="Gitblit WAR v${gb.version} (standard WAR webapp for servlet containers)" labels="Featured, Type-Package, OpSys-All" /> + + + diff --git a/distrib/federation.properties b/distrib/federation.properties new file mode 100644 index 00000000..e7e17f7c --- /dev/null +++ b/distrib/federation.properties @@ -0,0 +1,69 @@ +# +# Git Repository Settings +# + +# Base folder for repositories +# Use forward slashes even on Windows!! +# e.g. c:/gitrepos +# +# SINCE 0.5.0 +# RESTART REQUIRED +git.repositoriesFolder = git + +# Search the repositories folder subfolders for other repositories. +# Repositories MAY NOT be nested (i.e. one repository within another) +# but they may be grouped together in subfolders. +# e.g. c:/gitrepos/libraries/mylibrary.git +# c:/gitrepos/libraries/myotherlibrary.git +# +# SINCE 0.5.0 +git.searchRepositoriesSubfolders = true + +# Your federation name is used for federation status acknowledgments. If it is +# unset, and you elect to send a status acknowledgment, your Gitblit instance +# will be identified by its hostname, if available, else your internal ip address. +# The source Gitblit instance will also append your external IP address to your +# identification to differentiate multiple pulling systems behind a single proxy. +# +# SINCE 0.6.0 +federation.name = + +# Federation pull registrations +# Registrations are read once, at startup. +# +# RESTART REQUIRED +# +# frequency: +# The shortest frequency allowed is every 5 minutes +# Decimal frequency values are cast to integers +# Frequency values may be specified in mins, hours, or days +# Values that can not be parsed default to *federation.defaultFrequency* +# +# folder: +# if blank, the folder is *git.repositoriesFolder* +# if specified, the folder is relative to *git.repositoriesFolder* +# +# mergeAccounts: +# if true, remote accounts and their permissions are merged into your +# users.properties file +# +# notifyOnError: +# if true and the mail configuration is properly set, administrators will be +# notified by email of pull failures +# +# include and exclude: +# space-separated list of repositories to include or exclude from pull +# may be * wildcard to include or exclude all +# may use fuzzy match (e.g. org.eclipse.*) + +# +# (Nearly) Perfect Mirror example +# + +#federation.example1.url = https://go.gitblit.com +#federation.example1.token = 6f3b8a24bf970f17289b234284c94f43eb42f0e4 +#federation.example1.frequency = 120 mins +#federation.example1.folder = +#federation.example1.bare = true +#federation.example1.mirror = true +#federation.example1.mergeAccounts = true diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties index 734dddd0..e8acfa93 100644 --- a/distrib/gitblit.properties +++ b/distrib/gitblit.properties @@ -419,6 +419,7 @@ federation.sets = #federation.example1.token = 6f3b8a24bf970f17289b234284c94f43eb42f0e4 #federation.example1.frequency = 120 mins #federation.example1.folder = +#federation.example1.bare = true #federation.example1.mirror = true #federation.example1.mergeAccounts = true diff --git a/docs/01_setup.mkd b/docs/01_setup.mkd index 1c8db122..91900e1e 100644 --- a/docs/01_setup.mkd +++ b/docs/01_setup.mkd @@ -130,7 +130,7 @@ All repository settings are stored within the repository `.git/config` file unde accessRestriction = clone isFrozen = false showReadme = false - excludeFromFederation = false + federationStrategy = FEDERATE_THIS isFederated = false federationSets = diff --git a/docs/02_federation.mkd b/docs/02_federation.mkd index e5cc797f..e004a27a 100644 --- a/docs/02_federation.mkd +++ b/docs/02_federation.mkd @@ -37,7 +37,7 @@ Changing your *federation.passphrase* will break any registrations you have esta If you want your repositories (and optionally users accounts and settings) to be pulled by another Gitblit instance, you need to register your origin Gitblit instance with a pulling Gitblit instance by providing the url of your Gitblit instance and a federation token. -Gitblit generates the following federation tokens: +Gitblit generates the following standard federation tokens: %BEGINCODE% String allToken = SHA1(passphrase + "-ALL"); String usersAndRepositoriesToken = SHA1(passphrase + "-USERS_AND_REPOSITORIES"); @@ -52,9 +52,11 @@ Individual Gitblit repository configurations such as *description* and *accessRe If *federation.passphrase* has a non-empty value, the federation tokens are displayed in the log file and are visible, to administrators, in the web ui. +The three standard tokens grant access to ALL your non-excluded repositories. However, if you only want to specify different groups of repositories to be federated then you need to define *federation sets*. + #### Federation Sets -Federation Sets (*federation.sets*) are named groups of repositories. The Federation Sets are defined in `gitblit.properties` and are available for selection in the repository settings page. You can assign a repository to one or more sets and then distribute the token for the set. This allows you to grant federation pull access to a subset of your available repositories. Tokens for federation sets only grant pull access for the member repositories. +Federation Sets (*federation.sets*) are named groups of repositories. The Federation Sets are defined in `gitblit.properties` and are available for selection in the repository settings page. You can assign a repository to one or more sets and then distribute the federation token for the set. This allows you to grant federation pull access to a subset of your available repositories. Tokens for federation sets only grant pull access for the member repositories. ### Federation Proposals (Origin Gitblit Instance) @@ -91,7 +93,7 @@ You may view the details of a proposal by scrolling down to the bottom of the re ### Excluding Repositories (Origin Gitblit Instance) -You may exclude a repository from being pulled by a federated Gitblit instance by setting its *federation strategy* to EXCLUDE in the repository's settings page. +You may exclude a repository from being pulled by any federated Gitblit instance by setting its *federation strategy* to EXCLUDE in the repository's settings page. ### Excluding Repositories (Pulling Gitblit Instance) @@ -286,4 +288,25 @@ The repositories will be put in *git.repositoriesFolder*/example4. federation.example4.bare = true federation.example4.mirror = true federation.example4.exclude = * - federation.example4.include = somerepo.git \ No newline at end of file + federation.example4.include = somerepo.git + +## Federation Client + +Instead of setting up a full-blown pulling Gitblit instance, you can also use the [federation client](http://code.google.com/p/gitblit/downloads/detail?name=%FEDCLIENT%) command-line utility. This is a packaged subset of the federation feature in a smaller, simpler command-line only tool. + +The *federation client* relies on many of the same dependencies as Gitblit and will download them on first execution. + +### federation.properties +You may use the `federation.properties` file to configure one or more Gitblit instances that you want to pull from. This file is a subset of the standard `gitblit.properties` file. + +By default this tool does not daemonize itself; it executes and then quits. This allows you to use the native scheduling feature of your OS. Of course, if you'd rather use Gitblit's scheduler you may use that by specifying the `--daemon` parameter. + +### Command-Line Parameters +Instead of using `federation.properties` you may directly specify a Gitblit instance to pull from with command-line parameters. + + java -jar fedclient.jar --url https://go.gitblit.com --mirror --bare --token 123456789 + --repositoriesFolder c:/mymirror + + java -jar fedclient.jar --url https://go.gitblit.com --mirror --bare --token 123456789 + --repositoriesFolder c:/mymirror --daemon --frequency "24 hours" + diff --git a/docs/04_design.mkd b/docs/04_design.mkd index 70cea7c7..523d31b9 100644 --- a/docs/04_design.mkd +++ b/docs/04_design.mkd @@ -39,6 +39,7 @@ The following dependencies are automatically downloaded by Gitblit GO (or alread - [JUnit](http://junit.org) (Common Public License) - [commons-net](http://commons.apache.org/net) (Apache 2.0) - [ant-googlecode](http://code.google.com/p/ant-googlecode) (New BSD) +- [GenJar](http://genjar.sourceforge.net) (Apache 1.1) ## Building from Source [Eclipse](http://eclipse.org) is recommended for development as the project settings are preconfigured. diff --git a/src/com/gitblit/FederationClient.java b/src/com/gitblit/FederationClient.java new file mode 100644 index 00000000..daa9bfbf --- /dev/null +++ b/src/com/gitblit/FederationClient.java @@ -0,0 +1,133 @@ +/* + * Copyright 2011 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; +import com.gitblit.models.FederationModel; +import com.gitblit.utils.FederationUtils; +import com.gitblit.utils.StringUtils; + +/** + * Command-line client to pull federated Gitblit repositories. + * + * @author James Moger + * + */ +public class FederationClient { + + public static void main(String[] args) { + Params params = new Params(); + JCommander jc = new JCommander(params); + try { + jc.parse(args); + } catch (ParameterException t) { + usage(jc, t); + } + + IStoredSettings settings = new FileSettings(params.registrationsFile); + List registrations = new ArrayList(); + if (StringUtils.isEmpty(params.url)) { + registrations.addAll(FederationUtils.getFederationRegistrations(settings)); + } else { + if (StringUtils.isEmpty(params.token)) { + System.out.println("Must specify --token parameter!"); + System.exit(0); + } + FederationModel model = new FederationModel("Gitblit"); + model.url = params.url; + model.token = params.token; + model.mirror = params.mirror; + model.bare = params.bare; + model.frequency = params.frequency; + model.folder = ""; + registrations.add(model); + } + if (registrations.size() == 0) { + System.out.println("No Federation Registrations! Nothing to do."); + System.exit(0); + } + + System.out.println("Gitblit Federation Client v" + Constants.VERSION + " (" + Constants.VERSION_DATE + ")"); + + // command-line specified repositories folder + if (!StringUtils.isEmpty(params.repositoriesFolder)) { + settings.overrideSetting(Keys.git.repositoriesFolder, new File( + params.repositoriesFolder).getAbsolutePath()); + } + + // configure the Gitblit singleton for minimal, non-server operation + GitBlit.self().configureContext(settings, false); + FederationPullExecutor executor = new FederationPullExecutor(registrations, params.isDaemon); + executor.run(); + if (!params.isDaemon) { + System.out.println("Finished."); + System.exit(0); + } + } + + private static void usage(JCommander jc, ParameterException t) { + System.out.println(Constants.getGitBlitVersion()); + System.out.println(); + if (t != null) { + System.out.println(t.getMessage()); + System.out.println(); + } + + if (jc != null) { + jc.usage(); + } + System.exit(0); + } + + /** + * JCommander Parameters class for FederationClient. + */ + @Parameters(separators = " ") + private static class Params { + + @Parameter(names = { "--registrations" }, description = "Gitblit Federation Registrations File", required = false) + public String registrationsFile = "federation.properties"; + + @Parameter(names = { "--daemon" }, description = "Runs in daemon mode to schedule and pull repositories", required = false) + public boolean isDaemon; + + @Parameter(names = { "--url" }, description = "URL of Gitblit instance to mirror from", required = false) + public String url; + + @Parameter(names = { "--mirror" }, description = "Mirror repositories", required = false) + public boolean mirror; + + @Parameter(names = { "--bare" }, description = "Create bare repositories", required = false) + public boolean bare; + + @Parameter(names = { "--token" }, description = "Federation Token", required = false) + public String token; + + @Parameter(names = { "--frequency" }, description = "Period to wait between pull attempts (requires --daemon)", required = false) + public String frequency = "60 mins"; + + @Parameter(names = { "--repositoriesFolder" }, description = "Destination folder for cloned repositories", required = false) + public String repositoriesFolder; + + } +} diff --git a/src/com/gitblit/FederationClientLauncher.java b/src/com/gitblit/FederationClientLauncher.java new file mode 100644 index 00000000..80f5a3d9 --- /dev/null +++ b/src/com/gitblit/FederationClientLauncher.java @@ -0,0 +1,54 @@ +/* + * Copyright 2011 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit; + +import java.io.File; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import com.gitblit.build.Build; + +/** + * Downloads dependencies and launches command-line Federation client. + * + * @author James Moger + * + */ +public class FederationClientLauncher { + + public static void main(String[] args) { + // download federation client runtime dependencies + Build.federationClient(); + + File libFolder = new File("ext"); + List jars = Launcher.findJars(libFolder.getAbsoluteFile()); + + // sort the jars by name and then reverse the order so the newer version + // of the library gets loaded in the event that this is an upgrade + Collections.sort(jars); + Collections.reverse(jars); + for (File jar : jars) { + try { + Launcher.addJarFile(jar); + } catch (IOException e) { + + } + } + + FederationClient.main(args); + } +} diff --git a/src/com/gitblit/FederationPullExecutor.java b/src/com/gitblit/FederationPullExecutor.java index f27ea3c2..ef089d03 100644 --- a/src/com/gitblit/FederationPullExecutor.java +++ b/src/com/gitblit/FederationPullExecutor.java @@ -15,6 +15,8 @@ */ package com.gitblit; +import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT; + import java.io.File; import java.io.FileOutputStream; import java.net.InetAddress; @@ -23,14 +25,17 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.slf4j.Logger; @@ -57,6 +62,8 @@ public class FederationPullExecutor implements Runnable { private final List registrations; + private final boolean isDaemon; + /** * Constructor for specifying a single federation registration. This * constructor is used to schedule the next pull execution. @@ -64,7 +71,7 @@ public class FederationPullExecutor implements Runnable { * @param registration */ private FederationPullExecutor(FederationModel registration) { - this(Arrays.asList(registration)); + this(Arrays.asList(registration), true); } /** @@ -73,9 +80,13 @@ public class FederationPullExecutor implements Runnable { * on each registrations frequency setting. * * @param registrations + * @param isDaemon + * if true, registrations are rescheduled in perpetuity. if false, + * the federation pull operation is executed once. */ - public FederationPullExecutor(List registrations) { + public FederationPullExecutor(List registrations, boolean isDaemon) { this.registrations = registrations; + this.isDaemon = isDaemon; } /** @@ -109,7 +120,9 @@ public class FederationPullExecutor implements Runnable { "Failed to pull from federated gitblit ({0} @ {1})", registration.name, registration.url), t); } finally { - schedule(registration); + if (isDaemon) { + schedule(registration); + } } } } @@ -149,12 +162,25 @@ public class FederationPullExecutor implements Runnable { continue; } + // Determine local repository name String repositoryName; if (StringUtils.isEmpty(registrationFolder)) { repositoryName = repository.name; } else { repositoryName = registrationFolder + "/" + repository.name; } + + if (registration.bare) { + // bare repository, ensure .git suffix + if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) { + repositoryName += DOT_GIT_EXT; + } + } else { + // normal repository, strip .git suffix + if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) { + repositoryName = repositoryName.substring(0, repositoryName.indexOf(DOT_GIT_EXT)); + } + } // confirm that the origin of any pre-existing repository matches // the clone url @@ -164,8 +190,10 @@ public class FederationPullExecutor implements Runnable { StoredConfig config = existingRepository.getConfig(); config.load(); String origin = config.getString("remote", "origin", "url"); - fetchHead = JGitUtils.getCommit(existingRepository, "refs/remotes/origin/master") - .getName(); + RevCommit commit = JGitUtils.getCommit(existingRepository, "refs/remotes/origin/master"); + if (commit != null) { + fetchHead = commit.getName(); + } existingRepository.close(); if (!origin.startsWith(registration.url)) { logger.warn(MessageFormat @@ -181,6 +209,7 @@ public class FederationPullExecutor implements Runnable { Constants.FEDERATION_USER, registration.token); logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}", repository.name, registration.name, registration.url)); + CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name, cloneUrl, registration.bare, credentials); Repository r = GitBlit.self().getRepository(repositoryName); @@ -196,8 +225,9 @@ public class FederationPullExecutor implements Runnable { } else { // fetch and update boolean fetched = false; - String origin = JGitUtils.getCommit(r, "refs/remotes/origin/master").getName(); - fetched = !fetchHead.equals(origin); + RevCommit commit = JGitUtils.getCommit(r, "refs/remotes/origin/master"); + String origin = commit.getName(); + fetched = fetchHead == null || !fetchHead.equals(origin); if (registration.mirror) { // mirror @@ -225,6 +255,16 @@ public class FederationPullExecutor implements Runnable { // preserve local settings repository.isFrozen = rm.isFrozen; repository.federationStrategy = rm.federationStrategy; + + // merge federation sets + Set federationSets = new HashSet(); + if (rm.federationSets != null) { + federationSets.addAll(rm.federationSets); + } + if (repository.federationSets != null) { + federationSets.addAll(repository.federationSets); + } + repository.federationSets = new ArrayList(federationSets); } // only repositories that are actually _cloned_ from the origin // Gitblit repository are marked as federated. If the origin diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java index c2b214b4..62b93e7c 100644 --- a/src/com/gitblit/GitBlit.java +++ b/src/com/gitblit/GitBlit.java @@ -61,6 +61,7 @@ import com.gitblit.models.FederationModel; import com.gitblit.models.FederationProposal; import com.gitblit.models.RepositoryModel; import com.gitblit.models.UserModel; +import com.gitblit.utils.FederationUtils; import com.gitblit.utils.JGitUtils; import com.gitblit.utils.StringUtils; import com.google.gson.Gson; @@ -830,8 +831,8 @@ public class GitBlit implements ServletContextListener { // Schedule the federation executor List registrations = getFederationRegistrations(); if (registrations.size() > 0) { - scheduledExecutor.schedule(new FederationPullExecutor(registrations), 1, - TimeUnit.MINUTES); + FederationPullExecutor executor = new FederationPullExecutor(registrations, true); + scheduledExecutor.schedule(executor, 1, TimeUnit.MINUTES); } } @@ -843,79 +844,7 @@ public class GitBlit implements ServletContextListener { */ public List getFederationRegistrations() { if (federationRegistrations.isEmpty()) { - List keys = settings.getAllKeys(Keys.federation._ROOT); - keys.remove(Keys.federation.name); - keys.remove(Keys.federation.passphrase); - keys.remove(Keys.federation.allowProposals); - keys.remove(Keys.federation.proposalsFolder); - keys.remove(Keys.federation.defaultFrequency); - keys.remove(Keys.federation.sets); - Collections.sort(keys); - Map federatedModels = new HashMap(); - for (String key : keys) { - String value = key.substring(Keys.federation._ROOT.length() + 1); - List values = StringUtils.getStringsFromValue(value, "\\."); - String server = values.get(0); - if (!federatedModels.containsKey(server)) { - federatedModels.put(server, new FederationModel(server)); - } - String setting = values.get(1); - if (setting.equals("url")) { - // url of the origin Gitblit instance - federatedModels.get(server).url = settings.getString(key, ""); - } else if (setting.equals("token")) { - // token for the origin Gitblit instance - federatedModels.get(server).token = settings.getString(key, ""); - } else if (setting.equals("frequency")) { - // frequency of the pull operation - federatedModels.get(server).frequency = settings.getString(key, ""); - } else if (setting.equals("folder")) { - // destination folder of the pull operation - federatedModels.get(server).folder = settings.getString(key, ""); - } else if (setting.equals("bare")) { - // whether pulled repositories should be bare - federatedModels.get(server).bare = settings.getBoolean(key, true); - } else if (setting.equals("mirror")) { - // are the repositories to be true mirrors of the origin - federatedModels.get(server).mirror = settings.getBoolean(key, true); - } else if (setting.equals("mergeAccounts")) { - // merge remote accounts into local accounts - federatedModels.get(server).mergeAccounts = settings.getBoolean(key, false); - } else if (setting.equals("sendStatus")) { - // send a status acknowledgment to source Gitblit instance - // at end of git pull - federatedModels.get(server).sendStatus = settings.getBoolean(key, false); - } else if (setting.equals("notifyOnError")) { - // notify administrators on federation pull failures - federatedModels.get(server).notifyOnError = settings.getBoolean(key, false); - } else if (setting.equals("exclude")) { - // excluded repositories - federatedModels.get(server).exclusions = settings.getStrings(key); - } else if (setting.equals("include")) { - // included repositories - federatedModels.get(server).inclusions = settings.getStrings(key); - } - } - - // verify that registrations have a url and a token - for (FederationModel model : federatedModels.values()) { - if (StringUtils.isEmpty(model.url)) { - logger.warn(MessageFormat.format( - "Dropping federation registration {0}. Missing url.", model.name)); - continue; - } - if (StringUtils.isEmpty(model.token)) { - logger.warn(MessageFormat.format( - "Dropping federation registration {0}. Missing token.", model.name)); - continue; - } - // set default frequency if unspecified - if (StringUtils.isEmpty(model.frequency)) { - model.frequency = settings.getString(Keys.federation.defaultFrequency, - "60 mins"); - } - federationRegistrations.add(model); - } + federationRegistrations.addAll(FederationUtils.getFederationRegistrations(settings)); } return federationRegistrations; } @@ -1239,7 +1168,7 @@ public class GitBlit implements ServletContextListener { * * @param settings */ - public void configureContext(IStoredSettings settings) { + public void configureContext(IStoredSettings settings, boolean startFederation) { logger.info("Reading configuration from " + settings.toString()); this.settings = settings; repositoriesFolder = new File(settings.getString(Keys.git.repositoriesFolder, "git")); @@ -1268,13 +1197,15 @@ public class GitBlit implements ServletContextListener { loginService = new FileUserService(realmFile); } setUserService(loginService); - configureFederation(); mailExecutor = new MailExecutor(settings); if (mailExecutor.isReady()) { scheduledExecutor.scheduleAtFixedRate(mailExecutor, 1, 2, TimeUnit.MINUTES); } else { logger.warn("Mail server is not properly configured. Mail services disabled."); } + if (startFederation) { + configureFederation(); + } } /** @@ -1288,7 +1219,7 @@ public class GitBlit implements ServletContextListener { if (settings == null) { // Gitblit WAR is running in a servlet container WebXmlSettings webxmlSettings = new WebXmlSettings(contextEvent.getServletContext()); - configureContext(webxmlSettings); + configureContext(webxmlSettings, true); } } diff --git a/src/com/gitblit/GitBlitServer.java b/src/com/gitblit/GitBlitServer.java index d2164f18..039f59d3 100644 --- a/src/com/gitblit/GitBlitServer.java +++ b/src/com/gitblit/GitBlitServer.java @@ -237,7 +237,7 @@ public class GitBlitServer { // Setup the GitBlit context GitBlit gitblit = GitBlit.self(); - gitblit.configureContext(settings); + gitblit.configureContext(settings, true); rootContext.addEventListener(gitblit); try { diff --git a/src/com/gitblit/build/Build.java b/src/com/gitblit/build/Build.java index 40342269..684f2787 100644 --- a/src/com/gitblit/build/Build.java +++ b/src/com/gitblit/build/Build.java @@ -115,6 +115,19 @@ public class Build { // needed for site publishing downloadFromApache(MavenObject.COMMONSNET, BuildType.RUNTIME); } + + public static void federationClient() { + downloadFromApache(MavenObject.JCOMMANDER, BuildType.RUNTIME); + downloadFromApache(MavenObject.SERVLET, BuildType.RUNTIME); + downloadFromApache(MavenObject.MAIL, BuildType.RUNTIME); + downloadFromApache(MavenObject.SLF4JAPI, BuildType.RUNTIME); + downloadFromApache(MavenObject.SLF4LOG4J, BuildType.RUNTIME); + downloadFromApache(MavenObject.LOG4J, BuildType.RUNTIME); + downloadFromApache(MavenObject.GSON, BuildType.RUNTIME); + downloadFromApache(MavenObject.JSCH, BuildType.RUNTIME); + + downloadFromEclipse(MavenObject.JGIT, BuildType.RUNTIME); + } /** * Builds the Keys class based on the gitblit.properties file and inserts diff --git a/src/com/gitblit/models/FederationModel.java b/src/com/gitblit/models/FederationModel.java index efb1b322..d67ae561 100644 --- a/src/com/gitblit/models/FederationModel.java +++ b/src/com/gitblit/models/FederationModel.java @@ -73,6 +73,8 @@ public class FederationModel implements Serializable, Comparable getFederationRegistrations(IStoredSettings settings) { + List federationRegistrations = new ArrayList(); + List keys = settings.getAllKeys(Keys.federation._ROOT); + keys.remove(Keys.federation.name); + keys.remove(Keys.federation.passphrase); + keys.remove(Keys.federation.allowProposals); + keys.remove(Keys.federation.proposalsFolder); + keys.remove(Keys.federation.defaultFrequency); + keys.remove(Keys.federation.sets); + Collections.sort(keys); + Map federatedModels = new HashMap(); + for (String key : keys) { + String value = key.substring(Keys.federation._ROOT.length() + 1); + List values = StringUtils.getStringsFromValue(value, "\\."); + String server = values.get(0); + if (!federatedModels.containsKey(server)) { + federatedModels.put(server, new FederationModel(server)); + } + String setting = values.get(1); + if (setting.equals("url")) { + // url of the origin Gitblit instance + federatedModels.get(server).url = settings.getString(key, ""); + } else if (setting.equals("token")) { + // token for the origin Gitblit instance + federatedModels.get(server).token = settings.getString(key, ""); + } else if (setting.equals("frequency")) { + // frequency of the pull operation + federatedModels.get(server).frequency = settings.getString(key, ""); + } else if (setting.equals("folder")) { + // destination folder of the pull operation + federatedModels.get(server).folder = settings.getString(key, ""); + } else if (setting.equals("bare")) { + // whether pulled repositories should be bare + federatedModels.get(server).bare = settings.getBoolean(key, true); + } else if (setting.equals("mirror")) { + // are the repositories to be true mirrors of the origin + federatedModels.get(server).mirror = settings.getBoolean(key, true); + } else if (setting.equals("mergeAccounts")) { + // merge remote accounts into local accounts + federatedModels.get(server).mergeAccounts = settings.getBoolean(key, false); + } else if (setting.equals("sendStatus")) { + // send a status acknowledgment to source Gitblit instance + // at end of git pull + federatedModels.get(server).sendStatus = settings.getBoolean(key, false); + } else if (setting.equals("notifyOnError")) { + // notify administrators on federation pull failures + federatedModels.get(server).notifyOnError = settings.getBoolean(key, false); + } else if (setting.equals("exclude")) { + // excluded repositories + federatedModels.get(server).exclusions = settings.getStrings(key); + } else if (setting.equals("include")) { + // included repositories + federatedModels.get(server).inclusions = settings.getStrings(key); + } + } + + // verify that registrations have a url and a token + for (FederationModel model : federatedModels.values()) { + if (StringUtils.isEmpty(model.url)) { + LOGGER.warn(MessageFormat.format( + "Dropping federation registration {0}. Missing url.", model.name)); + continue; + } + if (StringUtils.isEmpty(model.token)) { + LOGGER.warn(MessageFormat.format( + "Dropping federation registration {0}. Missing token.", model.name)); + continue; + } + // set default frequency if unspecified + if (StringUtils.isEmpty(model.frequency)) { + model.frequency = settings.getString(Keys.federation.defaultFrequency, "60 mins"); + } + federationRegistrations.add(model); + } + return federationRegistrations; + } + /** * Sends a federation proposal to the Gitblit instance at remoteUrl * diff --git a/src/com/gitblit/utils/JGitUtils.java b/src/com/gitblit/utils/JGitUtils.java index faca9cb6..bfbc6240 100644 --- a/src/com/gitblit/utils/JGitUtils.java +++ b/src/com/gitblit/utils/JGitUtils.java @@ -37,8 +37,6 @@ import java.util.zip.ZipOutputStream; import org.eclipse.jgit.api.CloneCommand; import org.eclipse.jgit.api.FetchCommand; import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.PullCommand; -import org.eclipse.jgit.api.PullResult; import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.ResetCommand.ResetType; import org.eclipse.jgit.diff.DiffEntry; @@ -142,6 +140,7 @@ public class JGitUtils { * Encapsulates the result of cloning or pulling from a repository. */ public static class CloneResult { + public String name; public FetchResult fetchResult; public boolean createdRepository; } @@ -175,12 +174,22 @@ public class JGitUtils { * @return CloneResult * @throws Exception */ - public static CloneResult cloneRepository(File repositoriesFolder, String name, String fromUrl, boolean bare, - CredentialsProvider credentialsProvider) throws Exception { + public static CloneResult cloneRepository(File repositoriesFolder, String name, String fromUrl, + boolean bare, CredentialsProvider credentialsProvider) throws Exception { CloneResult result = new CloneResult(); - if (bare && !name.toLowerCase().endsWith(Constants.DOT_GIT_EXT)) { - name += Constants.DOT_GIT_EXT; + if (bare) { + // bare repository, ensure .git suffix + if (!name.toLowerCase().endsWith(Constants.DOT_GIT_EXT)) { + name += Constants.DOT_GIT_EXT; + } + } else { + // normal repository, strip .git suffix + if (name.toLowerCase().endsWith(Constants.DOT_GIT_EXT)) { + name = name.substring(0, name.indexOf(Constants.DOT_GIT_EXT)); + } } + result.name = name; + File folder = new File(repositoriesFolder, name); if (folder.exists()) { File gitDir = FileKey.resolve(new File(repositoriesFolder, name), FS.DETECTED); diff --git a/tests/com/gitblit/tests/GitBlitSuite.java b/tests/com/gitblit/tests/GitBlitSuite.java index 2908383e..52a1d0bf 100644 --- a/tests/com/gitblit/tests/GitBlitSuite.java +++ b/tests/com/gitblit/tests/GitBlitSuite.java @@ -73,7 +73,7 @@ public class GitBlitSuite extends TestSetup { @Override protected void setUp() throws Exception { FileSettings settings = new FileSettings("distrib/gitblit.properties"); - GitBlit.self().configureContext(settings); + GitBlit.self().configureContext(settings, true); FileUserService loginService = new FileUserService(new File("distrib/users.properties")); GitBlit.self().setUserService(loginService); diff --git a/tools/GenJar.jar b/tools/GenJar.jar new file mode 100644 index 00000000..6f225cb4 Binary files /dev/null and b/tools/GenJar.jar differ