Refined mirror federation feature. Documentation.

HEAD is now properly reset if mirroring is enabled.  Updated the
federation pull status enum to include no change and mirrored states.

distrib/gitblit.properties

@@ -391,6 +391,13 @@ federation.sets =
 #   if unspecified, the folder is *git.repositoriesFolder*
 #   if specified, the folder is relative to *git.repositoriesFolder*
 #
+# mirror:
+#   if true, each repository HEAD is reset to *origin/master* after each pull.
+#   The repository will be flagged *isFrozen* after the initial clone.
+#
+#   if false, each repository HEAD will point to the FETCH_HEAD of the initial
+#   clone from the origin until pushed to or otherwise manipulated.
+#
 # mergeAccounts:
 #   if true, remote accounts and their permissions are merged into your 
 #   users.properties file 
@@ -411,7 +418,8 @@ federation.sets =
 #federation.example1.url = https://go.gitblit.com
 #federation.example1.token = 6f3b8a24bf970f17289b234284c94f43eb42f0e4
 #federation.example1.frequency = 120 mins
-#federation.example1.folder = 
+#federation.example1.folder =
+#federation.example1.mirror = true 
 #federation.example1.mergeAccounts = true
 
 #

docs/02_federation.mkd

@@ -6,11 +6,11 @@ A Gitblit federation is basically an automated backup solution from one Gitblit
 
 If your Gitblit instance allows federation and it is properly registered with another Gitblit instance, each of the *non-excluded* repositories of your Gitblit instance can be mirrored, in their entirety, to the pulling Gitblit instance.  You may optionally allow pulling of user accounts and server settings.
 
-### Source Gitblit Instance Requirements
+### Origin Gitblit Instance Requirements
 
 - *git.enableGitServlet* must be true, all Git clone and pull requests are handled through Gitblit's JGit servlet
 - *federation.passphrase* must be non-empty
-- The Gitblit source instance must be http/https accessible by the pulling Gitblit instance.<br/>That may require configuring port-forwarding on your router and/or opening ports on your firewall.
+- The Gitblit origin instance must be http/https accessible by the pulling Gitblit instance.<br/>That may require configuring port-forwarding on your router and/or opening ports on your firewall.
 
 #### federation.passphrase
 
@@ -27,12 +27,12 @@ Changing your *federation.passphrase* will break any registrations you have esta
 
 ### Pulling Gitblit Instance Requirements
 
- - consider setting *federation.allowProposals=true* to facilitate the registration process from source Gitblit instances
+ - consider setting *federation.allowProposals=true* to facilitate the registration process from origin Gitblit instances
  - properly registered Gitblit instance including, at a minimum, url, *federation token*, and update frequency
 
 ### Controlling What Gets Pulled
 
-If you want your repositories (and optionally users accounts and settings) to be pulled by another Gitblit instance, you need to register your source Gitblit instance with a pulling Gitblit instance by providing the url of your Gitblit instance and a federation token.
+If you want your repositories (and optionally users accounts and settings) to be pulled by another Gitblit instance, you need to register your origin Gitblit instance with a pulling Gitblit instance by providing the url of your Gitblit instance and a federation token.
 
 Gitblit generates the following federation tokens:
 %BEGINCODE%
@@ -53,7 +53,7 @@ If *federation.passphrase* has a non-empty value, the federation tokens are disp
 
 Federation Sets (*federation.sets*) are named groups of repositories.  The Federation Sets are available for selection in the repository settings page.  You can assign a repository to one or more sets and then distribute the token for the set.  This allows you to grant federation pull access to a subset of your available repositories.  Tokens for federation sets only grant pull access for the member repositories.
 
-### Federation Proposals (Source Gitblit Instance)
+### Federation Proposals (Origin Gitblit Instance)
 
 Once you have properly setup your passphrase and can see your federation tokens, you are ready to share them with a pulling Gitblit instance.
  
@@ -84,7 +84,7 @@ If your Giblit instance has received a *federation proposal*, you will be alerte
 
 You may view the details of a proposal from scrolling down to the bottom of the repositories page and selecting a proposal.  Sample registration settings will be generated for you that you may copy & paste into either your `gitblit.properties` file or your `web.xml` file.
 
-### Excluding Repositories (Source Gitblit Instance)
+### Excluding Repositories (Origin Gitblit Instance)
 
 You may exclude a repository from being pulled by a federated Gitblit instance by setting its *federation strategy* in the repository's settings page.
 
@@ -106,13 +106,13 @@ You may exclude repositories to pull in a federation registration.  You may excl
     
 ### Tracking Status (Pulling Gitblit Instance)
 
-Below the repositories list on the repositories page you will find a section named *federation registrations*.  This section enumerates the other gitblit servers you have configured to periodically pull.  The *status* of the latest pull will be indicated on the left with a colored circle, similar to the status of an executed unit test or Hudson/Jenkins build.  You can drill into the details of the registration to view the status of the last pull from each repository available from that source Gitblit instance.  Additionally, you can specify the *federation.N.notifyOnError=true* flag, to be alerted via email of regressive status changes to individual registrations.
+Below the repositories list on the repositories page you will find a section named *federation registrations*.  This section enumerates the other gitblit servers you have configured to periodically pull.  The *status* of the latest pull will be indicated on the left with a colored circle, similar to the status of an executed unit test or Hudson/Jenkins build.  You can drill into the details of the registration to view the status of the last pull from each repository available from that origin Gitblit instance.  Additionally, you can specify the *federation.N.notifyOnError=true* flag, to be alerted via email of regressive status changes to individual registrations.
 
-### Tracking Status (Source Gitblit Instance)
+### Tracking Status (Origin Gitblit Instance)
 
-Source Gitblit instances can not directly track the success or failure status of Pulling Gitblit instances.  However, the Pulling Gitblit instance may elect to send a status acknowledgment (*federation.N.sendStatus=true*) to the source Gitblit server that indicates the per-repository status of the pull operation.  This is the same data that is displayed on the Pulling Gitblit instances ui.
+Origin Gitblit instances can not directly track the success or failure status of Pulling Gitblit instances.  However, the Pulling Gitblit instance may elect to send a status acknowledgment (*federation.N.sendStatus=true*) to the origin Gitblit server that indicates the per-repository status of the pull operation.  This is the same data that is displayed on the Pulling Gitblit instances ui.
 
-### How does it work? (Source Gitblit Instances)
+### How does it work? (Origin Gitblit Instances)
 
 A pulling Gitblit instance will periodically contact your Gitblit instance and will provide the token as proof that you have granted it federation access.  Your Gitblit instance will decide, based on the supplied token, if the requested data should be returned to the pulling Gitblit instance.  Gitblit data (user accounts, repository metadata, and server settings) are serialized as [JSON](http://json.org) using [google-gson](http://google-gson.googlecode.com) and returned to the pulling Gitblit instance.  Standard Git clone and pull operations are used to transfer commits.
 
@@ -144,7 +144,7 @@ By default all user accounts except the *admin* account are automatically pulled
 
 The pulling Gitblit instance will store a registration-specific `users.properties` file for the pulled user accounts and their repository permissions. This file is stored in the *federation.N.folder* folder.
 
-If you specify *federation.N.mergeAccounts=true*, then the user accounts from the source Gitblit instance will be integrated into the `users.properties` file of your Gitblit instance and allow sign-on of those users.
+If you specify *federation.N.mergeAccounts=true*, then the user accounts from the origin Gitblit instance will be integrated into the `users.properties` file of your Gitblit instance and allow sign-on of those users.
 
 **NOTE:**<br/>
 Upgrades from older Gitblit versions will not have the *#notfederated* role assigned to the *admin* account.  Without that role, your admin account WILL be transferred with an *ALL* or *USERS_AND_REPOSITORIES* token.<br/>
@@ -164,19 +164,19 @@ Gitblit does **not** detect conflict and it does **not** offer conflict resoluti
 
 If an object exists locally that has the same name as the remote object, it is assumed they are the same and the contents of the remote object are merged into the local object.  If you can not guarantee that this is the case, then you should not store any federated repositories directly in *git.repositoriesFolder* and you should not enable *mergeAccounts*.
 
-By default, federated repositories can not be pushed to, they are read-only by the *isFrozen* flag.  This flag is **ONLY** enforced by Gitblit's JGit servlet.  If you push to a federated repository after resetting the *isFrozen* flag or via some other Git access technique then you may break Gitblit's ability to continue pulling from the source repository.  If you are only pushing to a local branch then you might be safe.
+By default, federated repositories can not be pushed to, they are read-only by the *isFrozen* flag.  This flag is **ONLY** enforced by Gitblit's JGit servlet.  If you push to a federated repository after resetting the *isFrozen* flag or via some other Git access technique then you may break Gitblit's ability to continue pulling from the origin repository.  If you are only pushing to a local branch then you might be safe.
 
 ## Federation Pull Registration Keys
 
-<table>
+<table class="text">
 <tr><th>federation.N.url</th>
 <td>string</td>
-<td>the url of the source Gitblit instance</td>
+<td>the url of the origin Gitblit instance *(required)*</td>
 </tr>
 
 <tr><th>federation.N.token</th>
 <td>string</td>
-<td>the token provided by the source Gitblit instance</td>
+<td>the token provided by the origin Gitblit instance *(required)*</td>
 </tr>
 
 <tr><th>federation.N.frequency</th>
@@ -186,22 +186,22 @@ By default, federated repositories can not be pushed to, they are read-only by t
 
 <tr><th>federation.N.folder</th>
 <td>string</td>
-<td>the destination folder, relative to *git.repositoriesFolder*, for these repositories.  If empty, the repositories are put in *git.repositoriesFolder*.</td>
+<td>the destination folder, relative to *git.repositoriesFolder*, for these repositories.<br/>default is *git.repositoriesFolder*</td>
 </tr>
 
-<tr><th>federation.N.mergeAccounts</th>
+<tr><th>federation.N.mirror</th>
 <td>boolean</td>
-<td>merge the retrieved accounts into the `users.properties` of **this** Gitblit instance</td>
+<td>if **true** *(default)*, each repository HEAD is reset to *origin/master* after each pull.  The repository is flagged *isFrozen* after the initial clone.<p>If **false**, each repository HEAD will point to the FETCH_HEAD of the initial clone from the origin until pushed to or otherwise manipulated.</td>
 </tr>
 
-<tr><th>federation.N.sendStatus</th>
+<tr><th>federation.N.mergeAccounts</th>
 <td>boolean</td>
-<td>send the status of the federated pull to the source Gitblit instance</td>
+<td>if **true**, merge the retrieved accounts into the `users.properties` of **this** Gitblit instance.<br/>*default is false*</td>
 </tr>
 
-<tr><th>federation.N.freeze</th>
+<tr><th>federation.N.sendStatus</th>
 <td>boolean</td>
-<td>freeze the repository after the first pull, subsequent pulls respect the local *isFrozen* setting</td>
+<td>if **true**, send the status of the federated pull to the origin Gitblit instance.<br/>*default is false*</td>
 </tr>
 
 <tr><th>federation.N.include</th>
@@ -216,7 +216,7 @@ By default, federated repositories can not be pushed to, they are read-only by t
 
 <tr><th>federation.N.notifyOnError</th>
 <td>boolean</td>
-<td>send an email to the administrators on an error</td>
+<td>if **true**, send an email to the administrators on an error.<br/>*default is false*</td>
 </tr>
 </table>
 
@@ -226,8 +226,8 @@ These examples would be entered into the `gitblit.properties` file of the pullin
 
 #### (Nearly) Perfect Mirror Example
 
-This assumes that the *token* is the *ALL* token from the source gitblit instance.<br/>
-The repositories, example1_users.properties, and example1_gitblit.properties will be put in *git.repositoriesFolder* and the source user accounts will be merged into the local user accounts, including passwords and administrator status.  The mirror instance will also send a status acknowledgment at the end of the pull operation which will include the state of each repository pull (EXCLUDED, SKIPPED, PULLED).  This way the source Gitblit instance can monitor the health of its mirrors.
+This assumes that the *token* is the *ALL* token from the origin gitblit instance.<br/>
+The repositories, example1_users.properties, and example1_gitblit.properties will be put in *git.repositoriesFolder* and the origin user accounts will be merged into the local user accounts, including passwords and administrator status.  The mirror instance will also send a status acknowledgment at the end of the pull operation which will include the state of each repository pull (EXCLUDED, SKIPPED, PULLED).  This way the origin Gitblit instance can monitor the health of its mirrors.
 
 This example is considered *nearly* perfect because while the remote Gitblit's server settings are pulled and saved locally, they are not merged with your server settings so its not a true mirror, but its likely the mirror you'd want to configure.
 
@@ -235,9 +235,9 @@ This example is considered *nearly* perfect because while the remote Gitblit's s
     federation.example1.token = 6f3b8a24bf970f17289b234284c94f43eb42f0e4
     federation.example1.frequency = 120 mins
     federation.example1.folder = 
+    federation.example1.mirror = true
     federation.example1.mergeAccounts = true
     federation.example1.sendStatus = true
-    federation.example1.freeze = true
     
 #### Just Repositories Example
 
@@ -248,6 +248,7 @@ The repositories will be put in *git.repositoriesFolder*/example2.
     federation.example2.token = 6f3b8a24bf970f17289b234284c94f43eb42f0e4
     federation.example2.frequency = 120 mins
     federation.example2.folder = example2
+    federation.example2.mirror = true
     
 #### All-but-One Repository Example
 
@@ -258,6 +259,7 @@ The repositories will be put in *git.repositoriesFolder*/example3.
     federation.example3.token = 6f3b8a24bf970f17289b234284c94f43eb42f0e4
     federation.example3.frequency = 120 mins
     federation.example3.folder = example3
+    federation.example3.mirror = true
     federation.example3.exclude = somerepo.git
     
 #### Just One Repository Example
@@ -269,5 +271,6 @@ The repositories will be put in *git.repositoriesFolder*/example4.
     federation.example4.token = 6f3b8a24bf970f17289b234284c94f43eb42f0e4
     federation.example4.frequency = 120 mins
     federation.example4.folder = example4
+    federation.example4.mirror = true
     federation.example4.exclude = *
     federation.example4.include = somerepo.git

resources/markdown.css

@@ -64,4 +64,10 @@ div.markdown li {
 
 div.markdown em {
 	color: #b05000;
+}
+
+div.markdown table.text th, div.markdown table.text td {
+	vertical-align: top;
+	border-top: 1px solid #ccc;
+	padding:5px;
 }

src/com/gitblit/Constants.java

@@ -135,7 +135,7 @@ public class Constants {
 	 * Enumeration representing the statii of federation requests.
 	 */
 	public static enum FederationPullStatus {
-		PENDING, FAILED, SKIPPED, PULLED, EXCLUDED;
+		PENDING, FAILED, SKIPPED, NOCHANGE, PULLED, MIRRORED, EXCLUDED;
 
 		public static FederationPullStatus fromName(String name) {
 			for (FederationPullStatus type : values()) {

src/com/gitblit/FederationPullExecutor.java

@@ -28,6 +28,7 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
+import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.StoredConfig;
 import org.eclipse.jgit.transport.CredentialsProvider;
@@ -42,9 +43,9 @@ import com.gitblit.models.RepositoryModel;
 import com.gitblit.models.UserModel;
 import com.gitblit.utils.FederationUtils;
 import com.gitblit.utils.JGitUtils;
-import com.gitblit.utils.TimeUtils;
 import com.gitblit.utils.JGitUtils.CloneResult;
 import com.gitblit.utils.StringUtils;
+import com.gitblit.utils.TimeUtils;
 
 /**
  * FederationPullExecutor pulls repository updates and, optionally, user
@@ -115,7 +116,7 @@ public class FederationPullExecutor implements Runnable {
 
 	/**
 	 * Mirrors a repository and, optionally, the server's users, and/or
-	 * configuration settings from a remote Gitblit instance.
+	 * configuration settings from a origin Gitblit instance.
 	 * 
 	 * @param registration
 	 * @throws Exception
@@ -157,11 +158,14 @@ public class FederationPullExecutor implements Runnable {
 
 			// confirm that the origin of any pre-existing repository matches
 			// the clone url
+			String fetchHead = null;
 			Repository existingRepository = GitBlit.self().getRepository(repositoryName);
 			if (existingRepository != null) {
 				StoredConfig config = existingRepository.getConfig();
 				config.load();
 				String origin = config.getString("remote", "origin", "url");
+				fetchHead = JGitUtils.getCommit(existingRepository, "refs/remotes/origin/master")
+						.getName();
 				existingRepository.close();
 				if (!origin.startsWith(registration.url)) {
 					logger.warn(MessageFormat
@@ -181,16 +185,48 @@ public class FederationPullExecutor implements Runnable {
 					cloneUrl, credentials);
 			Repository r = GitBlit.self().getRepository(repositoryName);
 			RepositoryModel rm = GitBlit.self().getRepositoryModel(repositoryName);
+			repository.isFrozen = registration.mirror;
 			if (result.createdRepository) {
 				// default local settings
 				repository.federationStrategy = FederationStrategy.EXCLUDE;
-				repository.isFrozen = registration.freeze;
+				repository.isFrozen = registration.mirror;
+				repository.showRemoteBranches = !registration.mirror;
+				logger.info(MessageFormat.format("     cloning {0}", repository.name));
+				registration.updateStatus(repository, FederationPullStatus.MIRRORED);
 			} else {
+				// fetch and update
+				boolean fetched = false;
+				String origin = JGitUtils.getCommit(r, "refs/remotes/origin/master").getName();
+				fetched = !fetchHead.equals(origin);
+
+				if (registration.mirror) {
+					// mirror
+					if (fetched) {
+						// reset the local HEAD to origin/master
+						Ref ref = JGitUtils.resetHEAD(r, "origin/master");
+						logger.info(MessageFormat.format("     resetting HEAD of {0} to {1}",
+								repository.name, ref.getObjectId().getName()));
+						registration.updateStatus(repository, FederationPullStatus.MIRRORED);
+					} else {
+						// indicate no commits pulled
+						registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
+					}
+				} else {
+					// non-mirror
+					if (fetched) {
+						// indicate commits pulled to origin/master
+						registration.updateStatus(repository, FederationPullStatus.PULLED);
+					} else {
+						// indicate no commits pulled
+						registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
+					}
+				}
+
 				// preserve local settings
 				repository.isFrozen = rm.isFrozen;
 				repository.federationStrategy = rm.federationStrategy;
 			}
-			// only repositories that are actually _cloned_ from the source
+			// only repositories that are actually _cloned_ from the origin
 			// Gitblit repository are marked as federated. If the origin
 			// is from somewhere else, these repositories are not considered
 			// "federated" repositories.
@@ -198,7 +234,6 @@ public class FederationPullExecutor implements Runnable {
 
 			GitBlit.self().updateConfiguration(r, repository);
 			r.close();
-			registration.updateStatus(repository, FederationPullStatus.PULLED);
 		}
 
 		try {
@@ -212,7 +247,7 @@ public class FederationPullExecutor implements Runnable {
 				for (UserModel user : users) {
 					userService.updateUserModel(user.username, user);
 
-					// merge the remote permissions and remote accounts into
+					// merge the origin permissions and origin accounts into
 					// the user accounts of this Gitblit instance
 					if (registration.mergeAccounts) {
 						// reparent all repository permissions if the local
@@ -273,7 +308,7 @@ public class FederationPullExecutor implements Runnable {
 	}
 
 	/**
-	 * Sends a status acknowledgment to the source Gitblit instance. This
+	 * Sends a status acknowledgment to the origin Gitblit instance. This
 	 * includes the results of the federated pull.
 	 * 
 	 * @param registration
@@ -290,6 +325,7 @@ public class FederationPullExecutor implements Runnable {
 			federationName = addr.getHostName();
 		}
 		FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
+		logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
 	}
 
 	/**

src/com/gitblit/GitBlit.java

@@ -861,10 +861,10 @@ public class GitBlit implements ServletContextListener {
 				}
 				String setting = values.get(1);
 				if (setting.equals("url")) {
-					// url of the remote Gitblit instance
+					// url of the origin Gitblit instance
 					federatedModels.get(server).url = settings.getString(key, "");
 				} else if (setting.equals("token")) {
-					// token for the remote Gitblit instance
+					// token for the origin Gitblit instance
 					federatedModels.get(server).token = settings.getString(key, "");
 				} else if (setting.equals("frequency")) {
 					// frequency of the pull operation
@@ -872,9 +872,9 @@ public class GitBlit implements ServletContextListener {
 				} else if (setting.equals("folder")) {
 					// destination folder of the pull operation
 					federatedModels.get(server).folder = settings.getString(key, "");
-				} else if (setting.equals("freeze")) {
-					// set the repository to read-only after pull
-					federatedModels.get(server).freeze = settings.getBoolean(key, true);
+				} else if (setting.equals("mirror")) {
+					// are the repositories to be true mirrors of the origin
+					federatedModels.get(server).mirror = settings.getBoolean(key, true);
 				} else if (setting.equals("mergeAccounts")) {
 					// merge remote accounts into local accounts
 					federatedModels.get(server).mergeAccounts = settings.getBoolean(key, false);

src/com/gitblit/models/FederationModel.java

@@ -46,7 +46,7 @@ public class FederationModel implements Serializable, Comparable<FederationModel
 
 	public String folder;
 
-    public boolean freeze;
+    public boolean mirror;
 
 	public boolean mergeAccounts;
 

src/com/gitblit/utils/JGitUtils.java

@@ -37,6 +37,10 @@ import java.util.zip.ZipOutputStream;
 import org.eclipse.jgit.api.CloneCommand;
 import org.eclipse.jgit.api.FetchCommand;
 import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.PullCommand;
+import org.eclipse.jgit.api.PullResult;
+import org.eclipse.jgit.api.ResetCommand;
+import org.eclipse.jgit.api.ResetCommand.ResetType;
 import org.eclipse.jgit.diff.DiffEntry;
 import org.eclipse.jgit.diff.DiffEntry.ChangeType;
 import org.eclipse.jgit.diff.DiffFormatter;
@@ -229,7 +233,28 @@ public class JGitUtils {
 			fetch.setCredentialsProvider(credentialsProvider);
 		}
 		fetch.setRefSpecs(specs);
-		FetchResult result = fetch.call();
+		FetchResult fetchRes = fetch.call();
+		return fetchRes;
+	}
+
+	/**
+	 * Reset HEAD to the latest remote tracking commit.
+	 * 
+	 * @param repository
+	 * @param remoteRef
+	 *            the remote tracking reference (e.g. origin/master)
+	 * @return Ref
+	 * @throws Exception
+	 */
+	public static Ref resetHEAD(Repository repository, String remoteRef) throws Exception {
+		if (!remoteRef.startsWith(Constants.R_REMOTES)) {
+			remoteRef = Constants.R_REMOTES + remoteRef;
+		}
+		Git git = new Git(repository);
+		ResetCommand reset = git.reset();
+		reset.setMode(ResetType.SOFT);
+		reset.setRef(remoteRef);
+		Ref result = reset.call();
 		return result;
 	}
 

src/com/gitblit/wicket/WicketUtils.java

@@ -115,6 +115,7 @@ public class WicketUtils {
 	public static ContextImage getPullStatusImage(String wicketId, FederationPullStatus status) {
 		String filename = null;
 		switch (status) {
+		case MIRRORED:
 		case PULLED:
 			filename = "bullet_green.png";
 			break;
@@ -128,6 +129,7 @@ public class WicketUtils {
 			filename = "bullet_white.png";
 			break;
 		case PENDING:
+		case NOCHANGE:			
 		default:
 			filename = "bullet_black.png";
 		}

src/com/gitblit/wicket/pages/FederationProposalPage.java

@@ -79,7 +79,7 @@ public class FederationProposalPage extends BasePage {
 		sb.append(asParam(p, proposal.name, "frequency",
 				GitBlit.getString(Keys.federation.defaultFrequency, "60 mins")));
 		sb.append(asParam(p, proposal.name, "folder", proposal.name));
-		sb.append(asParam(p, proposal.name, "freeze", "true"));
+		sb.append(asParam(p, proposal.name, "mirror", "true"));
 		sb.append(asParam(p, proposal.name, "sendStatus", "true"));
 		sb.append(asParam(p, proposal.name, "notifyOnError", "true"));
 		sb.append(asParam(p, proposal.name, "exclude", ""));