gitblit/src/main/java/com/gitblit/FederationPullExecutor.java

/*
 * Copyright 2011 gitblit.com.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.gitblit;

import static org.eclipse.jgit.lib.Constants.DOT_GIT_EXT;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.StoredConfig;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.Constants.AccessPermission;
import com.gitblit.Constants.FederationPullStatus;
import com.gitblit.Constants.FederationStrategy;
import com.gitblit.GitBlitException.ForbiddenException;
import com.gitblit.models.FederationModel;
import com.gitblit.models.RefModel;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.TeamModel;
import com.gitblit.models.UserModel;
import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.FederationUtils;
import com.gitblit.utils.FileUtils;
import com.gitblit.utils.JGitUtils;
import com.gitblit.utils.JGitUtils.CloneResult;
import com.gitblit.utils.StringUtils;
import com.gitblit.utils.TimeUtils;

/**
 * FederationPullExecutor pulls repository updates and, optionally, user
 * accounts and server settings from registered Gitblit instances.
 */
public class FederationPullExecutor implements Runnable {

	private final Logger logger = LoggerFactory.getLogger(FederationPullExecutor.class);

	private final List<FederationModel> registrations;

	private final boolean isDaemon;

	/**
	 * Constructor for specifying a single federation registration. This
	 * constructor is used to schedule the next pull execution.
	 * 
	 * @param registration
	 */
	private FederationPullExecutor(FederationModel registration) {
		this(Arrays.asList(registration), true);
	}

	/**
	 * Constructor to specify a group of federation registrations. This is
	 * normally used at startup to pull and then schedule the next update based
	 * on each registrations frequency setting.
	 * 
	 * @param registrations
	 * @param isDaemon
	 *            if true, registrations are rescheduled in perpetuity. if
	 *            false, the federation pull operation is executed once.
	 */
	public FederationPullExecutor(List<FederationModel> registrations, boolean isDaemon) {
		this.registrations = registrations;
		this.isDaemon = isDaemon;
	}

	/**
	 * Run method for this pull executor.
	 */
	@Override
	public void run() {
		for (FederationModel registration : registrations) {
			FederationPullStatus was = registration.getLowestStatus();
			try {
				Date now = new Date(System.currentTimeMillis());
				pull(registration);
				sendStatusAcknowledgment(registration);
				registration.lastPull = now;
				FederationPullStatus is = registration.getLowestStatus();
				if (is.ordinal() < was.ordinal()) {
					// the status for this registration has downgraded
					logger.warn("Federation pull status of {0} is now {1}", registration.name,
							is.name());
					if (registration.notifyOnError) {
						String message = "Federation pull of " + registration.name + " @ "
								+ registration.url + " is now at " + is.name();
						GitBlit.self()
								.sendMailToAdministrators(
										"Pull Status of " + registration.name + " is " + is.name(),
										message);
					}
				}
			} catch (Throwable t) {
				logger.error(MessageFormat.format(
						"Failed to pull from federated gitblit ({0} @ {1})", registration.name,
						registration.url), t);
			} finally {
				if (isDaemon) {
					schedule(registration);
				}
			}
		}
	}

	/**
	 * Mirrors a repository and, optionally, the server's users, and/or
	 * configuration settings from a origin Gitblit instance.
	 * 
	 * @param registration
	 * @throws Exception
	 */
	private void pull(FederationModel registration) throws Exception {
		Map<String, RepositoryModel> repositories = FederationUtils.getRepositories(registration,
				true);
		String registrationFolder = registration.folder.toLowerCase().trim();
		// confirm valid characters in server alias
		Character c = StringUtils.findInvalidCharacter(registrationFolder);
		if (c != null) {
			logger.error(MessageFormat
					.format("Illegal character ''{0}'' in folder name ''{1}'' of federation registration {2}!",
							c, registrationFolder, registration.name));
			return;
		}
		File repositoriesFolder = new File(GitBlit.getString(Keys.git.repositoriesFolder, "git"));
		File registrationFolderFile = new File(repositoriesFolder, registrationFolder);
		registrationFolderFile.mkdirs();

		// Clone/Pull the repository
		for (Map.Entry<String, RepositoryModel> entry : repositories.entrySet()) {
			String cloneUrl = entry.getKey();
			RepositoryModel repository = entry.getValue();
			if (!repository.hasCommits) {
				logger.warn(MessageFormat.format(
						"Skipping federated repository {0} from {1} @ {2}. Repository is EMPTY.",
						repository.name, registration.name, registration.url));
				registration.updateStatus(repository, FederationPullStatus.SKIPPED);
				continue;
			}

			// Determine local repository name
			String repositoryName;
			if (StringUtils.isEmpty(registrationFolder)) {
				repositoryName = repository.name;
			} else {
				repositoryName = registrationFolder + "/" + repository.name;
			}

			if (registration.bare) {
				// bare repository, ensure .git suffix
				if (!repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
					repositoryName += DOT_GIT_EXT;
				}
			} else {
				// normal repository, strip .git suffix
				if (repositoryName.toLowerCase().endsWith(DOT_GIT_EXT)) {
					repositoryName = repositoryName.substring(0,
							repositoryName.indexOf(DOT_GIT_EXT));
				}
			}
			
			// confirm that the origin of any pre-existing repository matches
			// the clone url
			String fetchHead = null;
			Repository existingRepository = GitBlit.self().getRepository(repositoryName);
			
			if (existingRepository == null && GitBlit.self().isCollectingGarbage(repositoryName)) {
				logger.warn(MessageFormat.format("Skipping local repository {0}, busy collecting garbage", repositoryName));
				continue;
			}

			if (existingRepository != null) {
				StoredConfig config = existingRepository.getConfig();
				config.load();
				String origin = config.getString("remote", "origin", "url");
				RevCommit commit = JGitUtils.getCommit(existingRepository,
						org.eclipse.jgit.lib.Constants.FETCH_HEAD);
				if (commit != null) {
					fetchHead = commit.getName();
				}
				existingRepository.close();
				if (!origin.startsWith(registration.url)) {
					logger.warn(MessageFormat
							.format("Skipping federated repository {0} from {1} @ {2}. Origin does not match, consider EXCLUDING.",
									repository.name, registration.name, registration.url));
					registration.updateStatus(repository, FederationPullStatus.SKIPPED);
					continue;
				}
			}

			// clone/pull this repository
			CredentialsProvider credentials = new UsernamePasswordCredentialsProvider(
					Constants.FEDERATION_USER, registration.token);
			logger.info(MessageFormat.format("Pulling federated repository {0} from {1} @ {2}",
					repository.name, registration.name, registration.url));

			CloneResult result = JGitUtils.cloneRepository(registrationFolderFile, repository.name,
					cloneUrl, registration.bare, credentials);
			Repository r = GitBlit.self().getRepository(repositoryName);
			RepositoryModel rm = GitBlit.self().getRepositoryModel(repositoryName);
			repository.isFrozen = registration.mirror;
			if (result.createdRepository) {
				// default local settings
				repository.federationStrategy = FederationStrategy.EXCLUDE;
				repository.isFrozen = registration.mirror;
				repository.showRemoteBranches = !registration.mirror;
				logger.info(MessageFormat.format("     cloning {0}", repository.name));
				registration.updateStatus(repository, FederationPullStatus.MIRRORED);
			} else {
				// fetch and update
				boolean fetched = false;
				RevCommit commit = JGitUtils.getCommit(r, org.eclipse.jgit.lib.Constants.FETCH_HEAD);
				String newFetchHead = commit.getName();
				fetched = fetchHead == null || !fetchHead.equals(newFetchHead);

				if (registration.mirror) {
					// mirror
					if (fetched) {
						// update local branches to match the remote tracking branches
						for (RefModel ref : JGitUtils.getRemoteBranches(r, false, -1)) {
							if (ref.displayName.startsWith("origin/")) {
								String branch = org.eclipse.jgit.lib.Constants.R_HEADS
										+ ref.displayName.substring(ref.displayName.indexOf('/') + 1);
								String hash = ref.getReferencedObjectId().getName();
								
								JGitUtils.setBranchRef(r, branch, hash);
								logger.info(MessageFormat.format("     resetting {0} of {1} to {2}", branch,
										repository.name, hash));
							}
						}
						
						String newHead;
						if (StringUtils.isEmpty(repository.HEAD)) {
							newHead = newFetchHead;
						} else {
							newHead = repository.HEAD;
						}
						JGitUtils.setHEADtoRef(r, newHead);
						logger.info(MessageFormat.format("     resetting HEAD of {0} to {1}",
								repository.name, newHead));
						registration.updateStatus(repository, FederationPullStatus.MIRRORED);
					} else {
						// indicate no commits pulled
						registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
					}
				} else {
					// non-mirror
					if (fetched) {
						// indicate commits pulled to origin/master
						registration.updateStatus(repository, FederationPullStatus.PULLED);
					} else {
						// indicate no commits pulled
						registration.updateStatus(repository, FederationPullStatus.NOCHANGE);
					}
				}

				// preserve local settings
				repository.isFrozen = rm.isFrozen;
				repository.federationStrategy = rm.federationStrategy;

				// merge federation sets
				Set<String> federationSets = new HashSet<String>();
				if (rm.federationSets != null) {
					federationSets.addAll(rm.federationSets);
				}
				if (repository.federationSets != null) {
					federationSets.addAll(repository.federationSets);
				}
				repository.federationSets = new ArrayList<String>(federationSets);
				
				// merge indexed branches
				Set<String> indexedBranches = new HashSet<String>();
				if (rm.indexedBranches != null) {
					indexedBranches.addAll(rm.indexedBranches);
				}
				if (repository.indexedBranches != null) {
					indexedBranches.addAll(repository.indexedBranches);
				}
				repository.indexedBranches = new ArrayList<String>(indexedBranches);

			}
			// only repositories that are actually _cloned_ from the origin
			// Gitblit repository are marked as federated. If the origin
			// is from somewhere else, these repositories are not considered
			// "federated" repositories.
			repository.isFederated = cloneUrl.startsWith(registration.url);

			GitBlit.self().updateConfiguration(r, repository);
			r.close();
		}

		IUserService userService = null;

		try {
			// Pull USERS
			// TeamModels are automatically pulled because they are contained
			// within the UserModel. The UserService creates unknown teams
			// and updates existing teams.
			Collection<UserModel> users = FederationUtils.getUsers(registration);
			if (users != null && users.size() > 0) {
				File realmFile = new File(registrationFolderFile, registration.name + "_users.conf");
				realmFile.delete();
				userService = new ConfigUserService(realmFile);
				for (UserModel user : users) {
					userService.updateUserModel(user.username, user);

					// merge the origin permissions and origin accounts into
					// the user accounts of this Gitblit instance
					if (registration.mergeAccounts) {
						// reparent all repository permissions if the local
						// repositories are stored within subfolders
						if (!StringUtils.isEmpty(registrationFolder)) {
							if (user.permissions != null) {
								// pulling from >= 1.2 version
								Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
								user.permissions.clear();
								for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
									user.setRepositoryPermission(registrationFolder + "/" + entry.getKey(), entry.getValue());
								}
							} else {
								// pulling from <= 1.1 version
								List<String> permissions = new ArrayList<String>(user.repositories);
								user.repositories.clear();
								for (String permission : permissions) {
									user.addRepositoryPermission(registrationFolder + "/" + permission);
								}
							}
						}

						// insert new user or update local user
						UserModel localUser = GitBlit.self().getUserModel(user.username);
						if (localUser == null) {
							// create new local user
							GitBlit.self().updateUserModel(user.username, user, true);
						} else {
							// update repository permissions of local user
							if (user.permissions != null) {
								// pulling from >= 1.2 version
								Map<String, AccessPermission> copy = new HashMap<String, AccessPermission>(user.permissions);
								for (Map.Entry<String, AccessPermission> entry : copy.entrySet()) {
									localUser.setRepositoryPermission(entry.getKey(), entry.getValue());
								}
							} else {
								// pulling from <= 1.1 version
								for (String repository : user.repositories) {
									localUser.addRepositoryPermission(repository);
								}
							}
							localUser.password = user.password;
							localUser.canAdmin = user.canAdmin;
							GitBlit.self().updateUserModel(localUser.username, localUser, false);
						}

						for (String teamname : GitBlit.self().getAllTeamnames()) {
							TeamModel team = GitBlit.self().getTeamModel(teamname);
							if (user.isTeamMember(teamname) && !team.hasUser(user.username)) {
								// new team member
								team.addUser(user.username);
								GitBlit.self().updateTeamModel(teamname, team, false);
							} else if (!user.isTeamMember(teamname) && team.hasUser(user.username)) {
								// remove team member
								team.removeUser(user.username);
								GitBlit.self().updateTeamModel(teamname, team, false);
							}

							// update team repositories
							TeamModel remoteTeam = user.getTeam(teamname);
							if (remoteTeam != null) {
								if (remoteTeam.permissions != null) {
									// pulling from >= 1.2
									for (Map.Entry<String, AccessPermission> entry : remoteTeam.permissions.entrySet()){
										team.setRepositoryPermission(entry.getKey(), entry.getValue());
									}
									GitBlit.self().updateTeamModel(teamname, team, false);
								} else if(!ArrayUtils.isEmpty(remoteTeam.repositories)) {
									// pulling from <= 1.1
									team.addRepositoryPermissions(remoteTeam.repositories);
									GitBlit.self().updateTeamModel(teamname, team, false);
								}
							}
						}
					}
				}
			}
		} catch (ForbiddenException e) {
			// ignore forbidden exceptions
		} catch (IOException e) {
			logger.warn(MessageFormat.format(
					"Failed to retrieve USERS from federated gitblit ({0} @ {1})",
					registration.name, registration.url), e);
		}

		try {
			// Pull TEAMS
			// We explicitly pull these even though they are embedded in
			// UserModels because it is possible to use teams to specify
			// mailing lists or push scripts without specifying users.
			if (userService != null) {
				Collection<TeamModel> teams = FederationUtils.getTeams(registration);
				if (teams != null && teams.size() > 0) {
					for (TeamModel team : teams) {
						userService.updateTeamModel(team);
					}
				}
			}
		} catch (ForbiddenException e) {
			// ignore forbidden exceptions
		} catch (IOException e) {
			logger.warn(MessageFormat.format(
					"Failed to retrieve TEAMS from federated gitblit ({0} @ {1})",
					registration.name, registration.url), e);
		}

		try {
			// Pull SETTINGS
			Map<String, String> settings = FederationUtils.getSettings(registration);
			if (settings != null && settings.size() > 0) {
				Properties properties = new Properties();
				properties.putAll(settings);
				FileOutputStream os = new FileOutputStream(new File(registrationFolderFile,
						registration.name + "_" + Constants.PROPERTIES_FILE));
				properties.store(os, null);
				os.close();
			}
		} catch (ForbiddenException e) {
			// ignore forbidden exceptions
		} catch (IOException e) {
			logger.warn(MessageFormat.format(
					"Failed to retrieve SETTINGS from federated gitblit ({0} @ {1})",
					registration.name, registration.url), e);
		}

		try {
			// Pull SCRIPTS
			Map<String, String> scripts = FederationUtils.getScripts(registration);
			if (scripts != null && scripts.size() > 0) {
				for (Map.Entry<String, String> script : scripts.entrySet()) {
					String scriptName = script.getKey();
					if (scriptName.endsWith(".groovy")) {
						scriptName = scriptName.substring(0, scriptName.indexOf(".groovy"));
					}
					File file = new File(registrationFolderFile, registration.name + "_"
							+ scriptName + ".groovy");
					FileUtils.writeContent(file, script.getValue());
				}
			}
		} catch (ForbiddenException e) {
			// ignore forbidden exceptions
		} catch (IOException e) {
			logger.warn(MessageFormat.format(
					"Failed to retrieve SCRIPTS from federated gitblit ({0} @ {1})",
					registration.name, registration.url), e);
		}
	}

	/**
	 * Sends a status acknowledgment to the origin Gitblit instance. This
	 * includes the results of the federated pull.
	 * 
	 * @param registration
	 * @throws Exception
	 */
	private void sendStatusAcknowledgment(FederationModel registration) throws Exception {
		if (!registration.sendStatus) {
			// skip status acknowledgment
			return;
		}
		InetAddress addr = InetAddress.getLocalHost();
		String federationName = GitBlit.getString(Keys.federation.name, null);
		if (StringUtils.isEmpty(federationName)) {
			federationName = addr.getHostName();
		}
		FederationUtils.acknowledgeStatus(addr.getHostAddress(), registration);
		logger.info(MessageFormat.format("Pull status sent to {0}", registration.url));
	}

	/**
	 * Schedules the next check of the federated Gitblit instance.
	 * 
	 * @param registration
	 */
	private void schedule(FederationModel registration) {
		// schedule the next pull
		int mins = TimeUtils.convertFrequencyToMinutes(registration.frequency);
		registration.nextPull = new Date(System.currentTimeMillis() + (mins * 60 * 1000L));
		GitBlit.self().executor()
				.schedule(new FederationPullExecutor(registration), mins, TimeUnit.MINUTES);
		logger.info(MessageFormat.format(
				"Next pull of {0} @ {1} scheduled for {2,date,yyyy-MM-dd HH:mm}",
				registration.name, registration.url, registration.nextPull));
	}
}