FetchCommand: Add basic support for recursing into submodules

Extend FetchCommand to expose a new method, setRecurseSubmodules(mode),
which allows to set the mode to ON, OFF or ON_DEMAND.

After fetching a repository, its submodules are recursively fetched:

- When the mode is YES, submodules are always fetched.

- When the mode is NO, submodules are not fetched.

- When the mode is ON_DEMAND, submodules are only fetched when the
  parent repository receives an update of the submodule and the new
  revision is not already in the submodule.

The mode is determined in the following order of precedence:

- Value specified in the API call using setRecurseSubmodules.

- Value specified in the repository's config under the key
  submodule.name.fetchRecurseSubmodules

- Defaults to ON_DEMAND if neither of the previous is set.

Extend FetchResult to recursively include results for submodules, as
a map of the submodule path to an instance of FetchResult.

Test setup is based on testCloneRepositoryWithNestedSubmodules.

Change-Id: Ibc841683763307cb76e78e142e0da5b11b1add2a
Signed-off-by: David Pursehouse <david.pursehouse@gmail.com>

4 files changed, 219 insertions, 1 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java
index a9dca4d7bc..b365087888 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java
@@ -42,14 +42,17 @@
  */
 package org.eclipse.jgit.api;
 
+import java.io.IOException;
 import java.net.URISyntaxException;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.InvalidConfigurationException;
 import org.eclipse.jgit.api.errors.InvalidRemoteException;
 import org.eclipse.jgit.api.errors.JGitInternalException;
+import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.eclipse.jgit.errors.NoRemoteRepositoryException;
 import org.eclipse.jgit.errors.NotSupportedException;
 import org.eclipse.jgit.errors.TransportException;
@@ -60,6 +63,9 @@ import org.eclipse.jgit.lib.NullProgressMonitor;
 import org.eclipse.jgit.lib.ProgressMonitor;
 import org.eclipse.jgit.lib.Repository;
 import org.eclipse.jgit.lib.StoredConfig;
+import org.eclipse.jgit.lib.SubmoduleConfig.FetchRecurseSubmodulesMode;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.eclipse.jgit.submodule.SubmoduleWalk;
 import org.eclipse.jgit.transport.FetchResult;
 import org.eclipse.jgit.transport.RefSpec;
 import org.eclipse.jgit.transport.TagOpt;
@@ -74,7 +80,6 @@ import org.eclipse.jgit.transport.Transport;
  *      >Git documentation about Fetch</a>
  */
 public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
-
 	private String remote = Constants.DEFAULT_REMOTE_NAME;
 
 	private List<RefSpec> refSpecs;
@@ -91,6 +96,8 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
 
 	private TagOpt tagOption;
 
+	private FetchRecurseSubmodulesMode submoduleRecurseMode = null;
+
 	/**
 	 * @param repo
 	 */
@@ -99,6 +106,76 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
 		refSpecs = new ArrayList<>(3);
 	}
 
+	private FetchRecurseSubmodulesMode getRecurseMode(Repository repository,
+			String path) {
+		// Use the caller-specified mode, if set
+		if (submoduleRecurseMode != null) {
+			return submoduleRecurseMode;
+		}
+
+		// Fall back to submodule config, if set
+		FetchRecurseSubmodulesMode mode = repository.getConfig().getEnum(
+				FetchRecurseSubmodulesMode.values(),
+				ConfigConstants.CONFIG_SUBMODULE_SECTION, path,
+				ConfigConstants.CONFIG_KEY_FETCH_RECURSE_SUBMODULES, null);
+		if (mode != null) {
+			return mode;
+		}
+
+		// Default to on-demand mode
+		return FetchRecurseSubmodulesMode.ON_DEMAND;
+	}
+
+	private boolean isRecurseSubmodules() {
+		return submoduleRecurseMode != null
+				&& submoduleRecurseMode != FetchRecurseSubmodulesMode.NO;
+	}
+
+	private void fetchSubmodules(FetchResult results)
+			throws org.eclipse.jgit.api.errors.TransportException,
+			GitAPIException, InvalidConfigurationException {
+		try (SubmoduleWalk walk = new SubmoduleWalk(repo);
+				RevWalk revWalk = new RevWalk(repo)) {
+			// Walk over submodules in the parent repository's FETCH_HEAD.
+			walk.setTree(revWalk.parseTree(repo.resolve(Constants.FETCH_HEAD)));
+			while (walk.next()) {
+				Repository submoduleRepo = walk.getRepository();
+
+				// Skip submodules that don't exist locally (have not been
+				// cloned), are not registered in the .gitmodules file, or
+				// not registered in the parent repository's config.
+				if (submoduleRepo == null || walk.getModulesPath() == null
+						|| walk.getConfigUrl() == null) {
+					continue;
+				}
+
+				FetchRecurseSubmodulesMode recurseMode = getRecurseMode(
+						submoduleRepo, walk.getPath());
+
+				// When the fetch mode is "yes" we always fetch. When the mode
+				// is "on demand", we only fetch if the submodule's revision was
+				// updated to an object that is not currently present in the
+				// submodule.
+				if ((recurseMode == FetchRecurseSubmodulesMode.ON_DEMAND
+						&& !submoduleRepo.hasObject(walk.getObjectId()))
+						|| recurseMode == FetchRecurseSubmodulesMode.YES) {
+					FetchCommand f = new FetchCommand(submoduleRepo)
+							.setProgressMonitor(monitor).setTagOpt(tagOption)
+							.setCheckFetchedObjects(checkFetchedObjects)
+							.setRemoveDeletedRefs(isRemoveDeletedRefs())
+							.setThin(thin).setRefSpecs(refSpecs)
+							.setDryRun(dryRun)
+							.setRecurseSubmodules(recurseMode);
+					results.addSubmodule(walk.getPath(), f.call());
+				}
+			}
+		} catch (IOException e) {
+			throw new JGitInternalException(e.getMessage(), e);
+		} catch (ConfigInvalidException e) {
+			throw new InvalidConfigurationException(e.getMessage(), e);
+		}
+	}
+
 	/**
 	 * Executes the {@code fetch} command with all the options and parameters
 	 * collected by the setter methods of this class. Each instance of this
@@ -127,6 +204,11 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
 			configure(transport);
 
 			FetchResult result = transport.fetch(monitor, refSpecs);
+			if (!repo.isBare() && (!result.getTrackingRefUpdates().isEmpty()
+					|| isRecurseSubmodules())) {
+				fetchSubmodules(result);
+			}
+
 			return result;
 		} catch (NoRemoteRepositoryException e) {
 			throw new InvalidRemoteException(MessageFormat.format(
@@ -146,6 +228,20 @@ public class FetchCommand extends TransportCommand<FetchCommand, FetchResult> {
 	}
 
 	/**
+	 * Set the mode to be used for recursing into submodules.
+	 *
+	 * @param recurse
+	 * @return {@code this}
+	 * @since 4.7
+	 */
+	public FetchCommand setRecurseSubmodules(
+			FetchRecurseSubmodulesMode recurse) {
+		checkCallable();
+		submoduleRecurseMode = recurse;
+		return this;
+	}
+
+	/**
 	 * The remote (uri or name) used for the fetch operation. If no remote is
 	 * set, the default value of <code>Constants.DEFAULT_REMOTE_NAME</code> will
 	 * be used.
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
index cde41c2124..ff0d811ba9 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
@@ -374,4 +374,10 @@ public class ConfigConstants {
 	 * @since 4.6
 	 */
 	public static final String CONFIG_KEY_USEJGITBUILTIN = "useJGitBuiltin";
+
+	/**
+	 * The "fetchRecurseSubmodules" key
+	 * @since 4.7
+	 */
+	public static final String CONFIG_KEY_FETCH_RECURSE_SUBMODULES = "fetchRecurseSubmodules";
 }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SubmoduleConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SubmoduleConfig.java
new file mode 100644
index 0000000000..3126160c33
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SubmoduleConfig.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright (C) 2017, David Pursehouse <david.pursehouse@gmail.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.lib;
+
+/**
+ * Submodule section of a Git configuration file.
+ *
+ * @since 4.7
+ */
+public class SubmoduleConfig {
+
+	/**
+	 * Config values for submodule.[name].fetchRecurseSubmodules.
+	 */
+	public enum FetchRecurseSubmodulesMode implements Config.ConfigEnum {
+		/** Unconditionally recurse into all populated submodules. */
+		YES("true"), //$NON-NLS-1$
+
+		/**
+		 * Only recurse into a populated submodule when the superproject
+		 * retrieves a commit that updates the submodule's reference to a commit
+		 * that isn't already in the local submodule clone.
+		 */
+		ON_DEMAND("on-demand"), //$NON-NLS-1$
+
+		/** Completely disable recursion. */
+		NO("false"); //$NON-NLS-1$
+
+		private final String configValue;
+
+		private FetchRecurseSubmodulesMode(String configValue) {
+			this.configValue = configValue;
+		}
+
+		@Override
+		public String toConfigValue() {
+			return configValue;
+		}
+
+		@Override
+		public boolean matchConfigValue(String s) {
+			return configValue.equals(s);
+		}
+	}
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchResult.java
index 7df997c1e2..2667ec37ca 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchResult.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/FetchResult.java
@@ -48,7 +48,10 @@ package org.eclipse.jgit.transport;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Final status after a successful fetch from a remote repository.
@@ -58,12 +61,39 @@ import java.util.List;
 public class FetchResult extends OperationResult {
 	private final List<FetchHeadRecord> forMerge;
 
+	private final Map<String, FetchResult> submodules;
+
 	FetchResult() {
 		forMerge = new ArrayList<>();
+		submodules = new HashMap<>();
 	}
 
 	void add(final FetchHeadRecord r) {
 		if (!r.notForMerge)
 			forMerge.add(r);
 	}
+
+	/**
+	 * Add fetch results for a submodule.
+	 *
+	 * @param path
+	 *            the submodule path
+	 * @param result
+	 *            the fetch result
+	 * @since 4.7
+	 */
+	public void addSubmodule(String path, FetchResult result) {
+		submodules.put(path, result);
+	}
+
+	/**
+	 * Get fetch results for submodules.
+	 *
+	 * @return Fetch results for submodules as a map of submodule paths to fetch
+	 *         results.
+	 * @since 4.7
+	 */
+	public Map<String, FetchResult> submoduleResults() {
+		return Collections.unmodifiableMap(submodules);
+	}
 }