Ignore submodule on checkout instead of deleting it

The purpose of this commit is to prevent destruction of
submodules on checkout from a tree with a submodule to
another. For consistency we handle the reverse case too,
when we checkout a branch that has a submodule and the
submodule directory exists. And finally we ignore the
case where the submodule changes.

We do not update the submodules, we just try to ignore
them harder.

Bug: 356664
Change-Id: I202c695a57af99b13d0d7220803fd08def3d9b5e
Signed-off-by: Robin Rosenberg <robin.rosenberg@dewire.com>

org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java

@@ -414,9 +414,15 @@ public class DirCacheCheckout {
 			for (int i = removed.size() - 1; i >= 0; i--) {
 				String r = removed.get(i);
 				file = new File(repo.getWorkTree(), r);
-				if (!file.delete() && file.exists())
-					toBeDeleted.add(r);
-				else {
+				if (!file.delete() && file.exists()) {
+					// The list of stuff to delete comes from the index
+					// which will only contain a directory if it is
+					// a submodule, in which case we shall not attempt
+					// to delete it. A submodule is not empty, so it
+					// is safe to check this after a failed delete.
+					if (!file.isDirectory())
+						toBeDeleted.add(r);
+				} else {
 					if (!isSamePrefix(r, last))
 						removeEmptyParents(new File(repo.getWorkTree(), last));
 					last = r;
@@ -654,11 +660,14 @@ public class DirCacheCheckout {
 		if (i == null) {
 			// make sure not to overwrite untracked files
 			if (f != null) {
-				// a dirty worktree: the index is empty but we have a
-				// workingtree-file
-				if (mId == null || !mId.equals(f.getEntryObjectId())) {
-					conflict(name, null, h, m);
-					return;
+				// A submodule is not a file. We should ignore it
+				if (!FileMode.GITLINK.equals(m.getEntryFileMode())) {
+					// a dirty worktree: the index is empty but we have a
+					// workingtree-file
+					if (mId == null || !mId.equals(f.getEntryObjectId())) {
+						conflict(name, null, h, m);
+						return;
+					}
 				}
 			}
 
@@ -720,21 +729,33 @@ public class DirCacheCheckout {
 				 * </pre>
 				 */
 
-				if (hId.equals(iId)) {
-					if (f == null || f.isModified(dce, true))
+				if (dce.getFileMode() == FileMode.GITLINK) {
+					// Submodules that disappear from the checkout must
+					// be removed from the index, but not deleted from disk.
+					remove(name);
+				} else {
+					if (hId.equals(iId)) {
+						if (f == null || f.isModified(dce, true))
+							conflict(name, dce, h, m);
+						else
+							remove(name);
+					} else
 						conflict(name, dce, h, m);
-					else
-						remove(name);
-				} else
-					conflict(name, dce, h, m);
+				}
 			} else {
 				if (!hId.equals(mId) && !hId.equals(iId) && !mId.equals(iId))
 					conflict(name, dce, h, m);
 				else if (hId.equals(iId) && !mId.equals(iId)) {
-					if (dce != null && (f == null || f.isModified(dce, true)))
+					// For submodules just update the index with the new SHA-1
+					if (dce != null
+							&& FileMode.GITLINK.equals(dce.getFileMode())) {
+						update(name, mId, m.getEntryFileMode());
+					} else if (dce != null
+							&& (f == null || f.isModified(dce, true))) {
 						conflict(name, dce, h, m);
-					else
+					} else {
 						update(name, mId, m.getEntryFileMode());
+					}
 				} else {
 					keep(dce);
 				}