Add symlink support to JGit

The change includes comparing symbolic links between disk and index,
adding symbolic links to the index, creating/modifying links on
checkout. The behavior is controlled by the core.symlinks setting, just
as C Git does. When a new repository is created core.symlinks will be
set depending on the capabilities of the operating system and Java
runtime.

If core.symlinks is set to true, the assumption is that symlinks are
supported, which may result in runtime errors if this turns out not to
be the case.

Measuring the cost of jgit status on a repository with ~70000 files,
of which ~30000 are tracked reveals a penalty of about 10% for using
the Java7 (really NIO2) support module.

Bug: 354367
Change-Id: I12f0fdd9d26212324a586896ef7eb1f6ff89c39c
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

11 files changed, 156 insertions, 62 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java
index 11dfd1c585..29726146c3 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/BlameCommand.java
@@ -215,7 +215,7 @@ public class BlameCommand extends GitCommand<BlameResult> {
 						gen.push(null, dc.getEntry(entry).getObjectId());
 
 					File inTree = new File(repo.getWorkTree(), path);
-					if (inTree.isFile())
+					if (repo.getFS().isFile(inTree))
 						gen.push(null, new RawText(inTree));
 				}
 			}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java
index f273eafe1f..5967128113 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CleanCommand.java
@@ -53,6 +53,7 @@ import org.eclipse.jgit.api.errors.GitAPIException;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.errors.NoWorkTreeException;
 import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.util.FS;
 import org.eclipse.jgit.util.FileUtils;
 
 /**
@@ -100,13 +101,13 @@ public class CleanCommand extends GitCommand<Set<String>> {
 			Set<String> untrackedAndIgnoredDirs = new TreeSet<String>(
 					status.getUntrackedFolders());
 
+			FS fs = getRepository().getFS();
 			for (String p : status.getIgnoredNotInIndex()) {
 				File f = new File(repo.getWorkTree(), p);
-				if (f.isFile()) {
+				if (fs.isFile(f) || fs.isSymLink(f))
 					untrackedAndIgnoredFiles.add(p);
-				} else if (f.isDirectory()) {
+				else if (fs.isDirectory(f))
 					untrackedAndIgnoredDirs.add(p);
-				}
 			}
 
 			Set<String> filtered = filterFolders(untrackedAndIgnoredFiles,
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java
index e930c535e6..cb1e6cf147 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java
@@ -731,7 +731,7 @@ public class RebaseCommand extends GitCommand<RebaseResult> {
 				List<String> fileList = dco.getToBeDeleted();
 				for (String filePath : fileList) {
 					File fileToDelete = new File(repo.getWorkTree(), filePath);
-					if (fileToDelete.exists())
+					if (repo.getFS().exists(fileToDelete))
 						FileUtils.delete(fileToDelete, FileUtils.RECURSIVE
 								| FileUtils.RETRY);
 				}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java
index 3f1afd7cc0..79fe8d6f56 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java
@@ -60,6 +60,7 @@ import org.eclipse.jgit.errors.MissingObjectException;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -399,7 +400,6 @@ public class DirCacheCheckout {
 			MissingObjectException, IncorrectObjectTypeException,
 			CheckoutConflictException, IndexWriteException {
 		toBeDeleted.clear();
-
 		ObjectReader objectReader = repo.getObjectDatabase().newReader();
 		try {
 			if (headCommitTree != null)
@@ -425,13 +425,13 @@ public class DirCacheCheckout {
 			for (int i = removed.size() - 1; i >= 0; i--) {
 				String r = removed.get(i);
 				file = new File(repo.getWorkTree(), r);
-				if (!file.delete() && file.exists()) {
+				if (!file.delete() && repo.getFS().exists(file)) {
 					// The list of stuff to delete comes from the index
 					// which will only contain a directory if it is
 					// a submodule, in which case we shall not attempt
 					// to delete it. A submodule is not empty, so it
 					// is safe to check this after a failed delete.
-					if (!file.isDirectory())
+					if (!repo.getFS().isDirectory(file))
 						toBeDeleted.add(r);
 				} else {
 					if (last != null && !isSamePrefix(r, last))
@@ -583,9 +583,8 @@ public class DirCacheCheckout {
 		// represents the state for the merge iterator, the second last the
 		// state for the index iterator and the third last represents the state
 		// for the head iterator. The hexadecimal constant "F" stands for
-		// "file",
-		// an "D" stands for "directory" (tree), and a "0" stands for
-		// non-existing
+		// "file", a "D" stands for "directory" (tree), and a "0" stands for
+		// non-existing. Symbolic links and git links are treated as File here.
 		//
 		// Examples:
 		// ffMask == 0xFFD -> Head=File, Index=File, Merge=Tree
@@ -1117,35 +1116,45 @@ public class DirCacheCheckout {
 		ObjectLoader ol = or.open(entry.getObjectId());
 		File parentDir = f.getParentFile();
 		parentDir.mkdirs();
-		File tmpFile = File.createTempFile("._" + f.getName(), null, parentDir); //$NON-NLS-1$
-		WorkingTreeOptions opt = repo.getConfig().get(WorkingTreeOptions.KEY);
-		FileOutputStream rawChannel = new FileOutputStream(tmpFile);
-		OutputStream channel;
-		if (opt.getAutoCRLF() == AutoCRLF.TRUE)
-			channel = new AutoCRLFOutputStream(rawChannel);
-		else
-			channel = rawChannel;
-		try {
-			ol.copyTo(channel);
-		} finally {
-			channel.close();
-		}
 		FS fs = repo.getFS();
-		if (opt.isFileMode() && fs.supportsExecute()) {
-			if (FileMode.EXECUTABLE_FILE.equals(entry.getRawMode())) {
-				if (!fs.canExecute(tmpFile))
-					fs.setExecute(tmpFile, true);
-			} else {
-				if (fs.canExecute(tmpFile))
-					fs.setExecute(tmpFile, false);
+		WorkingTreeOptions opt = repo.getConfig().get(WorkingTreeOptions.KEY);
+		if (entry.getFileMode() == FileMode.SYMLINK
+				&& opt.getSymLinks() == SymLinks.TRUE) {
+			byte[] bytes = ol.getBytes();
+			String target = RawParseUtils.decode(bytes);
+			fs.createSymLink(f, target);
+			entry.setLength(bytes.length);
+			entry.setLastModified(fs.lastModified(f));
+		} else {
+			File tmpFile = File.createTempFile(
+					"._" + f.getName(), null, parentDir); //$NON-NLS-1$
+			FileOutputStream rawChannel = new FileOutputStream(tmpFile);
+			OutputStream channel;
+			if (opt.getAutoCRLF() == AutoCRLF.TRUE)
+				channel = new AutoCRLFOutputStream(rawChannel);
+			else
+				channel = rawChannel;
+			try {
+				ol.copyTo(channel);
+			} finally {
+				channel.close();
+			}
+			if (opt.isFileMode() && fs.supportsExecute()) {
+				if (FileMode.EXECUTABLE_FILE.equals(entry.getRawMode())) {
+					if (!fs.canExecute(tmpFile))
+						fs.setExecute(tmpFile, true);
+				} else {
+					if (fs.canExecute(tmpFile))
+						fs.setExecute(tmpFile, false);
+				}
+			}
+			try {
+				FileUtils.rename(tmpFile, f);
+			} catch (IOException e) {
+				throw new IOException(MessageFormat.format(
+						JGitText.get().couldNotWriteFile, tmpFile.getPath(),
+						f.getPath()));
 			}
-		}
-		try {
-			FileUtils.rename(tmpFile, f);
-		} catch (IOException e) {
-			throw new IOException(MessageFormat.format(
-					JGitText.get().couldNotWriteFile, tmpFile.getPath(),
-					f.getPath()));
 		}
 		entry.setLastModified(f.lastModified());
 		if (opt.getAutoCRLF() != AutoCRLF.FALSE)
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java
index 148781dfe9..4c420f693c 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java
@@ -64,6 +64,7 @@ import org.eclipse.jgit.internal.storage.file.ObjectDirectory.AlternateRepositor
 import org.eclipse.jgit.lib.BaseRepositoryBuilder;
 import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.RefDatabase;
@@ -293,6 +294,21 @@ public class FileRepository extends Repository {
 			fileMode = false;
 		}
 
+		SymLinks symLinks = SymLinks.FALSE;
+		if (getFS().supportsSymlinks()) {
+			File tmp = new File(getDirectory(), "tmplink"); //$NON-NLS-1$
+			try {
+				getFS().createSymLink(tmp, "target"); //$NON-NLS-1$
+				symLinks = null;
+				FileUtils.delete(tmp);
+			} catch (IOException e) {
+				// Normally a java.nio.file.FileSystemException
+			}
+		}
+		if (symLinks != null)
+			cfg.setString(ConfigConstants.CONFIG_CORE_SECTION, null,
+					ConfigConstants.CONFIG_KEY_SYMLINKS, symLinks.name()
+							.toLowerCase());
 		cfg.setInt(ConfigConstants.CONFIG_CORE_SECTION, null,
 				ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION, 0);
 		cfg.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
index fd22764b6a..bca79f2710 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java
@@ -123,6 +123,13 @@ public class ConfigConstants {
 	/** The "deltaBaseCacheLimit" key */
 	public static final String CONFIG_KEY_DELTA_BASE_CACHE_LIMIT = "deltaBaseCacheLimit";
 
+	/**
+	 * The "symlinks" key
+	 *
+	 * @since 3.3
+	 */
+	public static final String CONFIG_KEY_SYMLINKS = "symlinks";
+
 	/** The "streamFileThreshold" key */
 	public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = "streamFileThreshold";
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java
index 0fc3d4ad0e..18adb9aa13 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java
@@ -101,6 +101,18 @@ public class CoreConfig {
 
 	private final String excludesfile;
 
+	/**
+	 * Options for symlink handling
+	 *
+	 * @since 3.3
+	 */
+	public static enum SymLinks {
+		/** Checkout symbolic links as plain files */
+		FALSE,
+		/** Checkout symbolic links as links */
+		TRUE
+	}
+
 	private CoreConfig(final Config rc) {
 		compression = rc.getInt(ConfigConstants.CONFIG_CORE_SECTION,
 				ConfigConstants.CONFIG_KEY_COMPRESSION, DEFAULT_COMPRESSION);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java
index eca2f91bef..414746dc42 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java
@@ -86,6 +86,7 @@ import org.eclipse.jgit.treewalk.AbstractTreeIterator;
 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
 import org.eclipse.jgit.treewalk.NameConflictTreeWalk;
 import org.eclipse.jgit.treewalk.WorkingTreeIterator;
+import org.eclipse.jgit.util.FS;
 import org.eclipse.jgit.util.FileUtils;
 
 /**
@@ -255,11 +256,11 @@ public class ResolveMerger extends ThreeWayMerger {
 	}
 
 	private void createDir(File f) throws IOException {
-		if (!f.isDirectory() && !f.mkdirs()) {
+		if (!db.getFS().isDirectory(f) && !f.mkdirs()) {
 			File p = f;
-			while (p != null && !p.exists())
+			while (p != null && !db.getFS().exists(p))
 				p = p.getParentFile();
-			if (p == null || p.isDirectory())
+			if (p == null || db.getFS().isDirectory(p))
 				throw new IOException(JGitText.get().cannotCreateDirectory);
 			FileUtils.delete(p);
 			if (!f.mkdirs())
@@ -719,9 +720,10 @@ public class ResolveMerger extends ThreeWayMerger {
 				// support write operations
 				throw new UnsupportedOperationException();
 
+			FS fs = db.getFS();
 			of = new File(workTree, tw.getPathString());
 			File parentFolder = of.getParentFile();
-			if (!parentFolder.exists())
+			if (!fs.exists(parentFolder))
 				parentFolder.mkdirs();
 			fos = new FileOutputStream(of);
 			try {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java
index ce8a50c87f..cb919ecbcd 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java
@@ -46,6 +46,7 @@
 
 package org.eclipse.jgit.treewalk;
 
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -156,6 +157,8 @@ public class FileTreeIterator extends WorkingTreeIterator {
 
 		private long lastModified;
 
+		private FS fs;
+
 		/**
 		 * Create a new file entry.
 		 *
@@ -166,16 +169,26 @@ public class FileTreeIterator extends WorkingTreeIterator {
 		 */
 		public FileEntry(final File f, FS fs) {
 			file = f;
+			this.fs = fs;
 
-			if (f.isDirectory()) {
-				if (fs.exists(new File(f, Constants.DOT_GIT)))
-					mode = FileMode.GITLINK;
+			@SuppressWarnings("hiding")
+			FileMode mode = null;
+			try {
+				if (fs.isSymLink(f)) {
+					mode = FileMode.SYMLINK;
+				} else if (fs.isDirectory(f)) {
+					if (fs.exists(new File(f, Constants.DOT_GIT)))
+						mode = FileMode.GITLINK;
+					else
+						mode = FileMode.TREE;
+				} else if (fs.canExecute(file))
+					mode = FileMode.EXECUTABLE_FILE;
 				else
-					mode = FileMode.TREE;
-			} else if (fs.canExecute(file))
-				mode = FileMode.EXECUTABLE_FILE;
-			else
-				mode = FileMode.REGULAR_FILE;
+					mode = FileMode.REGULAR_FILE;
+			} catch (IOException e) {
+				mode = FileMode.MISSING;
+			}
+			this.mode = mode;
 		}
 
 		@Override
@@ -190,21 +203,35 @@ public class FileTreeIterator extends WorkingTreeIterator {
 
 		@Override
 		public long getLength() {
-			if (length < 0)
-				length = file.length();
+			if (length < 0) {
+				try {
+					length = fs.length(file);
+				} catch (IOException e) {
+					length = 0;
+				}
+			}
 			return length;
 		}
 
 		@Override
 		public long getLastModified() {
-			if (lastModified == 0)
-				lastModified = file.lastModified();
+			if (lastModified == 0) {
+				try {
+					lastModified = fs.lastModified(file);
+				} catch (IOException e) {
+					lastModified = 0;
+				}
+			}
 			return lastModified;
 		}
 
 		@Override
 		public InputStream openInputStream() throws IOException {
-			return new FileInputStream(file);
+			if (fs.isSymLink(file))
+				return new ByteArrayInputStream(fs.readSymLink(file).getBytes(
+						Constants.CHARACTER_ENCODING));
+			else
+				return new FileInputStream(file);
 		}
 
 		/**
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
index 75328c81c6..ac5198cd69 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java
@@ -74,6 +74,7 @@ import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.Constants;
 import org.eclipse.jgit.lib.CoreConfig;
 import org.eclipse.jgit.lib.CoreConfig.CheckStat;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 import org.eclipse.jgit.lib.FileMode;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectLoader;
@@ -203,6 +204,15 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 	}
 
 	/**
+	 * @return the repository this iterator works with
+	 *
+	 * @since 3.3
+	 */
+	public Repository getRepository() {
+		return repository;
+	}
+
+	/**
 	 * Define the matching {@link DirCacheIterator}, to optimize ObjectIds.
 	 *
 	 * Once the DirCacheIterator has been set this iterator must only be
@@ -252,14 +262,10 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 			}
 		}
 		switch (mode & FileMode.TYPE_MASK) {
+		case FileMode.TYPE_SYMLINK:
 		case FileMode.TYPE_FILE:
 			contentIdFromPtr = ptr;
 			return contentId = idBufferBlob(entries[ptr]);
-		case FileMode.TYPE_SYMLINK:
-			// Java does not support symbolic links, so we should not
-			// have reached this particular part of the walk code.
-			//
-			return zeroid;
 		case FileMode.TYPE_GITLINK:
 			contentIdFromPtr = ptr;
 			return contentId = idSubmodule(entries[ptr]);
@@ -723,8 +729,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
 			return false;
 
 		// Do not rely on filemode differences in case of symbolic links
-		if (FileMode.SYMLINK.equals(rawMode))
-			return false;
+		if (getOptions().getSymLinks() == SymLinks.FALSE)
+			if (FileMode.SYMLINK.equals(rawMode))
+				return false;
 
 		// Ignore the executable file bits if WorkingTreeOptions tell me to
 		// do so. Ignoring is done by setting the bits representing a
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java
index 15bfb917db..e5e22413d2 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeOptions.java
@@ -48,6 +48,7 @@ import org.eclipse.jgit.lib.ConfigConstants;
 import org.eclipse.jgit.lib.Config.SectionParser;
 import org.eclipse.jgit.lib.CoreConfig.AutoCRLF;
 import org.eclipse.jgit.lib.CoreConfig.CheckStat;
+import org.eclipse.jgit.lib.CoreConfig.SymLinks;
 
 /** Options used by the {@link WorkingTreeIterator}. */
 public class WorkingTreeOptions {
@@ -64,6 +65,8 @@ public class WorkingTreeOptions {
 
 	private final CheckStat checkStat;
 
+	private final SymLinks symlinks;
+
 	private WorkingTreeOptions(final Config rc) {
 		fileMode = rc.getBoolean(ConfigConstants.CONFIG_CORE_SECTION,
 				ConfigConstants.CONFIG_KEY_FILEMODE, true);
@@ -71,6 +74,8 @@ public class WorkingTreeOptions {
 				ConfigConstants.CONFIG_KEY_AUTOCRLF, AutoCRLF.FALSE);
 		checkStat = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null,
 				ConfigConstants.CONFIG_KEY_CHECKSTAT, CheckStat.DEFAULT);
+		symlinks = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null,
+				ConfigConstants.CONFIG_KEY_SYMLINKS, SymLinks.TRUE);
 	}
 
 	/** @return true if the execute bit on working files should be trusted. */
@@ -90,4 +95,12 @@ public class WorkingTreeOptions {
 	public CheckStat getCheckStat() {
 		return checkStat;
 	}
+
+	/**
+	 * @return how we handle symbolic links
+	 * @since 3.3
+	 */
+	public SymLinks getSymLinks() {
+		return symlinks;
+	}
 }