Merge branch 'stable-3.5' into stable-3.6

* stable-3.5:
  JGit v3.5.3.201412180710-r
  JGit v3.4.2.201412180340-r
  ObjectChecker: Disallow names potentially mapping to ".git" on HFS+
  ObjectChecker: Disallow Windows shortname "GIT~1"
  ObjectChecker: Disallow ".git." and ".git<space>"
  Always ignore case when forbidding .git in ObjectChecker
  DirCache: Refuse to read files with invalid paths
  DirCache: Replace isValidPath with DirCacheCheckout.checkValidPath
  Replace "a." with "a-" in unit tests
  Apache HttpClientConnection: replace calls to deprecated LocalFile()
  Fix two nits about DirCacheEntry constructors
  Detect buffering failures while writing rebase todo file
  Deprecate TemporaryBuffer.LocalFile without parent directory
  Switch FileHeader.extractFileLines to TemporaryBuffer.Heap
  AmazonS3: Buffer pushed pack content under $GIT_DIR
  DirCache: Buffer TREE extension to $GIT_DIR

Change-Id: Iee8acbaa9d4d9047b550641db1b8845d64530785
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

10 files changed, 192 insertions, 79 deletions

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java
index 645de2704e..98a1c8ca4b 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java
@@ -607,7 +607,8 @@ public class DirCache {
 		final LockFile tmp = myLock;
 		requireLocked(tmp);
 		try {
-			writeTo(new SafeBufferedOutputStream(tmp.getOutputStream()));
+			writeTo(liveFile.getParentFile(),
+					new SafeBufferedOutputStream(tmp.getOutputStream()));
 		} catch (IOException err) {
 			tmp.unlock();
 			throw err;
@@ -620,7 +621,7 @@ public class DirCache {
 		}
 	}
 
-	void writeTo(final OutputStream os) throws IOException {
+	void writeTo(File dir, final OutputStream os) throws IOException {
 		final MessageDigest foot = Constants.newMessageDigest();
 		final DigestOutputStream dos = new DigestOutputStream(os, foot);
 
@@ -670,14 +671,18 @@ public class DirCache {
 		}
 
 		if (writeTree) {
-			final TemporaryBuffer bb = new TemporaryBuffer.LocalFile();
-			tree.write(tmp, bb);
-			bb.close();
-
-			NB.encodeInt32(tmp, 0, EXT_TREE);
-			NB.encodeInt32(tmp, 4, (int) bb.length());
-			dos.write(tmp, 0, 8);
-			bb.writeTo(dos, null);
+			TemporaryBuffer bb = new TemporaryBuffer.LocalFile(dir, 5 << 20);
+			try {
+				tree.write(tmp, bb);
+				bb.close();
+
+				NB.encodeInt32(tmp, 0, EXT_TREE);
+				NB.encodeInt32(tmp, 4, (int) bb.length());
+				dos.write(tmp, 0, 8);
+				bb.writeTo(dos, null);
+			} finally {
+				bb.destroy();
+			}
 		}
 		writeIndexChecksum = foot.digest();
 		os.write(writeIndexChecksum);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java
index 692cc8fb58..015d9d6a85 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java
@@ -1291,7 +1291,9 @@ public class DirCacheCheckout {
 		try {
 			SystemReader.getInstance().checkPath(path);
 		} catch (CorruptObjectException e) {
-			throw new InvalidPathException(path);
+			InvalidPathException p = new InvalidPathException(path);
+			p.initCause(e);
+			throw p;
 		}
 	}
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java
index 65188c8f43..eef2e6d3c3 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java
@@ -56,6 +56,7 @@ import java.security.MessageDigest;
 import java.text.MessageFormat;
 import java.util.Arrays;
 
+import org.eclipse.jgit.errors.CorruptObjectException;
 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.Constants;
@@ -64,7 +65,6 @@ import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.util.IO;
 import org.eclipse.jgit.util.MutableInteger;
 import org.eclipse.jgit.util.NB;
-import org.eclipse.jgit.util.SystemReader;
 
 /**
  * A single file (or stage of a file) in a {@link DirCache}.
@@ -190,6 +190,16 @@ public class DirCacheEntry {
 			md.update((byte) 0);
 		}
 
+		try {
+			DirCacheCheckout.checkValidPath(toString(path));
+		} catch (InvalidPathException e) {
+			CorruptObjectException p =
+				new CorruptObjectException(e.getMessage());
+			if (e.getCause() != null)
+				p.initCause(e.getCause());
+			throw p;
+		}
+
 		// Index records are padded out to the next 8 byte alignment
 		// for historical reasons related to how C Git read the files.
 		//
@@ -203,7 +213,6 @@ public class DirCacheEntry {
 
 		if (mightBeRacilyClean(smudge_s, smudge_ns))
 			smudgeRacilyClean();
-
 	}
 
 	/**
@@ -217,7 +226,7 @@ public class DirCacheEntry {
 	 *             or DirCache file.
 	 */
 	public DirCacheEntry(final String newPath) {
-		this(Constants.encode(newPath));
+		this(Constants.encode(newPath), STAGE_0);
 	}
 
 	/**
@@ -266,11 +275,11 @@ public class DirCacheEntry {
 	 */
 	@SuppressWarnings("boxing")
 	public DirCacheEntry(final byte[] newPath, final int stage) {
-		if (!isValidPath(newPath))
-			throw new InvalidPathException(toString(newPath));
+		DirCacheCheckout.checkValidPath(toString(newPath));
 		if (stage < 0 || 3 < stage)
-			throw new IllegalArgumentException(MessageFormat.format(JGitText.get().invalidStageForPath
-					, stage, toString(newPath)));
+			throw new IllegalArgumentException(MessageFormat.format(
+					JGitText.get().invalidStageForPath,
+					stage, toString(newPath)));
 
 		info = new byte[INFO_LEN];
 		infoOffset = 0;
@@ -725,36 +734,6 @@ public class DirCacheEntry {
 		return Constants.CHARSET.decode(ByteBuffer.wrap(path)).toString();
 	}
 
-	static boolean isValidPath(final byte[] path) {
-		if (path.length == 0)
-			return false; // empty path is not permitted.
-
-		boolean componentHasChars = false;
-		for (final byte c : path) {
-			switch (c) {
-			case 0:
-				return false; // NUL is never allowed within the path.
-
-			case '/':
-				if (componentHasChars)
-					componentHasChars = false;
-				else
-					return false;
-				break;
-			case '\\':
-			case ':':
-				// Tree's never have a backslash in them, not even on Windows
-				// but even there we regard it as an invalid path
-				if (SystemReader.getInstance().isWindows())
-					return false;
-				//$FALL-THROUGH$
-			default:
-				componentHasChars = true;
-			}
-		}
-		return componentHasChars;
-	}
-
 	static int getMaximumInfoLength(boolean extended) {
 		return extended ? INFO_LEN_EXTENDED : INFO_LEN;
 	}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java
index 9fe54b0c5c..8435c9a64b 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java
@@ -492,13 +492,27 @@ public class ObjectChecker {
 					throw new CorruptObjectException("invalid name '..'");
 				break;
 			case 4:
-				if (isDotGit(raw, ptr + 1))
+				if (isGit(raw, ptr + 1))
+					throw new CorruptObjectException(String.format(
+							"invalid name '%s'",
+							RawParseUtils.decode(raw, ptr, end)));
+				break;
+			default:
+				if (end - ptr > 4 && isNormalizedGit(raw, ptr + 1, end))
 					throw new CorruptObjectException(String.format(
 							"invalid name '%s'",
 							RawParseUtils.decode(raw, ptr, end)));
 			}
+		} else if (isGitTilde1(raw, ptr, end)) {
+			throw new CorruptObjectException(String.format("invalid name '%s'",
+					RawParseUtils.decode(raw, ptr, end)));
 		}
 
+		if (macosx && isMacHFSGit(raw, ptr, end))
+			throw new CorruptObjectException(String.format(
+					"invalid name '%s' contains ignorable Unicode characters",
+					RawParseUtils.decode(raw, ptr, end)));
+
 		if (windows) {
 			// Windows ignores space and dot at end of file name.
 			if (raw[end - 1] == ' ' || raw[end - 1] == '.')
@@ -509,6 +523,88 @@ public class ObjectChecker {
 		}
 	}
 
+	// Mac's HFS+ folds permutations of ".git" and Unicode ignorable characters
+	// to ".git" therefore we should prevent such names
+	private static boolean isMacHFSGit(byte[] raw, int ptr, int end)
+			throws CorruptObjectException {
+		boolean ignorable = false;
+		byte[] git = new byte[] { '.', 'g', 'i', 't' };
+		int g = 0;
+		while (ptr < end) {
+			switch (raw[ptr]) {
+			case (byte) 0xe2: // http://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192
+				checkTruncatedIgnorableUTF8(raw, ptr, end);
+				switch (raw[ptr + 1]) {
+				case (byte) 0x80:
+					switch (raw[ptr + 2]) {
+					case (byte) 0x8c:	// U+200C 0xe2808c ZERO WIDTH NON-JOINER
+					case (byte) 0x8d:	// U+200D 0xe2808d ZERO WIDTH JOINER
+					case (byte) 0x8e:	// U+200E 0xe2808e LEFT-TO-RIGHT MARK
+					case (byte) 0x8f:	// U+200F 0xe2808f RIGHT-TO-LEFT MARK
+					case (byte) 0xaa:	// U+202A 0xe280aa LEFT-TO-RIGHT EMBEDDING
+					case (byte) 0xab:	// U+202B 0xe280ab RIGHT-TO-LEFT EMBEDDING
+					case (byte) 0xac:	// U+202C 0xe280ac POP DIRECTIONAL FORMATTING
+					case (byte) 0xad:	// U+202D 0xe280ad LEFT-TO-RIGHT OVERRIDE
+					case (byte) 0xae:	// U+202E 0xe280ae RIGHT-TO-LEFT OVERRIDE
+						ignorable = true;
+						ptr += 3;
+						continue;
+					default:
+						return false;
+					}
+				case (byte) 0x81:
+					switch (raw[ptr + 2]) {
+					case (byte) 0xaa:	// U+206A 0xe281aa INHIBIT SYMMETRIC SWAPPING
+					case (byte) 0xab:	// U+206B 0xe281ab ACTIVATE SYMMETRIC SWAPPING
+					case (byte) 0xac:	// U+206C 0xe281ac INHIBIT ARABIC FORM SHAPING
+					case (byte) 0xad:	// U+206D 0xe281ad ACTIVATE ARABIC FORM SHAPING
+					case (byte) 0xae:	// U+206E 0xe281ae NATIONAL DIGIT SHAPES
+					case (byte) 0xaf:	// U+206F 0xe281af NOMINAL DIGIT SHAPES
+						ignorable = true;
+						ptr += 3;
+						continue;
+					default:
+						return false;
+					}
+				}
+				break;
+			case (byte) 0xef: // http://www.utf8-chartable.de/unicode-utf8-table.pl?start=65024
+				checkTruncatedIgnorableUTF8(raw, ptr, end);
+				// U+FEFF 0xefbbbf ZERO WIDTH NO-BREAK SPACE
+				if ((raw[ptr + 1] == (byte) 0xbb)
+						&& (raw[ptr + 2] == (byte) 0xbf)) {
+					ignorable = true;
+					ptr += 3;
+					continue;
+				}
+				return false;
+			default:
+				if (g == 4)
+					return false;
+				if (raw[ptr++] != git[g++])
+					return false;
+			}
+		}
+		if (g == 4 && ignorable)
+			return true;
+		return false;
+	}
+
+	private static void checkTruncatedIgnorableUTF8(byte[] raw, int ptr, int end)
+			throws CorruptObjectException {
+		if ((ptr + 2) >= end)
+			throw new CorruptObjectException(MessageFormat.format(
+				"invalid name contains byte sequence ''{0}'' which is not a valid UTF-8 character",
+					toHexString(raw, ptr, end)));
+	}
+
+	private static String toHexString(byte[] raw, int ptr, int end) {
+		StringBuilder b = new StringBuilder("0x"); //$NON-NLS-1$
+		for (int i = ptr; i < end; i++)
+			b.append(String.format("%02x", Byte.valueOf(raw[i]))); //$NON-NLS-1$
+		return b.toString();
+	}
+
 	private static void checkNotWindowsDevice(byte[] raw, int ptr, int end)
 			throws CorruptObjectException {
 		switch (toLower(raw[ptr])) {
@@ -579,12 +675,36 @@ public class ObjectChecker {
 		return 1 <= c && c <= 31;
 	}
 
-	private boolean isDotGit(byte[] buf, int p) {
-		if (windows || macosx)
-			return toLower(buf[p]) == 'g'
-					&& toLower(buf[p + 1]) == 'i'
-					&& toLower(buf[p + 2]) == 't';
-		return buf[p] == 'g' && buf[p + 1] == 'i' && buf[p + 2] == 't';
+	private static boolean isGit(byte[] buf, int p) {
+		return toLower(buf[p]) == 'g'
+				&& toLower(buf[p + 1]) == 'i'
+				&& toLower(buf[p + 2]) == 't';
+	}
+
+	private static boolean isGitTilde1(byte[] buf, int p, int end) {
+		if (end - p != 5)
+			return false;
+		return toLower(buf[p]) == 'g' && toLower(buf[p + 1]) == 'i'
+				&& toLower(buf[p + 2]) == 't' && buf[p + 3] == '~'
+				&& buf[p + 4] == '1';
+	}
+
+	private static boolean isNormalizedGit(byte[] raw, int ptr, int end) {
+		if (isGit(raw, ptr)) {
+			int dots = 0;
+			boolean space = false;
+			int p = end - 1;
+			for (; (ptr + 2) < p; p--) {
+				if (raw[p] == '.')
+					dots++;
+				else if (raw[p] == ' ')
+					space = true;
+				else
+					break;
+			}
+			return p == ptr + 2 && (dots == 1 || space);
+		}
+		return false;
 	}
 
 	private static char toLower(byte b) {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java
index ef61e22032..4ebe5fedf3 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RebaseTodoFile.java
@@ -43,17 +43,17 @@
 
 package org.eclipse.jgit.lib;
 
-import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStreamWriter;
+import java.io.OutputStream;
 import java.util.LinkedList;
 import java.util.List;
 
 import org.eclipse.jgit.lib.RebaseTodoLine.Action;
 import org.eclipse.jgit.util.IO;
 import org.eclipse.jgit.util.RawParseUtils;
+import org.eclipse.jgit.util.io.SafeBufferedOutputStream;
 
 /**
  * Offers methods to read and write files formatted like the git-rebase-todo
@@ -216,9 +216,8 @@ public class RebaseTodoFile {
 	 */
 	public void writeRebaseTodoFile(String path, List<RebaseTodoLine> steps,
 			boolean append) throws IOException {
-		BufferedWriter fw = new BufferedWriter(new OutputStreamWriter(
-				new FileOutputStream(new File(repo.getDirectory(), path),
-						append), Constants.CHARACTER_ENCODING));
+		OutputStream fw = new SafeBufferedOutputStream(new FileOutputStream(
+				new File(repo.getDirectory(), path), append));
 		try {
 			StringBuilder sb = new StringBuilder();
 			for (RebaseTodoLine step : steps) {
@@ -232,8 +231,8 @@ public class RebaseTodoFile {
 					sb.append(" "); //$NON-NLS-1$
 					sb.append(step.getShortMessage().trim());
 				}
-				fw.write(sb.toString());
-				fw.newLine();
+				sb.append('\n');
+				fw.write(Constants.encode(sb.toString()));
 			}
 		} finally {
 			fw.close();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java
index 8171669bc8..534c827314 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/FileHeader.java
@@ -267,7 +267,7 @@ public class FileHeader extends DiffEntry {
 		final TemporaryBuffer[] tmp = new TemporaryBuffer[getParentCount() + 1];
 		try {
 			for (int i = 0; i < tmp.length; i++)
-				tmp[i] = new TemporaryBuffer.LocalFile();
+				tmp[i] = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
 			for (final HunkHeader h : getHunks())
 				h.extractFileLines(tmp);
 
@@ -281,11 +281,6 @@ public class FileHeader extends DiffEntry {
 			return r;
 		} catch (IOException ioe) {
 			throw new RuntimeException(JGitText.get().cannotConvertScriptToText, ioe);
-		} finally {
-			for (final TemporaryBuffer b : tmp) {
-				if (b != null)
-					b.destroy();
-			}
 		}
 	}
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java
index 7b48473523..383c1f8fef 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/Patch.java
@@ -139,14 +139,10 @@ public class Patch {
 	}
 
 	private static byte[] readFully(final InputStream is) throws IOException {
-		final TemporaryBuffer b = new TemporaryBuffer.LocalFile();
-		try {
-			b.copy(is);
-			b.close();
-			return b.toByteArray();
-		} finally {
-			b.destroy();
-		}
+		TemporaryBuffer b = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
+		b.copy(is);
+		b.close();
+		return b.toByteArray();
 	}
 
 	/**
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java
index 99d8b09d87..722bfc489d 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java
@@ -182,6 +182,9 @@ public class AmazonS3 {
 	/** Encryption algorithm, may be a null instance that provides pass-through. */
 	private final WalkEncryption encryption;
 
+	/** Directory for locally buffered content. */
+	private final File tmpDir;
+
 	/**
 	 * Create a new S3 client for the supplied user information.
 	 * <p>
@@ -251,6 +254,9 @@ public class AmazonS3 {
 		maxAttempts = Integer.parseInt(props.getProperty(
 				"httpclient.retry-max", "3")); //$NON-NLS-1$ //$NON-NLS-2$
 		proxySelector = ProxySelector.getDefault();
+
+		String tmp = props.getProperty("tmpdir"); //$NON-NLS-1$
+		tmpDir = tmp != null && tmp.length() > 0 ? new File(tmp) : null;
 	}
 
 	/**
@@ -452,7 +458,7 @@ public class AmazonS3 {
 			final ProgressMonitor monitor, final String monitorTask)
 			throws IOException {
 		final MessageDigest md5 = newMD5();
-		final TemporaryBuffer buffer = new TemporaryBuffer.LocalFile() {
+		final TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(tmpDir) {
 			@Override
 			public void close() throws IOException {
 				super.close();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportAmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportAmazonS3.java
index b3a55a581b..afaaa69a43 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportAmazonS3.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportAmazonS3.java
@@ -147,7 +147,11 @@ public class TransportAmazonS3 extends HttpTransport implements WalkTransport {
 			throws NotSupportedException {
 		super(local, uri);
 
-		s3 = new AmazonS3(loadProperties());
+		Properties props = loadProperties();
+		if (!props.contains("tmpdir") && local.getDirectory() != null) //$NON-NLS-1$
+			props.put("tmpdir", local.getDirectory().getPath()); //$NON-NLS-1$
+
+		s3 = new AmazonS3(props);
 		bucket = uri.getHost();
 
 		String p = uri.getPath();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java
index 88c32d2f53..10aade4e11 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/TemporaryBuffer.java
@@ -361,7 +361,12 @@ public abstract class TemporaryBuffer extends OutputStream {
 		 */
 		private File onDiskFile;
 
-		/** Create a new temporary buffer. */
+		/**
+		 * Create a new temporary buffer.
+		 *
+		 * @deprecated Use the {@code File} overload to supply a directory.
+		 */
+		@Deprecated
 		public LocalFile() {
 			this(null, DEFAULT_IN_CORE_LIMIT);
 		}
@@ -372,7 +377,9 @@ public abstract class TemporaryBuffer extends OutputStream {
 		 * @param inCoreLimit
 		 *            maximum number of bytes to store in memory. Storage beyond
 		 *            this limit will use the local file.
+		 * @deprecated Use the {@code File,int} overload to supply a directory.
 		 */
+		@Deprecated
 		public LocalFile(final int inCoreLimit) {
 			this(null, inCoreLimit);
 		}