Enhance FS.runProcess() to support stdin-redirection and binary data

In order to support filters in gitattributes FS.runProcess() is made
public. Support for stdin redirection has been added. Support for binary
data on stdin/stdout (as used be clean/smudge filters) has been added.

Change-Id: Ice2c152e9391368dc5748d7b825a838e3eb755f9
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RunExternalScriptTest.java

@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2015, Christian Halstrick <christian.halstrick@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.util;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.eclipse.jgit.junit.JGitTestUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+public class RunExternalScriptTest {
+	private ByteArrayOutputStream out;
+
+	private ByteArrayOutputStream err;
+
+	private String sep = System.getProperty("line.separator");
+
+	@Before
+	public void setUp() throws Exception {
+		out = new ByteArrayOutputStream();
+		err = new ByteArrayOutputStream();
+	}
+
+	@Test
+	public void testCopyStdIn() throws IOException, InterruptedException {
+		String inputStr = "a\nb\rc\r\nd";
+		File script = writeTempFile("cat -");
+		int rc = FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh", script.getPath()), out, err,
+				new ByteArrayInputStream(inputStr.getBytes()));
+		assertEquals(0, rc);
+		assertEquals(inputStr, new String(out.toByteArray()));
+		assertEquals("", new String(err.toByteArray()));
+	}
+
+	@Test
+	public void testCopyNullStdIn() throws IOException, InterruptedException {
+		File script = writeTempFile("cat -");
+		int rc = FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh", script.getPath()), out, err,
+				(InputStream) null);
+		assertEquals(0, rc);
+		assertEquals("", new String(out.toByteArray()));
+		assertEquals("", new String(err.toByteArray()));
+	}
+
+	@Test
+	public void testArguments() throws IOException, InterruptedException {
+		File script = writeTempFile("echo $#,$1,$2,$3,$4,$5,$6");
+		int rc = FS.DETECTED.runProcess(new ProcessBuilder("/bin/bash",
+				script.getPath(), "a", "b", "c"), out, err, (InputStream) null);
+		assertEquals(0, rc);
+		assertEquals("3,a,b,c,,,\n", new String(out.toByteArray()));
+		assertEquals("", new String(err.toByteArray()));
+	}
+
+	@Test
+	public void testRc() throws IOException, InterruptedException {
+		File script = writeTempFile("exit 3");
+		int rc = FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh", script.getPath(), "a", "b", "c"),
+				out, err, (InputStream) null);
+		assertEquals(3, rc);
+		assertEquals("", new String(out.toByteArray()));
+		assertEquals("", new String(err.toByteArray()));
+	}
+
+	@Test
+	public void testNullStdout() throws IOException, InterruptedException {
+		File script = writeTempFile("echo hi");
+		int rc = FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh", script.getPath()), null, err,
+				(InputStream) null);
+		assertEquals(0, rc);
+		assertEquals("", new String(out.toByteArray()));
+		assertEquals("", new String(err.toByteArray()));
+	}
+
+	@Test
+	public void testStdErr() throws IOException, InterruptedException {
+		File script = writeTempFile("echo hi >&2");
+		int rc = FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh", script.getPath()), null, err,
+				(InputStream) null);
+		assertEquals(0, rc);
+		assertEquals("", new String(out.toByteArray()));
+		assertEquals("hi" + sep, new String(err.toByteArray()));
+	}
+
+	@Test
+	public void testAllTogetherBin() throws IOException, InterruptedException {
+		String inputStr = "a\nb\rc\r\nd";
+		File script = writeTempFile("echo $#,$1,$2,$3,$4,$5,$6 >&2 ; cat -; exit 5");
+		int rc = FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh", script.getPath(), "a", "b", "c"),
+				out, err, new ByteArrayInputStream(inputStr.getBytes()));
+		assertEquals(5, rc);
+		assertEquals(inputStr, new String(out.toByteArray()));
+		assertEquals("3,a,b,c,,," + sep, new String(err.toByteArray()));
+	}
+
+	@Test(expected = IOException.class)
+	public void testWrongSh() throws IOException, InterruptedException {
+		File script = writeTempFile("cat -");
+		FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh-foo", script.getPath(), "a", "b",
+						"c"), out, err, (InputStream) null);
+	}
+
+	@Test
+	public void testWrongScript() throws IOException, InterruptedException {
+		File script = writeTempFile("cat-foo -");
+		int rc = FS.DETECTED.runProcess(
+				new ProcessBuilder("/bin/sh", script.getPath(), "a", "b", "c"),
+				out, err, (InputStream) null);
+		assertEquals(127, rc);
+	}
+
+	private File writeTempFile(String body) throws IOException {
+		File f = File.createTempFile("RunProcessTestScript_", "");
+		JGitTestUtil.write(f, body);
+		return f;
+	}
+}

org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java

@@ -44,15 +44,13 @@
 package org.eclipse.jgit.util;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import java.io.PrintStream;
-import java.io.PrintWriter;
 import java.nio.charset.Charset;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -864,52 +862,88 @@ public abstract class FS {
 	 * Runs the given process until termination, clearing its stdout and stderr
 	 * streams on-the-fly.
 	 *
-	 * @param hookProcessBuilder
-	 *            The process builder configured for this hook.
+	 * @param processBuilder
+	 *            The process builder configured for this process.
 	 * @param outRedirect
-	 *            A print stream on which to redirect the hook's stdout. Can be
-	 *            <code>null</code>, in which case the hook's standard output
-	 *            will be lost.
+	 *            A OutputStream on which to redirect the processes stdout. Can
+	 *            be <code>null</code>, in which case the processes standard
+	 *            output will be lost.
 	 * @param errRedirect
-	 *            A print stream on which to redirect the hook's stderr. Can be
-	 *            <code>null</code>, in which case the hook's standard error
-	 *            will be lost.
+	 *            A OutputStream on which to redirect the processes stderr. Can
+	 *            be <code>null</code>, in which case the processes standard
+	 *            error will be lost.
 	 * @param stdinArgs
 	 *            A string to pass on to the standard input of the hook. Can be
 	 *            <code>null</code>.
-	 * @return the exit value of this hook.
+	 * @return the exit value of this process.
 	 * @throws IOException
-	 *             if an I/O error occurs while executing this hook.
+	 *             if an I/O error occurs while executing this process.
 	 * @throws InterruptedException
 	 *             if the current thread is interrupted while waiting for the
 	 *             process to end.
-	 * @since 3.7
+	 * @since 4.2
 	 */
-	protected int runProcess(ProcessBuilder hookProcessBuilder,
+	public int runProcess(ProcessBuilder processBuilder,
 			OutputStream outRedirect, OutputStream errRedirect, String stdinArgs)
 			throws IOException, InterruptedException {
+		InputStream in = (stdinArgs == null) ? null : new ByteArrayInputStream(
+				stdinArgs.getBytes(Constants.CHARACTER_ENCODING));
+		return runProcess(processBuilder, outRedirect, errRedirect, in);
+	}
+
+	/**
+	 * Runs the given process until termination, clearing its stdout and stderr
+	 * streams on-the-fly.
+	 *
+	 * @param processBuilder
+	 *            The process builder configured for this process.
+	 * @param outRedirect
+	 *            An OutputStream on which to redirect the processes stdout. Can
+	 *            be <code>null</code>, in which case the processes standard
+	 *            output will be lost. If binary is set to <code>false</code>
+	 *            then it is expected that the process emits text data which
+	 *            should be processed line by line.
+	 * @param errRedirect
+	 *            An OutputStream on which to redirect the processes stderr. Can
+	 *            be <code>null</code>, in which case the processes standard
+	 *            error will be lost.
+	 * @param inRedirect
+	 *            An InputStream from which to redirect the processes stdin. Can
+	 *            be <code>null</code>, in which case the process doesn't get
+	 *            any data over stdin. If binary is set to
+	 *            <code>false</code> then it is expected that the process
+	 *            expects text data which should be processed line by line.
+	 * @return the return code of this process.
+	 * @throws IOException
+	 *             if an I/O error occurs while executing this process.
+	 * @throws InterruptedException
+	 *             if the current thread is interrupted while waiting for the
+	 *             process to end.
+	 * @since 4.2
+	 */
+	public int runProcess(ProcessBuilder processBuilder,
+			OutputStream outRedirect, OutputStream errRedirect,
+			InputStream inRedirect) throws IOException,
+			InterruptedException {
 		final ExecutorService executor = Executors.newFixedThreadPool(2);
 		Process process = null;
 		// We'll record the first I/O exception that occurs, but keep on trying
 		// to dispose of our open streams and file handles
 		IOException ioException = null;
 		try {
-			process = hookProcessBuilder.start();
+			process = processBuilder.start();
 			final Callable<Void> errorGobbler = new StreamGobbler(
 					process.getErrorStream(), errRedirect);
 			final Callable<Void> outputGobbler = new StreamGobbler(
 					process.getInputStream(), outRedirect);
 			executor.submit(errorGobbler);
 			executor.submit(outputGobbler);
-			if (stdinArgs != null) {
-				final PrintWriter stdinWriter = new PrintWriter(
-						process.getOutputStream());
-				stdinWriter.print(stdinArgs);
-				stdinWriter.flush();
-				// We are done with this hook's input. Explicitly close its
-				// stdin now to kick off any blocking read the hook might have.
-				stdinWriter.close();
+			OutputStream outputStream = process.getOutputStream();
+			if (inRedirect != null) {
+				new StreamGobbler(inRedirect, outputStream)
+						.call();
 			}
+			outputStream.close();
 			return process.waitFor();
 		} catch (IOException e) {
 			ioException = e;
@@ -1187,30 +1221,27 @@ public abstract class FS {
 	 * </p>
 	 */
 	private static class StreamGobbler implements Callable<Void> {
-		private final BufferedReader reader;
+		private InputStream in;
 
-		private final BufferedWriter writer;
+		private OutputStream out;
 
 		public StreamGobbler(InputStream stream, OutputStream output) {
-			this.reader = new BufferedReader(new InputStreamReader(stream));
-			if (output == null)
-				this.writer = null;
-			else
-				this.writer = new BufferedWriter(new OutputStreamWriter(output));
+			this.in = stream;
+			this.out = output;
 		}
 
 		public Void call() throws IOException {
 			boolean writeFailure = false;
-
-			String line = null;
-			while ((line = reader.readLine()) != null) {
-				// Do not try to write again after a failure, but keep reading
-				// as long as possible to prevent the input stream from choking.
-				if (!writeFailure && writer != null) {
+			byte buffer[] = new byte[4096];
+			int readBytes;
+			while ((readBytes = in.read(buffer)) != -1) {
+				// Do not try to write again after a failure, but keep
+				// reading as long as possible to prevent the input stream
+				// from choking.
+				if (!writeFailure && out != null) {
 					try {
-						writer.write(line);
-						writer.newLine();
-						writer.flush();
+						out.write(buffer, 0, readBytes);
+						out.flush();
 					} catch (IOException e) {
 						writeFailure = true;
 					}