LockFile: create OutputStream only when needed

Don't create the stream eagerly in lock(); that may cause JGit to
exceed OS or JVM limits on open file descriptors if many locks need
to be created, for instance when creating many refs. Instead create
the output stream only when one really needs to write something.

Bug: 573328
Change-Id: If9441ed40494d46f594a896d34a5c4f56f91ebf4
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>

3 files changed, 105 insertions, 43 deletions

diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
index feef39744b..2fa8713daa 100644
--- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
+++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
@@ -415,11 +415,14 @@ listingPacks=Listing packs
 localObjectsIncomplete=Local objects incomplete.
 localRefIsMissingObjects=Local ref {0} is missing object(s).
 localRepository=local repository
+lockAlreadyHeld=Lock on {0} already held
 lockCountMustBeGreaterOrEqual1=lockCount must be >= 1
 lockError=lock error: {0}
 lockFailedRetry=locking {0} failed after {1} retries
 lockOnNotClosed=Lock on {0} not closed.
 lockOnNotHeld=Lock on {0} not held.
+lockStreamClosed=Output to lock on {0} already closed
+lockStreamMultiple=Output to lock on {0} already opened
 logInconsistentFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: {} > {}, but diff = {}. Aborting measurement at resolution {}.
 logLargerFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: diff = {} > {} (last good value). Aborting measurement.
 logSmallerFiletime={}: got smaller file timestamp on {}, {}: {} < {}. Aborting measurement at resolution {}.
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
index 09fe03e065..ab9fc5c9bb 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
@@ -444,10 +444,13 @@ public class JGitText extends TranslationBundle {
 	/***/ public String localRefIsMissingObjects;
 	/***/ public String localRepository;
 	/***/ public String lockCountMustBeGreaterOrEqual1;
+	/***/ public String lockAlreadyHeld;
 	/***/ public String lockError;
 	/***/ public String lockFailedRetry;
 	/***/ public String lockOnNotClosed;
 	/***/ public String lockOnNotHeld;
+	/***/ public String lockStreamClosed;
+	/***/ public String lockStreamMultiple;
 	/***/ public String logInconsistentFiletimeDiff;
 	/***/ public String logLargerFiletimeDiff;
 	/***/ public String logSmallerFiletime;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java
index 2e0a6da3a4..ab407a6ae9 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
- * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org> and others
+ * Copyright (C) 2006-2021, Shawn O. Pearce <spearce@spearce.org> and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -96,11 +96,15 @@ public class LockFile {
 
 	private boolean haveLck;
 
-	FileOutputStream os;
+	private FileOutputStream os;
 
 	private boolean needSnapshot;
 
-	boolean fsync;
+	private boolean fsync;
+
+	private boolean isAppend;
+
+	private boolean written;
 
 	private FileSnapshot commitSnapshot;
 
@@ -127,6 +131,10 @@ public class LockFile {
 	 *             does not hold the lock.
 	 */
 	public boolean lock() throws IOException {
+		if (haveLck) {
+			throw new IllegalStateException(
+					MessageFormat.format(JGitText.get().lockAlreadyHeld, ref));
+		}
 		FileUtils.mkdirs(lck.getParentFile(), true);
 		try {
 			token = FS.DETECTED.createNewFileAtomic(lck);
@@ -134,18 +142,15 @@ public class LockFile {
 			LOG.error(JGitText.get().failedCreateLockFile, lck, e);
 			throw e;
 		}
-		if (token.isCreated()) {
+		boolean obtainedLock = token.isCreated();
+		if (obtainedLock) {
 			haveLck = true;
-			try {
-				os = new FileOutputStream(lck);
-			} catch (IOException ioe) {
-				unlock();
-				throw ioe;
-			}
+			isAppend = false;
+			written = false;
 		} else {
 			closeToken();
 		}
-		return haveLck;
+		return obtainedLock;
 	}
 
 	/**
@@ -158,12 +163,24 @@ public class LockFile {
 	 *             does not hold the lock.
 	 */
 	public boolean lockForAppend() throws IOException {
-		if (!lock())
+		if (!lock()) {
 			return false;
+		}
 		copyCurrentContent();
+		isAppend = true;
+		written = false;
 		return true;
 	}
 
+	// For tests only
+	boolean isLocked() {
+		return haveLck;
+	}
+
+	private FileOutputStream getStream() throws IOException {
+		return new FileOutputStream(lck, isAppend);
+	}
+
 	/**
 	 * Copy the current file content into the temporary file.
 	 * <p>
@@ -185,32 +202,31 @@ public class LockFile {
 	 */
 	public void copyCurrentContent() throws IOException {
 		requireLock();
-		try {
+		try (FileOutputStream out = getStream()) {
 			try (FileInputStream fis = new FileInputStream(ref)) {
 				if (fsync) {
 					FileChannel in = fis.getChannel();
 					long pos = 0;
 					long cnt = in.size();
 					while (0 < cnt) {
-						long r = os.getChannel().transferFrom(in, pos, cnt);
+						long r = out.getChannel().transferFrom(in, pos, cnt);
 						pos += r;
 						cnt -= r;
 					}
 				} else {
 					final byte[] buf = new byte[2048];
 					int r;
-					while ((r = fis.read(buf)) >= 0)
-						os.write(buf, 0, r);
+					while ((r = fis.read(buf)) >= 0) {
+						out.write(buf, 0, r);
+					}
 				}
+			} catch (FileNotFoundException fnfe) {
+				if (ref.exists()) {
+					throw fnfe;
+				}
+				// Don't worry about a file that doesn't exist yet, it
+				// conceptually has no current content to copy.
 			}
-		} catch (FileNotFoundException fnfe) {
-			if (ref.exists()) {
-				unlock();
-				throw fnfe;
-			}
-			// Don't worry about a file that doesn't exist yet, it
-			// conceptually has no current content to copy.
-			//
 		} catch (IOException | RuntimeException | Error ioe) {
 			unlock();
 			throw ioe;
@@ -253,18 +269,22 @@ public class LockFile {
 	 */
 	public void write(byte[] content) throws IOException {
 		requireLock();
-		try {
+		try (FileOutputStream out = getStream()) {
+			if (written) {
+				throw new IOException(MessageFormat
+						.format(JGitText.get().lockStreamClosed, ref));
+			}
 			if (fsync) {
-				FileChannel fc = os.getChannel();
+				FileChannel fc = out.getChannel();
 				ByteBuffer buf = ByteBuffer.wrap(content);
-				while (0 < buf.remaining())
+				while (0 < buf.remaining()) {
 					fc.write(buf);
+				}
 				fc.force(true);
 			} else {
-				os.write(content);
+				out.write(content);
 			}
-			os.close();
-			os = null;
+			written = true;
 		} catch (IOException | RuntimeException | Error ioe) {
 			unlock();
 			throw ioe;
@@ -283,36 +303,68 @@ public class LockFile {
 	public OutputStream getOutputStream() {
 		requireLock();
 
-		final OutputStream out;
-		if (fsync)
-			out = Channels.newOutputStream(os.getChannel());
-		else
-			out = os;
+		if (written || os != null) {
+			throw new IllegalStateException(MessageFormat
+					.format(JGitText.get().lockStreamMultiple, ref));
+		}
 
 		return new OutputStream() {
+
+			private OutputStream out;
+
+			private boolean closed;
+
+			private OutputStream get() throws IOException {
+				if (written) {
+					throw new IOException(MessageFormat
+							.format(JGitText.get().lockStreamMultiple, ref));
+				}
+				if (out == null) {
+					os = getStream();
+					if (fsync) {
+						out = Channels.newOutputStream(os.getChannel());
+					} else {
+						out = os;
+					}
+				}
+				return out;
+			}
+
 			@Override
 			public void write(byte[] b, int o, int n)
 					throws IOException {
-				out.write(b, o, n);
+				get().write(b, o, n);
 			}
 
 			@Override
 			public void write(byte[] b) throws IOException {
-				out.write(b);
+				get().write(b);
 			}
 
 			@Override
 			public void write(int b) throws IOException {
-				out.write(b);
+				get().write(b);
 			}
 
 			@Override
 			public void close() throws IOException {
+				if (closed) {
+					return;
+				}
+				closed = true;
 				try {
-					if (fsync)
-						os.getChannel().force(true);
-					out.close();
-					os = null;
+					if (written) {
+						throw new IOException(MessageFormat
+								.format(JGitText.get().lockStreamClosed, ref));
+					}
+					if (out != null) {
+						if (fsync) {
+							os.getChannel().force(true);
+						}
+						out.close();
+						os = null;
+					}
+					written = true;
 				} catch (IOException | RuntimeException | Error ioe) {
 					unlock();
 					throw ioe;
@@ -322,7 +374,7 @@ public class LockFile {
 	}
 
 	void requireLock() {
-		if (os == null) {
+		if (!haveLck) {
 			unlock();
 			throw new IllegalStateException(MessageFormat.format(JGitText.get().lockOnNotHeld, ref));
 		}
@@ -411,6 +463,8 @@ public class LockFile {
 		try {
 			FileUtils.rename(lck, ref, StandardCopyOption.ATOMIC_MOVE);
 			haveLck = false;
+			isAppend = false;
+			written = false;
 			closeToken();
 			return true;
 		} catch (IOException e) {
@@ -497,6 +551,8 @@ public class LockFile {
 				closeToken();
 			}
 		}
+		isAppend = false;
+		written = false;
 	}
 
 	/** {@inheritDoc} */