]> source.dussan.org Git - jgit.git/commitdiff
LockFile: create OutputStream only when needed 98/180198/4
authorThomas Wolf <thomas.wolf@paranor.ch>
Tue, 4 May 2021 21:48:56 +0000 (23:48 +0200)
committerMatthias Sohn <matthias.sohn@sap.com>
Fri, 7 May 2021 10:10:47 +0000 (12:10 +0200)
Don't create the stream eagerly in lock(); that may cause JGit to
exceed OS or JVM limits on open file descriptors if many locks need
to be created, for instance when creating many refs. Instead create
the output stream only when one really needs to write something.

Bug: 573328
Change-Id: If9441ed40494d46f594a896d34a5c4f56f91ebf4
Signed-off-by: Thomas Wolf <thomas.wolf@paranor.ch>
org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/LockFileTest.java
org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java

index 0f93749d9b6ddba79091f0cd12ad0ce2db57634d..509935dfb9e156e1446bb73d36396335b8f4edde 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2012, GitHub Inc. and others
+ * Copyright (C) 2012, 2021 GitHub Inc. and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -9,10 +9,16 @@
  */
 package org.eclipse.jgit.internal.storage.file;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.File;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.errors.LockFailedException;
@@ -49,4 +55,149 @@ public class LockFileTest extends RepositoryTestCase {
                        }
                }
        }
+
+       @Test
+       public void testLockTwice() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               lock.write("other".getBytes(StandardCharsets.US_ASCII));
+               lock.commit();
+               assertFalse(lock.isLocked());
+               checkFile(f, "other");
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               try (OutputStream out = lock.getOutputStream()) {
+                       out.write("second".getBytes(StandardCharsets.US_ASCII));
+               }
+               lock.commit();
+               assertFalse(lock.isLocked());
+               checkFile(f, "second");
+       }
+
+       @Test
+       public void testLockTwiceUnlock() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               lock.write("other".getBytes(StandardCharsets.US_ASCII));
+               lock.unlock();
+               assertFalse(lock.isLocked());
+               checkFile(f, "content");
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               try (OutputStream out = lock.getOutputStream()) {
+                       out.write("second".getBytes(StandardCharsets.US_ASCII));
+               }
+               lock.commit();
+               assertFalse(lock.isLocked());
+               checkFile(f, "second");
+       }
+
+       @Test
+       public void testLockWriteTwiceThrows1() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               lock.write("other".getBytes(StandardCharsets.US_ASCII));
+               assertThrows(Exception.class,
+                               () -> lock.write("second".getBytes(StandardCharsets.US_ASCII)));
+               lock.unlock();
+       }
+
+       @Test
+       public void testLockWriteTwiceThrows2() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               try (OutputStream out = lock.getOutputStream()) {
+                       out.write("other".getBytes(StandardCharsets.US_ASCII));
+               }
+               assertThrows(Exception.class,
+                               () -> lock.write("second".getBytes(StandardCharsets.US_ASCII)));
+               lock.unlock();
+       }
+
+       @Test
+       public void testLockWriteTwiceThrows3() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               lock.write("other".getBytes(StandardCharsets.US_ASCII));
+               assertThrows(Exception.class, () -> {
+                       try (OutputStream out = lock.getOutputStream()) {
+                               out.write("second".getBytes(StandardCharsets.US_ASCII));
+                       }
+               });
+               lock.unlock();
+       }
+
+       @Test
+       public void testLockWriteTwiceThrows4() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               try (OutputStream out = lock.getOutputStream()) {
+                       out.write("other".getBytes(StandardCharsets.US_ASCII));
+               }
+               assertThrows(Exception.class, () -> {
+                       try (OutputStream out = lock.getOutputStream()) {
+                               out.write("second".getBytes(StandardCharsets.US_ASCII));
+                       }
+               });
+               lock.unlock();
+       }
+
+       @Test
+       public void testLockUnclosedCommitThrows() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               try (OutputStream out = lock.getOutputStream()) {
+                       out.write("other".getBytes(StandardCharsets.US_ASCII));
+                       assertThrows(Exception.class, () -> lock.commit());
+               }
+       }
+
+       @Test
+       public void testLockNested() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               assertThrows(IllegalStateException.class, () -> lock.lock());
+               assertTrue(lock.isLocked());
+               lock.unlock();
+       }
+
+       @Test
+       public void testLockHeld() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lock());
+               assertTrue(lock.isLocked());
+               LockFile lock2 = new LockFile(f);
+               assertFalse(lock2.lock());
+               assertFalse(lock2.isLocked());
+               assertTrue(lock.isLocked());
+               lock.unlock();
+       }
+
+       @Test
+       public void testLockForAppend() throws Exception {
+               File f = writeTrashFile("somefile", "content");
+               LockFile lock = new LockFile(f);
+               assertTrue(lock.lockForAppend());
+               assertTrue(lock.isLocked());
+               lock.write("other".getBytes(StandardCharsets.US_ASCII));
+               lock.commit();
+               assertFalse(lock.isLocked());
+               checkFile(f, "contentother");
+       }
 }
index feef39744bd5b00cc23714c95b463602bff555a1..2fa8713daae10da07d7b32760035d739091cfb7d 100644 (file)
@@ -415,11 +415,14 @@ listingPacks=Listing packs
 localObjectsIncomplete=Local objects incomplete.
 localRefIsMissingObjects=Local ref {0} is missing object(s).
 localRepository=local repository
+lockAlreadyHeld=Lock on {0} already held
 lockCountMustBeGreaterOrEqual1=lockCount must be >= 1
 lockError=lock error: {0}
 lockFailedRetry=locking {0} failed after {1} retries
 lockOnNotClosed=Lock on {0} not closed.
 lockOnNotHeld=Lock on {0} not held.
+lockStreamClosed=Output to lock on {0} already closed
+lockStreamMultiple=Output to lock on {0} already opened
 logInconsistentFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: {} > {}, but diff = {}. Aborting measurement at resolution {}.
 logLargerFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: diff = {} > {} (last good value). Aborting measurement.
 logSmallerFiletime={}: got smaller file timestamp on {}, {}: {} < {}. Aborting measurement at resolution {}.
index 09fe03e06563b51b7de3487018ddf96d2af33b67..ab9fc5c9bb69d56ff2ce339ba3fd4f78a6b663a4 100644 (file)
@@ -444,10 +444,13 @@ public class JGitText extends TranslationBundle {
        /***/ public String localRefIsMissingObjects;
        /***/ public String localRepository;
        /***/ public String lockCountMustBeGreaterOrEqual1;
+       /***/ public String lockAlreadyHeld;
        /***/ public String lockError;
        /***/ public String lockFailedRetry;
        /***/ public String lockOnNotClosed;
        /***/ public String lockOnNotHeld;
+       /***/ public String lockStreamClosed;
+       /***/ public String lockStreamMultiple;
        /***/ public String logInconsistentFiletimeDiff;
        /***/ public String logLargerFiletimeDiff;
        /***/ public String logSmallerFiletime;
index 2e0a6da3a4f34ab390d3d1bbc2310677fd656d71..ab407a6ae97f530481cf2140ab49f0ceb2efd058 100644 (file)
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
- * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org> and others
+ * Copyright (C) 2006-2021, Shawn O. Pearce <spearce@spearce.org> and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -96,11 +96,15 @@ public class LockFile {
 
        private boolean haveLck;
 
-       FileOutputStream os;
+       private FileOutputStream os;
 
        private boolean needSnapshot;
 
-       boolean fsync;
+       private boolean fsync;
+
+       private boolean isAppend;
+
+       private boolean written;
 
        private FileSnapshot commitSnapshot;
 
@@ -127,6 +131,10 @@ public class LockFile {
         *             does not hold the lock.
         */
        public boolean lock() throws IOException {
+               if (haveLck) {
+                       throw new IllegalStateException(
+                                       MessageFormat.format(JGitText.get().lockAlreadyHeld, ref));
+               }
                FileUtils.mkdirs(lck.getParentFile(), true);
                try {
                        token = FS.DETECTED.createNewFileAtomic(lck);
@@ -134,18 +142,15 @@ public class LockFile {
                        LOG.error(JGitText.get().failedCreateLockFile, lck, e);
                        throw e;
                }
-               if (token.isCreated()) {
+               boolean obtainedLock = token.isCreated();
+               if (obtainedLock) {
                        haveLck = true;
-                       try {
-                               os = new FileOutputStream(lck);
-                       } catch (IOException ioe) {
-                               unlock();
-                               throw ioe;
-                       }
+                       isAppend = false;
+                       written = false;
                } else {
                        closeToken();
                }
-               return haveLck;
+               return obtainedLock;
        }
 
        /**
@@ -158,12 +163,24 @@ public class LockFile {
         *             does not hold the lock.
         */
        public boolean lockForAppend() throws IOException {
-               if (!lock())
+               if (!lock()) {
                        return false;
+               }
                copyCurrentContent();
+               isAppend = true;
+               written = false;
                return true;
        }
 
+       // For tests only
+       boolean isLocked() {
+               return haveLck;
+       }
+
+       private FileOutputStream getStream() throws IOException {
+               return new FileOutputStream(lck, isAppend);
+       }
+
        /**
         * Copy the current file content into the temporary file.
         * <p>
@@ -185,32 +202,31 @@ public class LockFile {
         */
        public void copyCurrentContent() throws IOException {
                requireLock();
-               try {
+               try (FileOutputStream out = getStream()) {
                        try (FileInputStream fis = new FileInputStream(ref)) {
                                if (fsync) {
                                        FileChannel in = fis.getChannel();
                                        long pos = 0;
                                        long cnt = in.size();
                                        while (0 < cnt) {
-                                               long r = os.getChannel().transferFrom(in, pos, cnt);
+                                               long r = out.getChannel().transferFrom(in, pos, cnt);
                                                pos += r;
                                                cnt -= r;
                                        }
                                } else {
                                        final byte[] buf = new byte[2048];
                                        int r;
-                                       while ((r = fis.read(buf)) >= 0)
-                                               os.write(buf, 0, r);
+                                       while ((r = fis.read(buf)) >= 0) {
+                                               out.write(buf, 0, r);
+                                       }
                                }
+                       } catch (FileNotFoundException fnfe) {
+                               if (ref.exists()) {
+                                       throw fnfe;
+                               }
+                               // Don't worry about a file that doesn't exist yet, it
+                               // conceptually has no current content to copy.
                        }
-               } catch (FileNotFoundException fnfe) {
-                       if (ref.exists()) {
-                               unlock();
-                               throw fnfe;
-                       }
-                       // Don't worry about a file that doesn't exist yet, it
-                       // conceptually has no current content to copy.
-                       //
                } catch (IOException | RuntimeException | Error ioe) {
                        unlock();
                        throw ioe;
@@ -253,18 +269,22 @@ public class LockFile {
         */
        public void write(byte[] content) throws IOException {
                requireLock();
-               try {
+               try (FileOutputStream out = getStream()) {
+                       if (written) {
+                               throw new IOException(MessageFormat
+                                               .format(JGitText.get().lockStreamClosed, ref));
+                       }
                        if (fsync) {
-                               FileChannel fc = os.getChannel();
+                               FileChannel fc = out.getChannel();
                                ByteBuffer buf = ByteBuffer.wrap(content);
-                               while (0 < buf.remaining())
+                               while (0 < buf.remaining()) {
                                        fc.write(buf);
+                               }
                                fc.force(true);
                        } else {
-                               os.write(content);
+                               out.write(content);
                        }
-                       os.close();
-                       os = null;
+                       written = true;
                } catch (IOException | RuntimeException | Error ioe) {
                        unlock();
                        throw ioe;
@@ -283,36 +303,68 @@ public class LockFile {
        public OutputStream getOutputStream() {
                requireLock();
 
-               final OutputStream out;
-               if (fsync)
-                       out = Channels.newOutputStream(os.getChannel());
-               else
-                       out = os;
+               if (written || os != null) {
+                       throw new IllegalStateException(MessageFormat
+                                       .format(JGitText.get().lockStreamMultiple, ref));
+               }
 
                return new OutputStream() {
+
+                       private OutputStream out;
+
+                       private boolean closed;
+
+                       private OutputStream get() throws IOException {
+                               if (written) {
+                                       throw new IOException(MessageFormat
+                                                       .format(JGitText.get().lockStreamMultiple, ref));
+                               }
+                               if (out == null) {
+                                       os = getStream();
+                                       if (fsync) {
+                                               out = Channels.newOutputStream(os.getChannel());
+                                       } else {
+                                               out = os;
+                                       }
+                               }
+                               return out;
+                       }
+
                        @Override
                        public void write(byte[] b, int o, int n)
                                        throws IOException {
-                               out.write(b, o, n);
+                               get().write(b, o, n);
                        }
 
                        @Override
                        public void write(byte[] b) throws IOException {
-                               out.write(b);
+                               get().write(b);
                        }
 
                        @Override
                        public void write(int b) throws IOException {
-                               out.write(b);
+                               get().write(b);
                        }
 
                        @Override
                        public void close() throws IOException {
+                               if (closed) {
+                                       return;
+                               }
+                               closed = true;
                                try {
-                                       if (fsync)
-                                               os.getChannel().force(true);
-                                       out.close();
-                                       os = null;
+                                       if (written) {
+                                               throw new IOException(MessageFormat
+                                                               .format(JGitText.get().lockStreamClosed, ref));
+                                       }
+                                       if (out != null) {
+                                               if (fsync) {
+                                                       os.getChannel().force(true);
+                                               }
+                                               out.close();
+                                               os = null;
+                                       }
+                                       written = true;
                                } catch (IOException | RuntimeException | Error ioe) {
                                        unlock();
                                        throw ioe;
@@ -322,7 +374,7 @@ public class LockFile {
        }
 
        void requireLock() {
-               if (os == null) {
+               if (!haveLck) {
                        unlock();
                        throw new IllegalStateException(MessageFormat.format(JGitText.get().lockOnNotHeld, ref));
                }
@@ -411,6 +463,8 @@ public class LockFile {
                try {
                        FileUtils.rename(lck, ref, StandardCopyOption.ATOMIC_MOVE);
                        haveLck = false;
+                       isAppend = false;
+                       written = false;
                        closeToken();
                        return true;
                } catch (IOException e) {
@@ -497,6 +551,8 @@ public class LockFile {
                                closeToken();
                        }
                }
+               isAppend = false;
+               written = false;
        }
 
        /** {@inheritDoc} */