add back 731 changes but increase gradle heap

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1922162 13f79535-47bb-0310-9956-ffa450edef68

3 files changed, 89 insertions, 10 deletions

diff --git a/gradle.properties b/gradle.properties
index b0b7a57395..26c6faab92 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,7 +1,8 @@
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-# Less than 2G definitely slows things down. -XX:+HeapDumpOnOutOfMemoryError
-org.gradle.jvmargs=-Xmx2G -Dfile.encoding=UTF-8
+# Less than 2G definitely slows things down. Increased to 3G due to OOM issues on ci-builds.apache.org.
+# -XX:+HeapDumpOnOutOfMemoryError
+org.gradle.jvmargs=-Xmx3G -Dfile.encoding=UTF-8
 
 # Activating will be much faster, but break the build of 'poi-ooxml-lite'
 org.gradle.caching=true
diff --git a/poi/src/main/java/org/apache/poi/poifs/filesystem/POIFSMiniStore.java b/poi/src/main/java/org/apache/poi/poifs/filesystem/POIFSMiniStore.java
index 18cdfd58f7..cb1929d0d6 100644
--- a/poi/src/main/java/org/apache/poi/poifs/filesystem/POIFSMiniStore.java
+++ b/poi/src/main/java/org/apache/poi/poifs/filesystem/POIFSMiniStore.java
@@ -247,20 +247,33 @@ public class POIFSMiniStore extends BlockStore {
      * based on full blocks used, not the data within the streams
      */
     void syncWithDataSource() throws IOException {
-        int blocksUsed = 0;
         for (BATBlock sbat : _sbat_blocks) {
             ByteBuffer block = _filesystem.getBlockAt(sbat.getOurBlockIndex());
             sbat.writeData(block);
-
-            if (!sbat.hasFreeSectors()) {
-                blocksUsed += _filesystem.getBigBlockSizeDetails().getBATEntriesPerBlock();
-            } else {
-                blocksUsed += sbat.getOccupiedSize();
-            }
         }
+
         // Set the size on the root in terms of the number of SBAT blocks
         // RootProperty.setSize does the sbat -> bytes conversion for us
-        _filesystem._get_property_table().getRoot().setSize(blocksUsed);
+        _filesystem._get_property_table().getRoot().setSize(computeSize());
+    }
+
+    /**
+     * Computes the size of the mini-stream (in number of mini blocks). The trailing
+     * unallocated mini blocks are ignored, the others are counted as allocated. This
+     * behaviour was checked with MSI files signed with signtool.
+     */
+    private int computeSize() {
+        int entriesPerBlock = _filesystem.getBigBlockSizeDetails().getBATEntriesPerBlock();
+        for (int sbatIndex = _sbat_blocks.size() - 1; sbatIndex >= 0; sbatIndex--) {
+            BATBlock sbat = _sbat_blocks.get(sbatIndex);
+            for (int miniBlockIndex = entriesPerBlock - 1; miniBlockIndex >= 0; miniBlockIndex--) {
+                if (sbat.getValueAt(miniBlockIndex) != POIFSConstants.UNUSED_BLOCK) {
+                    return (sbatIndex * entriesPerBlock) + miniBlockIndex + 1;
+                }
+            }
+        }
+
+        return 0;
     }
 
     @Override
diff --git a/poi/src/test/java/org/apache/poi/poifs/filesystem/TestPOIFSMiniStore.java b/poi/src/test/java/org/apache/poi/poifs/filesystem/TestPOIFSMiniStore.java
index 8d24e9f87b..6b85326a90 100644
--- a/poi/src/test/java/org/apache/poi/poifs/filesystem/TestPOIFSMiniStore.java
+++ b/poi/src/test/java/org/apache/poi/poifs/filesystem/TestPOIFSMiniStore.java
@@ -28,7 +28,9 @@ import java.nio.ByteBuffer;
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 
+import org.apache.commons.io.output.NullOutputStream;
 import org.apache.poi.POIDataSamples;
+import org.apache.poi.hpsf.ClassID;
 import org.apache.poi.poifs.common.POIFSConstants;
 import org.apache.poi.util.IOUtils;
 import org.junit.jupiter.api.Test;
@@ -470,4 +472,67 @@ final class TestPOIFSMiniStore {
         assertArrayEquals(data2B, r2);
         fs.close();
     }
+
+    /**
+     * Check the computation of the mini stream size when the mini FAT sectors contain unallocated mini sectors.
+     * https://github.com/apache/poi/pull/182
+     */
+    @Test
+    void testComputeSize() throws Exception {
+        try (POIFSFileSystem fs = new POIFSFileSystem()) {
+            fs.getPropertyTable().getRoot().setStorageClsid(new ClassID("000C108400000000C000000000000046")); // MSI storage class
+
+            // create 8 mini FAT sectors fully allocated
+            for (int i = 0; i < 8 * 128; i++) {
+                fs.getRoot().createDocument("Entry " + i, new ByteArrayInputStream(new byte[64]));
+            }
+
+            // Mini FAT Sector #1: Unallocate all the mini sectors
+            for (int i = 0; i < 128; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            // Mini FAT Sector #2: Unallocate 8 mini sectors at the beginning
+            for (int i = 128; i < 128 + 8; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            // Mini FAT Sector #3: Unallocate 4 mini sectors in the middle
+            for (int i = 2 * 128 + 64; i < 2 * 128 + 64 + 4; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            // Mini FAT Sector #4: Unallocate all the mini sectors
+            for (int i = 3 * 128; i < 4 * 128; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            // Mini FAT Sector #5: Unallocate 32 mini sectors at the end
+            for (int i = 5 * 128 - 32; i < 5 * 128; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            // Mini FAT Sector #6: Unallocate 64 mini sectors at the beginning and 16 mini sectors at the end
+            for (int i = 5 * 128; i < 5 * 128 + 64; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+            for (int i = 6 * 128 - 16; i < 6 * 128; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            // Mini FAT Sector #7: Unallocate all the mini sectors
+            for (int i = 6 * 128; i < 7 * 128; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            // Mini FAT Sector #8: Unallocate all the mini sectors
+            for (int i = 7 * 128; i < 8 * 128; i++) {
+                fs.getRoot().getEntry("Entry " + i).delete();
+            }
+
+            fs.writeFilesystem(NullOutputStream.INSTANCE);
+
+            assertEquals(48128, fs.getPropertyTable().getRoot().getSize(), "mini stream size");
+        }
+    }
 }