#66115 - Some Password protected XLS files are not read

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1901996 13f79535-47bb-0310-9956-ffa450edef68

9 files changed, 80 insertions, 40 deletions

diff --git a/poi-integration/src/test/java/org/apache/poi/stress/ExcInfo.java b/poi-integration/src/test/java/org/apache/poi/stress/ExcInfo.java
index c9241cb0f7..8d62ec300c 100644
--- a/poi-integration/src/test/java/org/apache/poi/stress/ExcInfo.java
+++ b/poi-integration/src/test/java/org/apache/poi/stress/ExcInfo.java
@@ -90,7 +90,7 @@ public class ExcInfo {
     public boolean isValid(String testName, String handler) {
         return
             !IGNORED_TESTS.equals(tests) &&
-            (tests == null || tests.contains(testName)) &&
-            (this.handler == null || this.handler.contains(handler));
+            (tests == null || (tests.contains(testName) && !tests.contains("!"+testName))) &&
+            (this.handler == null || (this.handler.contains(handler) && !this.handler.contains("!"+handler)));
     }
 }
diff --git a/poi/src/main/java/org/apache/poi/hssf/record/RecordInputStream.java b/poi/src/main/java/org/apache/poi/hssf/record/RecordInputStream.java
index 2084e6eb6b..a64d86b9d0 100644
--- a/poi/src/main/java/org/apache/poi/hssf/record/RecordInputStream.java
+++ b/poi/src/main/java/org/apache/poi/hssf/record/RecordInputStream.java
@@ -533,4 +533,9 @@ public final class RecordInputStream implements LittleEndianInput {
         ((InputStream)_dataInput).reset();
         _currentDataOffset = _markedDataOffset;
     }
+
+    @Internal
+    public boolean isEncrypted() {
+        return _dataInput instanceof Biff8DecryptingStream && ((Biff8DecryptingStream)_dataInput).isCurrentRecordEncrypted();
+    }
 }
diff --git a/poi/src/main/java/org/apache/poi/hssf/record/WriteAccessRecord.java b/poi/src/main/java/org/apache/poi/hssf/record/WriteAccessRecord.java
index 6c54028ffe..84061d0cda 100644
--- a/poi/src/main/java/org/apache/poi/hssf/record/WriteAccessRecord.java
+++ b/poi/src/main/java/org/apache/poi/hssf/record/WriteAccessRecord.java
@@ -17,11 +17,17 @@
 
 package org.apache.poi.hssf.record;
 
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.function.Supplier;
 
+import org.apache.poi.util.BitField;
+import org.apache.poi.util.BitFieldFactory;
 import org.apache.poi.util.GenericRecordUtil;
+import org.apache.poi.util.IOUtils;
 import org.apache.poi.util.LittleEndian;
 import org.apache.poi.util.LittleEndianOutput;
 import org.apache.poi.util.RecordFormatException;
@@ -36,10 +42,14 @@ import org.apache.poi.util.StringUtil;
 public final class WriteAccessRecord extends StandardRecord {
     public static final short sid = 0x005C;
 
+    private static final BitField UTF16FLAG = BitFieldFactory.getInstance(1);
+
     private static final byte PAD_CHAR = (byte) ' ';
     private static final int DATA_SIZE = 112;
+    private static final int STRING_SIZE = DATA_SIZE - 3;
+
     /** this record is always padded to a constant length */
-    private static final byte[] PADDING = new byte[DATA_SIZE];
+    private static final byte[] PADDING = new byte[STRING_SIZE];
     static {
         Arrays.fill(PADDING, PAD_CHAR);
     }
@@ -58,42 +68,57 @@ public final class WriteAccessRecord extends StandardRecord {
 
     public WriteAccessRecord(RecordInputStream in) {
         if (in.remaining() > DATA_SIZE) {
-            throw new RecordFormatException("Expected data size (" + DATA_SIZE + ") but got ("
-                    + in.remaining() + ")");
+            throw new RecordFormatException("Expected data size (" + DATA_SIZE + ") but got (" + in.remaining() + ")");
         }
-        // The string is always 112 characters (padded with spaces), therefore
+
+        // The string is always 109 characters (padded with spaces), therefore
         // this record can not be continued.
 
         int nChars = in.readUShort();
         int is16BitFlag = in.readUByte();
-        if (nChars > DATA_SIZE || (is16BitFlag & 0xFE) != 0) {
-            // String header looks wrong (probably missing)
-            // OOO doc says this is optional anyway.
-            // reconstruct data
-            byte[] data = new byte[3 + in.remaining()];
-            LittleEndian.putUShort(data, 0, nChars);
-            LittleEndian.putByte(data, 2, is16BitFlag);
-            in.readFully(data, 3, data.length-3);
-            String rawValue = new String(data, StringUtil.UTF8);
-            setUsername(rawValue.trim());
-            return;
-        }
-
-        String rawText;
-        if ((is16BitFlag & 0x01) == 0x00) {
-            rawText = StringUtil.readCompressedUnicode(in, nChars);
+        final byte[] data;
+        final Charset charset;
+        final int byteCnt;
+        if (nChars > STRING_SIZE || (is16BitFlag & 0xFE) != 0) {
+            // something is wrong - reconstruct data
+            if (in.isEncrypted()) {
+                // WPS Office seems to generate files with this record unencrypted (#66115)
+                // Libre Office/Excel can read those, but Excel will convert those back to encrypted
+                data = IOUtils.safelyAllocate(in.remaining(), STRING_SIZE);
+                in.readPlain(data, 0, data.length);
+                int i = data.length;
+                // PAD_CHAR is filled for every byte even for UTF16 strings
+                while (i>0 && data[i-1] == PAD_CHAR) {
+                    i--;
+                }
+                byteCnt = i;
+                // poor mans utf16 detection ...
+                charset = (data.length > 1 && data[1] == 0) ? StandardCharsets.UTF_16LE : StandardCharsets.ISO_8859_1;
+            } else {
+                // String header looks wrong (probably missing)
+                // OOO doc says this is optional anyway.
+                byteCnt = 3 + in.remaining();
+                data = IOUtils.safelyAllocate(byteCnt, DATA_SIZE);
+                LittleEndian.putUShort(data, 0, nChars);
+                LittleEndian.putByte(data, 2, is16BitFlag);
+                in.readFully(data, 3, byteCnt-3);
+                charset = StandardCharsets.UTF_8;
+            }
         } else {
-            rawText = StringUtil.readUnicodeLE(in, nChars);
-        }
-        field_1_username = rawText.trim();
-
-        // consume padding
-        int padSize = in.remaining();
-        while (padSize > 0) {
-            // in some cases this seems to be garbage (non spaces)
-            in.readUByte();
-            padSize--;
+            // the normal case ...
+            data = IOUtils.safelyAllocate(in.remaining(), STRING_SIZE);
+            in.readFully(data);
+            if (UTF16FLAG.isSet(is16BitFlag)) {
+                byteCnt = Math.min(nChars * 2, data.length);
+                charset = StandardCharsets.UTF_16LE;
+            } else {
+                byteCnt = Math.min(nChars, data.length);
+                charset = StandardCharsets.ISO_8859_1;
+            }
         }
+
+        String rawValue = new String(data, 0, byteCnt, charset);
+        setUsername(rawValue.trim());
     }
 
     /**
@@ -104,9 +129,8 @@ public final class WriteAccessRecord extends StandardRecord {
      */
     public void setUsername(String username) {
         boolean is16bit = StringUtil.hasMultibyte(username);
-        int encodedByteCount = 3 + username.length() * (is16bit ? 2 : 1);
-        int paddingSize = DATA_SIZE - encodedByteCount;
-        if (paddingSize < 0) {
+        int encodedByteCount = username.length() * (is16bit ? 2 : 1);
+        if (encodedByteCount > STRING_SIZE) {
             throw new IllegalArgumentException("Name is too long: " + username);
         }
 
@@ -131,14 +155,14 @@ public final class WriteAccessRecord extends StandardRecord {
 
         out.writeShort(username.length());
         out.writeByte(is16bit ? 0x01 : 0x00);
+
+        byte[] buf = PADDING.clone();
         if (is16bit) {
-            StringUtil.putUnicodeLE(username, out);
+            StringUtil.putUnicodeLE(username, buf, 0);
         } else {
-            StringUtil.putCompressedUnicode(username, out);
+            StringUtil.putCompressedUnicode(username, buf, 0);
         }
-        int encodedByteCount = 3 + username.length() * (is16bit ? 2 : 1);
-        int paddingSize = DATA_SIZE - encodedByteCount;
-        out.write(PADDING, 0, paddingSize);
+        out.write(buf);
     }
 
     @Override
diff --git a/poi/src/main/java/org/apache/poi/hssf/record/crypto/Biff8DecryptingStream.java b/poi/src/main/java/org/apache/poi/hssf/record/crypto/Biff8DecryptingStream.java
index d3dbf38ea2..74699d1bb1 100644
--- a/poi/src/main/java/org/apache/poi/hssf/record/crypto/Biff8DecryptingStream.java
+++ b/poi/src/main/java/org/apache/poi/hssf/record/crypto/Biff8DecryptingStream.java
@@ -26,6 +26,7 @@ import org.apache.poi.poifs.crypt.ChunkedCipherInputStream;
 import org.apache.poi.poifs.crypt.Decryptor;
 import org.apache.poi.poifs.crypt.EncryptionInfo;
 import org.apache.poi.util.IOUtils;
+import org.apache.poi.util.Internal;
 import org.apache.poi.util.LittleEndian;
 import org.apache.poi.util.LittleEndianConsts;
 import org.apache.poi.util.LittleEndianInput;
@@ -208,4 +209,8 @@ public final class Biff8DecryptingStream implements BiffHeaderInput, LittleEndia
         ccis.readPlain(b, off, len);
     }
 
+    @Internal
+    public boolean isCurrentRecordEncrypted() {
+        return !shouldSkipEncryptionOnCurrentRecord;
+    }
 }
diff --git a/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffDrawingToXml.java b/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffDrawingToXml.java
index 3613d1b68c..0b78830996 100644
--- a/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffDrawingToXml.java
+++ b/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffDrawingToXml.java
@@ -53,6 +53,7 @@ class TestBiffDrawingToXml extends BaseTestIteratingXLS {
         // HSSFWorkbook cannot open it as well
         excludes.put("43493.xls", RecordInputStream.LeftoverDataException.class);
         excludes.put("44958_1.xls", RecordInputStream.LeftoverDataException.class);
+        excludes.put("protected_66115.xls", EncryptedDocumentException.class);
         return excludes;
     }
 
diff --git a/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffViewer.java b/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffViewer.java
index 5013e45258..82ae713ab1 100644
--- a/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffViewer.java
+++ b/poi/src/test/java/org/apache/poi/hssf/dev/TestBiffViewer.java
@@ -20,6 +20,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Map;
 
+import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.util.RecordFormatException;
 
 class TestBiffViewer extends BaseTestIteratingXLS {
@@ -41,6 +42,7 @@ class TestBiffViewer extends BaseTestIteratingXLS {
 
         excludes.put("61300.xls", IndexOutOfBoundsException.class);
         excludes.put("poi-fuzz.xls", RecordFormatException.class);
+        excludes.put("protected_66115.xls", RecordFormatException.class);
 
         return excludes;
     }
diff --git a/poi/src/test/java/org/apache/poi/hssf/dev/TestEFBiffViewer.java b/poi/src/test/java/org/apache/poi/hssf/dev/TestEFBiffViewer.java
index bcd91e74d7..375bf8fa76 100644
--- a/poi/src/test/java/org/apache/poi/hssf/dev/TestEFBiffViewer.java
+++ b/poi/src/test/java/org/apache/poi/hssf/dev/TestEFBiffViewer.java
@@ -26,6 +26,7 @@ import org.apache.poi.hssf.eventusermodel.HSSFEventFactory;
 import org.apache.poi.hssf.eventusermodel.HSSFRequest;
 import org.apache.poi.hssf.record.RecordInputStream;
 import org.apache.poi.poifs.filesystem.POIFSFileSystem;
+import org.apache.poi.util.RecordFormatException;
 import org.junit.jupiter.api.Assertions;
 
 class TestEFBiffViewer extends BaseTestIteratingXLS {
@@ -37,6 +38,7 @@ class TestEFBiffViewer extends BaseTestIteratingXLS {
         excludes.put("51832.xls", EncryptedDocumentException.class);
         excludes.put("xor-encryption-abc.xls", EncryptedDocumentException.class);
         excludes.put("password.xls", EncryptedDocumentException.class);
+        excludes.put("protected_66115.xls", EncryptedDocumentException.class);
         // HSSFWorkbook cannot open it as well
         excludes.put("43493.xls", RecordInputStream.LeftoverDataException.class);
         excludes.put("44958_1.xls", RecordInputStream.LeftoverDataException.class);
diff --git a/poi/src/test/java/org/apache/poi/hssf/dev/TestFormulaViewer.java b/poi/src/test/java/org/apache/poi/hssf/dev/TestFormulaViewer.java
index 177871e25b..3b10498e69 100644
--- a/poi/src/test/java/org/apache/poi/hssf/dev/TestFormulaViewer.java
+++ b/poi/src/test/java/org/apache/poi/hssf/dev/TestFormulaViewer.java
@@ -44,6 +44,7 @@ class TestFormulaViewer extends BaseTestIteratingXLS {
         excludes.put("password.xls", EncryptedDocumentException.class);
         excludes.put("43493.xls", RecordInputStream.LeftoverDataException.class);  // HSSFWorkbook cannot open it as well
         excludes.put("44958_1.xls", RecordInputStream.LeftoverDataException.class);
+        excludes.put("protected_66115.xls", EncryptedDocumentException.class);
         return excludes;
     }
 
diff --git a/test-data/spreadsheet/stress.xls b/test-data/spreadsheet/stress.xls
index ecfae46d48..37ff2aae13 100644
--- a/test-data/spreadsheet/stress.xls
+++ b/test-data/spreadsheet/stress.xls