Handle encrypted files with known password in integration tests

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1797839 13f79535-47bb-0310-9956-ffa450edef68

src/integrationtest/org/apache/poi/TestAllFiles.java

@@ -34,6 +34,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
+import org.apache.poi.poifs.crypt.Decryptor;
 import org.apache.poi.stress.AbstractFileHandler;
 import org.apache.poi.stress.FileHandler;
 import org.apache.poi.stress.HDGFFileHandler;
@@ -53,6 +55,7 @@ import org.apache.poi.stress.XSSFFileHandler;
 import org.apache.poi.stress.XWPFFileHandler;
 import org.apache.tools.ant.DirectoryScanner;
 import org.junit.AssumptionViolatedException;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -88,6 +91,9 @@ public class TestAllFiles {
     private static final File ROOT_DIR = new File("test-data");
 
     static final String[] SCAN_EXCLUDES = new String[] { "**/.svn/**", "lost+found" };
+
+    private static final Map<String,String> FILE_PASSWORD;
+
     
     // map file extensions to the actual mappers
     static final Map<String, FileHandler> HANDLERS = new HashMap<String, FileHandler>();
@@ -200,6 +206,28 @@ public class TestAllFiles {
         HANDLERS.put("spreadsheet/BigSSTRecord2CR7", new NullFileHandler());
         HANDLERS.put("spreadsheet/BigSSTRecordCR", new NullFileHandler());
         HANDLERS.put("spreadsheet/test_properties1", new NullFileHandler());
+        
+        Map<String,String> passmap = new HashMap<String,String>();
+        passmap.put("slideshow/Password_Protected-hello.ppt", "hello");
+        passmap.put("slideshow/Password_Protected-56-hello.ppt", "hello");
+        passmap.put("slideshow/Password_Protected-np-hello.ppt", "hello");
+        passmap.put("slideshow/cryptoapi-proc2356.ppt", "crypto");
+        passmap.put("spreadsheet/xor-encryption-abc.xls", "abc");
+        passmap.put("spreadsheet/35897-type4.xls", "freedom");
+        passmap.put("spreadsheet/58616.xlsx", Decryptor.DEFAULT_PASSWORD);
+        passmap.put("spreadsheet/password.xls", "password");
+        passmap.put("spreadsheet/protected_passtika.xlsx", "tika");
+        passmap.put("document/bug53475-password-is-pass.docx", "pass");
+        passmap.put("document/bug53475-password-is-solrcell.docx", "solrcell");
+        passmap.put("document/password_password_cryptoapi.doc", "password");
+        passmap.put("document/password_tika_binaryrc4.doc", "tika");
+        passmap.put("poifs/protect.xlsx", Decryptor.DEFAULT_PASSWORD);
+        passmap.put("poifs/extenxls_pwd123.xlsx", "pwd123");
+        passmap.put("poifs/protected_agile.docx", Decryptor.DEFAULT_PASSWORD);
+        passmap.put("poifs/60320-protected.xlsx", "Test001!!");
+        passmap.put("poifs/protected_sha512.xlsx", "this is a test");
+        
+        FILE_PASSWORD = Collections.unmodifiableMap(passmap);
     }
 
     private static final Set<String> unmodifiableHashSet(String... a) {
@@ -228,25 +256,9 @@ public class TestAllFiles {
     );
 
     private static final Set<String> EXPECTED_FAILURES = unmodifiableHashSet(
-        // password protected files
-        "spreadsheet/password.xls",
-        "spreadsheet/protected_passtika.xlsx",
+        // password protected files without known password
         "spreadsheet/51832.xls",
         "document/PasswordProtected.doc",
-        "slideshow/Password_Protected-hello.ppt",
-        "slideshow/Password_Protected-56-hello.ppt",
-        "slideshow/Password_Protected-np-hello.ppt",
-        "slideshow/cryptoapi-proc2356.ppt",
-        //"document/bug53475-password-is-pass.docx",
-        //"document/bug53475-password-is-solrcell.docx",
-        "spreadsheet/xor-encryption-abc.xls",
-        "spreadsheet/35897-type4.xls",
-        //"poifs/protect.xlsx",
-        //"poifs/protected_sha512.xlsx",
-        //"poifs/extenxls_pwd123.xlsx",
-        //"poifs/protected_agile.docx",
-        "spreadsheet/58616.xlsx",
-        "poifs/60320-protected.xlsx",
 
         // TODO: fails XMLExportTest, is this ok?
         "spreadsheet/CustomXMLMapping-singleattributenamespace.xlsx",
@@ -368,6 +380,13 @@ public class TestAllFiles {
 
     @Parameter(value=1)
     public FileHandler handler;
+
+    @Before
+    public void setPassword() {
+        // this also removes the password for non encrypted files
+        String pass = TestAllFiles.FILE_PASSWORD.get(file);
+        Biff8EncryptionKey.setCurrentUserPassword(pass);
+    }
     
     @Test
     public void testAllFiles() throws Exception {
@@ -381,7 +400,6 @@ public class TestAllFiles {
                 handler instanceof OPCFileHandler;
         boolean ignoreHPSF = (handler instanceof HPSFFileHandler);
         
-        
         try {
             InputStream stream = new BufferedInputStream(new FileInputStream(inputFile), 64*1024);
             try {

src/integrationtest/org/apache/poi/stress/AbstractFileHandler.java

@@ -37,12 +37,8 @@ import org.apache.xmlbeans.XmlException;
 public abstract class AbstractFileHandler implements FileHandler {
     public static final Set<String> EXPECTED_EXTRACTOR_FAILURES = new HashSet<String>();
     static {
-        // password protected files
-        EXPECTED_EXTRACTOR_FAILURES.add("document/bug53475-password-is-pass.docx");
-        EXPECTED_EXTRACTOR_FAILURES.add("poifs/extenxls_pwd123.xlsx");
-        EXPECTED_EXTRACTOR_FAILURES.add("poifs/protect.xlsx");
-        EXPECTED_EXTRACTOR_FAILURES.add("poifs/protected_agile.docx");
-        EXPECTED_EXTRACTOR_FAILURES.add("poifs/protected_sha512.xlsx");
+        // password protected files without password
+    	// ... currently none ...
         
         // unsupported file-types, no supported OLE2 parts
         EXPECTED_EXTRACTOR_FAILURES.add("hmef/quick-winmail.dat");

src/integrationtest/org/apache/poi/stress/XSSFFileHandler.java

@@ -16,8 +16,9 @@
 ==================================================================== */
 package org.apache.poi.stress;
 
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.*;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assume.assumeFalse;
 
 import java.io.BufferedInputStream;
 import java.io.ByteArrayInputStream;
@@ -37,10 +38,14 @@ import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.TransformerException;
 
 import org.apache.poi.POIXMLException;
+import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
 import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
 import org.apache.poi.openxml4j.exceptions.OLE2NotOfficeXmlFileException;
 import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
 import org.apache.poi.openxml4j.opc.OPCPackage;
+import org.apache.poi.poifs.crypt.Decryptor;
+import org.apache.poi.poifs.crypt.EncryptionInfo;
+import org.apache.poi.poifs.filesystem.POIFSFileSystem;
 import org.apache.poi.util.IOUtils;
 import org.apache.poi.xssf.eventusermodel.XLSX2CSV;
 import org.apache.poi.xssf.eventusermodel.XSSFReader;
@@ -54,8 +59,9 @@ import org.xml.sax.SAXException;
 public class XSSFFileHandler extends SpreadsheetHandler {
     @Override
     public void handleFile(InputStream stream, String path) throws Exception {
-        // ignore password protected files
-        if (POIXMLDocumentHandler.isEncrypted(stream)) return;
+        // ignore password protected files if password is unknown
+        String pass = Biff8EncryptionKey.getCurrentUserPassword();
+        assumeFalse(pass == null && POIXMLDocumentHandler.isEncrypted(stream));
 
         final XSSFWorkbook wb;
 
@@ -63,11 +69,24 @@ public class XSSFFileHandler extends SpreadsheetHandler {
         {
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             IOUtils.copy(stream, out);
-            final byte[] bytes = out.toByteArray();
+            ByteArrayInputStream bytes = new ByteArrayInputStream(out.toByteArray());
 
-            checkXSSFReader(OPCPackage.open(new ByteArrayInputStream(bytes)));
-
-            wb = new XSSFWorkbook(new ByteArrayInputStream(bytes));
+            if (pass != null) {
+                POIFSFileSystem poifs = new POIFSFileSystem(bytes);
+                EncryptionInfo ei = new EncryptionInfo(poifs);
+                Decryptor dec = ei.getDecryptor();
+                boolean b = dec.verifyPassword(pass);
+                assertTrue("password mismatch", b);
+                InputStream is = dec.getDataStream(poifs);
+                out.reset();
+                IOUtils.copy(is, out);
+                is.close();
+                poifs.close();
+                bytes = new ByteArrayInputStream(out.toByteArray());
+            }
+            checkXSSFReader(OPCPackage.open(bytes));
+            bytes.reset();
+            wb = new XSSFWorkbook(bytes);
         }
 
         // use the combined handler for HSSF/XSSF

src/java/org/apache/poi/extractor/OLE2ExtractorFactory.java

@@ -26,17 +26,22 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.POIOLE2TextExtractor;
 import org.apache.poi.POITextExtractor;
 import org.apache.poi.hssf.OldExcelFormatException;
 import org.apache.poi.hssf.extractor.EventBasedExcelExtractor;
 import org.apache.poi.hssf.extractor.ExcelExtractor;
+import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
+import org.apache.poi.poifs.crypt.Decryptor;
+import org.apache.poi.poifs.crypt.EncryptionInfo;
 import org.apache.poi.poifs.filesystem.DirectoryEntry;
 import org.apache.poi.poifs.filesystem.DirectoryNode;
 import org.apache.poi.poifs.filesystem.Entry;
 import org.apache.poi.poifs.filesystem.NPOIFSFileSystem;
 import org.apache.poi.poifs.filesystem.OPOIFSFileSystem;
 import org.apache.poi.poifs.filesystem.POIFSFileSystem;
+import org.apache.poi.util.IOUtils;
 import org.apache.poi.util.POILogFactory;
 import org.apache.poi.util.POILogger;
 
@@ -167,9 +172,7 @@ public class OLE2ExtractorFactory {
      * Note that this won't check for embedded OOXML resources either, use
      *  {@link org.apache.poi.extractor.ExtractorFactory} for that.
      */
-    public static POITextExtractor createExtractor(DirectoryNode poifsDir)
-            throws IOException
-    {
+    public static POITextExtractor createExtractor(DirectoryNode poifsDir) throws IOException {
         // Look for certain entries in the stream, to figure it
         // out from
         for (String workbookName : WORKBOOK_DIR_ENTRY_NAMES) {
@@ -267,4 +270,29 @@ public class OLE2ExtractorFactory {
         }
         return e.toArray(new POITextExtractor[e.size()]);
     }
+
+    private static POITextExtractor createEncyptedOOXMLExtractor(DirectoryNode poifsDir)
+    throws IOException {
+        String pass = Biff8EncryptionKey.getCurrentUserPassword();
+        if (pass == null) {
+            pass = Decryptor.DEFAULT_PASSWORD;
+        }
+        
+        EncryptionInfo ei = new EncryptionInfo(poifsDir);
+        Decryptor dec = ei.getDecryptor();
+        InputStream is = null;
+        try {
+            if (!dec.verifyPassword(pass)) {
+                throw new EncryptedDocumentException("Invalid password specified - use Biff8EncryptionKey.setCurrentUserPassword() before calling extractor");
+            }
+            is = dec.getDataStream(poifsDir);
+            return createExtractor(is);
+        } catch (IOException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new IOException(e);
+        } finally {
+            IOUtils.closeQuietly(is);
+        }
+    }
 }

src/java/org/apache/poi/sl/usermodel/SlideShowFactory.java

@@ -76,13 +76,17 @@ public class SlideShowFactory {
 
         // If we get here, it isn't an encrypted PPTX file
         // So, treat it as a regular HSLF PPT one
+        boolean passwordSet = false;
         if (password != null) {
             Biff8EncryptionKey.setCurrentUserPassword(password);
+            passwordSet = true;
         }
         try {
             return createHSLFSlideShow(fs);
         } finally {
-            Biff8EncryptionKey.setCurrentUserPassword(null);
+            if (passwordSet) {
+                Biff8EncryptionKey.setCurrentUserPassword(null);
+            }
         }
     }
 

src/ooxml/java/org/apache/poi/extractor/ExtractorFactory.java

@@ -25,6 +25,7 @@ import java.io.PushbackInputStream;
 import java.util.ArrayList;
 import java.util.Iterator;
 
+import org.apache.poi.EncryptedDocumentException;
 import org.apache.poi.POIOLE2TextExtractor;
 import org.apache.poi.POITextExtractor;
 import org.apache.poi.POIXMLTextExtractor;
@@ -32,6 +33,7 @@ import org.apache.poi.hsmf.MAPIMessage;
 import org.apache.poi.hsmf.datatypes.AttachmentChunks;
 import org.apache.poi.hsmf.extractor.OutlookTextExtactor;
 import org.apache.poi.hssf.extractor.ExcelExtractor;
+import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
 import org.apache.poi.hwpf.extractor.WordExtractor;
 import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
 import org.apache.poi.openxml4j.opc.OPCPackage;
@@ -39,6 +41,8 @@ import org.apache.poi.openxml4j.opc.PackageAccess;
 import org.apache.poi.openxml4j.opc.PackagePart;
 import org.apache.poi.openxml4j.opc.PackageRelationshipCollection;
 import org.apache.poi.openxml4j.opc.PackageRelationshipTypes;
+import org.apache.poi.poifs.crypt.Decryptor;
+import org.apache.poi.poifs.crypt.EncryptionInfo;
 import org.apache.poi.poifs.filesystem.DirectoryEntry;
 import org.apache.poi.poifs.filesystem.DirectoryNode;
 import org.apache.poi.poifs.filesystem.DocumentFactoryHelper;
@@ -127,6 +131,9 @@ public class ExtractorFactory {
         NPOIFSFileSystem fs = null;
         try {
             fs = new NPOIFSFileSystem(f);
+            if (fs.getRoot().hasEntry(Decryptor.DEFAULT_POIFS_ENTRY)) {
+                return createEncyptedOOXMLExtractor(fs);
+            }
             POIOLE2TextExtractor extractor = createExtractor(fs);
             extractor.setFilesystem(fs);
             return extractor;
@@ -171,7 +178,9 @@ public class ExtractorFactory {
         }
 
         if (NPOIFSFileSystem.hasPOIFSHeader(inp)) {
-            return createExtractor(new NPOIFSFileSystem(inp));
+            NPOIFSFileSystem fs = new NPOIFSFileSystem(inp);
+            boolean isEncrypted = fs.getRoot().hasEntry(Decryptor.DEFAULT_POIFS_ENTRY); 
+            return isEncrypted ? createEncyptedOOXMLExtractor(fs) : createExtractor(fs);
         }
         if (DocumentFactoryHelper.hasOOXMLHeader(inp)) {
             return createExtractor(OPCPackage.open(inp));
@@ -397,4 +406,29 @@ public class ExtractorFactory {
     public static POITextExtractor[] getEmbededDocsTextExtractors(POIXMLTextExtractor ext) {
         throw new IllegalStateException("Not yet supported");
     }
+    
+    private static POIXMLTextExtractor createEncyptedOOXMLExtractor(NPOIFSFileSystem fs)
+    throws IOException {
+        String pass = Biff8EncryptionKey.getCurrentUserPassword();
+        if (pass == null) {
+            pass = Decryptor.DEFAULT_PASSWORD;
+        }
+        
+        EncryptionInfo ei = new EncryptionInfo(fs);
+        Decryptor dec = ei.getDecryptor();
+        InputStream is = null;
+        try {
+            if (!dec.verifyPassword(pass)) {
+                throw new EncryptedDocumentException("Invalid password specified - use Biff8EncryptionKey.setCurrentUserPassword() before calling extractor");
+            }
+            is = dec.getDataStream(fs);
+            return createExtractor(OPCPackage.open(is));
+        } catch (IOException e) {
+            throw e;
+        } catch (Exception e) {
+            throw new EncryptedDocumentException(e);
+        } finally {
+            IOUtils.closeQuietly(is);
+        }
+    }
 }

src/ooxml/java/org/apache/poi/ss/usermodel/WorkbookFactory.java

@@ -92,13 +92,17 @@ public class WorkbookFactory {
 
         // If we get here, it isn't an encrypted XLSX file
         // So, treat it as a regular HSSF XLS one
+        boolean passwordSet = false;
         if (password != null) {
             Biff8EncryptionKey.setCurrentUserPassword(password);
+            passwordSet = true;
         }
         try {
             return new HSSFWorkbook(root, true);
         } finally {
-            Biff8EncryptionKey.setCurrentUserPassword(null);
+            if (passwordSet) {
+                Biff8EncryptionKey.setCurrentUserPassword(null);
+            }
         }
     }