Add a number of tests for the dev-tools. The tests iterate over all .xls

files that are found under test-data, any newly added file or
code-change which breaks one of the dev-tools will cause the tests to
fail in the future. Known broken files can be excluded.

Also fixes two points where some files could cause BiffViewer to fail.

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1535885 13f79535-47bb-0310-9956-ffa450edef68

src/java/org/apache/poi/hssf/dev/BiffDrawingToXml.java

@@ -19,18 +19,23 @@
 
 package org.apache.poi.hssf.dev;
 
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.poi.ddf.EscherRecord;
 import org.apache.poi.hssf.model.InternalWorkbook;
-import org.apache.poi.hssf.record.*;
+import org.apache.poi.hssf.record.DrawingGroupRecord;
+import org.apache.poi.hssf.record.EscherAggregate;
 import org.apache.poi.hssf.usermodel.HSSFPatriarch;
 import org.apache.poi.hssf.usermodel.HSSFWorkbook;
 import org.apache.poi.poifs.filesystem.POIFSFileSystem;
 
-import java.io.*;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.List;
-
 /**
  * Utility for representing drawings contained in a binary Excel file as a XML tree
  *
@@ -132,7 +137,7 @@ public class BiffDrawingToXml {
         outputStream.close();
     }
 
-    public static void writeToFile(FileOutputStream fos, InputStream xlsWorkbook, boolean excludeWorkbookRecords, String[] params) throws IOException {
+    public static void writeToFile(OutputStream fos, InputStream xlsWorkbook, boolean excludeWorkbookRecords, String[] params) throws IOException {
         POIFSFileSystem fs = new POIFSFileSystem(xlsWorkbook);
         HSSFWorkbook workbook = new HSSFWorkbook(fs);
         InternalWorkbook internalWorkbook = getInternalWorkbook(workbook);

src/java/org/apache/poi/hssf/dev/BiffViewer.java

@@ -17,23 +17,20 @@
 
 package org.apache.poi.hssf.dev;
 
-import java.io.DataInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintStream;
-import java.io.Writer;
+import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.poi.hssf.record.*;
 import org.apache.poi.hssf.record.RecordInputStream.LeftoverDataException;
 import org.apache.poi.hssf.record.chart.*;
-import org.apache.poi.hssf.record.pivottable.*;
+import org.apache.poi.hssf.record.pivottable.DataItemRecord;
+import org.apache.poi.hssf.record.pivottable.ExtendedPivotTableViewFieldsRecord;
+import org.apache.poi.hssf.record.pivottable.PageItemRecord;
+import org.apache.poi.hssf.record.pivottable.StreamIDRecord;
+import org.apache.poi.hssf.record.pivottable.ViewDefinitionRecord;
+import org.apache.poi.hssf.record.pivottable.ViewFieldsRecord;
+import org.apache.poi.hssf.record.pivottable.ViewSourceRecord;
 import org.apache.poi.poifs.filesystem.POIFSFileSystem;
 import org.apache.poi.util.HexDump;
 import org.apache.poi.util.LittleEndian;
@@ -413,9 +410,8 @@ public final class BiffViewer {
 				boolean dumpInterpretedRecords = cmdArgs.shouldDumpRecordInterpretations();
 				boolean dumpHex = cmdArgs.shouldDumpBiffHex();
 				boolean zeroAlignHexDump = dumpInterpretedRecords;  // TODO - fix non-zeroAlign
-				BiffRecordListener recListener = new BiffRecordListener(dumpHex ? new OutputStreamWriter(ps) : null, zeroAlignHexDump, cmdArgs.suppressHeader());
-				is = new BiffDumpingStream(is, recListener);
-				createRecords(is, ps, recListener, dumpInterpretedRecords);
+				runBiffViewer(ps, is, dumpInterpretedRecords, dumpHex, zeroAlignHexDump,
+						cmdArgs.suppressHeader());
 			}
 			ps.close();
 		} catch (Exception e) {
@@ -423,6 +419,14 @@ public final class BiffViewer {
 		}
 	}
 
+	protected static void runBiffViewer(PrintStream ps, InputStream is,
+			boolean dumpInterpretedRecords, boolean dumpHex, boolean zeroAlignHexDump,
+			boolean suppressHeader) {
+		BiffRecordListener recListener = new BiffRecordListener(dumpHex ? new OutputStreamWriter(ps) : null, zeroAlignHexDump, suppressHeader);
+		is = new BiffDumpingStream(is, recListener);
+		createRecords(is, ps, recListener, dumpInterpretedRecords);
+	}
+
 	private static final class BiffRecordListener implements IBiffRecordListener {
 		private final Writer _hexDumpWriter;
 		private final List<String> _headers;
@@ -497,6 +501,7 @@ public final class BiffViewer {
 			_currentPos = 0;
 		}
 
+		@Override
 		public int read() throws IOException {
 			if (_currentPos >= _currentSize) {
 				fillNextBuffer();
@@ -510,6 +515,7 @@ public final class BiffViewer {
 			formatBufferIfAtEndOfRec();
 			return result;
 		}
+		@Override
 		public int read(byte[] b, int off, int len) throws IOException {
 			if (_currentPos >= _currentSize) {
 				fillNextBuffer();
@@ -532,6 +538,7 @@ public final class BiffViewer {
 			return result;
 		}
 
+		@Override
 		public int available() throws IOException {
 			return _currentSize - _currentPos + _is.available();
 		}
@@ -561,6 +568,7 @@ public final class BiffViewer {
 			int globalOffset = _overallStreamPos-_currentSize;
 			_listener.processRecord(globalOffset, _recordCounter, sid, dataSize, _data);
 		}
+		@Override
 		public void close() throws IOException {
 			_is.close();
 		}

src/java/org/apache/poi/hssf/dev/FormulaViewer.java

@@ -24,12 +24,12 @@ import org.apache.poi.hssf.model.HSSFFormulaParser;
 import org.apache.poi.hssf.record.FormulaRecord;
 import org.apache.poi.hssf.record.Record;
 import org.apache.poi.hssf.record.RecordFactory;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.poifs.filesystem.POIFSFileSystem;
 import org.apache.poi.ss.formula.ptg.ExpPtg;
 import org.apache.poi.ss.formula.ptg.FuncPtg;
 import org.apache.poi.ss.formula.ptg.OperationPtg;
 import org.apache.poi.ss.formula.ptg.Ptg;
-import org.apache.poi.hssf.usermodel.HSSFWorkbook;
-import org.apache.poi.poifs.filesystem.POIFSFileSystem;
 
 /**
  * FormulaViewer - finds formulas in a BIFF8 file and attempts to read them/display
@@ -68,7 +68,7 @@ public class FormulaViewer
 
         for (int k = 0; k < records.size(); k++)
         {
-            Record record = ( Record ) records.get(k);
+            Record record = records.get(k);
 
             if (record.getSid() == FormulaRecord.sid)
             {

src/java/org/apache/poi/hssf/record/ObjRecord.java

@@ -137,18 +137,22 @@ public final class ObjRecord extends Record {
 		return true;
 	}
 
+	@Override
 	public String toString() {
 		StringBuffer sb = new StringBuffer();
 
 		sb.append("[OBJ]\n");
-		for (int i = 0; i < subrecords.size(); i++) {
-			SubRecord record = subrecords.get(i);
-			sb.append("SUBRECORD: ").append(record.toString());
+		if(subrecords != null) {	// there are special cases where this can be, see comments in constructor above
+    		for (int i = 0; i < subrecords.size(); i++) {
+    			SubRecord record = subrecords.get(i);
+    			sb.append("SUBRECORD: ").append(record.toString());
+    		}
 		}
 		sb.append("[/OBJ]\n");
 		return sb.toString();
 	}
 	
+	@Override
 	public int getRecordSize() {
 		if (_uninterpretedData != null) {
 			return _uninterpretedData.length + 4;
@@ -170,6 +174,7 @@ public final class ObjRecord extends Record {
 		return size + 4;
 	}
 
+	@Override
 	public int serialize(int offset, byte[] data) {
 		int recSize = getRecordSize();
 		int dataSize = recSize - 4;
@@ -195,6 +200,7 @@ public final class ObjRecord extends Record {
 		return recSize;
 	}
 
+	@Override
 	public short getSid() {
 		return sid;
 	}
@@ -215,6 +221,7 @@ public final class ObjRecord extends Record {
 		return subrecords.add(o);
 	}
 
+	@Override
 	public Object clone() {
 		ObjRecord rec = new ObjRecord();
 

src/java/org/apache/poi/hssf/record/chart/CatLabRecord.java

@@ -83,7 +83,8 @@ public final class CatLabRecord extends StandardRecord {
 		buffer.append("    .wOffset =").append(HexDump.shortToHex(wOffset)).append('\n');
 		buffer.append("    .at      =").append(HexDump.shortToHex(at)).append('\n');
 		buffer.append("    .grbit   =").append(HexDump.shortToHex(grbit)).append('\n');
-		buffer.append("    .unused  =").append(HexDump.shortToHex(unused)).append('\n');
+		if(unused != null)
+			buffer.append("    .unused  =").append(HexDump.shortToHex(unused)).append('\n');
 
 		buffer.append("[/CATLAB]\n");
 		return buffer.toString();

src/testcases/org/apache/poi/hssf/dev/BaseXLSIteratingTest.java

@@ -0,0 +1,86 @@
+package org.apache.poi.hssf.dev;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+/**
+ * Base class for integration-style tests which iterate over all test-files
+ * and execute the same action to find out if any change breaks these applications.
+ */
+public abstract class BaseXLSIteratingTest {
+    protected static final OutputStream NULL_OUTPUT_STREAM = new NullOutputStream();
+
+	protected static final List<String> EXCLUDED = new ArrayList<String>();
+	protected static final List<String> SILENT_EXCLUDED = new ArrayList<String>();
+
+	@Test
+	public void testMain() throws Exception {
+		int count = runWithDir("test-data/spreadsheet");
+		count += runWithDir("test-data/hpsf");
+		
+		System.out.println("Had " + count + " files");
+	}
+
+	private int runWithDir(String dir) {
+		List<String> failed = new ArrayList<String>();
+
+		String[] files = new File(dir).list(new FilenameFilter() {
+			public boolean accept(File arg0, String arg1) {
+				return arg1.toLowerCase().endsWith(".xls");
+			}
+		});
+		
+		runWithArrayOfFiles(files, dir, failed);
+
+		assertTrue("Expected to have no failed except the ones excluded, but had: " + failed, 
+				failed.isEmpty());
+		
+		return files.length;
+	}
+
+	private void runWithArrayOfFiles(String[] files, String dir, List<String> failed) {
+		for(String file : files) {
+			try {
+				runOneFile(dir, file, failed);
+			} catch (Exception e) {
+				System.out.println("Failed: " + file);
+				if(SILENT_EXCLUDED.contains(file)) {
+					continue;
+				}
+
+				e.printStackTrace();
+				if(!EXCLUDED.contains(file)) {
+					failed.add(file);
+				}
+			}
+		}
+	}
+
+	abstract void runOneFile(String dir, String file, List<String> failed) throws Exception;
+
+	/**
+	 * Implementation of an OutputStream which does nothing, used
+	 * to redirect stdout to avoid spamming the console with output
+	 */
+	private static class NullOutputStream extends OutputStream {
+	    @Override
+	    public void write(byte[] b, int off, int len) {
+	    }
+
+	    @Override
+	    public void write(int b) {
+	    }
+
+	    @Override
+	    public void write(byte[] b) throws IOException {
+	    }
+	}
+}

src/testcases/org/apache/poi/hssf/dev/TestBiffDrawingToXml.java

@@ -0,0 +1,45 @@
+package org.apache.poi.hssf.dev;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestBiffDrawingToXml extends BaseXLSIteratingTest {
+	static {
+		// TODO: is it ok to fail these? 
+		// Look at the output of the test for the detailed stacktrace of the failures...
+//		EXCLUDED.add("password.xls"); 
+//		EXCLUDED.add("XRefCalc.xls"); 
+//		EXCLUDED.add("43493.xls");
+//		EXCLUDED.add("51832.xls"); 
+	};
+	
+	@Override
+	@Ignore("Not yet done, nearly all files fail with various errors, remove this method when done to use the one from the abstract base class!...")
+	@Test
+	public void testMain() throws Exception {
+	}
+	
+	@Override
+	void runOneFile(String dir, String file, List<String> failed)
+			throws Exception {
+		PrintStream save = System.out;
+		try {
+			//System.setOut(new PrintStream(TestBiffViewer.NULL_OUTPUT_STREAM));
+			// use a NullOutputStream to not write the bytes anywhere for best runtime 
+		    InputStream wb = new FileInputStream(new File(dir, file));
+		    try {
+		    	BiffDrawingToXml.writeToFile(NULL_OUTPUT_STREAM, wb, false, new String[] {});
+		    } finally {
+		    	wb.close();
+		    }
+		} finally {
+			System.setOut(save);
+		}
+	}
+}

src/testcases/org/apache/poi/hssf/dev/TestBiffViewer.java

@@ -0,0 +1,45 @@
+package org.apache.poi.hssf.dev;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.util.List;
+
+import org.apache.poi.poifs.filesystem.POIFSFileSystem;
+
+public class TestBiffViewer extends BaseXLSIteratingTest {
+	static {
+		// TODO: is it ok to fail these? 
+		// Look at the output of the test for the detailed stacktrace of the failures...
+		EXCLUDED.add("WORKBOOK_in_capitals.xls"); 	
+		EXCLUDED.add("password.xls"); 
+		EXCLUDED.add("NoGutsRecords.xls"); 
+		EXCLUDED.add("BOOK_in_capitals.xls"); 
+		EXCLUDED.add("XRefCalc.xls"); 
+		EXCLUDED.add("50833.xls"); 		// probably a problem in BiffViewer
+		EXCLUDED.add("43493.xls");
+		EXCLUDED.add("51832.xls"); 
+		EXCLUDED.add("OddStyleRecord.xls");		
+
+		SILENT_EXCLUDED.add("46904.xls"); 
+	};
+
+	@Override
+	void runOneFile(String dir, String file, List<String> failed) throws IOException {
+		FileInputStream inStream = new FileInputStream(new File(dir, file));
+		try {
+			POIFSFileSystem fs = new POIFSFileSystem(inStream);
+			InputStream is = fs.createDocumentInputStream("Workbook");
+			try {
+				// use a NullOutputStream to not write the bytes anywhere for best runtime 
+				BiffViewer.runBiffViewer(new PrintStream(NULL_OUTPUT_STREAM), is, true, true, true, false);
+			} finally {
+				is.close();
+			}
+		} finally {
+			inStream.close();
+		}
+	}
+}

src/testcases/org/apache/poi/hssf/dev/TestEFBiffViewer.java

@@ -0,0 +1,30 @@
+package org.apache.poi.hssf.dev;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+
+public class TestEFBiffViewer extends BaseXLSIteratingTest {
+	static {
+		// TODO: is it ok to fail these? 
+		// Look at the output of the test for the detailed stacktrace of the failures...
+		EXCLUDED.add("password.xls"); 
+		EXCLUDED.add("XRefCalc.xls"); 
+		EXCLUDED.add("43493.xls");
+		EXCLUDED.add("51832.xls"); 
+	};
+	
+	@Override
+	void runOneFile(String dir, String file, List<String> failed) throws IOException {
+		PrintStream save = System.out;
+		try {
+			// redirect standard out during the test to avoid spamming the console with output
+			System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
+
+			EFBiffViewer.main(new String[] { new File(dir, file).getAbsolutePath() });
+		} finally {
+			System.setOut(save);
+		}
+	}
+}

src/testcases/org/apache/poi/hssf/dev/TestFormulaViewer.java

@@ -0,0 +1,42 @@
+package org.apache.poi.hssf.dev;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.util.List;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestFormulaViewer extends BaseXLSIteratingTest {
+	static {
+		// TODO: is it ok to fail these? 
+		// Look at the output of the test for the detailed stacktrace of the failures...
+//		EXCLUDED.add("WORKBOOK_in_capitals.xls"); 	
+//		EXCLUDED.add("NoGutsRecords.xls"); 
+//		EXCLUDED.add("BOOK_in_capitals.xls"); 
+//		EXCLUDED.add("46904.xls"); 
+//		EXCLUDED.add("OddStyleRecord.xls");		
+	};
+	
+	@Override
+	@Ignore("Not yet done, nearly all files fail with various errors, remove this method when done to use the one from the abstract base class!...")
+	@Test
+	public void testMain() throws Exception {
+	}
+
+	@Override
+	void runOneFile(String dir, String file, List<String> failed) throws Exception {
+		PrintStream save = System.out;
+		try {
+			// redirect standard out during the test to avoid spamming the console with output
+			System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
+
+            FormulaViewer viewer = new FormulaViewer();
+            viewer.setFile(new File(dir, file).getAbsolutePath());
+            viewer.setList(true);
+            viewer.run();
+		} finally {
+			System.setOut(save);
+		}
+	}
+}

src/testcases/org/apache/poi/hssf/dev/TestReSave.java

@@ -0,0 +1,50 @@
+package org.apache.poi.hssf.dev;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.util.List;
+
+public class TestReSave extends BaseXLSIteratingTest {
+	static {
+		// TODO: is it ok to fail these? 
+		// Look at the output of the test for the detailed stacktrace of the failures...
+		EXCLUDED.add("password.xls"); 
+		EXCLUDED.add("43493.xls");
+		EXCLUDED.add("51832.xls"); 
+		EXCLUDED.add("49219.xls");
+		EXCLUDED.add("49931.xls");
+
+		SILENT_EXCLUDED.add("46904.xls"); 
+	};
+	
+	@Override
+	void runOneFile(String dir, String file, List<String> failed) throws Exception {
+		// avoid running on files leftover from previous failed runs
+		if(file.endsWith("-saved.xls")) {
+			return;
+		}
+
+		PrintStream save = System.out;
+		try {
+			// redirect standard out during the test to avoid spamming the console with output
+			System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
+
+			try {
+				ReSave.main(new String[] { new File(dir, file).getAbsolutePath() });
+    			try {
+        			// had one case where the re-saved could not be re-saved!
+        			ReSave.main(new String[] { new File(dir, file.replace(".xls", "-saved.xls")).getAbsolutePath() });
+    			} finally {
+    				// clean up the re-re-saved file
+    				new File(dir, file.replace(".xls", "-saved.xls").replace(".xls", "-saved.xls")).delete();
+    			}
+			} finally {
+				// clean up the re-saved file
+				new File(dir, file.replace(".xls", "-saved.xls")).delete();
+			}
+
+		} finally {
+			System.setOut(save);
+		}
+	}
+}

src/testcases/org/apache/poi/hssf/dev/TestRecordLister.java

@@ -0,0 +1,34 @@
+package org.apache.poi.hssf.dev;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+
+public class TestRecordLister extends BaseXLSIteratingTest {
+	static {
+		// TODO: is it ok to fail these? 
+		// Look at the output of the test for the detailed stacktrace of the failures...
+		EXCLUDED.add("WORKBOOK_in_capitals.xls"); 	
+		EXCLUDED.add("NoGutsRecords.xls"); 
+		EXCLUDED.add("BOOK_in_capitals.xls"); 
+		EXCLUDED.add("OddStyleRecord.xls");		
+
+		SILENT_EXCLUDED.add("46904.xls"); 
+	};
+	
+	@Override
+	void runOneFile(String dir, String file, List<String> failed) throws IOException {
+		PrintStream save = System.out;
+		try {
+			// redirect standard out during the test to avoid spamming the console with output
+			System.setOut(new PrintStream(NULL_OUTPUT_STREAM));
+
+			RecordLister viewer = new RecordLister();
+            viewer.setFile(new File(dir, file).getAbsolutePath());
+            viewer.run();
+		} finally {
+			System.setOut(save);
+		}
+	}
+}