From: Dominik Stadler Date: Tue, 29 Mar 2016 17:30:55 +0000 (+0000) Subject: Bug 58915: Add note about memory consumption and update changelog X-Git-Tag: REL_3_15_BETA2~391 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=349b14f8379ee7e81cebe77cb3a9ad932759b473;p=poi.git Bug 58915: Add note about memory consumption and update changelog git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1737025 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/.classpath b/.classpath index 5a7d21738b..ac2f3e8070 100644 --- a/.classpath +++ b/.classpath @@ -18,9 +18,9 @@ - - - + + + @@ -31,5 +31,5 @@ - + diff --git a/OOXMLLite.launch b/OOXMLLite.launch new file mode 100644 index 0000000000..e85d008217 --- /dev/null +++ b/OOXMLLite.launch @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/OOXMLPrettyPrint.launch b/OOXMLPrettyPrint.launch new file mode 100644 index 0000000000..d5420dc18f --- /dev/null +++ b/OOXMLPrettyPrint.launch @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/rerun.sh b/rerun.sh new file mode 100644 index 0000000000..980b91e1d8 --- /dev/null +++ b/rerun.sh @@ -0,0 +1 @@ +ant clean && ant compile-all jar test-all diff --git a/run.sh b/run.sh new file mode 100644 index 0000000000..14befa1cfc --- /dev/null +++ b/run.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# integration-tests fail: https://bitbucket.org/fakraemer/line-break-measurer-test/src/dc8e619063a4?at=master +#JAVA_HOME=/devtools/jdk1.6.0_45 + +#JAVA_HOME=/devtools/jdk1.7.0_76_x64 +JAVA_HOME=/devtools/jdk1.8.0_66_x64 +export JAVA_HOME + +ANT_HOME=/devtools/apache-ant-1.9.4 +export ANT_HOME + +PATH=$JAVA_HOME/bin:$ANT_HOME/bin:$PATH +export PATH + +FORREST_HOME=/devtools/apache-forrest-0.5.1-bin +export FORREST_HOME + +ANT_OPTS="-Xmx2048m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=D:/workspaces" +export ANT_OPTS + +rm -f build/*-testokfile.txt + +# -Dadditionaljar=lib/xercesImpl-2.6.1.jar +# -Dmain.docs.notRequired=true +nice -n 19 /devtools/apache-ant-1.9.6/bin/ant -Dcoverage.enabled=false $* diff --git a/sonar/run.sh b/sonar/run.sh new file mode 100644 index 0000000000..e8b9631029 --- /dev/null +++ b/sonar/run.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# -s /x1/jenkins/jenkins-master/catalina-base/temp/settings7837234259678677409.xml +mvn -B -f /workspaces/poi/sonar/pom.xml -Dmaven.repo.local=/workspaces/poi/sonar/repo clean install -U -DskipTests -fae -B && \ +mvn -f /workspaces/poi/sonar/pom.xml -e -B org.codehaus.mojo:sonar-maven-plugin:2.6:sonar -Dsonar.jdbc.driver=com.mysql.jdbc.Driver \ + "-Dsonar.jdbc.url=jdbc:mysql://192.168.0.64:3306/sonar?useUnicode=true&characterEncoding=utf8" \ + -Dsonar.host.url=http://localhost:9090 -Dmaven.repo.local=/workspaces/poi/sonar/repo diff --git a/src/integrationtest/integrationtest.iml b/src/integrationtest/integrationtest.iml new file mode 100644 index 0000000000..0566aa3491 --- /dev/null +++ b/src/integrationtest/integrationtest.iml @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/integrationtest/org/apache/poi/stress/HPSFFileHandler.java b/src/integrationtest/org/apache/poi/stress/HPSFFileHandler.java index 6a84201b45..69039e3c0e 100644 --- a/src/integrationtest/org/apache/poi/stress/HPSFFileHandler.java +++ b/src/integrationtest/org/apache/poi/stress/HPSFFileHandler.java @@ -39,17 +39,15 @@ public class HPSFFileHandler extends POIFSFileHandler { // a test-case to test this locally without executing the full TestAllFiles @Test public void test() throws Exception { - InputStream stream = new FileInputStream("test-data/hpsf/Test0313rur.adm"); + File file = new File("test-data/document/52372.doc"); + + InputStream stream = new FileInputStream(file); try { handleFile(stream); } finally { stream.close(); } - } - // a test-case to test this locally without executing the full TestAllFiles - @Test - public void testExtractor() throws Exception { - handleExtracting(new File("test-data/hpsf/TestBug44375.xls")); - } + handleExtracting(file); + } } diff --git a/src/integrationtest/run.sh b/src/integrationtest/run.sh new file mode 100644 index 0000000000..2676981f8f --- /dev/null +++ b/src/integrationtest/run.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# integration-tests fail: https://bitbucket.org/fakraemer/line-break-measurer-test/src/dc8e619063a4?at=master +#JAVA_HOME=/devtools/jdk1.6.0_45 + +#JAVA_HOME=/devtools/jdk1.7.0_76_x64 +JAVA_HOME=/devtools/jdk1.8.0_66_x64 +export JAVA_HOME + +ANT_HOME=/devtools/apache-ant-1.9.4 +export ANT_HOME + +PATH=$JAVA_HOME/bin:$ANT_HOME/bin:$PATH +export PATH + +FORREST_HOME=/devtools/apache-forrest-0.5.1-bin +export FORREST_HOME + +ANT_OPTS="-Xmx1524m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=D:/workspaces" +export ANT_OPTS + +#rm -f build/*-testokfile.txt + +# -Dadditionaljar=lib/xercesImpl-2.6.1.jar +# -Dmain.docs.notRequired=true +nice -n 19 /devtools/apache-ant-1.9.6/bin/ant -Dcoverage.enabled=false $* diff --git a/src/java/java.iml b/src/java/java.iml new file mode 100644 index 0000000000..cf5c2eee31 --- /dev/null +++ b/src/java/java.iml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/java/org/apache/poi/poifs/dev/POIFSDump.java b/src/java/org/apache/poi/poifs/dev/POIFSDump.java index f8c00365ac..1afb7bfefd 100644 --- a/src/java/org/apache/poi/poifs/dev/POIFSDump.java +++ b/src/java/org/apache/poi/poifs/dev/POIFSDump.java @@ -67,7 +67,7 @@ public class POIFSDump { is.close(); DirectoryEntry root = fs.getRoot(); - File file = new File(root.getName()); + File file = new File(new File(args[i]).getName() + "-" + root.getName()); file.mkdir(); dump(root, file); diff --git a/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFPivotCacheDefinition.java b/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFPivotCacheDefinition.java index c611e13e38..f5c65e92ab 100644 --- a/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFPivotCacheDefinition.java +++ b/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFPivotCacheDefinition.java @@ -111,6 +111,17 @@ public class XSSFPivotCacheDefinition extends POIXMLDocumentPart{ //Sets the pivotCacheDefinition tag xmlOptions.setSaveSyntheticDocumentElement(new QName(CTPivotCacheDefinition.type.getName(). getNamespaceURI(), "pivotCacheDefinition")); + + // ensure the fields have names + if(ctPivotCacheDefinition.getCacheFields() != null) { + CTCacheFields cFields = ctPivotCacheDefinition.getCacheFields(); + for(CTCacheField cf : cFields.getCacheFieldArray()) { + if(cf.getName() == null || cf.getName().isEmpty()) { + cf.setName("A"); + } + } + } + ctPivotCacheDefinition.save(out, xmlOptions); out.close(); } @@ -144,7 +155,8 @@ public class XSSFPivotCacheDefinition extends POIXMLDocumentPart{ cf.setNumFmtId(0); Cell cell = row.getCell(i); cell.setCellType(Cell.CELL_TYPE_STRING); - cf.setName(row.getCell(i).getStringCellValue()); + String stringCellValue = cell.getStringCellValue(); + cf.setName(stringCellValue); cf.addNewSharedItems(); } } diff --git a/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFSheet.java b/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFSheet.java index f0103719c1..d1edf6d0b5 100644 --- a/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFSheet.java +++ b/src/ooxml/java/org/apache/poi/xssf/usermodel/XSSFSheet.java @@ -2902,15 +2902,16 @@ public class XSSFSheet extends POIXMLDocumentPart implements Sheet { // we need to sort it in a way so the shifting does not mess up the structures, // i.e. when shifting down, start from down and go up, when shifting up, vice-versa SortedMap commentsToShift = new TreeMap(new Comparator() { - public int compare(XSSFComment o1, XSSFComment o2) { - int row1 = o1.getRow(); - int row2 = o2.getRow(); - - if(row1 == row2) { - // ordering is not important when row is equal, but don't return zero to still - // get multiple comments per row into the map - return o1.hashCode() - o2.hashCode(); - } + @Override + public int compare(XSSFComment o1, XSSFComment o2) { + int row1 = o1.getRow(); + int row2 = o2.getRow(); + + if(row1 == row2) { + // ordering is not important when row is equal, but don't return zero to still + // get multiple comments per row into the map + return o1.hashCode() - o2.hashCode(); + } // when shifting down, sort higher row-values first if(n > 0) { @@ -2986,6 +2987,78 @@ public class XSSFSheet extends POIXMLDocumentPart implements Sheet { map.put(r.getRowNum(), r); } _rows = map; + + // bug 57423: also re-order sheetdatamap + CTSheetData sheetData = worksheet.getSheetData(); + SortedMap ctmap = new TreeMap(); + for(CTRow row : sheetData.getRowArray()) { + System.out.println("Having row " + row.getR()); + ctmap.put(row.getR(), row); + } + /*Node domNode = sheetData.getDomNode(); + NodeList childNodes = domNode.getChildNodes(); + System.out.println("Had: " + childNodes.getLength() + ": " + childNodes); + for(int i = 0;i < childNodes.getLength();i++) { + Node item = childNodes.item(i); + System.out.println("Had: " + i + ": " + item); + }*/ + + //CTRow[] ctArray = new CTRow[ctmap.size()]; + /*int rowNum = 0; + int arrayIdx = 0; + for(CTRow row : ctmap.values()) { + if(row.getR() != (rowNum+1)) { + // XMLBeans releases any previous object in the setters, so we need to + // use new objects all the time + CTRow newRow = (CTRow) row.copy(); + + // we have to copy the Cells as well here as they are not included by + // the copy() and would be invalidated otherwise! + + XSSFRow xRow = _rows.get((int)row.getR()-1); + int cellCount = 0; + for (int cellNum = xRow.getFirstCellNum();cellNum < xRow.getLastCellNum();cellNum++) { + XSSFCell xssfCell = xRow.getCell(cellNum); + if(xssfCell == null) { + continue; + } + xssfCell.setCTCell(newRow.getCArray(cellCount)); + + cellCount++; + } + + CTCell[] cArray = new CTCell[cellCount]; + int i = 0; + for (int cellNum = xRow.getFirstCellNum();cellNum < xRow.getLastCellNum();cellNum++) { + XSSFCell xssfCell = xRow.getCell(cellNum); + if(xssfCell != null) { + cArray[i] = (CTCell) xssfCell.getCTCell().copy(); + i++; + + // we have to copy and re-create the XSSFCell here because the + // elements as otherwise setCArray below invalidates all the columns! + // see Bug 56170, XMLBeans seems to always release previous objects + // in the CArray, so we need to provide completely new ones here! + //_cells.put(entry.getKey(), new XSSFCell(this, cArray[i])); + xssfCell.setCTCell(cArray[cellNum]); + } + cellNum++; + } + + newRow.setCArray(cArray); + + sheetData.setRowArray(rowNum, row); + } + rowNum++; + }*/ + //sheetData.setRowArray(ctArray); + + /*CTRow row13 = (CTRow) sheetData.getRowArray(10).copy(); + CTRow row12 = (CTRow) sheetData.getRowArray(11).copy(); + sheetData.setRowArray(10, row12); + sheetData.setRowArray(11, row13);*/ + + //sheetData.getRowList() } private int shiftedRowNum(int startRow, int endRow, int n, int rownum) { @@ -3498,6 +3571,7 @@ public class XSSFSheet extends POIXMLDocumentPart implements Sheet { safeGetProtectionField().setSheet(false); } + /** * Enable or disable Autofilters locking. * This does not modify sheet protection status. @@ -3725,6 +3799,8 @@ public class XSSFSheet extends POIXMLDocumentPart implements Sheet { return dataValidationHelper; } + @Override + @SuppressWarnings("deprecation") //YK: getXYZArray() array accessors are deprecated in xmlbeans with JDK 1.5 support public List getDataValidations() { List xssfValidations = new ArrayList(); CTDataValidations dataValidations = this.worksheet.getDataValidations(); @@ -4100,6 +4176,7 @@ public class XSSFSheet extends POIXMLDocumentPart implements Sheet { return tables; } + @Override public int getColumnOutlineLevel(int columnIndex) { CTCol col = columnHelper.getColumn(columnIndex, false); if (col == null) { diff --git a/src/ooxml/testcases/org/apache/poi/xssf/usermodel/TestXSSFPivotTable.java b/src/ooxml/testcases/org/apache/poi/xssf/usermodel/TestXSSFPivotTable.java index 861984fd57..60d334667e 100644 --- a/src/ooxml/testcases/org/apache/poi/xssf/usermodel/TestXSSFPivotTable.java +++ b/src/ooxml/testcases/org/apache/poi/xssf/usermodel/TestXSSFPivotTable.java @@ -24,11 +24,12 @@ import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.AreaReference; import org.apache.poi.ss.util.CellReference; -import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPageField; -import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPageFields; -import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotFields; -import org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotTableDefinition; -import org.openxmlformats.schemas.spreadsheetml.x2006.main.STDataConsolidateFunction; +import org.apache.poi.xssf.XSSFTestDataSamples; +import org.openxmlformats.schemas.spreadsheetml.x2006.main.*; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.util.List; public class TestXSSFPivotTable extends TestCase { private XSSFPivotTable pivotTable; @@ -268,4 +269,46 @@ public class TestXSSFPivotTable extends TestCase { } fail(); } + + public void test58294() throws Exception { + XSSFWorkbook wb = new XSSFWorkbook ("C:\\temp\\test1.xlsx"); + XSSFSheet sheet = wb.getSheetAt(1); + XSSFSheet sheet0 = wb.getSheetAt(0); + sheet0.setActiveCell("A4"); + XSSFPivotTable pivotTable = sheet0.createPivotTable(new AreaReference("A3:H6"), new CellReference("A4"), sheet); + pivotTable.addRowLabel(1); + pivotTable.addRowLabel(3); + pivotTable.addRowLabel(5); + pivotTable.addColumnLabel(DataConsolidateFunction.SUM, 6, "Sum of days with hauls"); + pivotTable.addColumnLabel(DataConsolidateFunction.SUM, 7, "Sum of days site cutoff"); + //checkPivotTables(wb); + FileOutputStream fileOut = new FileOutputStream("c:\\temp\\test2new.xlsx"); + try { + wb.write(fileOut); + } finally { + fileOut.close(); + } + + XSSFWorkbook wbBack = XSSFTestDataSamples.writeOutAndReadBack(wb); + //checkPivotTables(wbBack); + + wb.close(); + } + + private void checkPivotTables(XSSFWorkbook wb) { + final List pivotTables = wb.getSheetAt(0).getPivotTables(); + assertNotNull(pivotTables); + assertEquals(3, pivotTables.size()); + final XSSFPivotTable pivotTable = pivotTables.get(2); + checkPivotTable(pivotTable); + } + + private void checkPivotTable(XSSFPivotTable pivotTableBack) { + assertNotNull(pivotTableBack.getPivotCacheDefinition()); + assertNotNull(pivotTableBack.getPivotCacheDefinition().getCTPivotCacheDefinition()); + final CTCacheFields cacheFields = pivotTableBack.getPivotCacheDefinition().getCTPivotCacheDefinition().getCacheFields(); + assertNotNull(cacheFields); + assertEquals(8, cacheFields.sizeOfCacheFieldArray()); + assertEquals("A", cacheFields.getCacheFieldList().get(0).getName()); + } } diff --git a/src/scratchpad/testcases/org/apache/poi/hwpf/converter/TestWordToConverterSuite.java b/src/scratchpad/testcases/org/apache/poi/hwpf/converter/TestWordToConverterSuite.java index 77a75a1937..6205e50b79 100644 --- a/src/scratchpad/testcases/org/apache/poi/hwpf/converter/TestWordToConverterSuite.java +++ b/src/scratchpad/testcases/org/apache/poi/hwpf/converter/TestWordToConverterSuite.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.FilenameFilter; import java.io.StringWriter; import java.util.Arrays; +import java.util.Collections; import java.util.List; import javax.xml.transform.OutputKeys; @@ -41,8 +42,7 @@ public class TestWordToConverterSuite /** * YK: a quick hack to exclude failing documents from the suite. */ - private static List failingFiles = Arrays - .asList( "ProblemExtracting.doc" ); + private static List failingFiles = Collections.singletonList("ProblemExtracting.doc"); public static Test suite() { diff --git a/src/testcases/org/apache/poi/hpsf/TestPropertySet.java b/src/testcases/org/apache/poi/hpsf/TestPropertySet.java new file mode 100644 index 0000000000..be8b16d4a6 --- /dev/null +++ b/src/testcases/org/apache/poi/hpsf/TestPropertySet.java @@ -0,0 +1,41 @@ +/* ==================================================================== + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +==================================================================== */ + +package org.apache.poi.hpsf; + +import org.apache.poi.POIDataSamples; +import org.apache.poi.poifs.filesystem.DocumentInputStream; +import org.apache.poi.poifs.filesystem.NPOIFSFileSystem; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertNotNull; + +public class TestPropertySet { + @Test + public void test52372() throws IOException, NoPropertySetStreamException, MarkUnsupportedException, UnexpectedPropertySetTypeException { + NPOIFSFileSystem poifs = new NPOIFSFileSystem(POIDataSamples.getDocumentInstance().getFile("52372.doc")); + + DocumentInputStream dis = poifs.createDocumentInputStream(SummaryInformation.DEFAULT_STREAM_NAME); + PropertySet ps = new PropertySet(dis); + SummaryInformation si = new SummaryInformation(ps); + dis.close(); + + assertNotNull(si); + } +} diff --git a/src/testcases/org/apache/poi/hssf/usermodel/TestCellStyle.java b/src/testcases/org/apache/poi/hssf/usermodel/TestCellStyle.java index 8d7fd012a6..151bed3c14 100644 --- a/src/testcases/org/apache/poi/hssf/usermodel/TestCellStyle.java +++ b/src/testcases/org/apache/poi/hssf/usermodel/TestCellStyle.java @@ -20,6 +20,7 @@ package org.apache.poi.hssf.usermodel; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.Calendar; import java.util.Date; @@ -544,4 +545,25 @@ public final class TestCellStyle extends TestCase { wb.close(); } + + + @Test + public void test58607() throws IOException { + HSSFWorkbook wb = new HSSFWorkbook(); + HSSFCellStyle style = wb.createCellStyle(); + + style.setDataFormat(wb.createDataFormat().getFormat("dd/MM/yyyy HH:mm:ss")); + + final Cell cell = wb.createSheet("test").createRow(0).createCell(0); + cell.setCellType(Cell.CELL_TYPE_NUMERIC); + cell.setCellStyle(style); + cell.setCellValue(new Date()); + + OutputStream out = new FileOutputStream("C:\\temp\\58607.xls"); + try { + wb.write(out); + } finally { + out.close(); + } + } } diff --git a/test-data/document/52372.doc b/test-data/document/52372.doc new file mode 100644 index 0000000000..ddaaf5ee9e Binary files /dev/null and b/test-data/document/52372.doc differ diff --git a/view.sh b/view.sh new file mode 100644 index 0000000000..68c74ad828 --- /dev/null +++ b/view.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +java -cp build/eclipse org.apache.poi.hssf.dev.BiffViewer --escher "$@"