From 51ce81924d206b66974d66045b654f36814219a5 Mon Sep 17 00:00:00 2001 From: PJ Fanning Date: Mon, 4 Oct 2021 20:34:47 +0000 Subject: [PATCH] [bug-65613] experimental SXSSF writeAvoidingTempFiles git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1893896 13f79535-47bb-0310-9956-ffa450edef68 --- .../poi/xssf/streaming/SXSSFWorkbook.java | 44 ++++++++--- .../xssf/DeferredSXSSFITestDataProvider.java | 14 ++++ .../streaming/TestDeferredSXSSFWorkbook.java | 78 +++++++++++++++++++ 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java b/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java index b915105629..f0421af8bd 100644 --- a/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java +++ b/poi-ooxml/src/main/java/org/apache/poi/xssf/streaming/SXSSFWorkbook.java @@ -17,13 +17,7 @@ package org.apache.poi.xssf.streaming; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; +import java.io.*; import java.nio.charset.StandardCharsets; import java.util.Enumeration; import java.util.HashMap; @@ -35,13 +29,16 @@ import java.util.NoSuchElementException; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.zip.Zip64Mode; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.poi.openxml4j.opc.OPCPackage; import org.apache.poi.openxml4j.util.ZipArchiveThresholdInputStream; import org.apache.poi.openxml4j.util.ZipEntrySource; import org.apache.poi.openxml4j.util.ZipFileZipEntrySource; +import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource; import org.apache.poi.openxml4j.util.ZipSecureFile; import org.apache.poi.ss.SpreadsheetVersion; import org.apache.poi.ss.formula.EvaluationWorkbook; @@ -403,8 +400,8 @@ public class SXSSFWorkbook implements Workbook { while (en.hasMoreElements()) { ZipArchiveEntry ze = en.nextElement(); ZipArchiveEntry zeOut = new ZipArchiveEntry(ze.getName()); - zeOut.setSize(ze.getSize()); - zeOut.setTime(ze.getTime()); + if (ze.getSize() >= 0) zeOut.setSize(ze.getSize()); + if (ze.getTime() >= 0) zeOut.setTime(ze.getTime()); zos.putArchiveEntry(zeOut); try (final InputStream is = zipEntrySource.getInputStream(ze)) { if (is instanceof ZipArchiveThresholdInputStream) { @@ -968,11 +965,38 @@ public class SXSSFWorkbook implements Workbook { } finally { deleted = tmplFile.delete(); } - if(!deleted) { + if (!deleted) { throw new IOException("Could not delete temporary file after processing: " + tmplFile); } } + /** + * Write out this workbook to an OutputStream. This (experimental) method avoids the temp file that + * {@link #write} creates but will use more memory as a result. Other SXSSF code can create temp files, + * so using this does not guarantee that there will be no temp file usage. + * + * @param stream - the java OutputStream you wish to write to + * @exception IOException if anything can't be written. + */ + @Beta + public void writeAvoidingTempFiles(OutputStream stream) throws IOException { + flushSheets(); + + //Save the template + try (UnsynchronizedByteArrayOutputStream bos = new UnsynchronizedByteArrayOutputStream()) { + _wb.write(bos); + + //Substitute the template entries with the generated sheet data files + try ( + InputStream is = bos.toInputStream(); + ZipInputStreamZipEntrySource source = new ZipInputStreamZipEntrySource( + new ZipArchiveThresholdInputStream(new ZipArchiveInputStream(is))) + ) { + injectData(source, stream); + } + } + } + protected void flushSheets() throws IOException { for (SXSSFSheet sheet : _xFromSxHash.values()) { diff --git a/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java b/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java index c35db12765..c915390dc2 100644 --- a/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java +++ b/poi-ooxml/src/test/java/org/apache/poi/xssf/DeferredSXSSFITestDataProvider.java @@ -78,6 +78,20 @@ public final class DeferredSXSSFITestDataProvider implements ITestDataProvider { } } + /** + * Returns an XSSFWorkbook since SXSSFWorkbook is write-only + */ + public XSSFWorkbook inMemoryWriteOutAndReadBack(SXSSFWorkbook wb) { + try (UnsynchronizedByteArrayOutputStream baos = new UnsynchronizedByteArrayOutputStream()) { + wb.writeAvoidingTempFiles(baos); + try (InputStream is = baos.toInputStream()) { + return new XSSFWorkbook(is); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override public DeferredSXSSFWorkbook createWorkbook() { DeferredSXSSFWorkbook wb = new DeferredSXSSFWorkbook(); diff --git a/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java b/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java index c07b59ca13..2623cfe9cc 100644 --- a/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java +++ b/poi-ooxml/src/test/java/org/apache/poi/xssf/streaming/TestDeferredSXSSFWorkbook.java @@ -187,6 +187,84 @@ public final class TestDeferredSXSSFWorkbook extends BaseTestXWorkbook { } } + @Test + void inMemoryWrite() throws IOException { + try (XSSFWorkbook xssfWb1 = new XSSFWorkbook()) { + xssfWb1.createSheet("S1"); + Sheet sheet = xssfWb1.createSheet("S2"); + Row row = sheet.createRow(1); + Cell cell = row.createCell(1); + cell.setCellValue("value 2_1_1"); + + try (DeferredSXSSFWorkbook wb1 = new DeferredSXSSFWorkbook(xssfWb1); + XSSFWorkbook xssfWb2 = DeferredSXSSFITestDataProvider.instance.writeOutAndReadBack(wb1)) { + assertTrue(wb1.dispose()); + + try (DeferredSXSSFWorkbook wb2 = new DeferredSXSSFWorkbook(xssfWb2)) { + // Add a row to the existing empty sheet + DeferredSXSSFSheet ssheet1 = wb2.getStreamingSheetAt(0); + ssheet1.setRowGenerator((ssxSheet) -> { + Row row1_1 = ssxSheet.createRow(1); + Cell cell1_1_1 = row1_1.createCell(1); + cell1_1_1.setCellValue("value 1_1_1"); + }); + + // Add a row to the existing non-empty sheet + DeferredSXSSFSheet ssheet2 = wb2.getStreamingSheetAt(1); + ssheet2.setRowGenerator((ssxSheet) -> { + Row row2_2 = ssxSheet.createRow(2); + Cell cell2_2_1 = row2_2.createCell(1); + cell2_2_1.setCellValue("value 2_2_1"); + }); + // Add a sheet with one row + DeferredSXSSFSheet ssheet3 = wb2.createSheet("S3"); + ssheet3.setRowGenerator((ssxSheet) -> { + Row row3_1 = ssxSheet.createRow(1); + Cell cell3_1_1 = row3_1.createCell(1); + cell3_1_1.setCellValue("value 3_1_1"); + }); + + try (XSSFWorkbook xssfWb3 = DeferredSXSSFITestDataProvider.instance.inMemoryWriteOutAndReadBack(wb2)) { + + assertEquals(3, xssfWb3.getNumberOfSheets()); + // Verify sheet 1 + XSSFSheet sheet1 = xssfWb3.getSheetAt(0); + assertEquals("S1", sheet1.getSheetName()); + assertEquals(1, sheet1.getPhysicalNumberOfRows()); + XSSFRow row1_1 = sheet1.getRow(1); + assertNotNull(row1_1); + XSSFCell cell1_1_1 = row1_1.getCell(1); + assertNotNull(cell1_1_1); + assertEquals("value 1_1_1", cell1_1_1.getStringCellValue()); + // Verify sheet 2 + XSSFSheet sheet2 = xssfWb3.getSheetAt(1); + assertEquals("S2", sheet2.getSheetName()); + assertEquals(2, sheet2.getPhysicalNumberOfRows()); + Row row2_1 = sheet2.getRow(1); + assertNotNull(row2_1); + Cell cell2_1_1 = row2_1.getCell(1); + assertNotNull(cell2_1_1); + assertEquals("value 2_1_1", cell2_1_1.getStringCellValue()); + XSSFRow row2_2 = sheet2.getRow(2); + assertNotNull(row2_2); + XSSFCell cell2_2_1 = row2_2.getCell(1); + assertNotNull(cell2_2_1); + assertEquals("value 2_2_1", cell2_2_1.getStringCellValue()); + // Verify sheet 3 + XSSFSheet sheet3 = xssfWb3.getSheetAt(2); + assertEquals("S3", sheet3.getSheetName()); + assertEquals(1, sheet3.getPhysicalNumberOfRows()); + XSSFRow row3_1 = sheet3.getRow(1); + assertNotNull(row3_1); + XSSFCell cell3_1_1 = row3_1.getCell(1); + assertNotNull(cell3_1_1); + assertEquals("value 3_1_1", cell3_1_1.getStringCellValue()); + } + } + } + } + } + @Test void sheetdataWriter() throws IOException { try (DeferredSXSSFWorkbook wb = new DeferredSXSSFWorkbook()) { -- 2.39.5