diff options
20 files changed, 194 insertions, 192 deletions
diff --git a/build.gradle b/build.gradle index 38f45d8684..8d40eb1152 100644 --- a/build.gradle +++ b/build.gradle @@ -120,6 +120,8 @@ subprojects { TEST9_SRC = 'src/test/java9' TEST9_OUT = "${buildDir}/classes/java9/test/" VERSIONS9 = 'META-INF/versions/9' + + NO_SCRATCHPAD = (findProperty("scratchpad.ignore") == "true") } tasks.withType(JavaCompile) { diff --git a/jenkins/create_jobs.groovy b/jenkins/create_jobs.groovy index 15ea158dc9..b8f490e272 100644 --- a/jenkins/create_jobs.groovy +++ b/jenkins/create_jobs.groovy @@ -63,7 +63,7 @@ def poijobs = [ ], [ name: 'POI-DSL-Gradle', trigger: triggerSundays, email: 'centic@apache.org', gradle: true ], - [ name: 'POI-DSL-no-scratchpad', trigger: triggerSundays, noScratchpad: true + [ name: 'POI-DSL-no-scratchpad', trigger: triggerSundays, noScratchpad: true, gradle: true ], [ name: 'POI-DSL-SonarQube', jdk: '1.11', trigger: 'H 7 * * *', maven: true, sonar: true, skipcigame: true, email: 'kiwiwings@apache.org', @@ -445,17 +445,9 @@ poijobs.each { poijob -> gradle { tasks('clean jenkins') useWrapper(true) - } - } else if (poijob.noScratchpad) { - ant { - targets(['clean', 'compile'] + (poijob.properties ?: [])) - prop('coverage.enabled', true) - antInstallation(antRT) - } - ant { - targets(['-Dscratchpad.ignore=true', 'jacocotask', 'test-all', 'testcoveragereport'] + (poijob.properties ?: [])) - prop('coverage.enabled', true) - antInstallation(antRT) + if (poijob.noScratchpad) { + switches('-Pscratchpad.ignore=true') + } } } else { ant { diff --git a/poi-examples/build.gradle b/poi-examples/build.gradle index fe8feea87f..4aff0b8a6b 100644 --- a/poi-examples/build.gradle +++ b/poi-examples/build.gradle @@ -28,20 +28,29 @@ sourceSets { dependencies { api project(':poi-ooxml') - api project(':poi-scratchpad') +// api project(':poi-scratchpad') implementation project(path: ':poi-ooxml', configuration: 'archives') implementation project(path: ':poi-ooxml-full', configuration: 'archives') - implementation project(path: ':poi-scratchpad', configuration: 'archives') + + if (NO_SCRATCHPAD) { + compileOnly project(path: ':poi-scratchpad', configuration: 'archives') + } else { + implementation project(path: ':poi-scratchpad', configuration: 'archives') + } implementation "org.apache.logging.log4j:log4j-core:${log4jVersion}" - testImplementation project(path: ':poi-ooxml', configuration: 'tests') + testImplementation(project(path: ':poi-ooxml', configuration: 'tests')) { + if (NO_SCRATCHPAD) { + exclude group: 'org.apache.poi', module: 'poi-scratchpad' + } + } testImplementation project(path: ':poi', configuration: 'tests') } final String MODULE_NAME = 'org.apache.poi.examples' final Pattern MODULE_REGEX = ~'\\.jar$' -final List MAIN_MODULE_PATH = sourceSets.main.runtimeClasspath.findAll{ it.path =~ MODULE_REGEX }.collect{ it.parent }.unique() +final List MODULE_COMPILE_PATH = sourceSets.main.compileClasspath.findAll{ it.path =~ MODULE_REGEX }.collect{ it.parent }.unique() task compileJava9(type: JavaCompile) { dependsOn 'compileJava', ':poi-ooxml:jar', ':poi-scratchpad:jar' @@ -53,7 +62,7 @@ task compileJava9(type: JavaCompile) { classpath = files() options.compilerArgs = [ '--patch-module', "${MODULE_NAME}=${sourceSets.main.output.classesDirs.asPath}", - '--module-path', files(MAIN_MODULE_PATH).asPath + '--module-path', files(MODULE_COMPILE_PATH).asPath ] } diff --git a/poi-examples/src/main/java9/module-info.java b/poi-examples/src/main/java9/module-info.java index 68b580e3ee..db16935d94 100644 --- a/poi-examples/src/main/java9/module-info.java +++ b/poi-examples/src/main/java9/module-info.java @@ -18,7 +18,7 @@ module org.apache.poi.examples { requires transitive org.apache.poi.ooxml; - requires transitive org.apache.poi.scratchpad; + requires static org.apache.poi.scratchpad; requires java.xml; exports org.apache.poi.examples.crypt; diff --git a/poi-excelant/build.gradle b/poi-excelant/build.gradle index ee7cd664c0..e22a80f1f2 100644 --- a/poi-excelant/build.gradle +++ b/poi-excelant/build.gradle @@ -38,14 +38,13 @@ dependencies { api 'org.apache.ant:ant:1.10.9' api project(':poi-ooxml') - api project(':poi-scratchpad') api project(path: ':poi-ooxml', configuration: 'archives') - api project(path: ':poi-scratchpad', configuration: 'archives') testImplementation project(path: ':poi-ooxml-lite-agent', configuration: 'archives') testImplementation project(path: ':poi', configuration: 'tests') - testImplementation project(path: ':poi-ooxml', configuration: 'tests') - testImplementation project(path: ':poi-scratchpad', configuration: 'tests') + testImplementation(project(path: ':poi-ooxml', configuration: 'tests')) { + exclude group: 'org.apache.poi', module: 'poi-scratchpad' + } testImplementation 'com.google.guava:guava:30.0-jre' testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.14.1' } diff --git a/poi-excelant/src/main/java9/module-info.java b/poi-excelant/src/main/java9/module-info.java index 43b64560a7..743b172ad1 100644 --- a/poi-excelant/src/main/java9/module-info.java +++ b/poi-excelant/src/main/java9/module-info.java @@ -19,7 +19,6 @@ module org.apache.poi.excelant { requires ant; requires org.apache.poi.ooxml; - requires org.apache.poi.scratchpad; exports org.apache.poi.ss.excelant; exports org.apache.poi.ss.excelant.util; diff --git a/poi-excelant/src/test/java9/module-info.java b/poi-excelant/src/test/java9/module-info.java index a26d9bd04d..97c40ae4b6 100644 --- a/poi-excelant/src/test/java9/module-info.java +++ b/poi-excelant/src/test/java9/module-info.java @@ -19,7 +19,6 @@ module org.apache.poi.excelant { requires ant; requires org.apache.poi.ooxml; - requires org.apache.poi.scratchpad; exports org.apache.poi.ss.excelant; exports org.apache.poi.ss.excelant.util; diff --git a/poi-integration/build.gradle b/poi-integration/build.gradle index 97a16100f0..28e961ed4a 100644 --- a/poi-integration/build.gradle +++ b/poi-integration/build.gradle @@ -44,13 +44,34 @@ dependencies { testImplementation 'org.apache.commons:commons-collections4:4.4' testImplementation 'com.google.guava:guava:30.0-jre' - testImplementation project(':poi-ooxml') - testImplementation project(':poi-scratchpad') - testImplementation project(':poi-examples') - testImplementation project(path:':poi-examples', configuration:'archives') + testImplementation(project(':poi-ooxml')) { + if (NO_SCRATCHPAD) { + exclude group: 'org.apache.poi', module: 'poi-scratchpad' + } + } + if (NO_SCRATCHPAD) { + testCompileOnly project(':poi-scratchpad') + testCompileOnly project(path:':poi-scratchpad', configuration:'tests') + } else { + testImplementation project(':poi-scratchpad') + testImplementation project(path:':poi-scratchpad', configuration:'tests') + } + testImplementation(project(':poi-examples')) { + if (NO_SCRATCHPAD) { + exclude group: 'org.apache.poi', module: 'poi-scratchpad' + } + } + testImplementation(project(path:':poi-examples', configuration:'archives')) { + if (NO_SCRATCHPAD) { + exclude group: 'org.apache.poi', module: 'poi-scratchpad' + } + } testImplementation project(path:':poi', configuration:'tests') - testImplementation project(path:':poi-ooxml', configuration:'tests') - testImplementation project(path:':poi-scratchpad', configuration:'tests') + testImplementation(project(path:':poi-ooxml', configuration:'tests')) { + if (NO_SCRATCHPAD) { + exclude group: 'org.apache.poi', module: 'poi-scratchpad' + } + } testImplementation project(path: ':poi-ooxml-lite-agent', configuration: 'archives') testImplementation 'org.apache.logging.log4j:log4j-slf4j-impl:2.14.1' } @@ -58,7 +79,8 @@ dependencies { final String MODULE_NAME = 'org.apache.poi.stress' final Pattern MODULE_NOT_REGEX = ~'((poi|poi-scratchpad|poi-ooxml)[/\\\\][^/\\\\]+$|batik-script)' final Pattern MODULE_REGEX = ~'\\.jar$' -final List TEST_MODULE_PATH = sourceSets.test.runtimeClasspath.findAll{ it.path =~ MODULE_REGEX && !(it.path =~ MODULE_NOT_REGEX) }.collect{ it.parent }.unique() +final List MODULE_COMPILE_PATH = sourceSets.test.compileClasspath.findAll{ it.path =~ MODULE_REGEX && !(it.path =~ MODULE_NOT_REGEX) }.collect{ it.parent }.unique() +final List MODULE_RUNTIME_PATH = sourceSets.test.runtimeClasspath.findAll{ it.path =~ MODULE_REGEX && !(it.path =~ MODULE_NOT_REGEX) }.collect{ it.parent }.unique() final String OOXML_LITE_AGENT = "../build/dist/maven/poi-ooxml-lite-agent/poi-ooxml-lite-agent-${project.version}.jar" final String OOXML_LITE_REPORT = '../build/ooxml-lite-report' @@ -79,7 +101,7 @@ task compileTest9(type: JavaCompile) { source = file(TEST9_SRC) options.compilerArgs = [ '--patch-module', "${MODULE_NAME}=${(sourceSets.main.output.classesDirs + sourceSets.test.output.classesDirs).asPath}", - '--module-path', files(TEST_MODULE_PATH).asPath + '--module-path', files(MODULE_COMPILE_PATH).asPath ] classpath = files() } @@ -141,9 +163,12 @@ test { if (JavaVersion.current() != JavaVersion.VERSION_1_8) { jvmArgs += [ '--add-modules', MODULE_NAME, - '--module-path', '../build/dist/maven/poi-integration-tests' + File.pathSeparator + files(TEST_MODULE_PATH).asPath, + '--module-path', '../build/dist/maven/poi-integration-tests' + File.pathSeparator + files(MODULE_RUNTIME_PATH).asPath, ] } + if (NO_SCRATCHPAD) { + systemProperty 'scratchpad.ignore', 'true' + } } } diff --git a/poi-integration/src/test/java/org/apache/poi/stress/FileHandlerFactory.java b/poi-integration/src/test/java/org/apache/poi/stress/FileHandlerFactory.java deleted file mode 100644 index 8be52b35a6..0000000000 --- a/poi-integration/src/test/java/org/apache/poi/stress/FileHandlerFactory.java +++ /dev/null @@ -1,120 +0,0 @@ -/* ==================================================================== - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -==================================================================== */ -package org.apache.poi.stress; - -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; - -public class FileHandlerFactory { - // map from patterns for mimetypes to the FileHandlers that should be able to - // work with that file - // use a Set<Pair> to have a defined order of applying the matches - private static final Map<Pattern, FileHandler> MIME_TYPES = new HashMap<>(); - static { - ////////////////// Word - - MIME_TYPES.put(Pattern.compile("application/vnd.ms-word.document.macroenabled.12"), new XWPFFileHandler()); - MIME_TYPES.put(Pattern.compile("application/vnd.ms-word.template.macroenabled.12"), new XWPFFileHandler()); - - // application/msword - MIME_TYPES.put(Pattern.compile(".*msword.*"), new HWPFFileHandler()); - // application/vnd.ms-word - MIME_TYPES.put(Pattern.compile(".*ms-word.*"), new HWPFFileHandler()); - - // application/vnd.openxmlformats-officedocument.wordprocessingml.document - MIME_TYPES.put(Pattern.compile(".*wordprocessingml.*"), new XWPFFileHandler()); - - ////////////////// Excel - MIME_TYPES.put(Pattern.compile("application/vnd.ms-excel.addin.macroEnabled.12"), new XSSFFileHandler()); - MIME_TYPES.put(Pattern.compile("application/vnd.ms-excel.sheet.binary.macroEnabled.12"), new XSSFFileHandler()); - - // application/msexcel - MIME_TYPES.put(Pattern.compile(".*msexcel.*"), new HSSFFileHandler()); - // application/vnd.ms-excel - MIME_TYPES.put(Pattern.compile(".*ms-excel.*"), new HSSFFileHandler()); - - // application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - MIME_TYPES.put(Pattern.compile(".*spreadsheetml.*"), new XSSFFileHandler()); - - ////////////////// Powerpoint - - // application/vnd.ms-powerpoint - MIME_TYPES.put(Pattern.compile("application/vnd.ms-powerpoint"), new HSLFFileHandler()); - // application/vnd.ms-officetheme - MIME_TYPES.put(Pattern.compile("application/vnd.ms-officetheme"), new HSLFFileHandler()); - - // application/vnd.openxmlformats-officedocument.presentationml.presentation - MIME_TYPES.put(Pattern.compile(".*presentationml.*"), new XSLFFileHandler()); - // application/vnd.ms-powerpoint.presentation.macroenabled.12 - MIME_TYPES.put(Pattern.compile("application/vnd.ms-powerpoint.presentation.macroenabled.12"), new XSLFFileHandler()); - // application/vnd.ms-powerpoint.slideshow.macroenabled.12 - MIME_TYPES.put(Pattern.compile("application/vnd.ms-powerpoint.slideshow.macroenabled.12"), new XSLFFileHandler()); - - ////////////////// Mail/TNEF - - // application/vnd.ms-tnef - MIME_TYPES.put(Pattern.compile(".*ms-tnef.*"), new HMEFFileHandler()); - - // application/vnd.ms-outlook - MIME_TYPES.put(Pattern.compile("application/vnd.ms-outlook"), new HSMFFileHandler()); - - ////////////////// Visio - - // application/vnd.visio - MIME_TYPES.put(Pattern.compile("application/vnd.visio.*"), new HDGFFileHandler()); - - // application/vnd.ms-visio.drawing - MIME_TYPES.put(Pattern.compile(".*vnd.ms-visio\\."), new XDGFFileHandler()); - - //application/vnd.ms-visio.viewer - MIME_TYPES.put(Pattern.compile(".*visio.*"), new HDGFFileHandler()); - - - ////////////////// Publisher - - // application/x-mspublisher - MIME_TYPES.put(Pattern.compile("application/x-mspublisher"), new HPBFFileHandler()); - - - ////////////////// Others - - // special type used by Tika - MIME_TYPES.put(Pattern.compile("application/x-tika-ooxml.*"), new OPCFileHandler()); - // special type used by Tika - MIME_TYPES.put(Pattern.compile("application/x-tika-msoffice.*"), new POIFSFileHandler()); - - // application/x-tika-old-excel - MIME_TYPES.put(Pattern.compile("application/x-tika-old-excel"), new POIFSFileHandler()); - - // application/vnd.openxmlformats-officedocument.drawingml.chart+xml - // ?!MIME_TYPES.put(Pattern.compile(".*drawingml.*"), ".dwg"); - - // application/vnd.openxmlformats-officedocument.vmlDrawing - // ?!MIME_TYPES.put(Pattern.compile(".*vmlDrawing.*"), ".dwg"); - } - - public static FileHandler getHandler(String mimeType) { - for(Map.Entry<Pattern,FileHandler> entry : MIME_TYPES.entrySet()) { - if(entry.getKey().matcher(mimeType).matches()) { - return entry.getValue(); - } - } - - return null; - } -} diff --git a/poi-integration/src/test/java/org/apache/poi/stress/FileHandlerKnown.java b/poi-integration/src/test/java/org/apache/poi/stress/FileHandlerKnown.java index aa3c827f1f..a438574610 100644 --- a/poi-integration/src/test/java/org/apache/poi/stress/FileHandlerKnown.java +++ b/poi-integration/src/test/java/org/apache/poi/stress/FileHandlerKnown.java @@ -22,29 +22,33 @@ import java.util.function.Supplier; @SuppressWarnings("unused") public enum FileHandlerKnown { - HDGF(HDGFFileHandler::new), - HMEF(HMEFFileHandler::new), - HPBF(HPBFFileHandler::new), - HPSF(HPSFFileHandler::new), - HSLF(HSLFFileHandler::new), - HSMF(HSMFFileHandler::new), - HSSF(HSSFFileHandler::new), - HWPF(HWPFFileHandler::new), - OPC(OPCFileHandler::new), - POIFS(POIFSFileHandler::new), - XDGF(XDGFFileHandler::new), - XSLF(XSLFFileHandler::new), - XSSFB(XSSFBFileHandler::new), - XSSF(XSSFFileHandler::new), - XWPF(XWPFFileHandler::new), - OWPF(OWPFFileHandler::new), - NULL(NullFileHandler::new) + HDGF, + HMEF, + HPBF, + HPSF, + HSLF, + HSMF, + HSSF, + HWPF, + OPC, + POIFS, + XDGF, + XSLF, + XSSFB, + XSSF, + XWPF, + OWPF, + NULL ; - public final Supplier<FileHandler> fileHandler; - - FileHandlerKnown(Supplier<FileHandler> fileHandler) { - this.fileHandler = fileHandler; + public FileHandler getHandler() { + try { + // Because of no-scratchpad handling, we need to resort to reflection here + String n = name().replace("NULL", "Null"); + return (FileHandler)Class.forName("org.apache.poi.stress." + n + "FileHandler").newInstance(); + } catch (Exception e) { + return new NullFileHandler(); + } } private static class NullFileHandler implements FileHandler { diff --git a/poi-integration/src/test/java/org/apache/poi/stress/HPSFFileHandler.java b/poi-integration/src/test/java/org/apache/poi/stress/HPSFFileHandler.java index 3caf166f31..274a3880e6 100644 --- a/poi-integration/src/test/java/org/apache/poi/stress/HPSFFileHandler.java +++ b/poi-integration/src/test/java/org/apache/poi/stress/HPSFFileHandler.java @@ -18,6 +18,8 @@ package org.apache.poi.stress; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeFalse; @@ -31,10 +33,12 @@ import java.util.Set; import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; import org.apache.poi.examples.hpsf.CopyCompare; +import org.apache.poi.extractor.POITextExtractor; import org.apache.poi.hpsf.DocumentSummaryInformation; import org.apache.poi.hpsf.HPSFPropertiesOnlyDocument; import org.apache.poi.hpsf.PropertySet; import org.apache.poi.hpsf.SummaryInformation; +import org.apache.poi.hpsf.extractor.HPSFPropertiesExtractor; import org.apache.poi.poifs.filesystem.DirectoryNode; import org.apache.poi.poifs.filesystem.DocumentInputStream; import org.apache.poi.poifs.filesystem.POIFSFileSystem; @@ -57,6 +61,35 @@ class HPSFFileHandler extends POIFSFileHandler { ); @Override + public void handleExtracting(File file) throws Exception { + if (!Boolean.getBoolean("scratchpad.ignore")) { + super.handleExtracting(file); + return; + } + + long length = file.length(); + long modified = file.lastModified(); + + try (POIFSFileSystem poifs = new POIFSFileSystem(file); + HPSFPropertiesExtractor extractor = new HPSFPropertiesExtractor(poifs)) { + + String fileAndParentName = file.getParentFile().getName() + "/" + file.getName(); + String relPath = file.getPath().replaceAll(".*test-data", "test-data").replace('\\', '/'); + + assertFalse(EXPECTED_EXTRACTOR_FAILURES.contains(fileAndParentName), + "Expected Extraction to fail for file " + relPath + " and handler " + this + ", but did not fail!"); + assertNotNull(extractor.getDocumentSummaryInformationText()); + assertNotNull(extractor.getSummaryInformationText()); + String text = extractor.getText(); + //System.out.println(text); + assertNotNull(text); + } + + assertEquals(length, file.length(), "File should not be modified by extractor"); + assertEquals(modified, file.lastModified(), "File should not be modified by extractor"); + } + + @Override public void handleFile(InputStream stream, String path) throws Exception { POIFSFileSystem poifs = new POIFSFileSystem(stream); HPSFPropertiesOnlyDocument hpsf = new HPSFPropertiesOnlyDocument(poifs); diff --git a/poi-integration/src/test/java/org/apache/poi/stress/StressMap.java b/poi-integration/src/test/java/org/apache/poi/stress/StressMap.java index f69fbfbf4f..cf533c5b4f 100644 --- a/poi-integration/src/test/java/org/apache/poi/stress/StressMap.java +++ b/poi-integration/src/test/java/org/apache/poi/stress/StressMap.java @@ -18,12 +18,14 @@ package org.apache.poi.stress; import java.io.File; import java.io.IOException; +import java.util.AbstractMap.SimpleEntry; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.BiConsumer; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -40,7 +42,8 @@ import org.apache.poi.ss.usermodel.WorkbookFactory; public class StressMap { private final MultiValuedMap<String, ExcInfo> exMap = new ArrayListValuedHashMap<>(); private final Map<String,String> handlerMap = new LinkedHashMap<>(); - + private final boolean SCRATCH_IGNORE = Boolean.getBoolean("scratchpad.ignore"); + private final Pattern SCRATCH_HANDLER = Pattern.compile("(HSLF|HWPF|HSMF|HMEF)"); public void load(File mapFile) throws IOException { try (Workbook wb = WorkbookFactory.create(mapFile)) { @@ -79,7 +82,6 @@ public class StressMap { handlerMap.clear(); - boolean IGNORE_SCRATCHPAD = Boolean.getBoolean("scratchpad.ignore"); boolean isFirst = true; for (Row row : sh) { if (isFirst) { @@ -87,7 +89,7 @@ public class StressMap { continue; } Cell cell = row.getCell(2); - if (IGNORE_SCRATCHPAD || cell == null || cell.getCellType() != CellType.STRING) { + if (SCRATCH_IGNORE || cell == null || cell.getCellType() != CellType.STRING) { cell = row.getCell(1); } handlerMap.put(row.getCell(0).getStringCellValue(), cell.getStringCellValue()); @@ -103,20 +105,38 @@ public class StressMap { exMap.clear(); Iterator<Row> iter = sh.iterator(); - List<BiConsumer<ExcInfo,String>> cols = initCols(iter.next()); + List<Map.Entry<String, BiConsumer<ExcInfo,String>>> cols = initCols(iter.next()); + + int idx = 0, handlerIdx = -1; + for (Map.Entry<String, BiConsumer<ExcInfo, String>> e : cols) { + if ("Handler".equals(e.getKey())) { + handlerIdx = idx; + } + idx++; + } while (iter.hasNext()) { + Row row = iter.next(); + + if (SCRATCH_IGNORE && handlerIdx > -1) { + String handler = row.getCell(handlerIdx).getStringCellValue(); + if (SCRATCH_HANDLER.matcher(handler).find()) { + // ignore exception of ignored files + continue; + } + } + ExcInfo info = new ExcInfo(); - for (Cell cell : iter.next()) { + for (Cell cell : row) { if (cell.getCellType() == CellType.STRING) { - cols.get(cell.getColumnIndex()).accept(info, cell.getStringCellValue()); + cols.get(cell.getColumnIndex()).getValue().accept(info, cell.getStringCellValue()); } } exMap.put(info.getFile(), info); } } - private static List<BiConsumer<ExcInfo,String>> initCols(Row row) { + private static List<Map.Entry<String, BiConsumer<ExcInfo,String>>> initCols(Row row) { Map<String,BiConsumer<ExcInfo,String>> m = new HashMap<>(); m.put("File", ExcInfo::setFile); m.put("Tests", ExcInfo::setTests); @@ -128,7 +148,7 @@ public class StressMap { return StreamSupport .stream(row.spliterator(), false) .map(Cell::getStringCellValue) - .map(v -> m.getOrDefault(v, (e,s) -> {})) + .map(v -> new SimpleEntry<>(v, m.getOrDefault(v, (e,s) -> {}))) .collect(Collectors.toList()); } diff --git a/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java b/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java index f35f3b27fb..9d803f8e06 100644 --- a/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java +++ b/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java @@ -66,7 +66,7 @@ import org.opentest4j.AssertionFailedError; * that we do not remove expected sanity checks. */ // also need to set JVM parameter: -Djunit.jupiter.execution.parallel.enabled=true -@Execution(ExecutionMode.CONCURRENT) +//@Execution(ExecutionMode.CONCURRENT) public class TestAllFiles { private static final String DEFAULT_TEST_DATA_PATH = "test-data"; public static final File ROOT_DIR = new File(System.getProperty("POI.testdata.path", DEFAULT_TEST_DATA_PATH)); @@ -78,6 +78,27 @@ public class TestAllFiles { "**/right-to-left.xlsx" //the threaded comments in this file cause XSSF clone to fail }; + // cheap workaround of skipping the few problematic files + public static final String[] SCAN_EXCLUDES_NOSCRATCHPAD = { + "**/.svn/**", + "lost+found", + "**/.git/**", + "**/right-to-left.xlsx", + "document/word2.doc", + "document/cpansearch.perl.org_src_tobyink_acme-rundoc-0.001_word-lib_hello_world.docm", + "hpsf/Test0313rur.adm", + "spreadsheet/43493.xls", + "spreadsheet/44958.xls", + "spreadsheet/44958_1.xls", + "spreadsheet/46904.xls", + "spreadsheet/51832.xls", + "spreadsheet/60284.xls", + "spreadsheet/testArraysAndTables.xls", + "spreadsheet/testEXCEL_3.xls", + "spreadsheet/testEXCEL_4.xls", + "poifs/unknown_properties.msg" + }; + private static final Set<String> EXPECTED_FAILURES = StressTestUtils.unmodifiableHashSet( "document/truncated62886.docx" ); @@ -86,10 +107,11 @@ public class TestAllFiles { StressMap sm = new StressMap(); sm.load(new File(ROOT_DIR, "spreadsheet/stress.xls")); + boolean noScratch = Boolean.getBoolean("scratchpad.ignore"); + DirectoryScanner scanner = new DirectoryScanner(); scanner.setBasedir(ROOT_DIR); - scanner.setExcludes(SCAN_EXCLUDES); - + scanner.setExcludes(noScratch ? SCAN_EXCLUDES_NOSCRATCHPAD : SCAN_EXCLUDES); scanner.scan(); final List<Arguments> result = new ArrayList<>(100); @@ -121,7 +143,7 @@ public class TestAllFiles { if (StressTestUtils.excludeFile(file, EXPECTED_FAILURES)) return; System.out.println("Running extractFiles on "+file); - FileHandler fileHandler = handler.fileHandler.get(); + FileHandler fileHandler = handler.getHandler(); assertNotNull(fileHandler, "Did not find a handler for file " + file); Executable exec = () -> fileHandler.handleExtracting(new File(ROOT_DIR, file)); verify(file, exec, exClass, exMessage, password); @@ -135,7 +157,7 @@ public class TestAllFiles { @MethodSource("handleFiles") void handleFile(String file, FileHandlerKnown handler, String password, Class<? extends Throwable> exClass, String exMessage) throws IOException { System.out.println("Running handleFiles on "+file); - FileHandler fileHandler = handler.fileHandler.get(); + FileHandler fileHandler = handler.getHandler(); assertNotNull(fileHandler, "Did not find a handler for file " + file); try (InputStream stream = new BufferedInputStream(new FileInputStream(new File(ROOT_DIR, file)), 64 * 1024)) { Executable exec = () -> fileHandler.handleFile(stream, file); @@ -151,7 +173,7 @@ public class TestAllFiles { @MethodSource("handleAdditionals") void handleAdditional(String file, FileHandlerKnown handler, String password, Class<? extends Throwable> exClass, String exMessage) { System.out.println("Running additionals on "+file); - FileHandler fileHandler = handler.fileHandler.get(); + FileHandler fileHandler = handler.getHandler(); assertNotNull(fileHandler, "Did not find a handler for file " + file); Executable exec = () -> fileHandler.handleAdditional(new File(ROOT_DIR, file)); verify(file, exec, exClass, exMessage, password); diff --git a/poi-integration/src/test/java9/module-info.java b/poi-integration/src/test/java9/module-info.java index 782edf74db..82eb9f6591 100644 --- a/poi-integration/src/test/java9/module-info.java +++ b/poi-integration/src/test/java9/module-info.java @@ -24,6 +24,7 @@ module org.apache.poi.stress { requires org.apache.commons.collections4; requires org.apache.poi.examples; + requires static org.apache.poi.scratchpad; exports org.apache.poi.stress; diff --git a/poi-ooxml/build.gradle b/poi-ooxml/build.gradle index 5d57783622..1179633f17 100644 --- a/poi-ooxml/build.gradle +++ b/poi-ooxml/build.gradle @@ -64,11 +64,12 @@ dependencies { api 'de.rototor.pdfbox:graphics2d:0.32' - - testImplementation project(':poi-scratchpad') + if (!NO_SCRATCHPAD) { + testImplementation project(':poi-scratchpad') + testImplementation project(path:':poi-scratchpad', configuration:'tests') + } testImplementation project(path:':poi', configuration:'tests') testImplementation project(path:':poi-ooxml-lite-agent', configuration: 'archives') - testImplementation project(path:':poi-scratchpad', configuration:'tests') testImplementation 'org.xmlunit:xmlunit-core:2.8.0' testImplementation 'org.reflections:reflections:0.9.12' testImplementation 'org.openjdk.jmh:jmh-core:1.26' @@ -219,6 +220,12 @@ test { systemProperties['junit.jupiter.execution.parallel.enabled'] = 'true' + if (NO_SCRATCHPAD) { + useJUnitPlatform { + excludeTags 'scratchpad.ignore' + } + } + doFirst { jvmArgs += [ "-javaagent:${OOXML_LITE_AGENT}=${OOXML_LITE_REPORT}|${OOXML_LITE_INCLUDES}", @@ -226,7 +233,7 @@ test { if (JavaVersion.current() != JavaVersion.VERSION_1_8) { jvmArgs += [ '--add-modules', MODULE_NAME, - '--module-path', '../build/dist/maven/poi-ooxml-tests' + java.io.File.pathSeparator + files(TEST_MODULE_PATH).asPath, + '--module-path', '../build/dist/maven/poi-ooxml-tests' + File.pathSeparator + files(TEST_MODULE_PATH).asPath, ] } } diff --git a/poi-ooxml/src/test/java/org/apache/poi/extractor/ooxml/TestExtractorFactory.java b/poi-ooxml/src/test/java/org/apache/poi/extractor/ooxml/TestExtractorFactory.java index 55e56107d6..a00e232e29 100644 --- a/poi-ooxml/src/test/java/org/apache/poi/extractor/ooxml/TestExtractorFactory.java +++ b/poi-ooxml/src/test/java/org/apache/poi/extractor/ooxml/TestExtractorFactory.java @@ -45,6 +45,7 @@ import org.apache.poi.poifs.filesystem.FileMagic; import org.apache.poi.poifs.filesystem.POIFSFileSystem; import org.apache.poi.xssf.extractor.XSSFEventBasedExcelExtractor; import org.apache.poi.xssf.extractor.XSSFExcelExtractor; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -54,6 +55,7 @@ import org.junit.jupiter.params.provider.ValueSource; /** * Test that the extractor factory plays nicely */ +@Tag("scratchpad.ignore") class TestExtractorFactory { private static final POIDataSamples ssTests = POIDataSamples.getSpreadSheetInstance(); diff --git a/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/dsig/TestSignatureInfo.java b/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/dsig/TestSignatureInfo.java index 3052903226..502a125613 100644 --- a/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/dsig/TestSignatureInfo.java +++ b/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/dsig/TestSignatureInfo.java @@ -160,6 +160,7 @@ import org.etsi.uri.x01903.v13.QualifyingPropertiesType; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -924,6 +925,7 @@ class TestSignatureInfo { @Test @DisabledOnJreEx("1.8.0_292") + @Tag("scratchpad.ignore") void testSignatureImage() throws Exception { initKeyPair(); diff --git a/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/tests/TestHxxFEncryption.java b/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/tests/TestHxxFEncryption.java index 6f4f27c486..d51ac67e3b 100644 --- a/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/tests/TestHxxFEncryption.java +++ b/poi-ooxml/src/test/java/org/apache/poi/poifs/crypt/tests/TestHxxFEncryption.java @@ -42,11 +42,13 @@ import org.apache.poi.poifs.crypt.EncryptionInfo; import org.apache.poi.poifs.crypt.EncryptionMode; import org.apache.poi.poifs.crypt.cryptoapi.CryptoAPIEncryptionHeader; import org.apache.poi.poifs.storage.RawDataUtil; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +@Tag("scratchpad.ignore") class TestHxxFEncryption { public static Stream<Arguments> data() throws IOException { final String base64 = diff --git a/poi-ooxml/src/test/java/org/apache/poi/ss/tests/usermodel/TestEmbedOLEPackage.java b/poi-ooxml/src/test/java/org/apache/poi/ss/tests/usermodel/TestEmbedOLEPackage.java index 4c085df879..183bab321f 100644 --- a/poi-ooxml/src/test/java/org/apache/poi/ss/tests/usermodel/TestEmbedOLEPackage.java +++ b/poi-ooxml/src/test/java/org/apache/poi/ss/tests/usermodel/TestEmbedOLEPackage.java @@ -63,8 +63,10 @@ import org.apache.poi.xssf.XSSFTestDataSamples; import org.apache.poi.xssf.usermodel.XSSFObjectData; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +@Tag("scratchpad.ignore") class TestEmbedOLEPackage { private static byte[] samplePPT, samplePPTX, samplePNG; diff --git a/poi-ooxml/src/test/java/org/apache/poi/xssf/usermodel/TestXSSFBugs.java b/poi-ooxml/src/test/java/org/apache/poi/xssf/usermodel/TestXSSFBugs.java index 25a16e0f69..cb736d599e 100644 --- a/poi-ooxml/src/test/java/org/apache/poi/xssf/usermodel/TestXSSFBugs.java +++ b/poi-ooxml/src/test/java/org/apache/poi/xssf/usermodel/TestXSSFBugs.java @@ -111,6 +111,7 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.usermodel.extensions.XSSFCellFill; import org.apache.xmlbeans.XmlException; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -3439,6 +3440,7 @@ public final class TestXSSFBugs extends BaseTestBugzillaIssues { @Test + @Tag("scratchpad.ignore") void testXLSXinPPT() throws Exception { assumeFalse(Boolean.getBoolean("scratchpad.ignore")); |