Use parallel build to speed up building and running tests

Enable parallel building and parallel test-execution
Provide a configuration to still build/test serially on CI
via CI_BUILD=TRUE gradle --no-parallel --max-workers=1 clean build
Adjust Jenkins DSL to set CI_BUILD to avoid problems in CI
Adjust TestAllFiles to not fail when parallel builds are enabled
Reduce memory settings to reduce requirements on build-environments
Add gradle plugin to list task-dependencies

Thanks to Andreas Reichel for the PR

Closes #275

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1894812 13f79535-47bb-0310-9956-ffa450edef68

diff --git a/build.gradle b/build.gradle
index 68a503520ebc79feaa188583317264e3d5b140e0..21b7b78cedd8592bb3a37dc93054b2c7804787cf 100644 (file)
--- a/build.gradle
+++ b/build.gradle
@@ -37,6 +37,7 @@ buildscript {
 
 plugins {
     id 'base'
+    id "com.dorongold.task-tree" version "2.1.0"
     id('org.nosphere.apache.rat') version '0.7.0'
     id 'distribution'
 }
@@ -51,6 +52,8 @@ if (project.hasProperty('enableSonar')) {
     apply plugin: 'org.sonarqube'
 }
 
+boolean isCIBuild = false;
+
 // For help converting an Ant build to a Gradle build, see
 // https://docs.gradle.org/current/userguide/ant.html
 
@@ -145,6 +148,7 @@ subprojects {
         options.encoding = 'UTF-8'
         options.compilerArgs << '-Xlint:unchecked'
         options.deprecation = true
+        options.incremental = true
 
         onlyIf {
             (name != "compileJava9" && name != "compileTest9") || JavaVersion.current() != JavaVersion.VERSION_1_8
@@ -258,7 +262,8 @@ subprojects {
 
         // set heap size for the test JVM(s)
         minHeapSize = "128m"
-        maxHeapSize = "1512m"
+        maxHeapSize = "1G"
+
 
         // Specifying the local via system properties did not work, so we set them this way
         jvmArgs << [
@@ -271,11 +276,38 @@ subprojects {
             '-Djavax.xml.stream.XMLInputFactory=com.sun.xml.internal.stream.XMLInputFactoryImpl',
             "-Dversion.id=${project.version}",
             '-ea',
-            '-Djunit.jupiter.execution.parallel.config.strategy=fixed',
-            '-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2'
             // -Xjit:verbose={compileStart|compileEnd},vlog=build/jit.log${no.jit.sherlock}   ... if ${isIBMVM}
         ]
 
+        // detect if running on Jenkins/CI
+        isCIBuild |= Boolean.valueOf(System.getenv("CI_BUILD"));
+
+        if (isCIBuild) {
+               jvmArgs += [
+                   // Strictly serial
+                   // '-Djunit.jupiter.execution.parallel.enabled=false',
+
+                   // OR parallel on 2 threads
+                               '-Djunit.jupiter.execution.parallel.config.strategy=fixed',
+                               '-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2'
+                       ]
+                       maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+               } else {
+                       jvmArgs += [
+                           '-Djunit.jupiter.execution.parallel.enabled=true',
+                           '-Djunit.jupiter.execution.parallel.config.strategy=dynamic',
+
+                // this setting breaks the test builds, do not use it!
+                               //'-Djunit.jupiter.execution.parallel.mode.default=concurrent'
+                       ]
+
+                        // Explicitly defining the maxParallelForks was always slower than not setting it
+                        // So we leave this to Gradle itself, which seems to be very smart
+                        // maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+                        // maxParallelForks = Math.max( Runtime.runtime.availableProcessors() - 1, 1 )
+
+        }
+
         // show standard out and standard error of the test JVM(s) on the console
         //testLogging.showStandardStreams = true
 

diff --git a/gradle.properties b/gradle.properties
index 4f4e5ef86a1f5b18a24b38cad0f77b25dc93319f..b7042fcd7788e57516d41b9ac3d39d86bc78bcb5 100644 (file)
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,3 +1,15 @@
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx4096m
\ No newline at end of file
+# Less than 2G definitely slows things down. -XX:+HeapDumpOnOutOfMemoryError
+org.gradle.jvmargs=-Xmx2G -XX:MaxPermSize=512m -Dfile.encoding=UTF-8
+
+# Activating will be much faster, but break the build of 'poi-ooxml-lite'
+# @todo: look into poi-ooxml-lite task generateModuleInfo and enforce running whatever is needed before
+org.gradle.caching=false
+
+# Modularise your project and enable parallel build
+org.gradle.parallel=true
+
+# Enable configure on demand.
+org.gradle.configureondemand=true
+

diff --git a/jenkins/create_jobs.groovy b/jenkins/create_jobs.groovy
index c8397fa1cb8278189c1ad622aef54e8eff21374c..761e1af34ae60095bb281cc89267c253f146d58b 100644 (file)
--- a/jenkins/create_jobs.groovy
+++ b/jenkins/create_jobs.groovy
@@ -262,6 +262,7 @@ poijobs.each { poijob ->
         label(slaves)
         environmentVariables {
             env('LANG', 'en_US.UTF-8')
+            env('CI_BUILD', 'TRUE')
             if(jdkKey == '1.10') {
                 // when using JDK 9/10 for running Ant, we need to provide more modules for the forbidden-api-checks task
                 // on JDK 11 and newer there is no such module any more, so do not add it here

diff --git a/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java b/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java
index 30338f0f8b90afd4917da1e0414d2556a92dd4f6..524ec0a2f21024278f5cc68c928badfb3ba953b1 100644 (file)
--- a/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java
+++ b/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java
@@ -36,6 +36,8 @@ import java.util.stream.Stream;
 import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
 import org.apache.tools.ant.DirectoryScanner;
 import org.junit.jupiter.api.function.Executable;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -66,7 +68,7 @@ import org.opentest4j.AssertionFailedError;
  *  that we do not remove expected sanity checks.
  */
 // also need to set JVM parameter: -Djunit.jupiter.execution.parallel.enabled=true
-//@Execution(ExecutionMode.CONCURRENT)
+@Execution(ExecutionMode.CONCURRENT)
 public class TestAllFiles {
     private static final String DEFAULT_TEST_DATA_PATH = "test-data";
     public static final File ROOT_DIR = new File(System.getProperty("POI.testdata.path", DEFAULT_TEST_DATA_PATH));
@@ -117,6 +119,11 @@ public class TestAllFiles {
 
         final List<Arguments> result = new ArrayList<>(100);
         for (String file : scanner.getIncludedFiles()) {
+                       // avoid running on files leftover from previous failed runs
+                       if(file.endsWith("-saved.xls") || file.endsWith("TestHPSFWritingFunctionality.doc")) {
+                               continue;
+                       }
+
             for (FileHandlerKnown handler : sm.getHandler(file)) {
                 ExcInfo info1 = sm.getExcInfo(file, testName, handler);
                 if (info1 == null || info1.isValid(testName, handler.name())) {

diff --git a/poi/build.gradle b/poi/build.gradle
index 21b4627d3ea613415feee2c703cc1e12fdef271d..fbf2ba4725cfcf3f829c02c3ba63a4d3069fbadd 100644 (file)
--- a/poi/build.gradle
+++ b/poi/build.gradle
@@ -183,6 +183,7 @@ javadocJar {
 }
 
 sourcesJar {
+    dependsOn generateVersionJava
     metaInf {
         from("$projectDir/../legal/LICENSE")
         from("$projectDir/../legal/NOTICE")

diff --git a/settings.gradle b/settings.gradle
index 7d799fd6065df3ceab53caf01a27324d0eeac028..2d4eef563fa129fd806161dee991771dd8c4a3a6 100644 (file)
--- a/settings.gradle
+++ b/settings.gradle
@@ -1,5 +1,4 @@
 rootProject.name = 'poi'
 
 include 'poi', 'poi-ooxml-full', 'poi-ooxml-lite-agent', 'poi-scratchpad',
-        'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration',
-        'poi-ooxml-lite'
\ No newline at end of file
+        'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration' , 'poi-ooxml-lite'
\ No newline at end of file