Output test results in XML reports instead of to standard output

This is a workaround for a problem with the integrated TeamCity JUnit
listener which is not thread safe. Without this the test results are
completely mixed up in TeamCity. For more information, see
http://youtrack.jetbrains.com/issue/TW-33176.

All standard JUnit test reports is put in result/reports. For uitest
there are three report directories: result/reports-tb2,
result/reports-tb3 and result/reports-integration to be able to clean
the directories before running a given type of tests.

This change is based on the separate fixes by John.

Change-Id: I2f0dee416f6745544636f2f3dcea0d0832ff8076

common.xml

@@ -305,16 +305,20 @@
         <property name="test.src" location="${result.dir}/../tests/src" />
         <property name="test.classes" location="${result.dir}/tests/classes" />
 
+        <!-- Ensure report dir is empty before running tests -->
+        <property name="report.dir" location="${result.dir}/reports" />
+        <delete dir="${report.dir}" />
+        <mkdir dir="${report.dir}" />
 
-        <junit printsummary="withOutAndErr" fork="yes">
-            <formatter usefile="false" type="plain" />
+        <junit showoutput="no" printsummary="no" fork="yes">
+            <formatter type="xml" />
             <jvmarg value="-ea" />
             <classpath location="${test.classes}" />
             <classpath location="${classes}" />
             <classpath refid="classpath.compile.custom" />
             <classpath refid="classpath.test.dependencies" />
 
-            <batchtest fork="yes">
+            <batchtest fork="yes" todir="${report.dir}">
                 <fileset dir="${test.src}">
                     <exclude name="**/Abstract*" />
                     <exclude name="**/Mock*" />
@@ -363,7 +367,12 @@
         <ivy:cachepath pathid="classpath.test.dependencies" conf="test" />
     </target>
 
-    <target name="clean">
+    <target name="remove-test-reports">
+        <fail unless="result.dir" message="No result.dir parameter given" />
+        <delete dir="${result.dir}/reports" />
+    </target>
+
+    <target name="clean" depends="remove-test-reports">
         <fail unless="result.dir" message="No result.dir parameter given" />
         <delete dir="${result.dir}" />
     </target>

uitest/build.xml

@@ -150,8 +150,6 @@
         <echo>Checkstyle is disabled for uitest for now</echo>
     </target>
     <target name="test" depends="checkstyle">
-        <!--<antcall target="common.test.run" /> -->
-        <echo>WHAT? No JUnit tests for ${module.name}!</echo>
     </target>
 
     <target name="test-testbench" depends="clean-testbench-errors" description="Run all TestBench based tests, including server tests">

uitest/integration_tests.xml

@@ -27,6 +27,9 @@
     <!-- Base url where the testable application is deployed -->
     <property name="deployment.url" value="http://${test.integration.server}:8080" />
 
+
+    <property name="report.dir" location="${integration_test.dir}/result/reports-integration" />
+
     <!-- ssh host values -->
     <property name="ant.hub" value="${test.integration.antfile}" />
     <property name="user" value="${test.integration.user}" />
@@ -56,6 +59,7 @@
             <property name="testfiles" value="${testfiles}" />
             <property name="test-output-dir" value="${integration_test.dir}/result/integration-test-output/${server-name}" />
             <property name="retries" value="0" />
+            <property name="report.qualifier" value="integration-tb2/${server-name}" />
 
             <fileset dir="." includes="test.xml" />
         </subant>
@@ -69,6 +73,7 @@
             <property name="server.start.succeeded" value="1" />
             <property name="testfiles" value="${testfiles-theme}" />
             <property name="test-output-dir" value="${integration_test.dir}/result/integration-test-output/${server-name}" />
+            <property name="report.qualifier" value="integration-theme-tb2/${server-name}" />
 
             <fileset dir="." includes="test.xml" />
         </subant>
@@ -87,6 +92,7 @@
             <property name="browsers" value="${test_browsers}" />
             <property name="testfiles" value="${test-GAE}" />
             <property name="test-output-dir" value="../build/integration-test-gae-output" />
+            <property name="report.qualifier" value="integration-gae-tb2/${server-name}" />
 
             <fileset dir="." includes="test.xml" />
         </subant>
@@ -105,16 +111,19 @@
         <fail unless="server-name" message="Server name must be defined in server-name" />
         <fail unless="deployment.url" message="Deplyoment url must be defined in deployment.url" />
         <fail unless="com.vaadin.testbench.screenshot.directory" message="Screenshot directory must be defined in com.vaadin.testbench.screenshot.directory" />
-
-        <junit printsummary="withOutAndErr" fork="yes">
-            <formatter usefile="false" type="plain" />
+	<property name="server.report.dir" location="${report.dir}/integration-test-tb3/${server-name}" />
+         <!-- The junit task does not create the report dir... -->
+        <mkdir dir="${server.report.dir}" />
+          
+        <junit showoutput="no" printsummary="no" fork="yes">
+            <formatter type="xml" />
             <classpath refid="classpath.tb3" />
 
             <jvmarg value="-Dcom.vaadin.testbench.screenshot.directory=${com.vaadin.testbench.screenshot.directory}" />
             <jvmarg value="-Ddeployment.url=${deployment.url}" />
             <jvmarg value="-Dserver-name=${server-name}" />
             <jvmarg value="-Djava.awt.headless=true" />
-            <test name="${junit.test.suite}" />
+            <test name="${junit.test.suite}" todir="${server.report.dir}" />
         </junit>
     </target>
 
@@ -360,6 +369,9 @@
         <property name="passphrase" value="${passphrase}" />
         <fail unless="sshkey.file" message="You must define an ssh.keyfile parameter" />
         <fail unless="com.vaadin.testbench.screenshot.directory" message="You must define a com.vaadin.testbench.screenshot.directory parameter" />
+        <delete dir="${report.dir}" />
+        <mkdir dir="${report.dir}" />
+
         <parallel>
             <antcontrib:trycatch property="tried">
                 <try>

uitest/tb3test.xml

@@ -2,6 +2,7 @@
 <project name="tb3test" xmlns:antcontrib="antlib:net.sf.antcontrib" xmlns:ivy="antlib:org.apache.ivy.ant" basedir=".">
 
     <dirname property="tb3test.dir" file="${ant.file.tb3test}" />
+    <property name="report.dir" location="${tb3test.dir}/result/reports-tb3" />
 
     <ivy:resolve file="${tb3test.dir}/ivy.xml" conf="build, build-provided" />
     <ivy:cachepath pathid="classpath.tb3.lib" conf="build, build-provided" />
@@ -19,13 +20,16 @@
     <target name="run-tb3-suite">
         <fail unless="junit.test.suite" message="Define suite to run using junit.test.suite" />
         <fail unless="com.vaadin.testbench.screenshot.directory" message="Define screenshot directory using -Dcom.vaadin.testbench.screenshot.directory" />
-        <junit printsummary="withOutAndErr" fork="yes">
-            <formatter usefile="false" type="plain" />
+        <delete dir="${report.dir}" />
+        <mkdir dir="${report.dir}" />
+
+        <junit showoutput="no" printsummary="no" fork="yes">
+            <formatter type="xml" />
             <classpath refid="classpath.tb3" />
 
             <jvmarg value="-Dcom.vaadin.testbench.screenshot.directory=${com.vaadin.testbench.screenshot.directory}" />
             <jvmarg value="-Djava.awt.headless=true" />
-            <test name="${junit.test.suite}" />
+            <test name="${junit.test.suite}" todir="${report.dir}" />
         </junit>
 
     </target>

uitest/test.xml

@@ -3,7 +3,6 @@
 
     <include file="../common.xml" />
     <dirname property="test.xml.dir" file="${ant.file.Run Vaadin Testbench Tests}" />
-
     <!-- ================================================================== -->
     <!-- Configuration -->
     <!-- ================================================================== -->
@@ -84,6 +83,11 @@
     </target>
 
     <target name="run-tb2-tests" depends="check-parameters,compile-tests">
+        <property name="report.qualifier" value="tb2" />
+        <property name="report.dir" location="${test.xml.dir}/result/reports-${report.qualifier}" />
+        
+        <delete dir="${report.dir}" />
+        <mkdir dir="${report.dir}" />
         <fileset dir="${test-output-dir}" id="tests-fileset">
             <include name="**/**.java" />
         </fileset>
@@ -103,13 +107,16 @@
 
     <!-- This target runs the generated and compiled junit tests -->
     <target name="execute-tests">
-        <junit fork="yes" printsummary="withOutAndErr" maxmemory="96m">
+        <!-- Junit does not create the report dir so we need to ensure it
+        exists -->
+        <mkdir dir="${report.dir}" />
+        <junit fork="yes" showoutput="no" printsummary="no" maxmemory="96m">
             <classpath>
                 <path refid="classpath" />
                 <pathelement path="${class-dir}" />
             </classpath>
 
-            <formatter usefile="false" type="plain" />
+            <formatter type="xml" />
 
             <jvmarg value="-Dcom.vaadin.testbench.tester.host=${com.vaadin.testbench.tester.host}" />
             <jvmarg value="-Dcom.vaadin.testbench.deployment.url=${com.vaadin.testbench.deployment.url}" />
@@ -129,7 +136,7 @@
             <jvmarg value="-Dcom.vaadin.testbench.screenshot.reference.debug=${com.vaadin.testbench.screenshot.reference.debug}" />
             <jvmarg value="-Dcom.vaadin.testbench.screenshot.cursor=${com.vaadin.testbench.screenshot.cursor}" />
 
-            <batchtest>
+            <batchtest todir="${report.dir}">
                 <filelist dir="${test-output-dir}" files="${target}" />
             </batchtest>
         </junit>