Refactored build scripts to support TB2, TB3 + integration tests (#12572)

* Main build configuration triggers unit tests for all modules and uitest/build.xml testbench tests in parallel
* uitest/build.xml triggers Jetty startup and integration (server) tests in parallel. After the server has started, TB2 and TB3 tests are run in parallel.
* Server integration tests for servlet containers are run using TB3 and the com.vaadin.tests.tb3.ServletIntegrationTests test suite.
* Portlet integration tests are still run using TB2 test scripts

Change-Id: Ie6bffd4e68b4889074e9c470faa3c65f923e55c4

7 files changed, 411 insertions, 69 deletions

diff --git a/build.xml b/build.xml
index 5f748d8f40..bef7f7a2f3 100644
--- a/build.xml
+++ b/build.xml
@@ -43,19 +43,16 @@
     <target name="test" depends="buildorder">
         <subant buildpathref="build-path" target="test" />
     </target>
+
     <target name="test-all" depends="buildorder">
         <property name="war.file" location="result/artifacts/${vaadin.version}/vaadin-uitest/vaadin-uitest-${vaadin.version}.war" />
+
         <parallel>
-            <sequential>
-                <!-- Sleep before running integration tests so testbench 
-                    tests have time to compile and start -->
-                <sleep minutes="4" />
-                <ant antfile="uitest/integration_tests.xml" target="integration-test-all" inheritall="false" inheritrefs="false">
-                    <property name="demo.war" value="${war.file}" />
-                </ant>
-            </sequential>
+            <!-- JUnit tests, can be run without a server -->
             <subant buildpathref="build-path" target="test" />
-            <ant antfile="uitest/test.xml" target="test-package">
+
+            <!-- Server deployment + TestBench tests  -->
+            <ant antfile="uitest/build.xml" target="test-testbench">
                 <property name="war.file" location="${war.file}" />
             </ant>
         </parallel>
diff --git a/uitest/build.xml b/uitest/build.xml
index bd0f49ae1e..76b75e9203 100644
--- a/uitest/build.xml
+++ b/uitest/build.xml
@@ -9,6 +9,7 @@
 
     <!-- global properties -->
     <property name="module.name" value="vaadin-uitest" />
+    <property name="uitest.dir" location="${vaadin.basedir}/uitest" />
     <property name="result.dir" value="result" />
     <property name="result.war" location="${result.dir}/lib/${module.name}-${vaadin.version}.war" />
 
@@ -107,6 +108,11 @@
             <path refid="classpath.runtime.dependencies" />
         </copy>
 
+        <delete>
+            <!-- Avoid including some potentially conflicting jars in the war -->
+            <fileset dir="${deps.dir}" includes="jetty-*.jar" />
+            <fileset dir="${deps.dir}" includes="servlet-api-*.jar" />
+        </delete>
 
         <!-- Ensure filtered webcontent files are available -->
         <antcall target="common.filter.webcontent" />
@@ -148,4 +154,53 @@
         <echo>WHAT? No JUnit tests for ${module.name}!</echo>
     </target>
 
+    <target name="test-testbench" depends="clean-testbench-errors" description="Run all TestBench based tests, including server tests">
+        <parallel>
+            <daemons>
+                <!-- Start server -->
+                <ant antfile="${uitest.dir}/vaadin-server.xml" inheritall="true" inheritrefs="true" target="deploy-and-start" />
+            </daemons>
+            <sequential>
+                <!-- Server tests -->
+
+                <!-- Sleep before running integration tests so testbench 2 
+                        tests have time to compile and start -->
+                <sleep minutes="4" />
+                <ant antfile="${uitest.dir}/integration_tests.xml" target="integration-test-all" inheritall="false" inheritrefs="false">
+                    <property name="demo.war" value="${war.file}" />
+                </ant>
+            </sequential>
+            <sequential>
+                <!-- Wait for server to start -->
+                <ant antfile="${uitest.dir}/vaadin-server.xml" target="wait-for-startup" />
+
+                <!-- Run all different kinds of TestBench tests in parallel -->
+                <parallel>
+
+                    <!-- Legacy TestBench 2 tests -->
+                    <sequential>
+                        <ant antfile="${uitest.dir}/test.xml" target="tb2-tests" />
+                        <echo message="TestBench 2 tests complete" />
+                    </sequential>
+
+                    <!-- TestBench 3 tests -->
+                    <sequential>
+                        <ant antfile="${uitest.dir}/tb3test.xml" target="run-all-tb3-tests" inheritall="true" />
+                        <echo message="TestBench 3 tests complete" />
+                    </sequential>
+                </parallel>
+            </sequential>
+        </parallel>
+    </target>
+
+    <target name="clean-testbench-errors">
+        <fail unless="com.vaadin.testbench.screenshot.directory" message="Define screenshot directory using -Dcom.vaadin.testbench.screenshot.directory" />
+        <mkdir dir="${com.vaadin.testbench.screenshot.directory}/errors" />
+        <delete>
+            <fileset dir="${com.vaadin.testbench.screenshot.directory}/errors">
+                <include name="*" />
+            </fileset>
+        </delete>
+    </target>
+
 </project>
diff --git a/uitest/integration_tests.xml b/uitest/integration_tests.xml
index 78e053991e..f1b2070bc8 100644
--- a/uitest/integration_tests.xml
+++ b/uitest/integration_tests.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 
-<project xmlns:antcontrib="antlib:net.sf.antcontrib" name="Vaadin Integration Tests" basedir="." default="integration-test-all">
+<project xmlns:antcontrib="antlib:net.sf.antcontrib" xmlns:ivy="antlib:org.apache.ivy.ant" name="Vaadin Integration Tests" basedir="." default="integration-test-all">
 
     <!-- Import common targets -->
     <import file="../common.xml" />
@@ -32,33 +32,22 @@
     <property name="user" value="${test.integration.user}" />
     <property name="passphrase" value="" />
 
+    <ivy:resolve file="ivy.xml" conf="build, build-provided" />
+    <ivy:cachepath pathid="classpath.tb3.lib" conf="build, build-provided" />
+    <path id="classpath.tb3">
+        <path refid="classpath.tb3.lib" />
+        <path location="result/classes" />
+    </path>
+
     <!-- Upload war to deploy to ssh host -->
     <target name="integration-test-upload-demo">
         <scp file="${demo.war}" todir="${user}@${test.integration.server}:integration-tests/servers/demo.war" keyfile="${sshkey.file}" passphrase="${passphrase}" />
     </target>
 
     <!-- Run basic integration test test -->
-    <target name="integration-test-servlet">
-        <fileset dir="integration-testscripts" id="html-test-files" includes="integration-test-${server-name}-servlet.html" />
-        <pathconvert pathsep=" " property="testfiles" refid="html-test-files" />
-        <subant target="run-tests" failonerror="false" antfile="test.xml">
-            <property name="com.vaadin.testbench.lib.dir" value="${com.vaadin.testbench.lib.dir}" />
-            <property name="com.vaadin.testbench.tester.host" value="${com.vaadin.testbench.tester.host}" />
-            <property name="com.vaadin.testbench.deployment.url" value="${deployment.url}" />
-            <property name="server.start.succeeded" value="1" />
-            <property name="browsers" value="${test_browsers}" />
-            <property name="testfiles" value="${testfiles}" />
-            <property name="test-output-dir" value="${integration_test.dir}/result/integration-test-output/${server-name}" />
-            <property name="retries" value="0" />
-
-            <fileset dir="." includes="test.xml" />
-        </subant>
-    </target>
-
-    <target name="integration-test-push-servlet">
-        <fileset dir="integration-testscripts" id="html-test-files" includes="integration-test-${server-name}-push-servlet.html" />
-        <pathconvert pathsep=" " property="testfiles" refid="html-test-files" />
-        <subant target="run-tests" failonerror="false" antfile="test.xml">
+    <target name="legacy-integration-test">
+        <fail unless="testfiles" message="You need to specify the files to run using the 'testfiles' property" />
+        <subant target="run-tb2-tests" failonerror="false" antfile="test.xml">
             <property name="com.vaadin.testbench.lib.dir" value="${com.vaadin.testbench.lib.dir}" />
             <property name="com.vaadin.testbench.tester.host" value="${com.vaadin.testbench.tester.host}" />
             <property name="com.vaadin.testbench.deployment.url" value="${deployment.url}" />
@@ -73,7 +62,7 @@
     </target>
 
     <target name="integration-test-theme">
-        <subant target="run-tests" failonerror="false" antfile="test.xml">
+        <subant target="run-tb2-tests" failonerror="false" antfile="test.xml">
             <property name="com.vaadin.testbench.lib.dir" value="${com.vaadin.testbench.lib.dir}" />
             <property name="com.vaadin.testbench.tester.host" value="${com.vaadin.testbench.tester.host}" />
             <property name="com.vaadin.testbench.deployment.url" value="${deployment.url}" />
@@ -90,7 +79,7 @@
         <fileset dir="integration-testscripts" id="html-test-files" includes="GoogleAppEngine/integration-test-GAE.html" />
         <pathconvert pathsep=" " property="test-GAE" refid="html-test-files" />
 
-        <subant target="run-tests" failonerror="false" antfile="test.xml">
+        <subant target="run-tb2-tests" failonerror="false" antfile="test.xml">
             <property name="com.vaadin.testbench.lib.dir" value="${com.vaadin.testbench.lib.dir}" />
             <property name="com.vaadin.testbench.tester.host" value="${com.vaadin.testbench.tester.host}" />
             <property name="com.vaadin.testbench.deployment.url" value="http://vaadin-integration-test.appspot.com/" />
@@ -107,6 +96,27 @@
         <sshexec host="${test.integration.server}" username="${user}" keyfile="${sshkey.file}" command="ant -f ${ant.hub} deploy-to-GAE" />
     </target>
 
+    <target name="run-tb3-servlet-test">
+        <antcall target="run-tb3-test" inheritall="true">
+            <param name="junit.test.suite" value="com.vaadin.tests.tb3.ServletIntegrationTests" />
+        </antcall>
+    </target>
+    <target name="run-tb3-test">
+        <fail unless="server-name" message="Server name must be defined in server-name" />
+        <fail unless="deployment.url" message="Deplyoment url must be defined in deployment.url" />
+        <fail unless="com.vaadin.testbench.screenshot.directory" message="Screenshot directory must be defined in com.vaadin.testbench.screenshot.directory" />
+
+        <junit printsummary="withOutAndErr" fork="yes">
+            <formatter usefile="false" type="plain" />
+            <classpath refid="classpath.tb3" />
+
+            <jvmarg value="-Dcom.vaadin.testbench.screenshot.directory=${com.vaadin.testbench.screenshot.directory}" />
+            <jvmarg value="-Ddeployment.url=${deployment.url}" />
+            <jvmarg value="-Dserver-name=${server-name}" />
+            <jvmarg value="-Djava.awt.headless=true" />
+            <test name="${junit.test.suite}" />
+        </junit>
+    </target>
 
     <target name="integration-test-tomcat7">
         <antcall target="run-generic-integration-test">
@@ -333,7 +343,7 @@
 
     <!-- Upload demo, clean error screenshots and test deployment on all 
         servers -->
-    <target name="integration-test-all">
+    <target name="integration-test-all" unless="tests.integration.skip">
         <property name="passphrase" value="${passphrase}" />
         <fail unless="sshkey.file" message="You must define an ssh.keyfile parameter" />
         <fail unless="com.vaadin.testbench.screenshot.directory" message="You must define a com.vaadin.testbench.screenshot.directory parameter" />
@@ -341,7 +351,8 @@
             <antcontrib:trycatch property="tried">
                 <try>
                     <!-- Still running GAE test from the old server which 
-                        requires its own lock -->
+                    requires its own lock -->
+
                     <echo message="Getting lock" />
                     <antcall target="integration-test-get-lock" />
                     <echo message="Got lock" />
@@ -387,7 +398,6 @@
             <antcall target="integration-test-tomcat6" />
             <antcall target="integration-test-tomcat7" />
             <antcall target="integration-test-websphere8" />
-
         </parallel>
 
     </target>
@@ -446,31 +456,24 @@
                     </condition>
                 </fail>
 
-                <copy file="integration-testscripts/common/integration_test.tpl" tofile="integration-testscripts/integration-test-${target-server}-servlet.html" overwrite="true" />
-                <antcall target="integration-test-servlet">
-                    <param name="server-name" value="${target-server}" />
-                    <param name="deployment.url" value="http://${target-host}:${target-port}" />
-                </antcall>
-
-                <!-- Run theme tests in all browsers if there's a property 
-                    with the test files -->
                 <antcontrib:if>
-                    <isset property="testfiles-theme" />
+                    <isset property="testfiles" />
                     <antcontrib:then>
-                        <antcall target="integration-test-theme">
+                        <echo>Starting legacy (TB2) test for ${target-server}</echo>
+                        <antcall target="legacy-integration-test">
                             <param name="server-name" value="${target-server}" />
                             <param name="deployment.url" value="http://${target-host}:${target-port}" />
                         </antcall>
                     </antcontrib:then>
+                    <antcontrib:else>
+                        <echo>Starting TB3 test for ${target-server}</echo>
+                        <antcall target="run-tb3-servlet-test">
+                            <param name="server-name" value="${target-server}" />
+                            <param name="deployment.url" value="http://${target-host}:${target-port}" />
+                        </antcall>
+                    </antcontrib:else>
                 </antcontrib:if>
 
-                <!-- Run integration tests with push -->
-                <copy file="integration-testscripts/common/integration_push_test.tpl" tofile="integration-testscripts/integration-test-${target-server}-push-servlet.html" overwrite="true" />
-                <antcall target="integration-test-push-servlet">
-                    <param name="server-name" value="${target-server}" />
-                    <param name="deployment.url" value="http://${target-host}:${target-port}" />
-                </antcall>
-
                 <!-- Run theme tests in all browsers if there's a property 
                     with the test files -->
                 <antcontrib:if>
diff --git a/uitest/src/com/vaadin/tests/tb3/AllTB3Tests.java b/uitest/src/com/vaadin/tests/tb3/AllTB3Tests.java
new file mode 100644
index 0000000000..bd9027bec2
--- /dev/null
+++ b/uitest/src/com/vaadin/tests/tb3/AllTB3Tests.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.model.InitializationError;
+
+import com.vaadin.tests.tb3.AllTB3Tests.AllTB3TestsSuite;
+
+/**
+ * Test consisting of all TB3 tests except integration tests (classes extending
+ * AbstractTB3Test, excludes package com.vaadin.test.integration).
+ * 
+ * @author Vaadin Ltd
+ */
+@RunWith(AllTB3TestsSuite.class)
+public class AllTB3Tests {
+
+    public static class AllTB3TestsSuite extends TB3TestSuite {
+
+        public AllTB3TestsSuite(Class<?> klass) throws InitializationError {
+            super(klass, AbstractTB3Test.class, "com.vaadin.tests",
+                    new String[] { "com.vaadin.tests.integration" });
+        }
+
+    }
+
+}
diff --git a/uitest/src/com/vaadin/tests/tb3/TB3TestSuite.java b/uitest/src/com/vaadin/tests/tb3/TB3TestSuite.java
new file mode 100644
index 0000000000..dc187000d2
--- /dev/null
+++ b/uitest/src/com/vaadin/tests/tb3/TB3TestSuite.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Modifier;
+import java.net.JarURLConnection;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.JarEntry;
+
+import org.junit.runners.Suite;
+import org.junit.runners.model.InitializationError;
+
+/**
+ * Test suite which consists of all the TB3 tests passed in the constructor.
+ * Runs the tests in parallel using a {@link ParallelScheduler}
+ * 
+ * @author Vaadin Ltd
+ */
+public class TB3TestSuite extends Suite {
+
+    public TB3TestSuite(Class<?> klass,
+            Class<? extends AbstractTB3Test> baseClass, String basePackage,
+            String[] ignorePackages) throws InitializationError {
+        super(klass, findTests(baseClass, basePackage, ignorePackages));
+        setScheduler(new ParallelScheduler());
+    }
+
+    /**
+     * Traverses the directory on the classpath (inside or outside a Jar file)
+     * specified by 'basePackage'. Collects all classes inside the location
+     * which can be assigned to 'baseClass' except for classes inside packages
+     * listed in 'ignoredPackages'.
+     * 
+     * @param baseClass
+     * @param basePackage
+     * @param ignorePackages
+     * @return
+     */
+    private static Class<?>[] findTests(
+            Class<? extends AbstractTB3Test> baseClass, String basePackage,
+            String[] ignorePackages) {
+        try {
+            List<?> l = findClasses(baseClass, basePackage, ignorePackages);
+            return l.toArray(new Class[] {});
+        } catch (IOException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    /**
+     * Traverses the directory on the classpath (inside or outside a Jar file)
+     * specified by 'basePackage'. Collects all classes inside the location
+     * which can be assigned to 'baseClass' except for classes inside packages
+     * listed in 'ignoredPackages'.
+     * 
+     * @param baseClass
+     * @param basePackage
+     * @param ignoredPackages
+     * @return
+     * @throws IOException
+     */
+    private static <T> List<Class<? extends T>> findClasses(Class<T> baseClass,
+            String basePackage, String[] ignoredPackages) throws IOException {
+        List<Class<? extends T>> classes = new ArrayList<Class<? extends T>>();
+        String basePackageDirName = "/" + basePackage.replace('.', '/');
+        URL location = baseClass.getResource(basePackageDirName);
+        if (location.getProtocol().equals("file")) {
+            try {
+                File f = new File(location.toURI());
+                if (!f.exists()) {
+                    throw new IOException("Directory " + f.toString()
+                            + " does not exist");
+                }
+                findPackages(f, basePackage, baseClass, classes,
+                        ignoredPackages);
+            } catch (URISyntaxException e) {
+                throw new IOException(e.getMessage());
+            }
+        } else if (location.getProtocol().equals("jar")) {
+            JarURLConnection juc = (JarURLConnection) location.openConnection();
+            findClassesInJar(juc, basePackage, baseClass, classes);
+        }
+
+        Collections.sort(classes, new Comparator<Class<? extends T>>() {
+
+            @Override
+            public int compare(Class<? extends T> o1, Class<? extends T> o2) {
+                return o1.getName().compareTo(o2.getName());
+            }
+
+        });
+        return classes;
+    }
+
+    /**
+     * Traverses the given directory and collects all classes which are inside
+     * the given 'javaPackage' and can be assigned to the given 'baseClass'. The
+     * found classes are added to 'result'.
+     * 
+     * @param parent
+     *            The directory to traverse
+     * @param javaPackage
+     *            The java package which 'parent' contains
+     * @param baseClass
+     *            The class which the target classes extend
+     * @param result
+     *            The collection to which found classes are added
+     * @param ignoredPackages
+     *            A collection of packages (including sub packages) to ignore
+     */
+    private static <T> void findPackages(File parent, String javaPackage,
+            Class<T> baseClass, Collection<Class<? extends T>> result,
+            String[] ignoredPackages) {
+        for (String ignoredPackage : ignoredPackages) {
+            if (javaPackage.equals(ignoredPackage)) {
+                return;
+            }
+        }
+
+        for (File file : parent.listFiles()) {
+            if (file.isDirectory()) {
+                findPackages(file, javaPackage + "." + file.getName(),
+                        baseClass, result, ignoredPackages);
+            } else if (file.getName().endsWith(".class")) {
+                String fullyQualifiedClassName = javaPackage + "."
+                        + file.getName().replace(".class", "");
+                addClassIfMatches(result, fullyQualifiedClassName, baseClass);
+            }
+        }
+
+    }
+
+    /**
+     * Traverses a Jar file using the given connection and collects all classes
+     * which are inside the given 'javaPackage' and can be assigned to the given
+     * 'baseClass'. The found classes are added to 'result'.
+     * 
+     * @param javaPackage
+     *            The java package containing the classes (classes may be in a
+     *            sub package)
+     * @param baseClass
+     *            The class which the target classes extend
+     * @param result
+     *            The collection to which found classes are added
+     * @throws IOException
+     */
+    private static <T> void findClassesInJar(JarURLConnection juc,
+            String javaPackage, Class<T> baseClass,
+            Collection<Class<? extends T>> result) throws IOException {
+        String javaPackageDir = javaPackage.replace('.', '/');
+        Enumeration<JarEntry> ent = juc.getJarFile().entries();
+        while (ent.hasMoreElements()) {
+            JarEntry e = ent.nextElement();
+            if (e.getName().endsWith(".class")
+                    && e.getName().startsWith(javaPackageDir)) {
+                String fullyQualifiedClassName = e.getName().replace('/', '.')
+                        .replace(".class", "");
+                addClassIfMatches(result, fullyQualifiedClassName, baseClass);
+            }
+        }
+    }
+
+    /**
+     * Verifies that the class represented by 'fullyQualifiedClassName' can be
+     * loaded, assigned to 'baseClass' and is not an abstract or anonymous
+     * class.
+     * 
+     * @param result
+     *            The collection to add to
+     * @param fullyQualifiedClassName
+     *            The candidate class
+     * @param baseClass
+     *            The class 'fullyQualifiedClassName' should be assignable to
+     */
+    @SuppressWarnings("unchecked")
+    private static <T> void addClassIfMatches(
+            Collection<Class<? extends T>> result,
+            String fullyQualifiedClassName, Class<T> baseClass) {
+        try {
+            // Try to load the class
+
+            Class<?> c = Class.forName(fullyQualifiedClassName);
+            if (!baseClass.isAssignableFrom(c)) {
+                return;
+            }
+            if (!Modifier.isAbstract(c.getModifiers()) && !c.isAnonymousClass()) {
+                result.add((Class<? extends T>) c);
+            }
+        } catch (Exception e) {
+            // Could ignore that class cannot be loaded
+            e.printStackTrace();
+        } catch (LinkageError e) {
+            // Ignore. Client side classes will at least throw LinkageErrors
+        }
+
+    }
+
+}
\ No newline at end of file
diff --git a/uitest/tb3test.xml b/uitest/tb3test.xml
new file mode 100644
index 0000000000..92008ff9f3
--- /dev/null
+++ b/uitest/tb3test.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0"?>
+<project name="tb3test" xmlns:antcontrib="antlib:net.sf.antcontrib" xmlns:ivy="antlib:org.apache.ivy.ant" basedir=".">
+
+    <dirname property="tb3test.dir" file="${ant.file.tb3test}" />
+
+    <ivy:resolve file="${tb3test.dir}/ivy.xml" conf="build, build-provided" />
+    <ivy:cachepath pathid="classpath.tb3.lib" conf="build, build-provided" />
+    <path id="classpath.tb3">
+        <path refid="classpath.tb3.lib" />
+        <path location="${tb3test.dir}/result/classes" />
+    </path>
+
+    <target name="run-all-tb3-tests" description="Run all the TB3 tests (except server tests) in the project">
+        <antcall target="run-tb3-suite">
+            <param name="junit.test.suite" value="com.vaadin.tests.tb3.AllTB3Tests" />
+        </antcall>
+    </target>
+
+    <target name="run-tb3-suite">
+        <fail unless="junit.test.suite" message="Define suite to run using junit.test.suite" />
+        <fail unless="com.vaadin.testbench.screenshot.directory" message="Define screenshot directory using -Dcom.vaadin.testbench.screenshot.directory" />
+        <junit printsummary="withOutAndErr" fork="yes">
+            <formatter usefile="false" type="plain" />
+            <classpath refid="classpath.tb3" />
+
+            <jvmarg value="-Dcom.vaadin.testbench.screenshot.directory=${com.vaadin.testbench.screenshot.directory}" />
+            <jvmarg value="-Djava.awt.headless=true" />
+            <test name="${junit.test.suite}" />
+        </junit>
+
+    </target>
+
+</project>
diff --git a/uitest/test.xml b/uitest/test.xml
index 3baccb4117..dd6964e59c 100644
--- a/uitest/test.xml
+++ b/uitest/test.xml
@@ -83,7 +83,7 @@
         <fail unless="com.vaadin.testbench.deployment.url" message="The 'com.vaadin.testbench.deployment.url' property must be defined." />
     </target>
 
-    <target name="run-tests" depends="compile-tests">
+    <target name="run-tb2-tests" depends="check-parameters,compile-tests">
         <fileset dir="${test-output-dir}" id="tests-fileset">
             <include name="**/**.java" />
         </fileset>
@@ -161,12 +161,10 @@
     <!-- ================================================================== -->
 
     <!-- The default target. -->
-    <target name="run-and-clean-up" depends="check-parameters,remove-error-screens,run-tests" />
+    <target name="run-and-clean-up" depends="check-parameters,remove-error-screens,run-tb2-tests" />
 
-
-    <!-- Starts the server and runs all TestBench tests -->
-    <target name="test-package">
-        <fail unless="war.file" message="No 'war.file' parameter given." />
+    <!-- Runs all TestBench 2 tests -->
+    <target name="tb2-tests">
         <property name="test-output-dir" location="${test.xml.dir}/result/testbench-junit-classes" />
         <property name="retries" value="2" />
 
@@ -177,15 +175,6 @@
         <property name="com.vaadin.testbench.screenshot.block.error" value="0.025" />
         <property name="com.vaadin.testbench.debug" value="false" />
 
-        <parallel>
-            <daemons>
-                <ant antfile="${test.xml.dir}/vaadin-server.xml" inheritall="true" inheritrefs="true" target="deploy-and-start" />
-            </daemons>
-            <sequential>
-                <ant antfile="${test.xml.dir}/vaadin-server.xml" target="wait-for-startup" />
-                <antcall inheritall="true" inheritrefs="true" target="run-and-clean-up" />
-                <echo message="All TestBench tests have been run" />
-            </sequential>
-        </parallel>
+        <antcall inheritall="true" inheritrefs="true" target="run-tb2-tests" />
     </target>
 </project>