Added categories inclusion and exclusion feature to TB3TestSuite.

Added test category annotation.
Added default values for properties.
Added categories to tests.

Change-Id: I04259f1e56a75eb8c9834c125f44cb375196ddf8

15 files changed, 130 insertions, 16 deletions

diff --git a/uitest/src/com/vaadin/tests/annotations/TestCategory.java b/uitest/src/com/vaadin/tests/annotations/TestCategory.java
new file mode 100644
index 0000000000..c48df2bf8c
--- /dev/null
+++ b/uitest/src/com/vaadin/tests/annotations/TestCategory.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.vaadin.tests.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+@Inherited
+public @interface TestCategory {
+
+    String value();
+
+}
diff --git a/uitest/src/com/vaadin/tests/push/BarInUIDLTest.java b/uitest/src/com/vaadin/tests/push/BarInUIDLTest.java
index 840d653e1e..cd716dcaa3 100644
--- a/uitest/src/com/vaadin/tests/push/BarInUIDLTest.java
+++ b/uitest/src/com/vaadin/tests/push/BarInUIDLTest.java
@@ -19,8 +19,10 @@ import org.junit.Assert;
 import org.junit.Test;
 import org.openqa.selenium.WebElement;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public class BarInUIDLTest extends MultiBrowserTest {
     @Test
     public void sendBarInUIDL() {
diff --git a/uitest/src/com/vaadin/tests/push/BasicPushTest.java b/uitest/src/com/vaadin/tests/push/BasicPushTest.java
index e03262ca7e..670876e0f4 100644
--- a/uitest/src/com/vaadin/tests/push/BasicPushTest.java
+++ b/uitest/src/com/vaadin/tests/push/BasicPushTest.java
@@ -19,9 +19,11 @@ import org.junit.Assert;
 import org.junit.Test;
 import org.openqa.selenium.WebElement;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.AbstractTB3Test;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public abstract class BasicPushTest extends MultiBrowserTest {
 
     @Test
diff --git a/uitest/src/com/vaadin/tests/push/EnableDisablePushTest.java b/uitest/src/com/vaadin/tests/push/EnableDisablePushTest.java
index 275b6d5b53..2e14f9459b 100644
--- a/uitest/src/com/vaadin/tests/push/EnableDisablePushTest.java
+++ b/uitest/src/com/vaadin/tests/push/EnableDisablePushTest.java
@@ -20,8 +20,10 @@ import static org.junit.Assert.assertEquals;
 import org.junit.Test;
 import org.openqa.selenium.WebElement;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public class EnableDisablePushTest extends MultiBrowserTest {
     @Test
     public void testEnablePushWhenUsingPolling() throws Exception {
diff --git a/uitest/src/com/vaadin/tests/push/IdlePushChannelTest.java b/uitest/src/com/vaadin/tests/push/IdlePushChannelTest.java
index 4dcc8a680d..95a916f72d 100644
--- a/uitest/src/com/vaadin/tests/push/IdlePushChannelTest.java
+++ b/uitest/src/com/vaadin/tests/push/IdlePushChannelTest.java
@@ -18,8 +18,10 @@ package com.vaadin.tests.push;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public abstract class IdlePushChannelTest extends MultiBrowserTest {
 
     private static final int SEVEN_MINUTES_IN_MS = 7 * 60 * 1000;
diff --git a/uitest/src/com/vaadin/tests/push/PushErrorHandlingTest.java b/uitest/src/com/vaadin/tests/push/PushErrorHandlingTest.java
index a4b94b0573..8dbf91f9ee 100644
--- a/uitest/src/com/vaadin/tests/push/PushErrorHandlingTest.java
+++ b/uitest/src/com/vaadin/tests/push/PushErrorHandlingTest.java
@@ -20,8 +20,10 @@ import org.junit.Test;
 import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public class PushErrorHandlingTest extends MultiBrowserTest {
 
     @Test
diff --git a/uitest/src/com/vaadin/tests/push/PushFromInitTest.java b/uitest/src/com/vaadin/tests/push/PushFromInitTest.java
index 4101de29cf..15453fc054 100644
--- a/uitest/src/com/vaadin/tests/push/PushFromInitTest.java
+++ b/uitest/src/com/vaadin/tests/push/PushFromInitTest.java
@@ -19,8 +19,10 @@ import org.junit.Test;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.support.ui.ExpectedCondition;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public class PushFromInitTest extends MultiBrowserTest {
     @Test
     public void testPushFromInit() {
diff --git a/uitest/src/com/vaadin/tests/push/PushLargeDataStreamingTest.java b/uitest/src/com/vaadin/tests/push/PushLargeDataStreamingTest.java
index 0d71c21118..14dc5208ef 100644
--- a/uitest/src/com/vaadin/tests/push/PushLargeDataStreamingTest.java
+++ b/uitest/src/com/vaadin/tests/push/PushLargeDataStreamingTest.java
@@ -19,8 +19,10 @@ import org.junit.Test;
 import org.openqa.selenium.By;
 import org.openqa.selenium.support.ui.ExpectedConditions;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public class PushLargeDataStreamingTest extends MultiBrowserTest {
 
     @Test
diff --git a/uitest/src/com/vaadin/tests/push/TogglePushTest.java b/uitest/src/com/vaadin/tests/push/TogglePushTest.java
index 1867f4a63a..49110c79dc 100644
--- a/uitest/src/com/vaadin/tests/push/TogglePushTest.java
+++ b/uitest/src/com/vaadin/tests/push/TogglePushTest.java
@@ -19,8 +19,10 @@ import org.junit.Assert;
 import org.junit.Test;
 import org.openqa.selenium.WebElement;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public class TogglePushTest extends MultiBrowserTest {
 
     @Test
diff --git a/uitest/src/com/vaadin/tests/push/TrackMessageSizeUITest.java b/uitest/src/com/vaadin/tests/push/TrackMessageSizeUITest.java
index f904675b5e..4aab0f867a 100644
--- a/uitest/src/com/vaadin/tests/push/TrackMessageSizeUITest.java
+++ b/uitest/src/com/vaadin/tests/push/TrackMessageSizeUITest.java
@@ -18,13 +18,15 @@ package com.vaadin.tests.push;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+@TestCategory("push")
 public class TrackMessageSizeUITest extends MultiBrowserTest {
     @Test
     public void runTests() {
         openTestURL();
-        Assert.assertEquals("1. All tests run",
-                vaadinElementById("Log_row_0").getText());
+        Assert.assertEquals("1. All tests run", vaadinElementById("Log_row_0")
+                .getText());
     }
 }
\ No newline at end of file
diff --git a/uitest/src/com/vaadin/tests/tb3/MultiBrowserTestWithProxy.java b/uitest/src/com/vaadin/tests/tb3/MultiBrowserTestWithProxy.java
index 1a1e507a4a..d600b5fef2 100755
--- a/uitest/src/com/vaadin/tests/tb3/MultiBrowserTestWithProxy.java
+++ b/uitest/src/com/vaadin/tests/tb3/MultiBrowserTestWithProxy.java
@@ -24,7 +24,9 @@ import org.junit.Before;
 import com.jcraft.jsch.JSch;
 import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.Session;
+import com.vaadin.tests.annotations.TestCategory;
 
+@TestCategory("push")
 public abstract class MultiBrowserTestWithProxy extends MultiBrowserTest {
 
     private static AtomicInteger availablePort = new AtomicInteger(2000);
diff --git a/uitest/src/com/vaadin/tests/tb3/TB3Runner.java b/uitest/src/com/vaadin/tests/tb3/TB3Runner.java
index 18a2b69025..8b536858e5 100644
--- a/uitest/src/com/vaadin/tests/tb3/TB3Runner.java
+++ b/uitest/src/com/vaadin/tests/tb3/TB3Runner.java
@@ -35,6 +35,7 @@ import org.junit.runners.model.InitializationError;
 import org.junit.runners.model.Statement;
 import org.openqa.selenium.remote.DesiredCapabilities;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.AbstractTB3Test.BrowserUtil;
 import com.vaadin.tests.tb3.AbstractTB3Test.RunLocally;
 
@@ -77,14 +78,20 @@ public class TB3Runner extends BlockJUnit4ClassRunner {
 
         try {
             AbstractTB3Test testClassInstance = getTestClassInstance();
-            Collection<DesiredCapabilities> desiredCapabilites = getDesiredCapabilities(testClassInstance);
+            Collection<DesiredCapabilities> desiredCapabilities = getDesiredCapabilities(testClassInstance);
 
             TestNameSuffix testNameSuffixProperty = findAnnotation(
                     testClassInstance.getClass(), TestNameSuffix.class);
 
             for (FrameworkMethod m : getTestMethods()) {
-                if (desiredCapabilites.size() > 0) {
-                    for (DesiredCapabilities capabilities : desiredCapabilites) {
+                // No browsers available for this test, so we need to
+                // wrap the test method inside IgnoredTestMethod.
+                // This will add @Ignore annotation to it.
+                if (desiredCapabilities.size() <= 0
+                        || categoryIsExcludedOrNotExcplicitlyIncluded()) {
+                    tests.add(new IgnoredTestMethod(m.getMethod()));
+                } else {
+                    for (DesiredCapabilities capabilities : desiredCapabilities) {
                         TB3Method method = new TB3Method(m.getMethod(),
                                 capabilities);
                         if (testNameSuffixProperty != null) {
@@ -94,12 +101,6 @@ public class TB3Runner extends BlockJUnit4ClassRunner {
                         }
                         tests.add(method);
                     }
-
-                } else {
-                    // No browsers available for this test, so we need to
-                    // wrap the test method inside IgnoredTestMethod.
-                    // This will add @Ignore annotation to it.
-                    tests.add(new IgnoredTestMethod(m.getMethod()));
                 }
             }
         } catch (Exception e) {
@@ -109,6 +110,60 @@ public class TB3Runner extends BlockJUnit4ClassRunner {
         return tests;
     }
 
+    private boolean categoryIsExcludedOrNotExcplicitlyIncluded() {
+        Class<?> c = getTestClass().getJavaClass();
+
+        if (categoryIsExcluded(c)) {
+            return true;
+        }
+
+        if (explicitInclusionIsUsed()) {
+            return !categoryIsIncluded(c);
+        }
+
+        return false;
+    }
+
+    private boolean categoryIsIncluded(Class<?> c) {
+        String include = System.getProperty("categories.include");
+        if (include != null && include.trim().length() > 0) {
+            return hasCategoryFor(c, include.toLowerCase().trim());
+        }
+
+        return false;
+    }
+
+    private static boolean explicitInclusionIsUsed() {
+        String include = System.getProperty("categories.include");
+
+        return include != null && include.trim().length() > 0;
+    }
+
+    private static boolean categoryIsExcluded(Class<?> c) {
+        String exclude = System.getProperty("categories.exclude");
+        if (exclude != null && exclude.trim().length() > 0) {
+            return hasCategoryFor(c, exclude.toLowerCase().trim());
+        }
+
+        return false;
+    }
+
+    private static boolean hasCategoryFor(Class<?> c, String searchString) {
+        if (hasCategory(c)) {
+            return searchString.contains(getCategory(c).toLowerCase());
+        }
+
+        return false;
+    }
+
+    private static boolean hasCategory(Class<?> c) {
+        return c.getAnnotation(TestCategory.class) != null;
+    }
+
+    private static String getCategory(Class<?> c) {
+        return c.getAnnotation(TestCategory.class).value();
+    }
+
     private List<FrameworkMethod> getTestMethods() {
         return getTestClass().getAnnotatedMethods(Test.class);
     }
@@ -150,16 +205,16 @@ public class TB3Runner extends BlockJUnit4ClassRunner {
         for (DesiredCapabilities d : desiredCapabilites) {
             String browserName = (d.getBrowserName() + d.getVersion())
                     .toLowerCase();
-            if (include != null) {
-                if (include.toLowerCase().contains(browserName)) {
+            if (include != null && include.trim().length() > 0) {
+                if (include.trim().toLowerCase().contains(browserName)) {
                     filteredCapabilities.add(d);
                 }
             } else {
                 filteredCapabilities.add(d);
             }
 
-            if (exclude != null) {
-                if (exclude.toLowerCase().contains(browserName)) {
+            if (exclude != null && exclude.trim().length() > 0) {
+                if (exclude.trim().toLowerCase().contains(browserName)) {
                     filteredCapabilities.remove(d);
                 }
             }
diff --git a/uitest/src/com/vaadin/tests/tb3/TB3TestSuite.java b/uitest/src/com/vaadin/tests/tb3/TB3TestSuite.java
index 5878e8ab15..60bdb53083 100644
--- a/uitest/src/com/vaadin/tests/tb3/TB3TestSuite.java
+++ b/uitest/src/com/vaadin/tests/tb3/TB3TestSuite.java
@@ -246,7 +246,7 @@ public class TB3TestSuite extends Suite {
         if (c.getAnnotation(ExcludeFromSuite.class) != null) {
             return false;
         }
+
         return true;
     }
-
 }
\ No newline at end of file
diff --git a/uitest/src/com/vaadin/tests/tb3/WebsocketTest.java b/uitest/src/com/vaadin/tests/tb3/WebsocketTest.java
index e9ef11957c..69a06a561a 100644
--- a/uitest/src/com/vaadin/tests/tb3/WebsocketTest.java
+++ b/uitest/src/com/vaadin/tests/tb3/WebsocketTest.java
@@ -25,6 +25,7 @@ import java.util.List;
 
 import org.openqa.selenium.remote.DesiredCapabilities;
 
+import com.vaadin.tests.annotations.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest.Browser;
 
 /**
@@ -33,6 +34,7 @@ import com.vaadin.tests.tb3.MultiBrowserTest.Browser;
  * 
  * @author Vaadin Ltd
  */
+@TestCategory("push")
 public abstract class WebsocketTest extends PrivateTB3Configuration {
     private static List<DesiredCapabilities> websocketBrowsers = new ArrayList<DesiredCapabilities>();
     static {
diff --git a/uitest/tb3test.xml b/uitest/tb3test.xml
index ec78b9c9d2..6e0f25a8f7 100644
--- a/uitest/tb3test.xml
+++ b/uitest/tb3test.xml
@@ -3,6 +3,10 @@
 
     <dirname property="tb3test.dir" file="${ant.file.tb3test}" />
     <property name="report.dir" location="${tb3test.dir}/result/reports-tb3" />
+    <property name="browsers.include" value="" />
+    <property name="browsers.exclude" value="" />
+    <property name="categories.include" value="" />
+    <property name="categories.exclude" value="" />
 
     <ivy:resolve file="${tb3test.dir}/ivy.xml" conf="build, build-provided" />
     <ivy:cachepath pathid="classpath.tb3.lib" conf="build, build-provided" />
@@ -31,6 +35,8 @@
             <jvmarg value="-Djava.awt.headless=true" />
             <jvmarg value="-Dbrowsers.include=${browsers.include}" />
             <jvmarg value="-Dbrowsers.exclude=${browsers.exclude}" />
+            <jvmarg value="-Dcategories.include=${categories.include}" />
+            <jvmarg value="-Dcategories.exclude=${categories.exclude}" />
             <test name="${junit.test.suite}" todir="${report.dir}" />
         </junit>