ParameterizedTB3Runner for adding permutations using a static getter

* Generates all screenshot names based on the test names

Change-Id: If8733238a88a71085ade5d12cc2372fb85b4d994

6 files changed, 216 insertions, 92 deletions

diff --git a/uitest/src/com/vaadin/tests/application/CriticalNotificationsTestBase.java b/uitest/src/com/vaadin/tests/application/CriticalNotificationsTest.java
index f3813fce50..03564cbaf7 100644
--- a/uitest/src/com/vaadin/tests/application/CriticalNotificationsTestBase.java
+++ b/uitest/src/com/vaadin/tests/application/CriticalNotificationsTest.java
@@ -22,48 +22,7 @@ import com.vaadin.testbench.elements.CheckBoxElement;
 import com.vaadin.testbench.elements.NotificationElement;
 import com.vaadin.tests.tb3.MultiBrowserThemeTest;
 
-public abstract class CriticalNotificationsTestBase extends
-        MultiBrowserThemeTest {
-
-    public static class ValoCriticalNotificationsTest extends
-            CriticalNotificationsTestBase {
-        @Override
-        protected String getTheme() {
-            return "valo";
-        }
-    }
-
-    public static class ReindeerCriticalNotificationsTest extends
-            CriticalNotificationsTestBase {
-        @Override
-        protected String getTheme() {
-            return "reindeer";
-        }
-    }
-
-    public static class RunoCriticalNotificationsTest extends
-            CriticalNotificationsTestBase {
-        @Override
-        protected String getTheme() {
-            return "runo";
-        }
-    }
-
-    public static class ChameleonCriticalNotificationsTest extends
-            CriticalNotificationsTestBase {
-        @Override
-        protected String getTheme() {
-            return "chameleon";
-        }
-    }
-
-    public static class BaseCriticalNotificationsTest extends
-            CriticalNotificationsTestBase {
-        @Override
-        protected String getTheme() {
-            return "base";
-        }
-    }
+public class CriticalNotificationsTest extends MultiBrowserThemeTest {
 
     @Test
     public void internalError() throws Exception {
diff --git a/uitest/src/com/vaadin/tests/integration/ParameterizedTB3Runner.java b/uitest/src/com/vaadin/tests/integration/ParameterizedTB3Runner.java
new file mode 100644
index 0000000000..59870a59a8
--- /dev/null
+++ b/uitest/src/com/vaadin/tests/integration/ParameterizedTB3Runner.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2000-2014 Vaadin Ltd.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.vaadin.tests.integration;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+
+import com.vaadin.tests.tb3.TB3Runner;
+
+/**
+ * TestBench test runner which supports static @Parameters annotated methods
+ * providing parameters for the corresponding setter.
+ * <p>
+ * {@code @Parameters public static Collection<String> getThemes() } creates one
+ * permutation for each value returned by {@code getThemes()}. The value is
+ * automatically assigned to the test instance using {@code setTheme(String)}
+ * before invoking the test method
+ * 
+ * @author Vaadin Ltd
+ */
+public class ParameterizedTB3Runner extends TB3Runner {
+
+    public ParameterizedTB3Runner(Class<?> klass) throws InitializationError {
+        super(klass);
+    }
+
+    @Override
+    protected List<FrameworkMethod> computeTestMethods() {
+        List<FrameworkMethod> methods = super.computeTestMethods();
+
+        Map<Method, Collection<String>> parameters = new LinkedHashMap<Method, Collection<String>>();
+
+        // Find all @Parameters methods and invoke them to find out permutations
+
+        for (Method m : getTestClass().getJavaClass().getMethods()) {
+            Parameters p = m.getAnnotation(Parameters.class);
+            if (p == null) {
+                continue;
+            }
+
+            if (!m.getName().startsWith("get") || !m.getName().endsWith("s")) {
+                throw new IllegalStateException(
+                        "Method "
+                                + m.getName()
+                                + " is annotated with @Parameter but is not named getSomeThings() as it should");
+            }
+
+            if (m.getParameterTypes().length != 0) {
+                throw new IllegalStateException(
+                        "Method "
+                                + m.getName()
+                                + " annotated with @Parameter should not have any arguments");
+            }
+
+            if (!Modifier.isStatic(m.getModifiers())) {
+                throw new IllegalStateException("Method " + m.getName()
+                        + " annotated with @Parameter must be static");
+            }
+
+            // getThemes -> setTheme
+            String setter = "set" + m.getName().substring("get".length());
+            setter = setter.substring(0, setter.length() - 1);
+            // property = property.substring(0, 1).toLowerCase()
+            // + property.substring(1);
+
+            Method setterMethod;
+            try {
+                setterMethod = getTestClass().getJavaClass().getMethod(setter,
+                        String.class);
+            } catch (Exception e) {
+                throw new IllegalStateException("No setter " + setter
+                        + " found in "
+                        + getTestClass().getJavaClass().getName(), e);
+            }
+
+            Collection<String> values;
+            try {
+                values = (Collection<String>) m.invoke(null);
+                if (!values.isEmpty()) {
+                    // Ignore any empty collections to allow e.g. intergration
+                    // tests to use "/demo" path by default without adding that
+                    // to the screnshot name
+                    parameters.put(setterMethod, values);
+                }
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+
+        // Add method permutations for all @Parameters
+        for (Method setter : parameters.keySet()) {
+            List<FrameworkMethod> newMethods = new ArrayList<FrameworkMethod>();
+            for (FrameworkMethod m : methods) {
+
+                if (!(m instanceof TBMethod)) {
+                    System.err.println("Unknown method type: "
+                            + m.getClass().getName());
+                    newMethods.add(m);
+                    continue;
+                }
+
+                // testFoo
+                // testBar
+                // ->
+                // testFoo[valo]
+                // testFoo[runo]
+                // testBar[valo]
+                // testBar[runo]
+
+                for (final String value : parameters.get(setter)) {
+                    newMethods.add(new TBMethodWithBefore((TBMethod) m, setter,
+                            value));
+                }
+            }
+            // Update methods so next parameters will use all expanded methods
+            methods = newMethods;
+        }
+        return methods;
+    }
+
+    public static class TBMethodWithBefore extends TBMethod {
+
+        private Method setter;
+        private String value;
+        private TBMethod parent;
+
+        public TBMethodWithBefore(TBMethod m, Method setter, String value) {
+            super(m.getMethod(), m.getCapabilities());
+            parent = m;
+            this.setter = setter;
+            this.value = value;
+        }
+
+        @Override
+        public Object invokeExplosively(Object target, Object... params)
+                throws Throwable {
+            setter.invoke(target, value);
+            return parent.invokeExplosively(target, params);
+        }
+
+        @Override
+        public String getName() {
+            return parent.getName() + "[" + value + "]";
+        };
+
+    }
+}
diff --git a/uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java b/uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java
index c1afe4a370..c5bdffb7a6 100644
--- a/uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java
+++ b/uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java
@@ -37,6 +37,7 @@ import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHttpEntityEnclosingRequest;
 import org.junit.Assert;
 import org.junit.Rule;
+import org.junit.rules.TestName;
 import org.junit.runner.RunWith;
 import org.openqa.selenium.By;
 import org.openqa.selenium.Dimension;
@@ -98,6 +99,9 @@ import elemental.json.impl.JsonUtil;
 public abstract class AbstractTB3Test extends ParallelTest {
 
     @Rule
+    public TestName testName = new TestName();
+
+    @Rule
     public RetryOnFail retry = new RetryOnFail();
 
     /**
diff --git a/uitest/src/com/vaadin/tests/tb3/MultiBrowserTest.java b/uitest/src/com/vaadin/tests/tb3/MultiBrowserTest.java
index c53209d1dc..23ead80fce 100644
--- a/uitest/src/com/vaadin/tests/tb3/MultiBrowserTest.java
+++ b/uitest/src/com/vaadin/tests/tb3/MultiBrowserTest.java
@@ -20,8 +20,6 @@ import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.List;
 
-import org.junit.Rule;
-import org.junit.rules.TestName;
 import org.openqa.selenium.ie.InternetExplorerDriver;
 import org.openqa.selenium.remote.DesiredCapabilities;
 
@@ -46,9 +44,6 @@ import com.vaadin.testbench.parallel.BrowserUtil;
  */
 public abstract class MultiBrowserTest extends PrivateTB3Configuration {
 
-    @Rule
-    public TestName testName = new TestName();
-
     protected List<DesiredCapabilities> getBrowsersSupportingWebSocket() {
         // No WebSocket support in IE8-9 and PhantomJS
         return getBrowserCapabilities(Browser.IE10, Browser.IE11,
diff --git a/uitest/src/com/vaadin/tests/tb3/MultiBrowserThemeTest.java b/uitest/src/com/vaadin/tests/tb3/MultiBrowserThemeTest.java
index 0964e6eb65..9a02b29bc2 100644
--- a/uitest/src/com/vaadin/tests/tb3/MultiBrowserThemeTest.java
+++ b/uitest/src/com/vaadin/tests/tb3/MultiBrowserThemeTest.java
@@ -16,18 +16,34 @@
 package com.vaadin.tests.tb3;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized.Parameters;
 import org.openqa.selenium.remote.DesiredCapabilities;
 
+import com.vaadin.tests.integration.ParameterizedTB3Runner;
+
 /**
  * Test which uses theme returned by {@link #getTheme()} for running the test
  */
+@RunWith(ParameterizedTB3Runner.class)
 public abstract class MultiBrowserThemeTest extends MultiBrowserTest {
 
-    protected abstract String getTheme();
+    private String theme;
+
+    public void setTheme(String theme) {
+        this.theme = theme;
+    }
+
+    @Parameters
+    public static Collection<String> getThemes() {
+        return Arrays.asList(new String[] { "valo", "reindeer", "runo",
+                "chameleon", "base" });
+    }
 
     @Override
     protected boolean requireWindowFocusForIE() {
@@ -37,7 +53,7 @@ public abstract class MultiBrowserThemeTest extends MultiBrowserTest {
     @Override
     protected void openTestURL(Class<?> uiClass, String... parameters) {
         Set<String> params = new HashSet<String>(Arrays.asList(parameters));
-        params.add("theme=" + getTheme());
+        params.add("theme=" + theme);
         super.openTestURL(uiClass, params.toArray(new String[params.size()]));
     }
 
diff --git a/uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java b/uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java
index 6b8fb1741b..78266e9a9d 100644
--- a/uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java
+++ b/uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java
@@ -381,8 +381,7 @@ public abstract class ScreenshotTB3Test extends AbstractTB3Test {
      *         fails
      */
     private String getScreenshotFailureName() {
-        return getScreenshotBaseName() + "_"
-                + getUniqueIdentifier(getDesiredCapabilities())
+        return getScreenshotBaseName() + "_" + getUniqueIdentifier(null)
                 + "-failure.png";
     }
 
@@ -418,52 +417,34 @@ public abstract class ScreenshotTB3Test extends AbstractTB3Test {
      */
     private String getScreenshotReferenceName(String identifier,
             Integer versionOverride) {
-        String uniqueBrowserIdentifier;
-        if (versionOverride == null) {
-            uniqueBrowserIdentifier = getUniqueIdentifier(getDesiredCapabilities());
-        } else {
-            uniqueBrowserIdentifier = getUniqueIdentifier(
-                    getDesiredCapabilities(), "" + versionOverride);
-        }
-
-        // WindowMaximizeRestoreTest_Windows_InternetExplorer_8_window-1-moved-maximized-restored.png
         return getScreenshotReferenceDirectory() + File.separator
-                + getScreenshotBaseName() + "_" + uniqueBrowserIdentifier + "_"
-                + identifier + ".png";
+                + getScreenshotBaseName() + "_"
+                + getUniqueIdentifier(versionOverride) + "_" + identifier
+                + ".png";
     }
 
-    /**
-     * Returns a string which uniquely (enough) identifies this browser. Used
-     * mainly in screenshot names.
-     * 
-     * @param capabilities
-     * @param versionOverride
-     * 
-     * @return a unique string for each browser
-     */
-    private String getUniqueIdentifier(DesiredCapabilities capabilities,
-            String versionOverride) {
-        return getUniqueIdentifier(BrowserUtil.getPlatform(capabilities),
-                BrowserUtil.getBrowserIdentifier(capabilities), versionOverride);
-    }
+    private String getUniqueIdentifier(Integer versionOverride) {
+        String testNameAndParameters = testName.getMethodName();
+        // runTest-wildfly9-nginx[Windows_Firefox_24][/buffering/demo][valo]
 
-    /**
-     * Returns a string which uniquely (enough) identifies this browser. Used
-     * mainly in screenshot names.
-     * 
-     * @param capabilities
-     * 
-     * @return a unique string for each browser
-     */
-    private String getUniqueIdentifier(DesiredCapabilities capabilities) {
-        return getUniqueIdentifier(BrowserUtil.getPlatform(capabilities),
-                BrowserUtil.getBrowserIdentifier(capabilities),
-                capabilities.getVersion());
-    }
+        String parameters = testNameAndParameters.substring(
+                testNameAndParameters.indexOf("[") + 1,
+                testNameAndParameters.length() - 1);
+        // Windows_Firefox_24][/buffering/demo][valo
+
+        parameters = parameters.replace("][", "_");
+        // Windows_Firefox_24_/buffering/demo_valo
 
-    private String getUniqueIdentifier(String platform, String browser,
-            String version) {
-        return platform + "_" + browser + "_" + version;
+        parameters = parameters.replace("/", "");
+        // Windows_Firefox_24_bufferingdemo_valo
+
+        if (versionOverride != null) {
+            // Windows_Firefox_17_-buffering-demo_valo
+            parameters = parameters.replaceFirst("_"
+                    + getDesiredCapabilities().getVersion(), "_"
+                    + versionOverride);
+        }
+        return parameters;
     }
 
     /**