-; Location where vaadin-testbench jar can be found
-com.vaadin.testbench.lib.dir=<enter location of testbench here>
-
-; Deployment url to use for testing. Context path must be /
-com.vaadin.testbench.deployment.url=http://<enter your ip here>:8888/
+;
+; For both TestBench 2 and 3
+;
; Location of the screenshot directory.
; This is the directory that contains the "references" directory
com.vaadin.testbench.screenshot.directory=<enter the full path to the screenshots directory, parent of "references" directory>
-; Run the whole test even if
+
+;
+; For only TestBench 2
+;
+
+; Location where TestBench 2 jar can be found
+com.vaadin.testbench.lib.dir=<enter location of testbench here>
+
+; Deployment url to use for testing. Context path must be /
+com.vaadin.testbench.deployment.url=http://<enter your ip here>:8888/
+
+; Run the whole test even if a screenshot comparison fails
com.vaadin.testbench.screenshot.softfail=true
; Screen capture at the end if the test fails
; Uncomment to limit to certain browsers or override in launch configuration
; browsers=winxp-opera10
-; Claim that the server has started succesfully. Needed for the tests to run
+; Claim that the server has started succesfully. Needed for TB2 tests to be executed
server.start.succeeded=1
-test-output-dir=../build/test-output
\ No newline at end of file
+; Directory where temporary Java classes are created
+test-output-dir=../build/test-output
+++ /dev/null
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head profile="http://selenium-ide.openqa.org/profiles/test-case">
-<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
-<link rel="selenium.base" href="" />
-<title>New Test</title>
-</head>
-<body>
-<table cellpadding="1" cellspacing="1" border="1">
-<thead>
-<tr><td rowspan="1" colspan="3">New Test</td></tr>
-</thead><tbody>
-<tr>
- <td>open</td>
- <td>/run/com.vaadin.tests.components.label.LabelModes?restartApplication</td>
- <td></td>
-</tr>
-<tr>
- <td>screenCapture</td>
- <td></td>
- <td>labelmodes</td>
-</tr>
-
-</tbody></table>
-</body>
-</html>
import com.vaadin.shared.ui.label.ContentMode;
import com.vaadin.tests.components.ComponentTestCase;
+import com.vaadin.tests.tb3.SimpleMultiBrowserTest;
import com.vaadin.ui.Label;
public class LabelModes extends ComponentTestCase<Label> {
+ public static class LabelModesTest extends SimpleMultiBrowserTest {
+ @Override
+ public void test() throws Exception {
+ compareScreen("labelmodes");
+ }
+
+ }
+
@Override
protected Class<Label> getTestClass() {
return Label.class;
--- /dev/null
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+import org.junit.After;
+import org.junit.Before;
+import org.openqa.selenium.Platform;
+import org.openqa.selenium.WebDriver;
+import org.openqa.selenium.WebElement;
+import org.openqa.selenium.remote.BrowserType;
+import org.openqa.selenium.remote.DesiredCapabilities;
+import org.openqa.selenium.remote.RemoteWebDriver;
+
+import com.vaadin.server.LegacyApplication;
+import com.vaadin.testbench.By;
+import com.vaadin.testbench.TestBench;
+import com.vaadin.testbench.TestBenchTestCase;
+import com.vaadin.ui.UI;
+
+/**
+ * Base class for TestBench 3+ tests. All TB3+ tests in the project should
+ * extend this class.
+ *
+ * Provides:
+ * <ul>
+ * <li>Helpers for browser selection</li>
+ * <li>Hub connection setup and teardown</li>
+ * <li>Automatic opening of a given test on the development server using
+ * {@link #getUIClass()} or by automatically finding an enclosing UI class</li>
+ * <li>Generic helpers for creating TB3+ tests</li>
+ * <li>Automatic URL generation based on needed features, e.g.
+ * {@link #isDebug()}, {@link #isPushEnabled()}</li>
+ * </ul>
+ *
+ * @author Vaadin Ltd
+ */
+public abstract class AbstractTB3Test extends TestBenchTestCase {
+ /**
+ * Height of the screenshots we want to capture
+ */
+ private static final int SCREENSHOT_HEIGHT = 850;
+
+ /**
+ * Width of the screenshots we want to capture
+ */
+ private static final int SCREENSHOT_WIDTH = 1500;
+
+ private DesiredCapabilities desiredCapabilities;
+ {
+ // Default browser to run on unless setDesiredCapabilities is called
+ desiredCapabilities = BrowserUtil.firefox(24);
+ }
+
+ /**
+ * Connect to the hub using a remote web driver, set the canvas size and
+ * opens the initial URL as specified by {@link #getTestUrl()}
+ *
+ * @throws MalformedURLException
+ */
+ @Before
+ public void setup() throws MalformedURLException {
+ DesiredCapabilities capabilities = getDesiredCapabilities();
+ driver = TestBench.createDriver(new RemoteWebDriver(
+ new URL(getHubURL()), capabilities));
+ int w = SCREENSHOT_WIDTH;
+ int h = SCREENSHOT_HEIGHT;
+
+ if (BrowserUtil.isIE8(capabilities)) {
+ // IE8 gets size wrong, who would have guessed...
+ w += 4;
+ h += 4;
+ }
+ try {
+ testBench().resizeViewPortTo(w, h);
+ } catch (UnsupportedOperationException e) {
+ // Opera does not support this...
+ }
+
+ String testUrl = getTestUrl();
+ if (testUrl != null) {
+ driver.get(testUrl);
+ }
+ }
+
+ /**
+ * Returns the full URL to be opened when the test starts.
+ *
+ * @return the full URL to open or null to not open any URL automatically
+ */
+ protected String getTestUrl() {
+ String baseUrl = getBaseURL();
+ if (baseUrl.endsWith("/")) {
+ baseUrl = baseUrl.substring(0, baseUrl.length() - 1);
+ }
+
+ return baseUrl + getDeploymentPath();
+ }
+
+ /**
+ *
+ * @return the location (URL) of the TB hub
+ */
+ protected String getHubURL() {
+ return "http://" + getHubHostname() + ":4444/wd/hub";
+ }
+
+ /**
+ * Used for building the hub URL to use for the test
+ *
+ * @return the host name of the TestBench hub
+ */
+ protected abstract String getHubHostname();
+
+ /**
+ * Used to determine what URL to initially open for the test
+ *
+ * @return the host name of development server
+ */
+ protected abstract String getDeploymentHostname();
+
+ /**
+ * Used to determine which capabilities should be used when setting up a
+ * {@link WebDriver} for this test. Typically set by a test runner or left
+ * at its default (Firefox 24). If you want to run a test on a single
+ * browser other than Firefox 24 you can override this method.
+ *
+ * @return the requested browser capabilities
+ */
+ protected DesiredCapabilities getDesiredCapabilities() {
+ return desiredCapabilities;
+ }
+
+ /**
+ * Sets the requested browser capabilities (typically browser name and
+ * version)
+ *
+ * @param desiredCapabilities
+ */
+ public void setDesiredCapabilities(DesiredCapabilities desiredCapabilities) {
+ this.desiredCapabilities = desiredCapabilities;
+ }
+
+ /**
+ * Shuts down the driver after the test has been completed
+ *
+ * @throws Exception
+ */
+ @After
+ public void tearDown() throws Exception {
+ if (driver != null) {
+ driver.quit();
+ }
+ driver = null;
+ }
+
+ /**
+ * Finds a Vaadin element based on the part of a TB3 style locator following
+ * the :: (e.g.
+ * vaadin=runLabelModes::PID_Scheckboxaction-Enabled/domChild[0] ->
+ * PID_Scheckboxaction-Enabled/domChild[0]).
+ *
+ * @param vaadinLocator
+ * The part following :: of the vaadin locator string
+ * @return
+ */
+ protected WebElement vaadinElement(String vaadinLocator) {
+ String base = getApplicationId(getDeploymentPath());
+
+ base += "::";
+
+ return driver.findElement(By.vaadin(base + vaadinLocator));
+ }
+
+ /**
+ * Find a Vaadin element based on its id given using Component.setId
+ *
+ * @param id
+ * The id to locate
+ * @return
+ */
+ public WebElement vaadinElementById(String id) {
+ return vaadinElement("PID_S" + id);
+ }
+
+ /**
+ * Returns the path that should be used for the test. The path contains the
+ * full path (appended to hostname+port) and must start with a slash.
+ *
+ * @return The path to open automatically when the test starts
+ */
+ protected String getDeploymentPath() {
+ Class<?> uiClass = getUIClass();
+ if (uiClass != null) {
+ return getDeploymentPath(uiClass);
+ }
+ throw new IllegalArgumentException("Unable to determine path for "
+ + getClass().getCanonicalName());
+
+ }
+
+ /**
+ * Returns the UI class the current test is connected to. Uses the enclosing
+ * class if the test class is a static inner class to a UI class.
+ *
+ * Test which are not enclosed by a UI class must implement this method and
+ * return the UI class they want to test.
+ *
+ * Note that this method will update the test name to the enclosing class to
+ * be compatible with TB2 screenshot naming
+ *
+ * @return the UI class the current test is connected to
+ */
+ protected Class<?> getUIClass() {
+ Class<?> enclosingClass = getClass().getEnclosingClass();
+ if (enclosingClass != null) {
+ return enclosingClass;
+ }
+ return null;
+ }
+
+ /**
+ * Determines whether to run the test in debug mode (with the debug console
+ * open) or not
+ *
+ * @return true to run with the debug window open, false by default
+ */
+ protected boolean isDebug() {
+ return false;
+ }
+
+ /**
+ * Determines whether to run the test with push enabled (using /run-push) or
+ * not. Note that push tests can and should typically be created using @Push
+ * on the UI instead of overriding this method
+ *
+ * @return true to use push in the test, false to use whatever UI specifies
+ */
+ protected boolean isPushEnabled() {
+ return false;
+ }
+
+ /**
+ * Returns the path for the given UI class when deployed on the test server.
+ * The path contains the full path (appended to hostname+port) and must
+ * start with a slash.
+ *
+ * This method takes into account {@link #isPushEnabled()} and
+ * {@link #isDebug()} when the path is generated.
+ *
+ * @param uiClass
+ * @return The path to the given UI class
+ */
+ private String getDeploymentPath(Class<?> uiClass) {
+ String runPath = "/run";
+ if (isPushEnabled()) {
+ runPath = "/run-push";
+ }
+
+ if (UI.class.isAssignableFrom(uiClass)) {
+ return runPath + "/" + uiClass.getCanonicalName()
+ + (isDebug() ? "?debug" : "");
+ } else if (LegacyApplication.class.isAssignableFrom(uiClass)) {
+ return runPath + "/" + uiClass.getCanonicalName()
+ + "?restartApplication" + (isDebug() ? "&debug" : "");
+ } else {
+ throw new IllegalArgumentException(
+ "Unable to determine path for enclosing class "
+ + uiClass.getCanonicalName());
+ }
+ }
+
+ /**
+ * Used to determine what URL to initially open for the test
+ *
+ * @return The base URL for the test. Does not include a trailing slash.
+ */
+ protected String getBaseURL() {
+ return "http://" + getDeploymentHostname() + ":8888";
+ }
+
+ /**
+ * Generates the application id based on the URL in a way compatible with
+ * VaadinServletService.
+ *
+ * @param pathWithQueryParameters
+ * The path part of the URL, possibly still containing query
+ * parameters
+ * @return The application ID string used in Vaadin locators
+ */
+ private String getApplicationId(String pathWithQueryParameters) {
+ // Remove any possible URL parameters
+ String pathWithoutQueryParameters = pathWithQueryParameters.replaceAll(
+ "\\?.*", "");
+ if ("".equals(pathWithoutQueryParameters)) {
+ return "ROOT";
+ }
+
+ // Retain only a-z and numbers
+ return pathWithoutQueryParameters.replaceAll("[^a-zA-Z0-9]", "");
+ }
+
+ /**
+ * Helper method for sleeping X ms in a test. Catches and ignores
+ * InterruptedExceptions
+ *
+ * @param timeoutMillis
+ * Number of ms to wait
+ */
+ protected void sleep(int timeoutMillis) {
+ try {
+ Thread.sleep(timeoutMillis);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Provides helper method for selecting the browser to run on
+ *
+ * @author Vaadin Ltd
+ */
+ public static class BrowserUtil {
+ /**
+ * Gets the capabilities for Safari of the given version
+ *
+ * @param version
+ * the major version
+ * @return an object describing the capabilities required for running a
+ * test on the given Safari version
+ */
+ protected static DesiredCapabilities safari(int version) {
+ DesiredCapabilities c = DesiredCapabilities.safari();
+ c.setVersion("" + version);
+ return c;
+ }
+
+ /**
+ * Gets the capabilities for Chrome of the given version
+ *
+ * @param version
+ * the major version
+ * @return an object describing the capabilities required for running a
+ * test on the given Chrome version
+ */
+ protected static DesiredCapabilities chrome(int version) {
+ DesiredCapabilities c = DesiredCapabilities.chrome();
+ c.setVersion("" + version);
+ c.setPlatform(Platform.XP);
+ return c;
+ }
+
+ /**
+ * Gets the capabilities for Opera of the given version
+ *
+ * @param version
+ * the major version
+ * @return an object describing the capabilities required for running a
+ * test on the given Opera version
+ */
+ protected static DesiredCapabilities opera(int version) {
+ DesiredCapabilities c = DesiredCapabilities.opera();
+ c.setVersion("" + version);
+ c.setPlatform(Platform.XP);
+ return c;
+ }
+
+ /**
+ * Gets the capabilities for Firefox of the given version
+ *
+ * @param version
+ * the major version
+ * @return an object describing the capabilities required for running a
+ * test on the given Firefox version
+ */
+ protected static DesiredCapabilities firefox(int version) {
+ DesiredCapabilities c = DesiredCapabilities.firefox();
+ c.setVersion("" + version);
+ c.setPlatform(Platform.XP);
+ return c;
+ }
+
+ /**
+ * Gets the capabilities for Internet Explorer of the given version
+ *
+ * @param version
+ * the major version
+ * @return an object describing the capabilities required for running a
+ * test on the given Internet Explorer version
+ */
+ protected static DesiredCapabilities ie(int version) {
+ DesiredCapabilities c = DesiredCapabilities.internetExplorer();
+ c.setVersion("" + version);
+ return c;
+ }
+
+ /**
+ * Checks if the given capabilities refer to Internet Explorer 8
+ *
+ * @param capabilities
+ * @return true if the capabilities refer to IE8, false otherwise
+ */
+ public static boolean isIE8(DesiredCapabilities capabilities) {
+ return BrowserType.IE.equals(capabilities.getBrowserName())
+ && "8".equals(capabilities.getVersion());
+ }
+
+ /**
+ * Returns a human readable identifier of the given browser. Used for
+ * test naming and screenshots
+ *
+ * @param capabilities
+ * @return a human readable string describing the capabilities
+ */
+ public static String getBrowserIdentifier(
+ DesiredCapabilities capabilities) {
+ String browserName = capabilities.getBrowserName();
+
+ if (BrowserType.IE.equals(browserName)) {
+ return "InternetExplorer";
+ } else if (BrowserType.FIREFOX.equals(browserName)) {
+ return "Firefox";
+ } else if (BrowserType.CHROME.equals(browserName)) {
+ return "Chrome";
+ } else if (BrowserType.SAFARI.equals(browserName)) {
+ return "Safari";
+ } else if (BrowserType.OPERA.equals(browserName)) {
+ return "Opera";
+ }
+
+ return browserName;
+ }
+
+ /**
+ * Returns a human readable identifier of the platform described by the
+ * given capabilities. Used mainly for screenshots
+ *
+ * @param capabilities
+ * @return a human readable string describing the platform
+ */
+ public static String getPlatform(DesiredCapabilities capabilities) {
+ if (capabilities.getPlatform() == Platform.WIN8
+ || capabilities.getPlatform() == Platform.WINDOWS
+ || capabilities.getPlatform() == Platform.VISTA
+ || capabilities.getPlatform() == Platform.XP) {
+ return "Windows";
+ } else if (capabilities.getPlatform() == Platform.MAC) {
+ return "Mac";
+ }
+ return capabilities.getPlatform().toString();
+ }
+
+ /**
+ * Returns a string which uniquely (enough) identifies this browser.
+ * Used mainly in screenshot names.
+ *
+ * @param capabilities
+ *
+ * @return a unique string for each browser
+ */
+ public static String getUniqueIdentifier(
+ DesiredCapabilities capabilities) {
+ return getUniqueIdentifier(getPlatform(capabilities),
+ getBrowserIdentifier(capabilities),
+ capabilities.getVersion());
+ }
+
+ /**
+ * Returns a string which uniquely (enough) identifies this browser.
+ * Used mainly in screenshot names.
+ *
+ * @param capabilities
+ *
+ * @return a unique string for each browser
+ */
+ public static String getUniqueIdentifier(
+ DesiredCapabilities capabilities, String versionOverride) {
+ return getUniqueIdentifier(getPlatform(capabilities),
+ getBrowserIdentifier(capabilities), versionOverride);
+ }
+
+ private static String getUniqueIdentifier(String platform,
+ String browser, String version) {
+ return platform + "_" + browser + "_" + version;
+ }
+
+ }
+
+ /**
+ * Called by the test runner whenever there is an exception in the test that
+ * will cause termination of the test
+ *
+ * @param t
+ * the throwable which caused the termination
+ */
+ public void onUncaughtException(Throwable t) {
+ // Do nothing by default
+
+ }
+}
--- /dev/null
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized.Parameters;
+import org.openqa.selenium.remote.DesiredCapabilities;
+
+/**
+ * Base class for tests which should be run on all supported browsers. The test
+ * is automatically launched for multiple browsers in parallel by the test
+ * runner.
+ *
+ * Sub classes can, but typically should not, restrict the browsers used by
+ * implementing a
+ *
+ * <pre>
+ * @Parameters
+ * public static Collection<DesiredCapabilities> getBrowsersForTest() {
+ * }
+ * </pre>
+ *
+ * @author Vaadin Ltd
+ */
+@RunWith(value = TB3Runner.class)
+public abstract class MultiBrowserTest extends PrivateTB3Configuration {
+
+ private static List<DesiredCapabilities> allBrowsers = new ArrayList<DesiredCapabilities>();
+ private static List<DesiredCapabilities> websocketBrowsers = new ArrayList<DesiredCapabilities>();
+ static {
+ allBrowsers.add(BrowserUtil.ie(8));
+ allBrowsers.add(BrowserUtil.ie(9));
+ allBrowsers.add(BrowserUtil.ie(10));
+ allBrowsers.add(BrowserUtil.firefox(17));
+ // Uncomment once we have the capability to run on Safari 6
+ // allBrowsers.add(safari(6));
+ allBrowsers.add(BrowserUtil.chrome(29));
+ allBrowsers.add(BrowserUtil.opera(12));
+
+ websocketBrowsers.addAll(allBrowsers);
+ websocketBrowsers.remove(BrowserUtil.ie(8));
+ websocketBrowsers.remove(BrowserUtil.ie(9));
+ }
+
+ @Parameters
+ public static Collection<DesiredCapabilities> getBrowsersForTest() {
+ return getAllBrowsers();
+ }
+
+ public static Collection<DesiredCapabilities> getAllBrowsers() {
+ return Collections.unmodifiableCollection(allBrowsers);
+ }
+
+ /**
+ * @return A subset of {@link #getAllBrowsers()} including only those which
+ * support websockets
+ */
+ public static Collection<DesiredCapabilities> getWebsocketBrowsers() {
+ return Collections.unmodifiableCollection(websocketBrowsers);
+ }
+
+}
--- /dev/null
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.junit.runners.model.RunnerScheduler;
+
+/**
+ * JUnit scheduler capable of running multiple tets in parallel. Each test is
+ * run in its own thread. Uses an {@link ExecutorService} to manage the threads.
+ *
+ * @author Vaadin Ltd
+ */
+public class ParallelScheduler implements RunnerScheduler {
+ private final List<Future<Object>> fResults = new ArrayList<Future<Object>>();
+
+ private final ExecutorService fService = Executors.newCachedThreadPool();
+
+ @Override
+ public void schedule(final Runnable childStatement) {
+ fResults.add(fService.submit(new Callable<Object>() {
+ @Override
+ public Object call() throws Exception {
+ childStatement.run();
+ return null;
+ }
+ }));
+ }
+
+ @Override
+ public void finished() {
+ for (Future<Object> each : fResults) {
+ try {
+ each.get();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright 2000-2013 Vaadind Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.util.Enumeration;
+import java.util.Properties;
+
+/**
+ * Provides values for parameters which depend on where the test is run.
+ * Parameters should be configured in work/eclipse-run-selected-test.properties.
+ * A template is available in uitest/.
+ *
+ * @author Vaadin Ltd
+ */
+public abstract class PrivateTB3Configuration extends ScreenshotTB3Test {
+ private static final String HOSTNAME_PROPERTY = "com.vaadin.testbench.deployment.hostname";
+ private final Properties properties = new Properties();
+
+ public PrivateTB3Configuration() {
+ File file = new File("work", "eclipse-run-selected-test.properties");
+ if (file.exists()) {
+ try {
+ properties.load(new FileInputStream(file));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private String getProperty(String name) {
+ String property = properties.getProperty(name);
+ if (property == null) {
+ property = System.getProperty(name);
+ }
+
+ return property;
+ }
+
+ @Override
+ protected String getScreenshotDirectory() {
+ String screenshotDirectory = getProperty("com.vaadin.testbench.screenshot.directory");
+ if (screenshotDirectory == null) {
+ throw new RuntimeException(
+ "No screenshot directory defined. Use -Dcom.vaadin.testbench.screenshot.directory=<path>");
+ }
+ return screenshotDirectory;
+ }
+
+ @Override
+ protected String getHubHostname() {
+ return "tb3-hub.intra.itmill.com";
+ }
+
+ @Override
+ protected String getDeploymentHostname() {
+ String hostName = getProperty(HOSTNAME_PROPERTY);
+
+ if (hostName == null || "".equals(hostName)) {
+ hostName = findAutoHostname();
+ }
+
+ return hostName;
+ }
+
+ /**
+ * Tries to automatically determine the IP address of the machine the test
+ * is running on.
+ *
+ * @return An IP address of one of the network interfaces in the machine.
+ * @throws RuntimeException
+ * if there was an error or no IP was found
+ */
+ private String findAutoHostname() {
+ try {
+ Enumeration<NetworkInterface> interfaces = NetworkInterface
+ .getNetworkInterfaces();
+ while (interfaces.hasMoreElements()) {
+ NetworkInterface current = interfaces.nextElement();
+ if (!current.isUp() || current.isLoopback()
+ || current.isVirtual()) {
+ continue;
+ }
+ Enumeration<InetAddress> addresses = current.getInetAddresses();
+ while (addresses.hasMoreElements()) {
+ InetAddress current_addr = addresses.nextElement();
+ if (current_addr.isLoopbackAddress()) {
+ continue;
+ }
+ String hostAddress = current_addr.getHostAddress();
+ if (hostAddress.startsWith("192.168.")) {
+ return hostAddress;
+ }
+ }
+ }
+ } catch (SocketException e) {
+ throw new RuntimeException("Could not enumerate ");
+ }
+
+ throw new RuntimeException(
+ "No compatible (192.168.*) ip address found.");
+ }
+
+}
--- /dev/null
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.imageio.ImageIO;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.openqa.selenium.OutputType;
+import org.openqa.selenium.TakesScreenshot;
+import org.openqa.selenium.remote.DesiredCapabilities;
+
+import com.vaadin.testbench.Parameters;
+import com.vaadin.testbench.commands.TestBenchCommands;
+
+/**
+ * Base class which provides functionality for tests which use the automatic
+ * screenshot comparison function.
+ *
+ * @author Vaadin Ltd
+ */
+public abstract class ScreenshotTB3Test extends AbstractTB3Test {
+
+ private String screenshotBaseName;
+
+ @Rule
+ public TestRule watcher = new TestWatcher() {
+
+ @Override
+ protected void starting(org.junit.runner.Description description) {
+ Class<?> testClass = description.getTestClass();
+ // Runner adds [BrowserName] which we do not want to use in the
+ // screenshot name
+ String testMethod = description.getMethodName();
+ testMethod = testMethod.replaceAll("\\[.*\\]", "");
+
+ String className = testClass.getSimpleName();
+ if (testClass.getEnclosingClass() != null) {
+ className = testClass.getEnclosingClass().getSimpleName();
+ }
+
+ screenshotBaseName = className + "-" + testMethod;
+ };
+ };
+
+ /**
+ * Contains a list of screenshot identifiers for which
+ * {@link #compareScreen(String)} has failed during the test
+ */
+ private List<String> screenshotFailures = new ArrayList<String>();
+
+ /**
+ * Defines TestBench screen comparison parameters before each test run
+ */
+ @Before
+ public void setupScreenComparisonParameters() {
+ Parameters.setScreenshotErrorDirectory(getScreenshotErrorDirectory());
+ Parameters
+ .setScreenshotReferenceDirectory(getScreenshotReferenceDirectory());
+ }
+
+ /**
+ * Grabs a screenshot and compares with the reference image with the given
+ * identifier. Supports alternative references and will succeed if the
+ * screenshot matches at least one of the references.
+ *
+ * In case of a failed comparison this method stores the grabbed screenshots
+ * in the error directory as defined by
+ * {@link #getScreenshotErrorDirectory()}. It will also generate a html file
+ * in the same directory, comparing the screenshot with the first found
+ * reference.
+ *
+ * @param identifier
+ * @throws IOException
+ */
+ protected void compareScreen(String identifier) throws IOException {
+ if (identifier == null || identifier.isEmpty()) {
+ throw new IllegalArgumentException("Empty identifier not supported");
+ }
+
+ File mainReference = getScreenshotReferenceFile(identifier);
+
+ List<File> alternativeFiles = findReferenceAlternatives(mainReference);
+ List<File> failedReferenceAlternatives = new ArrayList<File>();
+
+ for (File file : alternativeFiles) {
+ if (testBench(driver).compareScreen(file)) {
+ break;
+ } else {
+ failedReferenceAlternatives.add(file);
+ }
+ }
+
+ File referenceToKeep = null;
+ if (failedReferenceAlternatives.size() != alternativeFiles.size()) {
+ // Matched one comparison but not all, remove all error images +
+ // HTML files
+ } else {
+ // All comparisons failed, keep the main error image + HTML
+ screenshotFailures.add(mainReference.getName());
+ referenceToKeep = mainReference;
+ }
+
+ // Remove all PNG/HTML files we no longer need (failed alternative
+ // references or all error files (PNG/HTML) if comparison succeeded)
+ for (File failedAlternative : failedReferenceAlternatives) {
+ File failurePng = getErrorFileFromReference(failedAlternative);
+ if (failedAlternative != referenceToKeep) {
+ // Delete png + HTML
+ String htmlFileName = failurePng.getName().replace(".png",
+ ".html");
+ File failureHtml = new File(failurePng.getParentFile(),
+ htmlFileName);
+
+ failurePng.delete();
+ failureHtml.delete();
+ }
+ }
+ }
+
+ /**
+ *
+ * @param referenceFile
+ * The reference image file (in the directory defined by
+ * {@link #getScreenshotReferenceDirectory()})
+ * @return the file name of the file generated in the directory defined by
+ * {@link #getScreenshotErrorDirectory()} if comparison with the
+ * given reference image fails.
+ */
+ private File getErrorFileFromReference(File referenceFile) {
+ return new File(referenceFile.getAbsolutePath().replace(
+ getScreenshotReferenceDirectory(),
+ getScreenshotErrorDirectory()));
+ }
+
+ /**
+ * Finds alternative references for the given files
+ *
+ * @param reference
+ * @return all references which should be considered when comparing with the
+ * given files, including the given reference
+ */
+ private List<File> findReferenceAlternatives(File reference) {
+ List<File> files = new ArrayList<File>();
+ files.add(reference);
+
+ File screenshotDir = reference.getParentFile();
+ String name = reference.getName();
+ // Remove ".png"
+ String nameBase = name.substring(0, name.length() - 4);
+ for (int i = 1;; i++) {
+ File file = new File(screenshotDir, nameBase + "_" + i + ".png");
+ if (file.exists()) {
+ files.add(file);
+ } else {
+ break;
+ }
+ }
+
+ return files;
+ }
+
+ /**
+ * @param testName
+ * @return the reference file name to use for the given browser, as
+ * described by {@literal capabilities}, and identifier
+ */
+ private File getScreenshotReferenceFile(String identifier) {
+ DesiredCapabilities capabilities = getDesiredCapabilities();
+
+ String originalName = getScreenshotReferenceName(identifier);
+ File exactVersionFile = new File(originalName);
+ if (exactVersionFile.exists()) {
+ return exactVersionFile;
+ }
+
+ String browserVersion = capabilities.getVersion();
+
+ if (browserVersion.matches("\\d+")) {
+ for (int version = Integer.parseInt(browserVersion); version > 0; version--) {
+ String fileName = getScreenshotReferenceName(identifier,
+ version);
+ File oldVersionFile = new File(fileName);
+ if (oldVersionFile.exists()) {
+ return oldVersionFile;
+ }
+ }
+ }
+
+ return exactVersionFile;
+ }
+
+ /**
+ * @return the base directory of 'reference' and 'errors' screenshots
+ */
+ protected abstract String getScreenshotDirectory();
+
+ /**
+ * @return the directory where reference images are stored (the 'reference'
+ * folder inside the screenshot directory)
+ */
+ private String getScreenshotReferenceDirectory() {
+ return getScreenshotDirectory() + "/reference";
+ }
+
+ /**
+ * @return the directory where comparison error images should be created
+ * (the 'errors' folder inside the screenshot directory)
+ */
+ private String getScreenshotErrorDirectory() {
+ return getScreenshotDirectory() + "/errors";
+ }
+
+ /**
+ * Checks if any screenshot comparisons failures occurred during the test
+ * and combines all comparison errors into one exception
+ *
+ * @throws IOException
+ * If there were failures during the test
+ */
+ @After
+ public void checkCompareFailures() throws IOException {
+ if (!screenshotFailures.isEmpty()) {
+ throw new IOException(
+ "The following screenshots did not match the reference: "
+ + screenshotFailures.toString());
+ }
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * com.vaadin.tests.tb3.AbstractTB3Test#onUncaughtException(java.lang.Throwable
+ * )
+ */
+ @Override
+ public void onUncaughtException(Throwable cause) {
+ super.onUncaughtException(cause);
+ // Grab a "failure" screenshot and store in the errors folder for later
+ // analysis
+ try {
+ TestBenchCommands testBench = testBench();
+ if (testBench != null) {
+ testBench.disableWaitForVaadin();
+ }
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
+ try {
+ if (driver != null) {
+ BufferedImage screenshotImage = ImageIO
+ .read(new ByteArrayInputStream(
+ ((TakesScreenshot) driver)
+ .getScreenshotAs(OutputType.BYTES)));
+ ImageIO.write(screenshotImage, "png", new File(
+ getScreenshotFailureName()));
+ }
+ } catch (Throwable t) {
+ t.printStackTrace();
+ }
+
+ }
+
+ /**
+ * @return the name of a "failure" image which is stored in the folder
+ * defined by {@link #getScreenshotErrorDirectory()} when the test
+ * fails
+ */
+ private String getScreenshotFailureName() {
+ return getScreenshotErrorBaseName() + "-failure.png";
+ }
+
+ /**
+ * @return the base name used for screenshots. This is the first part of the
+ * screenshot file name, typically created as "testclass-testmethod"
+ */
+ public String getScreenshotBaseName() {
+ return screenshotBaseName;
+ }
+
+ /**
+ * Returns the name of the reference file based on the given parameters.
+ *
+ * @param testName
+ * @param capabilities
+ * @param identifier
+ * @return the full path of the reference
+ */
+ private String getScreenshotReferenceName(String identifier) {
+ return getScreenshotReferenceName(identifier, null);
+ }
+
+ /**
+ * Returns the name of the reference file based on the given parameters. The
+ * version given in {@literal capabilities} is used unless it is overridden
+ * by the {@literal versionOverride} parameter.
+ *
+ * @param testName
+ * @param capabilities
+ * @param identifier
+ * @return the full path of the reference
+ */
+ private String getScreenshotReferenceName(String identifier,
+ Integer versionOverride) {
+ String uniqueBrowserIdentifier;
+ if (versionOverride == null) {
+ uniqueBrowserIdentifier = BrowserUtil
+ .getUniqueIdentifier(getDesiredCapabilities());
+ } else {
+ uniqueBrowserIdentifier = BrowserUtil.getUniqueIdentifier(
+ getDesiredCapabilities(), "" + versionOverride);
+ }
+
+ // WindowMaximizeRestoreTest_Windows_InternetExplorer_8_window-1-moved-maximized-restored.png
+ return getScreenshotReferenceDirectory() + "/"
+ + getScreenshotBaseName() + "_" + uniqueBrowserIdentifier + "_"
+ + identifier + ".png";
+ }
+
+ /**
+ * Returns the base name of the screenshot in the error directory. This is a
+ * name so that all files matching {@link #getScreenshotErrorBaseName()}*
+ * are owned by this test instance (taking into account
+ * {@link #getDesiredCapabilities()}) and can safely be removed before
+ * running this test.
+ */
+ private String getScreenshotErrorBaseName() {
+ return getScreenshotReferenceName("dummy", null).replace(
+ getScreenshotReferenceDirectory(),
+ getScreenshotErrorDirectory()).replace("_dummy.png", "");
+ }
+
+ /**
+ * Removes any old screenshots related to this test from the errors
+ * directory before running the test
+ */
+ @Before
+ public void cleanErrorDirectory() {
+ // Remove any screenshots for this test from the error directory
+ // before running it. Leave unrelated files as-is
+ File errorDirectory = new File(getScreenshotErrorDirectory());
+
+ // Create errors directory if it does not exist
+ if (!errorDirectory.exists()) {
+ errorDirectory.mkdirs();
+ }
+
+ final String errorBase = getScreenshotErrorBaseName()
+ .replace("\\", "/");
+ File[] files = errorDirectory.listFiles(new FileFilter() {
+
+ @Override
+ public boolean accept(File pathname) {
+ String thisFile = pathname.getAbsolutePath().replace("\\", "/");
+ if (thisFile.startsWith(errorBase)) {
+ return true;
+ }
+ return false;
+ }
+ });
+ for (File f : files) {
+ f.delete();
+ }
+ }
+}
--- /dev/null
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ *
+ */
+package com.vaadin.tests.tb3;
+
+import org.junit.Test;
+
+/**
+ * A simple version of {@link MultiBrowserTest} which allows only one test
+ * method ({@link #test()}). Uses only the enclosing class name as test
+ * identifier (i.e. excludes "-test").
+ *
+ * This class is only provided as a helper for converting existing TB2 tests
+ * without renaming all screenshots. All new TB3+ tests should extend
+ * {@link MultiBrowserTest} directly instead of this.
+ *
+ * @author Vaadin Ltd
+ */
+public abstract class SimpleMultiBrowserTest extends MultiBrowserTest {
+
+ @Test
+ public abstract void test() throws Exception;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.vaadin.tests.tb3.ScreenshotTB3Test#getScreenshotBaseName()
+ */
+ @Override
+ public String getScreenshotBaseName() {
+ return super.getScreenshotBaseName().replaceFirst("-test$", "");
+ }
+}
--- /dev/null
+/*
+ * Copyright 2000-2013 Vaadin Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.vaadin.tests.tb3;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+import org.openqa.selenium.remote.DesiredCapabilities;
+
+import com.vaadin.tests.tb3.AbstractTB3Test.BrowserUtil;
+
+/**
+ * This runner is loosely based on FactoryTestRunner by Ted Young
+ * (http://tedyoung.me/2011/01/23/junit-runtime-tests-custom-runners/). The
+ * generated test names give information about the parameters used (unlike
+ * {@link Parameterized}).
+ *
+ * @since 7.1
+ */
+public class TB3Runner extends BlockJUnit4ClassRunner {
+
+ public TB3Runner(Class<?> klass) throws InitializationError {
+ super(klass);
+ setScheduler(new ParallelScheduler());
+ }
+
+ @Override
+ protected List<FrameworkMethod> computeTestMethods() {
+ List<FrameworkMethod> tests = new LinkedList<FrameworkMethod>();
+
+ // Find all methods in our test class marked with @Parameters.
+ for (FrameworkMethod method : getTestClass().getAnnotatedMethods(
+ Parameters.class)) {
+ // Make sure the Parameters method is static
+ if (!Modifier.isStatic(method.getMethod().getModifiers())) {
+ throw new IllegalArgumentException("@Parameters " + method
+ + " must be static.");
+ }
+
+ // Execute the method (statically)
+ Object params;
+ try {
+ params = method.getMethod().invoke(
+ getTestClass().getJavaClass());
+ } catch (Throwable t) {
+ throw new RuntimeException("Could not run test factory method "
+ + method.getName(), t);
+ }
+
+ // Did the factory return an array? If so, make it a list.
+ if (params.getClass().isArray()) {
+ params = Arrays.asList((Object[]) params);
+ }
+
+ // Did the factory return a scalar object? If so, put it in a list.
+ if (!(params instanceof Iterable<?>)) {
+ params = Collections.singletonList(params);
+ }
+
+ // For each object returned by the factory.
+ for (Object param : (Iterable<?>) params) {
+ if (!(param instanceof DesiredCapabilities)) {
+ throw new RuntimeException("Unexpected parameter type "
+ + param.getClass().getName()
+ + " when expecting DesiredCapabilities");
+ }
+ DesiredCapabilities capabilities = (DesiredCapabilities) param;
+ // Find any methods marked with @Test.
+ for (FrameworkMethod m : getTestClass().getAnnotatedMethods(
+ Test.class)) {
+ tests.add(new TB3Method(m.getMethod(), capabilities));
+ }
+ }
+ }
+
+ return tests;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.junit.runners.BlockJUnit4ClassRunner#withBefores(org.junit.runners
+ * .model.FrameworkMethod, java.lang.Object,
+ * org.junit.runners.model.Statement)
+ */
+ @Override
+ protected Statement withBefores(final FrameworkMethod method,
+ final Object target, Statement statement) {
+ if (!(method instanceof TB3Method)) {
+ throw new RuntimeException("Unexpected method type "
+ + method.getClass().getName() + ", expected TB3Method");
+ }
+ final TB3Method tb3method = (TB3Method) method;
+
+ // setDesiredCapabilities before running the real @Befores (which use
+ // capabilities)
+
+ final Statement realBefores = super.withBefores(method, target,
+ statement);
+ return new Statement() {
+
+ @Override
+ public void evaluate() throws Throwable {
+ ((AbstractTB3Test) target)
+ .setDesiredCapabilities(tb3method.capabilities);
+ try {
+ realBefores.evaluate();
+ } catch (Throwable t) {
+ // Give the test a chance to e.g. produce an error
+ // screenshot before failing the test by re-throwing the
+ // exception
+ ((AbstractTB3Test) target).onUncaughtException(t);
+ throw t;
+ }
+ }
+ };
+ }
+
+ private static class TB3Method extends FrameworkMethod {
+ private DesiredCapabilities capabilities;
+
+ public TB3Method(Method method, DesiredCapabilities capabilities) {
+ super(method);
+ this.capabilities = capabilities;
+ }
+
+ @Override
+ public Object invokeExplosively(final Object target, Object... params)
+ throws Throwable {
+ // Executes the test method with the supplied parameters
+ return super.invokeExplosively(target);
+ }
+
+ @Override
+ public String getName() {
+ return String.format("%s[%s]", getMethod().getName(),
+ BrowserUtil.getUniqueIdentifier(capabilities));
+ }
+
+ }
+}
projects / vaadin-framework.git / commitdiff