diff options
| author | Artur Signell <artur@vaadin.com> | 2013-09-16 08:50:16 +0300 |
|---|---|---|
| committer | Vaadin Code Review <review@vaadin.com> | 2013-09-23 12:35:15 +0000 |
| commit | 7f7dc316e3593bc4823f2cbc8e6f4814f233ce03 (patch) | |
| tree | 1b31381f39a8948d6fcc1de50dadf8860c3c9857 /uitest | |
| parent | c7aaa0413f63d938791e760b23fb3f609f285aa0 (diff) | |
| download | vaadin-framework-7f7dc316e3593bc4823f2cbc8e6f4814f233ce03.tar.gz vaadin-framework-7f7dc316e3593bc4823f2cbc8e6f4814f233ce03.zip | |
Base files for TB3 tests (#12572)
* Converted LabelModes to TB3 for validation
Change-Id: Ic9e69d46623a16986961bdc8cc050b375622a91d
Diffstat (limited to 'uitest')
10 files changed, 1415 insertions, 35 deletions
diff --git a/uitest/eclipse-run-selected-test.properties b/uitest/eclipse-run-selected-test.properties index f6cb2551e9..cbd1ab1cef 100644 --- a/uitest/eclipse-run-selected-test.properties +++ b/uitest/eclipse-run-selected-test.properties @@ -1,14 +1,23 @@ -; Location where vaadin-testbench jar can be found -com.vaadin.testbench.lib.dir=<enter location of testbench here> - -; Deployment url to use for testing. Context path must be / -com.vaadin.testbench.deployment.url=http://<enter your ip here>:8888/ +; +; For both TestBench 2 and 3 +; ; Location of the screenshot directory. ; This is the directory that contains the "references" directory com.vaadin.testbench.screenshot.directory=<enter the full path to the screenshots directory, parent of "references" directory> -; Run the whole test even if + +; +; For only TestBench 2 +; + +; Location where TestBench 2 jar can be found +com.vaadin.testbench.lib.dir=<enter location of testbench here> + +; Deployment url to use for testing. Context path must be / +com.vaadin.testbench.deployment.url=http://<enter your ip here>:8888/ + +; Run the whole test even if a screenshot comparison fails com.vaadin.testbench.screenshot.softfail=true ; Screen capture at the end if the test fails @@ -23,7 +32,8 @@ com.vaadin.testbench.screenshot.cursor=true ; Uncomment to limit to certain browsers or override in launch configuration ; browsers=winxp-opera10 -; Claim that the server has started succesfully. Needed for the tests to run +; Claim that the server has started succesfully. Needed for TB2 tests to be executed server.start.succeeded=1 -test-output-dir=../build/test-output
\ No newline at end of file +; Directory where temporary Java classes are created +test-output-dir=../build/test-output diff --git a/uitest/src/com/vaadin/tests/components/label/LabelModes.html b/uitest/src/com/vaadin/tests/components/label/LabelModes.html deleted file mode 100644 index 356688b456..0000000000 --- a/uitest/src/com/vaadin/tests/components/label/LabelModes.html +++ /dev/null @@ -1,27 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> -<head profile="http://selenium-ide.openqa.org/profiles/test-case"> -<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> -<link rel="selenium.base" href="" /> -<title>New Test</title> -</head> -<body> -<table cellpadding="1" cellspacing="1" border="1"> -<thead> -<tr><td rowspan="1" colspan="3">New Test</td></tr> -</thead><tbody> -<tr> - <td>open</td> - <td>/run/com.vaadin.tests.components.label.LabelModes?restartApplication</td> - <td></td> -</tr> -<tr> - <td>screenCapture</td> - <td></td> - <td>labelmodes</td> -</tr> - -</tbody></table> -</body> -</html> diff --git a/uitest/src/com/vaadin/tests/components/label/LabelModes.java b/uitest/src/com/vaadin/tests/components/label/LabelModes.java index e5bc539f36..1959447a4b 100644 --- a/uitest/src/com/vaadin/tests/components/label/LabelModes.java +++ b/uitest/src/com/vaadin/tests/components/label/LabelModes.java @@ -2,10 +2,19 @@ package com.vaadin.tests.components.label; import com.vaadin.shared.ui.label.ContentMode; import com.vaadin.tests.components.ComponentTestCase; +import com.vaadin.tests.tb3.SimpleMultiBrowserTest; import com.vaadin.ui.Label; public class LabelModes extends ComponentTestCase<Label> { + public static class LabelModesTest extends SimpleMultiBrowserTest { + @Override + public void test() throws Exception { + compareScreen("labelmodes"); + } + + } + @Override protected Class<Label> getTestClass() { return Label.class; diff --git a/uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java b/uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java new file mode 100644 index 0000000000..f27fff5873 --- /dev/null +++ b/uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java @@ -0,0 +1,516 @@ +/* + * Copyright 2000-2013 Vaadin Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.vaadin.tests.tb3; + +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.After; +import org.junit.Before; +import org.openqa.selenium.Platform; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.remote.BrowserType; +import org.openqa.selenium.remote.DesiredCapabilities; +import org.openqa.selenium.remote.RemoteWebDriver; + +import com.vaadin.server.LegacyApplication; +import com.vaadin.testbench.By; +import com.vaadin.testbench.TestBench; +import com.vaadin.testbench.TestBenchTestCase; +import com.vaadin.ui.UI; + +/** + * Base class for TestBench 3+ tests. All TB3+ tests in the project should + * extend this class. + * + * Provides: + * <ul> + * <li>Helpers for browser selection</li> + * <li>Hub connection setup and teardown</li> + * <li>Automatic opening of a given test on the development server using + * {@link #getUIClass()} or by automatically finding an enclosing UI class</li> + * <li>Generic helpers for creating TB3+ tests</li> + * <li>Automatic URL generation based on needed features, e.g. + * {@link #isDebug()}, {@link #isPushEnabled()}</li> + * </ul> + * + * @author Vaadin Ltd + */ +public abstract class AbstractTB3Test extends TestBenchTestCase { + /** + * Height of the screenshots we want to capture + */ + private static final int SCREENSHOT_HEIGHT = 850; + + /** + * Width of the screenshots we want to capture + */ + private static final int SCREENSHOT_WIDTH = 1500; + + private DesiredCapabilities desiredCapabilities; + { + // Default browser to run on unless setDesiredCapabilities is called + desiredCapabilities = BrowserUtil.firefox(24); + } + + /** + * Connect to the hub using a remote web driver, set the canvas size and + * opens the initial URL as specified by {@link #getTestUrl()} + * + * @throws MalformedURLException + */ + @Before + public void setup() throws MalformedURLException { + DesiredCapabilities capabilities = getDesiredCapabilities(); + driver = TestBench.createDriver(new RemoteWebDriver( + new URL(getHubURL()), capabilities)); + int w = SCREENSHOT_WIDTH; + int h = SCREENSHOT_HEIGHT; + + if (BrowserUtil.isIE8(capabilities)) { + // IE8 gets size wrong, who would have guessed... + w += 4; + h += 4; + } + try { + testBench().resizeViewPortTo(w, h); + } catch (UnsupportedOperationException e) { + // Opera does not support this... + } + + String testUrl = getTestUrl(); + if (testUrl != null) { + driver.get(testUrl); + } + } + + /** + * Returns the full URL to be opened when the test starts. + * + * @return the full URL to open or null to not open any URL automatically + */ + protected String getTestUrl() { + String baseUrl = getBaseURL(); + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.substring(0, baseUrl.length() - 1); + } + + return baseUrl + getDeploymentPath(); + } + + /** + * + * @return the location (URL) of the TB hub + */ + protected String getHubURL() { + return "http://" + getHubHostname() + ":4444/wd/hub"; + } + + /** + * Used for building the hub URL to use for the test + * + * @return the host name of the TestBench hub + */ + protected abstract String getHubHostname(); + + /** + * Used to determine what URL to initially open for the test + * + * @return the host name of development server + */ + protected abstract String getDeploymentHostname(); + + /** + * Used to determine which capabilities should be used when setting up a + * {@link WebDriver} for this test. Typically set by a test runner or left + * at its default (Firefox 24). If you want to run a test on a single + * browser other than Firefox 24 you can override this method. + * + * @return the requested browser capabilities + */ + protected DesiredCapabilities getDesiredCapabilities() { + return desiredCapabilities; + } + + /** + * Sets the requested browser capabilities (typically browser name and + * version) + * + * @param desiredCapabilities + */ + public void setDesiredCapabilities(DesiredCapabilities desiredCapabilities) { + this.desiredCapabilities = desiredCapabilities; + } + + /** + * Shuts down the driver after the test has been completed + * + * @throws Exception + */ + @After + public void tearDown() throws Exception { + if (driver != null) { + driver.quit(); + } + driver = null; + } + + /** + * Finds a Vaadin element based on the part of a TB3 style locator following + * the :: (e.g. + * vaadin=runLabelModes::PID_Scheckboxaction-Enabled/domChild[0] -> + * PID_Scheckboxaction-Enabled/domChild[0]). + * + * @param vaadinLocator + * The part following :: of the vaadin locator string + * @return + */ + protected WebElement vaadinElement(String vaadinLocator) { + String base = getApplicationId(getDeploymentPath()); + + base += "::"; + + return driver.findElement(By.vaadin(base + vaadinLocator)); + } + + /** + * Find a Vaadin element based on its id given using Component.setId + * + * @param id + * The id to locate + * @return + */ + public WebElement vaadinElementById(String id) { + return vaadinElement("PID_S" + id); + } + + /** + * Returns the path that should be used for the test. The path contains the + * full path (appended to hostname+port) and must start with a slash. + * + * @return The path to open automatically when the test starts + */ + protected String getDeploymentPath() { + Class<?> uiClass = getUIClass(); + if (uiClass != null) { + return getDeploymentPath(uiClass); + } + throw new IllegalArgumentException("Unable to determine path for " + + getClass().getCanonicalName()); + + } + + /** + * Returns the UI class the current test is connected to. Uses the enclosing + * class if the test class is a static inner class to a UI class. + * + * Test which are not enclosed by a UI class must implement this method and + * return the UI class they want to test. + * + * Note that this method will update the test name to the enclosing class to + * be compatible with TB2 screenshot naming + * + * @return the UI class the current test is connected to + */ + protected Class<?> getUIClass() { + Class<?> enclosingClass = getClass().getEnclosingClass(); + if (enclosingClass != null) { + return enclosingClass; + } + return null; + } + + /** + * Determines whether to run the test in debug mode (with the debug console + * open) or not + * + * @return true to run with the debug window open, false by default + */ + protected boolean isDebug() { + return false; + } + + /** + * Determines whether to run the test with push enabled (using /run-push) or + * not. Note that push tests can and should typically be created using @Push + * on the UI instead of overriding this method + * + * @return true to use push in the test, false to use whatever UI specifies + */ + protected boolean isPushEnabled() { + return false; + } + + /** + * Returns the path for the given UI class when deployed on the test server. + * The path contains the full path (appended to hostname+port) and must + * start with a slash. + * + * This method takes into account {@link #isPushEnabled()} and + * {@link #isDebug()} when the path is generated. + * + * @param uiClass + * @return The path to the given UI class + */ + private String getDeploymentPath(Class<?> uiClass) { + String runPath = "/run"; + if (isPushEnabled()) { + runPath = "/run-push"; + } + + if (UI.class.isAssignableFrom(uiClass)) { + return runPath + "/" + uiClass.getCanonicalName() + + (isDebug() ? "?debug" : ""); + } else if (LegacyApplication.class.isAssignableFrom(uiClass)) { + return runPath + "/" + uiClass.getCanonicalName() + + "?restartApplication" + (isDebug() ? "&debug" : ""); + } else { + throw new IllegalArgumentException( + "Unable to determine path for enclosing class " + + uiClass.getCanonicalName()); + } + } + + /** + * Used to determine what URL to initially open for the test + * + * @return The base URL for the test. Does not include a trailing slash. + */ + protected String getBaseURL() { + return "http://" + getDeploymentHostname() + ":8888"; + } + + /** + * Generates the application id based on the URL in a way compatible with + * VaadinServletService. + * + * @param pathWithQueryParameters + * The path part of the URL, possibly still containing query + * parameters + * @return The application ID string used in Vaadin locators + */ + private String getApplicationId(String pathWithQueryParameters) { + // Remove any possible URL parameters + String pathWithoutQueryParameters = pathWithQueryParameters.replaceAll( + "\\?.*", ""); + if ("".equals(pathWithoutQueryParameters)) { + return "ROOT"; + } + + // Retain only a-z and numbers + return pathWithoutQueryParameters.replaceAll("[^a-zA-Z0-9]", ""); + } + + /** + * Helper method for sleeping X ms in a test. Catches and ignores + * InterruptedExceptions + * + * @param timeoutMillis + * Number of ms to wait + */ + protected void sleep(int timeoutMillis) { + try { + Thread.sleep(timeoutMillis); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** + * Provides helper method for selecting the browser to run on + * + * @author Vaadin Ltd + */ + public static class BrowserUtil { + /** + * Gets the capabilities for Safari of the given version + * + * @param version + * the major version + * @return an object describing the capabilities required for running a + * test on the given Safari version + */ + protected static DesiredCapabilities safari(int version) { + DesiredCapabilities c = DesiredCapabilities.safari(); + c.setVersion("" + version); + return c; + } + + /** + * Gets the capabilities for Chrome of the given version + * + * @param version + * the major version + * @return an object describing the capabilities required for running a + * test on the given Chrome version + */ + protected static DesiredCapabilities chrome(int version) { + DesiredCapabilities c = DesiredCapabilities.chrome(); + c.setVersion("" + version); + c.setPlatform(Platform.XP); + return c; + } + + /** + * Gets the capabilities for Opera of the given version + * + * @param version + * the major version + * @return an object describing the capabilities required for running a + * test on the given Opera version + */ + protected static DesiredCapabilities opera(int version) { + DesiredCapabilities c = DesiredCapabilities.opera(); + c.setVersion("" + version); + c.setPlatform(Platform.XP); + return c; + } + + /** + * Gets the capabilities for Firefox of the given version + * + * @param version + * the major version + * @return an object describing the capabilities required for running a + * test on the given Firefox version + */ + protected static DesiredCapabilities firefox(int version) { + DesiredCapabilities c = DesiredCapabilities.firefox(); + c.setVersion("" + version); + c.setPlatform(Platform.XP); + return c; + } + + /** + * Gets the capabilities for Internet Explorer of the given version + * + * @param version + * the major version + * @return an object describing the capabilities required for running a + * test on the given Internet Explorer version + */ + protected static DesiredCapabilities ie(int version) { + DesiredCapabilities c = DesiredCapabilities.internetExplorer(); + c.setVersion("" + version); + return c; + } + + /** + * Checks if the given capabilities refer to Internet Explorer 8 + * + * @param capabilities + * @return true if the capabilities refer to IE8, false otherwise + */ + public static boolean isIE8(DesiredCapabilities capabilities) { + return BrowserType.IE.equals(capabilities.getBrowserName()) + && "8".equals(capabilities.getVersion()); + } + + /** + * Returns a human readable identifier of the given browser. Used for + * test naming and screenshots + * + * @param capabilities + * @return a human readable string describing the capabilities + */ + public static String getBrowserIdentifier( + DesiredCapabilities capabilities) { + String browserName = capabilities.getBrowserName(); + + if (BrowserType.IE.equals(browserName)) { + return "InternetExplorer"; + } else if (BrowserType.FIREFOX.equals(browserName)) { + return "Firefox"; + } else if (BrowserType.CHROME.equals(browserName)) { + return "Chrome"; + } else if (BrowserType.SAFARI.equals(browserName)) { + return "Safari"; + } else if (BrowserType.OPERA.equals(browserName)) { + return "Opera"; + } + + return browserName; + } + + /** + * Returns a human readable identifier of the platform described by the + * given capabilities. Used mainly for screenshots + * + * @param capabilities + * @return a human readable string describing the platform + */ + public static String getPlatform(DesiredCapabilities capabilities) { + if (capabilities.getPlatform() == Platform.WIN8 + || capabilities.getPlatform() == Platform.WINDOWS + || capabilities.getPlatform() == Platform.VISTA + || capabilities.getPlatform() == Platform.XP) { + return "Windows"; + } else if (capabilities.getPlatform() == Platform.MAC) { + return "Mac"; + } + return capabilities.getPlatform().toString(); + } + + /** + * Returns a string which uniquely (enough) identifies this browser. + * Used mainly in screenshot names. + * + * @param capabilities + * + * @return a unique string for each browser + */ + public static String getUniqueIdentifier( + DesiredCapabilities capabilities) { + return getUniqueIdentifier(getPlatform(capabilities), + getBrowserIdentifier(capabilities), + capabilities.getVersion()); + } + + /** + * Returns a string which uniquely (enough) identifies this browser. + * Used mainly in screenshot names. + * + * @param capabilities + * + * @return a unique string for each browser + */ + public static String getUniqueIdentifier( + DesiredCapabilities capabilities, String versionOverride) { + return getUniqueIdentifier(getPlatform(capabilities), + getBrowserIdentifier(capabilities), versionOverride); + } + + private static String getUniqueIdentifier(String platform, + String browser, String version) { + return platform + "_" + browser + "_" + version; + } + + } + + /** + * Called by the test runner whenever there is an exception in the test that + * will cause termination of the test + * + * @param t + * the throwable which caused the termination + */ + public void onUncaughtException(Throwable t) { + // Do nothing by default + + } +} diff --git a/uitest/src/com/vaadin/tests/tb3/MultiBrowserTest.java b/uitest/src/com/vaadin/tests/tb3/MultiBrowserTest.java new file mode 100644 index 0000000000..5a1d07c0c0 --- /dev/null +++ b/uitest/src/com/vaadin/tests/tb3/MultiBrowserTest.java @@ -0,0 +1,81 @@ +/* + * Copyright 2000-2013 Vaadin Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.vaadin.tests.tb3; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized.Parameters; +import org.openqa.selenium.remote.DesiredCapabilities; + +/** + * Base class for tests which should be run on all supported browsers. The test + * is automatically launched for multiple browsers in parallel by the test + * runner. + * + * Sub classes can, but typically should not, restrict the browsers used by + * implementing a + * + * <pre> + * @Parameters + * public static Collection<DesiredCapabilities> getBrowsersForTest() { + * } + * </pre> + * + * @author Vaadin Ltd + */ +@RunWith(value = TB3Runner.class) +public abstract class MultiBrowserTest extends PrivateTB3Configuration { + + private static List<DesiredCapabilities> allBrowsers = new ArrayList<DesiredCapabilities>(); + private static List<DesiredCapabilities> websocketBrowsers = new ArrayList<DesiredCapabilities>(); + static { + allBrowsers.add(BrowserUtil.ie(8)); + allBrowsers.add(BrowserUtil.ie(9)); + allBrowsers.add(BrowserUtil.ie(10)); + allBrowsers.add(BrowserUtil.firefox(17)); + // Uncomment once we have the capability to run on Safari 6 + // allBrowsers.add(safari(6)); + allBrowsers.add(BrowserUtil.chrome(29)); + allBrowsers.add(BrowserUtil.opera(12)); + + websocketBrowsers.addAll(allBrowsers); + websocketBrowsers.remove(BrowserUtil.ie(8)); + websocketBrowsers.remove(BrowserUtil.ie(9)); + } + + @Parameters + public static Collection<DesiredCapabilities> getBrowsersForTest() { + return getAllBrowsers(); + } + + public static Collection<DesiredCapabilities> getAllBrowsers() { + return Collections.unmodifiableCollection(allBrowsers); + } + + /** + * @return A subset of {@link #getAllBrowsers()} including only those which + * support websockets + */ + public static Collection<DesiredCapabilities> getWebsocketBrowsers() { + return Collections.unmodifiableCollection(websocketBrowsers); + } + +} diff --git a/uitest/src/com/vaadin/tests/tb3/ParallelScheduler.java b/uitest/src/com/vaadin/tests/tb3/ParallelScheduler.java new file mode 100644 index 0000000000..f8013169fa --- /dev/null +++ b/uitest/src/com/vaadin/tests/tb3/ParallelScheduler.java @@ -0,0 +1,60 @@ +/* + * Copyright 2000-2013 Vaadin Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.vaadin.tests.tb3; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.junit.runners.model.RunnerScheduler; + +/** + * JUnit scheduler capable of running multiple tets in parallel. Each test is + * run in its own thread. Uses an {@link ExecutorService} to manage the threads. + * + * @author Vaadin Ltd + */ +public class ParallelScheduler implements RunnerScheduler { + private final List<Future<Object>> fResults = new ArrayList<Future<Object>>(); + + private final ExecutorService fService = Executors.newCachedThreadPool(); + + @Override + public void schedule(final Runnable childStatement) { + fResults.add(fService.submit(new Callable<Object>() { + @Override + public Object call() throws Exception { + childStatement.run(); + return null; + } + })); + } + + @Override + public void finished() { + for (Future<Object> each : fResults) { + try { + each.get(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/uitest/src/com/vaadin/tests/tb3/PrivateTB3Configuration.java b/uitest/src/com/vaadin/tests/tb3/PrivateTB3Configuration.java new file mode 100644 index 0000000000..3d7dead928 --- /dev/null +++ b/uitest/src/com/vaadin/tests/tb3/PrivateTB3Configuration.java @@ -0,0 +1,123 @@ +/* + * Copyright 2000-2013 Vaadind Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.vaadin.tests.tb3; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.Enumeration; +import java.util.Properties; + +/** + * Provides values for parameters which depend on where the test is run. + * Parameters should be configured in work/eclipse-run-selected-test.properties. + * A template is available in uitest/. + * + * @author Vaadin Ltd + */ +public abstract class PrivateTB3Configuration extends ScreenshotTB3Test { + private static final String HOSTNAME_PROPERTY = "com.vaadin.testbench.deployment.hostname"; + private final Properties properties = new Properties(); + + public PrivateTB3Configuration() { + File file = new File("work", "eclipse-run-selected-test.properties"); + if (file.exists()) { + try { + properties.load(new FileInputStream(file)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + private String getProperty(String name) { + String property = properties.getProperty(name); + if (property == null) { + property = System.getProperty(name); + } + + return property; + } + + @Override + protected String getScreenshotDirectory() { + String screenshotDirectory = getProperty("com.vaadin.testbench.screenshot.directory"); + if (screenshotDirectory == null) { + throw new RuntimeException( + "No screenshot directory defined. Use -Dcom.vaadin.testbench.screenshot.directory=<path>"); + } + return screenshotDirectory; + } + + @Override + protected String getHubHostname() { + return "tb3-hub.intra.itmill.com"; + } + + @Override + protected String getDeploymentHostname() { + String hostName = getProperty(HOSTNAME_PROPERTY); + + if (hostName == null || "".equals(hostName)) { + hostName = findAutoHostname(); + } + + return hostName; + } + + /** + * Tries to automatically determine the IP address of the machine the test + * is running on. + * + * @return An IP address of one of the network interfaces in the machine. + * @throws RuntimeException + * if there was an error or no IP was found + */ + private String findAutoHostname() { + try { + Enumeration<NetworkInterface> interfaces = NetworkInterface + .getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + NetworkInterface current = interfaces.nextElement(); + if (!current.isUp() || current.isLoopback() + || current.isVirtual()) { + continue; + } + Enumeration<InetAddress> addresses = current.getInetAddresses(); + while (addresses.hasMoreElements()) { + InetAddress current_addr = addresses.nextElement(); + if (current_addr.isLoopbackAddress()) { + continue; + } + String hostAddress = current_addr.getHostAddress(); + if (hostAddress.startsWith("192.168.")) { + return hostAddress; + } + } + } + } catch (SocketException e) { + throw new RuntimeException("Could not enumerate "); + } + + throw new RuntimeException( + "No compatible (192.168.*) ip address found."); + } + +} diff --git a/uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java b/uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java new file mode 100644 index 0000000000..645d9cd0cb --- /dev/null +++ b/uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java @@ -0,0 +1,392 @@ +/* + * Copyright 2000-2013 Vaadin Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.vaadin.tests.tb3; + +import java.awt.image.BufferedImage; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import javax.imageio.ImageIO; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.rules.TestWatcher; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.remote.DesiredCapabilities; + +import com.vaadin.testbench.Parameters; +import com.vaadin.testbench.commands.TestBenchCommands; + +/** + * Base class which provides functionality for tests which use the automatic + * screenshot comparison function. + * + * @author Vaadin Ltd + */ +public abstract class ScreenshotTB3Test extends AbstractTB3Test { + + private String screenshotBaseName; + + @Rule + public TestRule watcher = new TestWatcher() { + + @Override + protected void starting(org.junit.runner.Description description) { + Class<?> testClass = description.getTestClass(); + // Runner adds [BrowserName] which we do not want to use in the + // screenshot name + String testMethod = description.getMethodName(); + testMethod = testMethod.replaceAll("\\[.*\\]", ""); + + String className = testClass.getSimpleName(); + if (testClass.getEnclosingClass() != null) { + className = testClass.getEnclosingClass().getSimpleName(); + } + + screenshotBaseName = className + "-" + testMethod; + }; + }; + + /** + * Contains a list of screenshot identifiers for which + * {@link #compareScreen(String)} has failed during the test + */ + private List<String> screenshotFailures = new ArrayList<String>(); + + /** + * Defines TestBench screen comparison parameters before each test run + */ + @Before + public void setupScreenComparisonParameters() { + Parameters.setScreenshotErrorDirectory(getScreenshotErrorDirectory()); + Parameters + .setScreenshotReferenceDirectory(getScreenshotReferenceDirectory()); + } + + /** + * Grabs a screenshot and compares with the reference image with the given + * identifier. Supports alternative references and will succeed if the + * screenshot matches at least one of the references. + * + * In case of a failed comparison this method stores the grabbed screenshots + * in the error directory as defined by + * {@link #getScreenshotErrorDirectory()}. It will also generate a html file + * in the same directory, comparing the screenshot with the first found + * reference. + * + * @param identifier + * @throws IOException + */ + protected void compareScreen(String identifier) throws IOException { + if (identifier == null || identifier.isEmpty()) { + throw new IllegalArgumentException("Empty identifier not supported"); + } + + File mainReference = getScreenshotReferenceFile(identifier); + + List<File> alternativeFiles = findReferenceAlternatives(mainReference); + List<File> failedReferenceAlternatives = new ArrayList<File>(); + + for (File file : alternativeFiles) { + if (testBench(driver).compareScreen(file)) { + break; + } else { + failedReferenceAlternatives.add(file); + } + } + + File referenceToKeep = null; + if (failedReferenceAlternatives.size() != alternativeFiles.size()) { + // Matched one comparison but not all, remove all error images + + // HTML files + } else { + // All comparisons failed, keep the main error image + HTML + screenshotFailures.add(mainReference.getName()); + referenceToKeep = mainReference; + } + + // Remove all PNG/HTML files we no longer need (failed alternative + // references or all error files (PNG/HTML) if comparison succeeded) + for (File failedAlternative : failedReferenceAlternatives) { + File failurePng = getErrorFileFromReference(failedAlternative); + if (failedAlternative != referenceToKeep) { + // Delete png + HTML + String htmlFileName = failurePng.getName().replace(".png", + ".html"); + File failureHtml = new File(failurePng.getParentFile(), + htmlFileName); + + failurePng.delete(); + failureHtml.delete(); + } + } + } + + /** + * + * @param referenceFile + * The reference image file (in the directory defined by + * {@link #getScreenshotReferenceDirectory()}) + * @return the file name of the file generated in the directory defined by + * {@link #getScreenshotErrorDirectory()} if comparison with the + * given reference image fails. + */ + private File getErrorFileFromReference(File referenceFile) { + return new File(referenceFile.getAbsolutePath().replace( + getScreenshotReferenceDirectory(), + getScreenshotErrorDirectory())); + } + + /** + * Finds alternative references for the given files + * + * @param reference + * @return all references which should be considered when comparing with the + * given files, including the given reference + */ + private List<File> findReferenceAlternatives(File reference) { + List<File> files = new ArrayList<File>(); + files.add(reference); + + File screenshotDir = reference.getParentFile(); + String name = reference.getName(); + // Remove ".png" + String nameBase = name.substring(0, name.length() - 4); + for (int i = 1;; i++) { + File file = new File(screenshotDir, nameBase + "_" + i + ".png"); + if (file.exists()) { + files.add(file); + } else { + break; + } + } + + return files; + } + + /** + * @param testName + * @return the reference file name to use for the given browser, as + * described by {@literal capabilities}, and identifier + */ + private File getScreenshotReferenceFile(String identifier) { + DesiredCapabilities capabilities = getDesiredCapabilities(); + + String originalName = getScreenshotReferenceName(identifier); + File exactVersionFile = new File(originalName); + if (exactVersionFile.exists()) { + return exactVersionFile; + } + + String browserVersion = capabilities.getVersion(); + + if (browserVersion.matches("\\d+")) { + for (int version = Integer.parseInt(browserVersion); version > 0; version--) { + String fileName = getScreenshotReferenceName(identifier, + version); + File oldVersionFile = new File(fileName); + if (oldVersionFile.exists()) { + return oldVersionFile; + } + } + } + + return exactVersionFile; + } + + /** + * @return the base directory of 'reference' and 'errors' screenshots + */ + protected abstract String getScreenshotDirectory(); + + /** + * @return the directory where reference images are stored (the 'reference' + * folder inside the screenshot directory) + */ + private String getScreenshotReferenceDirectory() { + return getScreenshotDirectory() + "/reference"; + } + + /** + * @return the directory where comparison error images should be created + * (the 'errors' folder inside the screenshot directory) + */ + private String getScreenshotErrorDirectory() { + return getScreenshotDirectory() + "/errors"; + } + + /** + * Checks if any screenshot comparisons failures occurred during the test + * and combines all comparison errors into one exception + * + * @throws IOException + * If there were failures during the test + */ + @After + public void checkCompareFailures() throws IOException { + if (!screenshotFailures.isEmpty()) { + throw new IOException( + "The following screenshots did not match the reference: " + + screenshotFailures.toString()); + } + + } + + /* + * (non-Javadoc) + * + * @see + * com.vaadin.tests.tb3.AbstractTB3Test#onUncaughtException(java.lang.Throwable + * ) + */ + @Override + public void onUncaughtException(Throwable cause) { + super.onUncaughtException(cause); + // Grab a "failure" screenshot and store in the errors folder for later + // analysis + try { + TestBenchCommands testBench = testBench(); + if (testBench != null) { + testBench.disableWaitForVaadin(); + } + } catch (Throwable t) { + t.printStackTrace(); + } + try { + if (driver != null) { + BufferedImage screenshotImage = ImageIO + .read(new ByteArrayInputStream( + ((TakesScreenshot) driver) + .getScreenshotAs(OutputType.BYTES))); + ImageIO.write(screenshotImage, "png", new File( + getScreenshotFailureName())); + } + } catch (Throwable t) { + t.printStackTrace(); + } + + } + + /** + * @return the name of a "failure" image which is stored in the folder + * defined by {@link #getScreenshotErrorDirectory()} when the test + * fails + */ + private String getScreenshotFailureName() { + return getScreenshotErrorBaseName() + "-failure.png"; + } + + /** + * @return the base name used for screenshots. This is the first part of the + * screenshot file name, typically created as "testclass-testmethod" + */ + public String getScreenshotBaseName() { + return screenshotBaseName; + } + + /** + * Returns the name of the reference file based on the given parameters. + * + * @param testName + * @param capabilities + * @param identifier + * @return the full path of the reference + */ + private String getScreenshotReferenceName(String identifier) { + return getScreenshotReferenceName(identifier, null); + } + + /** + * Returns the name of the reference file based on the given parameters. The + * version given in {@literal capabilities} is used unless it is overridden + * by the {@literal versionOverride} parameter. + * + * @param testName + * @param capabilities + * @param identifier + * @return the full path of the reference + */ + private String getScreenshotReferenceName(String identifier, + Integer versionOverride) { + String uniqueBrowserIdentifier; + if (versionOverride == null) { + uniqueBrowserIdentifier = BrowserUtil + .getUniqueIdentifier(getDesiredCapabilities()); + } else { + uniqueBrowserIdentifier = BrowserUtil.getUniqueIdentifier( + getDesiredCapabilities(), "" + versionOverride); + } + + // WindowMaximizeRestoreTest_Windows_InternetExplorer_8_window-1-moved-maximized-restored.png + return getScreenshotReferenceDirectory() + "/" + + getScreenshotBaseName() + "_" + uniqueBrowserIdentifier + "_" + + identifier + ".png"; + } + + /** + * Returns the base name of the screenshot in the error directory. This is a + * name so that all files matching {@link #getScreenshotErrorBaseName()}* + * are owned by this test instance (taking into account + * {@link #getDesiredCapabilities()}) and can safely be removed before + * running this test. + */ + private String getScreenshotErrorBaseName() { + return getScreenshotReferenceName("dummy", null).replace( + getScreenshotReferenceDirectory(), + getScreenshotErrorDirectory()).replace("_dummy.png", ""); + } + + /** + * Removes any old screenshots related to this test from the errors + * directory before running the test + */ + @Before + public void cleanErrorDirectory() { + // Remove any screenshots for this test from the error directory + // before running it. Leave unrelated files as-is + File errorDirectory = new File(getScreenshotErrorDirectory()); + + // Create errors directory if it does not exist + if (!errorDirectory.exists()) { + errorDirectory.mkdirs(); + } + + final String errorBase = getScreenshotErrorBaseName() + .replace("\\", "/"); + File[] files = errorDirectory.listFiles(new FileFilter() { + + @Override + public boolean accept(File pathname) { + String thisFile = pathname.getAbsolutePath().replace("\\", "/"); + if (thisFile.startsWith(errorBase)) { + return true; + } + return false; + } + }); + for (File f : files) { + f.delete(); + } + } +} diff --git a/uitest/src/com/vaadin/tests/tb3/SimpleMultiBrowserTest.java b/uitest/src/com/vaadin/tests/tb3/SimpleMultiBrowserTest.java new file mode 100644 index 0000000000..a7ade3f9f7 --- /dev/null +++ b/uitest/src/com/vaadin/tests/tb3/SimpleMultiBrowserTest.java @@ -0,0 +1,49 @@ +/* + * Copyright 2000-2013 Vaadin Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * + */ +package com.vaadin.tests.tb3; + +import org.junit.Test; + +/** + * A simple version of {@link MultiBrowserTest} which allows only one test + * method ({@link #test()}). Uses only the enclosing class name as test + * identifier (i.e. excludes "-test"). + * + * This class is only provided as a helper for converting existing TB2 tests + * without renaming all screenshots. All new TB3+ tests should extend + * {@link MultiBrowserTest} directly instead of this. + * + * @author Vaadin Ltd + */ +public abstract class SimpleMultiBrowserTest extends MultiBrowserTest { + + @Test + public abstract void test() throws Exception; + + /* + * (non-Javadoc) + * + * @see com.vaadin.tests.tb3.ScreenshotTB3Test#getScreenshotBaseName() + */ + @Override + public String getScreenshotBaseName() { + return super.getScreenshotBaseName().replaceFirst("-test$", ""); + } +} diff --git a/uitest/src/com/vaadin/tests/tb3/TB3Runner.java b/uitest/src/com/vaadin/tests/tb3/TB3Runner.java new file mode 100644 index 0000000000..510d200ffa --- /dev/null +++ b/uitest/src/com/vaadin/tests/tb3/TB3Runner.java @@ -0,0 +1,167 @@ +/* + * Copyright 2000-2013 Vaadin Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.vaadin.tests.tb3; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.junit.Test; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.Statement; +import org.openqa.selenium.remote.DesiredCapabilities; + +import com.vaadin.tests.tb3.AbstractTB3Test.BrowserUtil; + +/** + * This runner is loosely based on FactoryTestRunner by Ted Young + * (http://tedyoung.me/2011/01/23/junit-runtime-tests-custom-runners/). The + * generated test names give information about the parameters used (unlike + * {@link Parameterized}). + * + * @since 7.1 + */ +public class TB3Runner extends BlockJUnit4ClassRunner { + + public TB3Runner(Class<?> klass) throws InitializationError { + super(klass); + setScheduler(new ParallelScheduler()); + } + + @Override + protected List<FrameworkMethod> computeTestMethods() { + List<FrameworkMethod> tests = new LinkedList<FrameworkMethod>(); + + // Find all methods in our test class marked with @Parameters. + for (FrameworkMethod method : getTestClass().getAnnotatedMethods( + Parameters.class)) { + // Make sure the Parameters method is static + if (!Modifier.isStatic(method.getMethod().getModifiers())) { + throw new IllegalArgumentException("@Parameters " + method + + " must be static."); + } + + // Execute the method (statically) + Object params; + try { + params = method.getMethod().invoke( + getTestClass().getJavaClass()); + } catch (Throwable t) { + throw new RuntimeException("Could not run test factory method " + + method.getName(), t); + } + + // Did the factory return an array? If so, make it a list. + if (params.getClass().isArray()) { + params = Arrays.asList((Object[]) params); + } + + // Did the factory return a scalar object? If so, put it in a list. + if (!(params instanceof Iterable<?>)) { + params = Collections.singletonList(params); + } + + // For each object returned by the factory. + for (Object param : (Iterable<?>) params) { + if (!(param instanceof DesiredCapabilities)) { + throw new RuntimeException("Unexpected parameter type " + + param.getClass().getName() + + " when expecting DesiredCapabilities"); + } + DesiredCapabilities capabilities = (DesiredCapabilities) param; + // Find any methods marked with @Test. + for (FrameworkMethod m : getTestClass().getAnnotatedMethods( + Test.class)) { + tests.add(new TB3Method(m.getMethod(), capabilities)); + } + } + } + + return tests; + } + + /* + * (non-Javadoc) + * + * @see + * org.junit.runners.BlockJUnit4ClassRunner#withBefores(org.junit.runners + * .model.FrameworkMethod, java.lang.Object, + * org.junit.runners.model.Statement) + */ + @Override + protected Statement withBefores(final FrameworkMethod method, + final Object target, Statement statement) { + if (!(method instanceof TB3Method)) { + throw new RuntimeException("Unexpected method type " + + method.getClass().getName() + ", expected TB3Method"); + } + final TB3Method tb3method = (TB3Method) method; + + // setDesiredCapabilities before running the real @Befores (which use + // capabilities) + + final Statement realBefores = super.withBefores(method, target, + statement); + return new Statement() { + + @Override + public void evaluate() throws Throwable { + ((AbstractTB3Test) target) + .setDesiredCapabilities(tb3method.capabilities); + try { + realBefores.evaluate(); + } catch (Throwable t) { + // Give the test a chance to e.g. produce an error + // screenshot before failing the test by re-throwing the + // exception + ((AbstractTB3Test) target).onUncaughtException(t); + throw t; + } + } + }; + } + + private static class TB3Method extends FrameworkMethod { + private DesiredCapabilities capabilities; + + public TB3Method(Method method, DesiredCapabilities capabilities) { + super(method); + this.capabilities = capabilities; + } + + @Override + public Object invokeExplosively(final Object target, Object... params) + throws Throwable { + // Executes the test method with the supplied parameters + return super.invokeExplosively(target); + } + + @Override + public String getName() { + return String.format("%s[%s]", getMethod().getName(), + BrowserUtil.getUniqueIdentifier(capabilities)); + } + + } +} |