Use new TestBench and ReferenceNameGenerator (#10963)

This patch contains the first part of screenshots being renamed.

4 files changed, 47 insertions, 426 deletions

diff --git a/uitest/src/test/java/com/vaadin/tests/components/table/TableTooManyColumnsTest.java b/uitest/src/test/java/com/vaadin/tests/components/table/TableTooManyColumnsTest.java
index f5e078ab9f..cd28863d2d 100644
--- a/uitest/src/test/java/com/vaadin/tests/components/table/TableTooManyColumnsTest.java
+++ b/uitest/src/test/java/com/vaadin/tests/components/table/TableTooManyColumnsTest.java
@@ -25,7 +25,7 @@ public class TableTooManyColumnsTest extends MultiBrowserTest {
         TestBenchElementCommands scrollable = testBenchElement(menu);
         scrollable.scroll(3000);
 
-        compareScreen(getScreenshotBaseName());
+        compareScreen("init");
     }
 
 }
diff --git a/uitest/src/test/java/com/vaadin/tests/components/upload/UploadTitleWithTooltipTest.java b/uitest/src/test/java/com/vaadin/tests/components/upload/UploadTitleWithTooltipTest.java
index 250bfca8e6..be61634c28 100644
--- a/uitest/src/test/java/com/vaadin/tests/components/upload/UploadTitleWithTooltipTest.java
+++ b/uitest/src/test/java/com/vaadin/tests/components/upload/UploadTitleWithTooltipTest.java
@@ -33,7 +33,7 @@ public class UploadTitleWithTooltipTest extends TooltipTest {
 
         checkTooltip(input, "tooltip");
 
-        compareScreen(getScreenshotBaseName());
+        compareScreen("init");
     }
 
 }
diff --git a/uitest/src/test/java/com/vaadin/tests/tb3/AbstractTB3Test.java b/uitest/src/test/java/com/vaadin/tests/tb3/AbstractTB3Test.java
index a18573fc04..5e6f866b7f 100644
--- a/uitest/src/test/java/com/vaadin/tests/tb3/AbstractTB3Test.java
+++ b/uitest/src/test/java/com/vaadin/tests/tb3/AbstractTB3Test.java
@@ -25,6 +25,7 @@ import org.apache.http.HttpResponse;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.message.BasicHttpEntityEnclosingRequest;
 import org.junit.Rule;
+import org.junit.rules.ExternalResource;
 import org.junit.rules.TestName;
 import org.junit.runner.RunWith;
 import org.openqa.selenium.By;
@@ -45,7 +46,6 @@ import org.openqa.selenium.remote.HttpCommandExecutor;
 import org.openqa.selenium.remote.RemoteWebDriver;
 import org.openqa.selenium.support.ui.ExpectedCondition;
 import org.openqa.selenium.support.ui.ExpectedConditions;
-import org.openqa.selenium.support.ui.WebDriverWait;
 
 import com.vaadin.server.LegacyApplication;
 import com.vaadin.server.UIProvider;
@@ -87,6 +87,17 @@ public abstract class AbstractTB3Test extends ParallelTest {
     @Rule
     public TestName testName = new TestName();
 
+    /*
+     * Rule for closing the application after test is done.
+     */
+    @Rule
+    public ExternalResource rule = new ExternalResource() {
+        @Override
+        protected void after() {
+            closeApplication();
+        }
+    };
+
     /**
      * Height of the screenshots we want to capture
      */
diff --git a/uitest/src/test/java/com/vaadin/tests/tb3/ScreenshotTB3Test.java b/uitest/src/test/java/com/vaadin/tests/tb3/ScreenshotTB3Test.java
index 9a14fccb4a..455930bd14 100644
--- a/uitest/src/test/java/com/vaadin/tests/tb3/ScreenshotTB3Test.java
+++ b/uitest/src/test/java/com/vaadin/tests/tb3/ScreenshotTB3Test.java
@@ -1,25 +1,17 @@
 package com.vaadin.tests.tb3;
 
-import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.commons.io.FileUtils;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TestRule;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
-import org.openqa.selenium.Platform;
+import org.openqa.selenium.HasCapabilities;
 import org.openqa.selenium.WebDriver;
 import org.openqa.selenium.WebElement;
-import org.openqa.selenium.remote.DesiredCapabilities;
 
-import com.vaadin.testbench.Parameters;
-import com.vaadin.testbench.ScreenshotOnFailureRule;
+import com.vaadin.testbench.TestBenchElement;
 import com.vaadin.testbench.screenshot.ImageFileUtil;
 
 /**
@@ -30,59 +22,32 @@ import com.vaadin.testbench.screenshot.ImageFileUtil;
  */
 public abstract class ScreenshotTB3Test extends AbstractTB3Test {
 
-    @Rule
-    public ScreenshotOnFailureRule screenshotOnFailure = new ScreenshotOnFailureRule(
-            this, true) {
-
-        @Override
-        protected void failed(Throwable throwable, Description description) {
-            super.failed(throwable, description);
-            closeApplication();
-        }
-
-        @Override
-        protected void succeeded(Description description) {
-            super.succeeded(description);
-            closeApplication();
-        }
-
-        @Override
-        protected File getErrorScreenshotFile(Description description) {
-            return ImageFileUtil
-                    .getErrorScreenshotFile(getScreenshotFailureName());
-        };
-    };
-
-    private String screenshotBaseName;
-
-    @Rule
-    public TestRule watcher = new TestWatcher() {
-
-        @Override
-        protected void starting(org.junit.runner.Description description) {
-            Class<?> testClass = description.getTestClass();
-            // Runner adds [BrowserName] which we do not want to use in the
-            // screenshot name
-            String testMethod = description.getMethodName();
-            testMethod = testMethod.replaceAll("\\[.*\\]", "");
-
-            String className = testClass.getSimpleName();
-            screenshotBaseName = className + "-" + testMethod;
-        }
-    };
-
     /**
      * Contains a list of screenshot identifiers for which
      * {@link #compareScreen(String)} has failed during the test
      */
-    private List<String> screenshotFailures;
+    private List<String> screenshotFailures = new ArrayList<>();
 
-    /**
-     * Defines TestBench screen comparison parameters before each test run
-     */
-    @Before
-    public void setupScreenComparisonParameters() {
-        screenshotFailures = new ArrayList<>();
+    @Override
+    public void setDriver(WebDriver driver) {
+        super.setDriver(driver);
+
+        // After setting up the driver, we can set the reference name generator.
+        String testClassName = getClass().getSimpleName();
+        testBench().setReferenceNameGenerator((identifier, capabilities) -> {
+            // Make sure error screenshot directory exists.
+            String browserFolder = capabilities.getBrowserName().toLowerCase()
+                    .replaceAll(" ", "");
+            Paths.get(ImageFileUtil.getScreenshotErrorDirectory(),
+                    browserFolder).toFile().mkdirs();
+            return Paths
+                    .get(browserFolder,
+                            String.format("%s-%s%s", testClassName,
+                                    testName.getMethodName().replace('[', '_')
+                                            .replace(']', '_'),
+                                    identifier))
+                    .toString();
+        });
     }
 
     /**
@@ -100,213 +65,20 @@ public abstract class ScreenshotTB3Test extends AbstractTB3Test {
      * @throws IOException
      */
     protected void compareScreen(String identifier) throws IOException {
-        compareScreen(null, identifier);
+        if (!testBench().compareScreen(identifier)) {
+            screenshotFailures.add(testBench().getReferenceNameGenerator()
+                    .generateName(identifier,
+                            ((HasCapabilities) getDriver()).getCapabilities()));
+        }
     }
 
     protected void compareScreen(WebElement element, String identifier)
             throws IOException {
-        if (identifier == null || identifier.isEmpty()) {
-            throw new IllegalArgumentException(
-                    "Empty identifier not supported");
-        }
-
-        File mainReference = getScreenshotReferenceFile(identifier);
-
-        List<File> referenceFiles = findReferenceAndAlternatives(mainReference);
-        List<File> failedReferenceFiles = new ArrayList<>();
-
-        for (File referenceFile : referenceFiles) {
-            boolean match = false;
-            if (element == null) {
-                // Full screen
-                match = testBench(driver).compareScreen(referenceFile);
-            } else {
-                // Only the element
-                match = customTestBench.compareScreen(element, referenceFile);
-            }
-            if (match) {
-                // There might be failure files because of retries in TestBench.
-                deleteFailureFiles(getErrorFileFromReference(referenceFile));
-                break;
-            } else {
-                failedReferenceFiles.add(referenceFile);
-            }
-        }
-
-        File referenceToKeep = null;
-        if (failedReferenceFiles.size() == referenceFiles.size()) {
-            // Ensure we use the correct browser version (e.g. if running IE11
-            // and only an IE 10 reference was available, then mainReference
-            // will be for IE 10, not 11)
-            String originalName = getScreenshotReferenceName(identifier);
-            File exactVersionFile = new File(originalName);
-
-            if (!exactVersionFile.equals(mainReference)) {
-                // Rename png+html to have the correct version
-                File correctPng = getErrorFileFromReference(exactVersionFile);
-                File producedPng = getErrorFileFromReference(mainReference);
-                File correctHtml = htmlFromPng(correctPng);
-                File producedHtml = htmlFromPng(producedPng);
-
-                producedPng.renameTo(correctPng);
-                producedHtml.renameTo(correctHtml);
-                referenceToKeep = exactVersionFile;
-                screenshotFailures.add(exactVersionFile.getName());
-            } else {
-                // All comparisons failed, keep the main error image + HTML
-                screenshotFailures.add(mainReference.getName());
-                referenceToKeep = mainReference;
-            }
-        }
-
-        // Remove all PNG/HTML files we no longer need (failed alternative
-        // references or all error files (PNG/HTML) if comparison succeeded)
-        for (File failedAlternative : failedReferenceFiles) {
-            File failurePng = getErrorFileFromReference(failedAlternative);
-            if (failedAlternative != referenceToKeep) {
-                // Delete png + HTML
-                deleteFailureFiles(failurePng);
-            }
-        }
-        if (referenceToKeep != null) {
-            File errorPng = getErrorFileFromReference(referenceToKeep);
-            enableAutoswitch(new File(errorPng.getParentFile(),
-                    errorPng.getName() + ".html"));
-        }
-    }
-
-    private CustomTestBenchCommandExecutor customTestBench = null;
-
-    @Override
-    public void setDriver(WebDriver driver) {
-        super.setDriver(driver);
-
-        // Set custom command executor
-        customTestBench = new CustomTestBenchCommandExecutor(getDriver());
-    }
-
-    private void enableAutoswitch(File htmlFile)
-            throws FileNotFoundException, IOException {
-        if (htmlFile == null || !htmlFile.exists()) {
-            return;
+        if (!((TestBenchElement) element).compareScreen(identifier)) {
+            screenshotFailures.add(testBench().getReferenceNameGenerator()
+                    .generateName(identifier,
+                            ((HasCapabilities) getDriver()).getCapabilities()));
         }
-
-        String html = FileUtils.readFileToString(htmlFile);
-
-        html = html.replace("body onclick=\"",
-                "body onclick=\"clearInterval(autoSwitch);");
-        html = html.replace("</script>",
-                ";autoSwitch=setInterval(switchImage,500);</script>");
-
-        FileUtils.writeStringToFile(htmlFile, html);
-    }
-
-    private void deleteFailureFiles(File failurePng) {
-        File failureHtml = htmlFromPng(failurePng);
-
-        failurePng.delete();
-        failureHtml.delete();
-    }
-
-    /**
-     * Returns a new File which points to a .html file instead of the given .png
-     * file
-     *
-     * @param png
-     * @return
-     */
-    private static File htmlFromPng(File png) {
-        return new File(png.getParentFile(),
-                png.getName().replaceAll("\\.png$", ".png.html"));
-    }
-
-    /**
-     *
-     * @param referenceFile
-     *            The reference image file (in the directory defined by
-     *            {@link #getScreenshotReferenceDirectory()})
-     * @return the file name of the file generated in the directory defined by
-     *         {@link #getScreenshotErrorDirectory()} if comparison with the
-     *         given reference image fails.
-     */
-    private File getErrorFileFromReference(File referenceFile) {
-
-        String absolutePath = referenceFile.getAbsolutePath();
-        String screenshotReferenceDirectory = Parameters
-                .getScreenshotReferenceDirectory();
-        String screenshotErrorDirectory = Parameters
-                .getScreenshotErrorDirectory();
-        // We throw an exception to safeguard against accidental reference
-        // deletion. See (#14446)
-        if (!absolutePath.contains(screenshotReferenceDirectory)) {
-            throw new IllegalStateException(
-                    "Reference screenshot not in reference directory. Screenshot path: '"
-                            + absolutePath + "', directory path: '"
-                            + screenshotReferenceDirectory + "'");
-        }
-        return new File(absolutePath.replace(screenshotReferenceDirectory,
-                screenshotErrorDirectory));
-    }
-
-    /**
-     * Finds alternative references for the given files
-     *
-     * @param reference
-     * @return all references which should be considered when comparing with the
-     *         given files, including the given reference
-     */
-    private List<File> findReferenceAndAlternatives(File reference) {
-        List<File> files = new ArrayList<>();
-        files.add(reference);
-
-        File screenshotDir = reference.getParentFile();
-        String name = reference.getName();
-        // Remove ".png"
-        String nameBase = name.substring(0, name.length() - 4);
-        for (int i = 1;; i++) {
-            File file = new File(screenshotDir, nameBase + "_" + i + ".png");
-            if (file.exists()) {
-                files.add(file);
-            } else {
-                break;
-            }
-        }
-
-        return files;
-    }
-
-    /**
-     * @param testName
-     * @return the reference file name to use for the given browser, as
-     *         described by {@literal capabilities}, and identifier
-     */
-    private File getScreenshotReferenceFile(String identifier) {
-        DesiredCapabilities capabilities = getDesiredCapabilities();
-
-        String originalName = getScreenshotReferenceName(identifier);
-        File exactVersionFile = new File(originalName);
-        if (exactVersionFile.exists()) {
-            return exactVersionFile;
-        }
-
-        String browserVersion = capabilities.getVersion();
-
-        // compare against screenshots for this version and older
-        // default such that if no particular version is requested, compare with
-        // any version
-        int maxVersion = 100;
-        if (browserVersion.matches("\\d+")) {
-            maxVersion = Integer.parseInt(browserVersion);
-        }
-        for (int version = maxVersion; version > 0; version--) {
-            String fileName = getScreenshotReferenceName(identifier, version);
-            File oldVersionFile = new File(fileName);
-            if (oldVersionFile.exists()) {
-                return oldVersionFile;
-            }
-        }
-
-        return exactVersionFile;
     }
 
     /**
@@ -323,167 +95,5 @@ public abstract class ScreenshotTB3Test extends AbstractTB3Test {
                     "The following screenshots did not match the reference: "
                             + screenshotFailures.toString());
         }
-
-    }
-
-    /**
-     * @return the name of a "failure" image which is stored in the folder
-     *         defined by {@link #getScreenshotErrorDirectory()} when the test
-     *         fails
-     */
-    private String getScreenshotFailureName() {
-        return getScreenshotBaseName() + "_" + getUniqueIdentifier(null, null)
-                + "-failure.png";
-    }
-
-    /**
-     * @return the base name used for screenshots. This is the first part of the
-     *         screenshot file name, typically created as "testclass-testmethod"
-     */
-    public String getScreenshotBaseName() {
-        return screenshotBaseName;
-    }
-
-    /**
-     * Returns the name of the reference file based on the given parameters.
-     *
-     * @param testName
-     * @param capabilities
-     * @param identifier
-     * @return the full path of the reference
-     */
-    private String getScreenshotReferenceName(String identifier) {
-        return getScreenshotReferenceName(identifier, null);
-    }
-
-    /**
-     * Returns the name of the reference file based on the given parameters.
-     * This method takes in a {@code versionOverride} parameter to find a
-     * specific version in the reference name. If the {@link Platform} defined
-     * in the {@link DesiredCapabilities} is {@code ANY}, this method will
-     * attempt different platforms in the reference file names if needed.
-     *
-     * @param identifier
-     * @param versionOverride
-     * @return the full path of the reference
-     */
-    private String getScreenshotReferenceName(String identifier,
-            Integer versionOverride) {
-        String fileName = getScreenshotReferenceName(identifier,
-                versionOverride, null);
-        File refFile = new File(fileName);
-        if (!refFile.exists()
-                && getDesiredCapabilities().getPlatform() == Platform.ANY) {
-            for (Platform p : Platform.values()) {
-                String tmpName = getScreenshotReferenceName(identifier,
-                        versionOverride, p);
-                if (new File(tmpName).exists()) {
-                    return tmpName;
-                }
-            }
-        }
-        return fileName;
-    }
-
-    /**
-     * Returns the name of the reference file based on the given parameters.This
-     * method takes in {@code versionOverride} and {@code platformOverride}
-     * parameters.
-     *
-     * @param identifier
-     * @param versionOverride
-     * @param platformOverride
-     * @return the full path of the reference
-     */
-    private String getScreenshotReferenceName(String identifier,
-            Integer versionOverride, Platform platformOverride) {
-        return Parameters.getScreenshotReferenceDirectory() + File.separator
-                + getScreenshotBaseName() + "_"
-                + getUniqueIdentifier(versionOverride, platformOverride) + "_"
-                + identifier + ".png";
-    }
-
-    private String getUniqueIdentifier(Integer versionOverride,
-            Platform platformOverride) {
-        String testNameAndParameters = testName.getMethodName();
-        // runTest-wildfly9-nginx[Windows_Firefox_24][/buffering/demo][valo]
-
-        String parameters = testNameAndParameters.substring(
-                testNameAndParameters.indexOf("[") + 1,
-                testNameAndParameters.length() - 1);
-        // Windows_Firefox_24][/buffering/demo][valo
-
-        parameters = parameters.replace("][", "_");
-        // Windows_Firefox_24_/buffering/demo_valo
-
-        parameters = parameters.replace("/", "");
-        // Windows_Firefox_24_bufferingdemo_valo
-
-        if (versionOverride != null) {
-            // Windows_Firefox_17_bufferingdemo_valo
-            int indexOfBrowser = parameters.indexOf("_") + 1;
-            parameters = parameters.substring(0, indexOfBrowser)
-                    + parameters.substring(indexOfBrowser).replaceFirst(
-                            "_" + getDesiredCapabilities().getVersion(),
-                            "_" + versionOverride);
-        }
-
-        if (platformOverride != null) {
-            // LINUX_Firefox_17_bufferingdemo_valo
-            parameters = getPlatformName(platformOverride)
-                    + parameters.substring(parameters.indexOf("_"));
-        }
-
-        return parameters;
-    }
-
-    private String getPlatformName(Platform platform) {
-        switch (platform) {
-        case WINDOWS:
-            // Reference file names have Windows instead of WINDOWS
-            return "Windows";
-        default:
-            return platform.name();
-        }
-    }
-
-    /**
-     * Returns the base name of the screenshot in the error directory. This is a
-     * name so that all files matching {@link #getScreenshotErrorBaseName()}*
-     * are owned by this test instance (taking into account
-     * {@link #getDesiredCapabilities()}) and can safely be removed before
-     * running this test.
-     */
-    private String getScreenshotErrorBaseName() {
-        return getScreenshotReferenceName("dummy", null)
-                .replace(Parameters.getScreenshotReferenceDirectory(),
-                        Parameters.getScreenshotErrorDirectory())
-                .replace("_dummy.png", "");
-    }
-
-    /**
-     * Removes any old screenshots related to this test from the errors
-     * directory before running the test
-     */
-    @Before
-    public void cleanErrorDirectory() {
-        // Remove any screenshots for this test from the error directory
-        // before running it. Leave unrelated files as-is
-        File errorDirectory = new File(
-                Parameters.getScreenshotErrorDirectory());
-
-        // Create errors directory if it does not exist
-        if (!errorDirectory.exists()) {
-            errorDirectory.mkdirs();
-        }
-
-        final String errorBase = getScreenshotErrorBaseName();
-        File[] files = errorDirectory.listFiles(pathname -> {
-            String thisFile = pathname.getAbsolutePath();
-            return thisFile.startsWith(errorBase);
-        });
-        for (File f : files) {
-            f.delete();
-        }
     }
 }