vaadin-framework/uitest/src/test/java/com/vaadin/tests/tb3/ScreenshotTB3Test.java

/*
 * Copyright 2000-2016 Vaadin Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package com.vaadin.tests.tb3;

import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.rules.TestRule;
import org.junit.rules.TestWatcher;
import org.junit.runner.Description;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.remote.DesiredCapabilities;

import com.vaadin.testbench.Parameters;
import com.vaadin.testbench.ScreenshotOnFailureRule;
import com.vaadin.testbench.screenshot.ImageFileUtil;

/**
 * Base class which provides functionality for tests which use the automatic
 * screenshot comparison function.
 *
 * @author Vaadin Ltd
 */
public abstract class ScreenshotTB3Test extends AbstractTB3Test {

    @Rule
    public ScreenshotOnFailureRule screenshotOnFailure = new ScreenshotOnFailureRule(
            this, true) {

        @Override
        protected void failed(Throwable throwable, Description description) {
            super.failed(throwable, description);
            closeApplication();
        }

        @Override
        protected void succeeded(Description description) {
            super.succeeded(description);
            closeApplication();
        }

        @Override
        protected File getErrorScreenshotFile(Description description) {
            return ImageFileUtil
                    .getErrorScreenshotFile(getScreenshotFailureName());
        };
    };

    private String screenshotBaseName;

    @Rule
    public TestRule watcher = new TestWatcher() {

        @Override
        protected void starting(org.junit.runner.Description description) {
            Class<?> testClass = description.getTestClass();
            // Runner adds [BrowserName] which we do not want to use in the
            // screenshot name
            String testMethod = description.getMethodName();
            testMethod = testMethod.replaceAll("\\[.*\\]", "");

            String className = testClass.getSimpleName();
            screenshotBaseName = className + "-" + testMethod;
        }
    };

    /**
     * Contains a list of screenshot identifiers for which
     * {@link #compareScreen(String)} has failed during the test
     */
    private List<String> screenshotFailures;

    /**
     * Defines TestBench screen comparison parameters before each test run
     */
    @Before
    public void setupScreenComparisonParameters() {
        screenshotFailures = new ArrayList<>();

        Parameters.setScreenshotErrorDirectory(getScreenshotErrorDirectory());
        Parameters.setScreenshotReferenceDirectory(
                getScreenshotReferenceDirectory());
    }

    /**
     * Grabs a screenshot and compares with the reference image with the given
     * identifier. Supports alternative references and will succeed if the
     * screenshot matches at least one of the references.
     *
     * In case of a failed comparison this method stores the grabbed screenshots
     * in the error directory as defined by
     * {@link #getScreenshotErrorDirectory()}. It will also generate a html file
     * in the same directory, comparing the screenshot with the first found
     * reference.
     *
     * @param identifier
     * @throws IOException
     */
    protected void compareScreen(String identifier) throws IOException {
        compareScreen(null, identifier);
    }

    protected void compareScreen(WebElement element, String identifier)
            throws IOException {
        if (identifier == null || identifier.isEmpty()) {
            throw new IllegalArgumentException(
                    "Empty identifier not supported");
        }

        File mainReference = getScreenshotReferenceFile(identifier);

        List<File> referenceFiles = findReferenceAndAlternatives(mainReference);
        List<File> failedReferenceFiles = new ArrayList<>();

        for (File referenceFile : referenceFiles) {
            boolean match = false;
            if (element == null) {
                // Full screen
                match = testBench(driver).compareScreen(referenceFile);
            } else {
                // Only the element
                match = customTestBench(driver).compareScreen(element,
                        referenceFile);
            }
            if (match) {
                // There might be failure files because of retries in TestBench.
                deleteFailureFiles(getErrorFileFromReference(referenceFile));
                break;
            } else {
                failedReferenceFiles.add(referenceFile);
            }
        }

        File referenceToKeep = null;
        if (failedReferenceFiles.size() == referenceFiles.size()) {
            // Ensure we use the correct browser version (e.g. if running IE11
            // and only an IE 10 reference was available, then mainReference
            // will be for IE 10, not 11)
            String originalName = getScreenshotReferenceName(identifier);
            File exactVersionFile = new File(originalName);

            if (!exactVersionFile.equals(mainReference)) {
                // Rename png+html to have the correct version
                File correctPng = getErrorFileFromReference(exactVersionFile);
                File producedPng = getErrorFileFromReference(mainReference);
                File correctHtml = htmlFromPng(correctPng);
                File producedHtml = htmlFromPng(producedPng);

                producedPng.renameTo(correctPng);
                producedHtml.renameTo(correctHtml);
                referenceToKeep = exactVersionFile;
                screenshotFailures.add(exactVersionFile.getName());
            } else {
                // All comparisons failed, keep the main error image + HTML
                screenshotFailures.add(mainReference.getName());
                referenceToKeep = mainReference;
            }
        }

        // Remove all PNG/HTML files we no longer need (failed alternative
        // references or all error files (PNG/HTML) if comparison succeeded)
        for (File failedAlternative : failedReferenceFiles) {
            File failurePng = getErrorFileFromReference(failedAlternative);
            if (failedAlternative != referenceToKeep) {
                // Delete png + HTML
                deleteFailureFiles(failurePng);
            }
        }
        if (referenceToKeep != null) {
            File errorPng = getErrorFileFromReference(referenceToKeep);
            enableAutoswitch(new File(errorPng.getParentFile(),
                    errorPng.getName() + ".html"));
        }
    }

    private CustomTestBenchCommandExecutor customTestBench = null;

    private CustomTestBenchCommandExecutor customTestBench(WebDriver driver) {
        if (customTestBench == null) {
            customTestBench = new CustomTestBenchCommandExecutor(driver);
        }

        return customTestBench;
    }

    private void enableAutoswitch(File htmlFile)
            throws FileNotFoundException, IOException {
        if (htmlFile == null || !htmlFile.exists()) {
            return;
        }

        String html = FileUtils.readFileToString(htmlFile);

        html = html.replace("body onclick=\"",
                "body onclick=\"clearInterval(autoSwitch);");
        html = html.replace("</script>",
                ";autoSwitch=setInterval(switchImage,500);</script>");

        FileUtils.writeStringToFile(htmlFile, html);
    }

    private void deleteFailureFiles(File failurePng) {
        File failureHtml = htmlFromPng(failurePng);

        failurePng.delete();
        failureHtml.delete();
    }

    /**
     * Returns a new File which points to a .html file instead of the given .png
     * file
     *
     * @param png
     * @return
     */
    private static File htmlFromPng(File png) {
        return new File(png.getParentFile(),
                png.getName().replaceAll("\\.png$", ".png.html"));
    }

    /**
     *
     * @param referenceFile
     *            The reference image file (in the directory defined by
     *            {@link #getScreenshotReferenceDirectory()})
     * @return the file name of the file generated in the directory defined by
     *         {@link #getScreenshotErrorDirectory()} if comparison with the
     *         given reference image fails.
     */
    private File getErrorFileFromReference(File referenceFile) {

        String absolutePath = referenceFile.getAbsolutePath();
        String screenshotReferenceDirectory = getScreenshotReferenceDirectory();
        String screenshotErrorDirectory = getScreenshotErrorDirectory();
        // We throw an exception to safeguard against accidental reference
        // deletion. See (#14446)
        if (!absolutePath.contains(screenshotReferenceDirectory)) {
            throw new IllegalStateException(
                    "Reference screenshot not in reference directory. Screenshot path: '"
                            + absolutePath + "', directory path: '"
                            + screenshotReferenceDirectory + "'");
        }
        return new File(absolutePath.replace(screenshotReferenceDirectory,
                screenshotErrorDirectory));
    }

    /**
     * Finds alternative references for the given files
     *
     * @param reference
     * @return all references which should be considered when comparing with the
     *         given files, including the given reference
     */
    private List<File> findReferenceAndAlternatives(File reference) {
        List<File> files = new ArrayList<>();
        files.add(reference);

        File screenshotDir = reference.getParentFile();
        String name = reference.getName();
        // Remove ".png"
        String nameBase = name.substring(0, name.length() - 4);
        for (int i = 1;; i++) {
            File file = new File(screenshotDir, nameBase + "_" + i + ".png");
            if (file.exists()) {
                files.add(file);
            } else {
                break;
            }
        }

        return files;
    }

    /**
     * @param testName
     * @return the reference file name to use for the given browser, as
     *         described by {@literal capabilities}, and identifier
     */
    private File getScreenshotReferenceFile(String identifier) {
        DesiredCapabilities capabilities = getDesiredCapabilities();

        String originalName = getScreenshotReferenceName(identifier);
        File exactVersionFile = new File(originalName);
        if (exactVersionFile.exists()) {
            return exactVersionFile;
        }

        String browserVersion = capabilities.getVersion();

        // compare against screenshots for this version and older
        // default such that if no particular version is requested, compare with
        // any version
        int maxVersion = 100;
        if (browserVersion.matches("\\d+")) {
            maxVersion = Integer.parseInt(browserVersion);
        }
        for (int version = maxVersion; version > 0; version--) {
            String fileName = getScreenshotReferenceName(identifier, version);
            File oldVersionFile = new File(fileName);
            if (oldVersionFile.exists()) {
                return oldVersionFile;
            }
        }

        return exactVersionFile;
    }

    /**
     * @return the base directory of 'reference' and 'errors' screenshots
     */
    protected abstract String getScreenshotDirectory();

    /**
     * @return the base directory of 'reference' and 'errors' screenshots with a
     *         trailing file separator
     */
    private String getScreenshotDirectoryWithTrailingSeparator() {
        String screenshotDirectory = getScreenshotDirectory();
        if (!screenshotDirectory.endsWith(File.separator)) {
            screenshotDirectory += File.separator;
        }
        return screenshotDirectory;
    }

    /**
     * @return the directory where reference images are stored (the 'reference'
     *         folder inside the screenshot directory)
     */
    private String getScreenshotReferenceDirectory() {
        return getScreenshotDirectoryWithTrailingSeparator() + "reference";
    }

    /**
     * @return the directory where comparison error images should be created
     *         (the 'errors' folder inside the screenshot directory)
     */
    private String getScreenshotErrorDirectory() {
        return getScreenshotDirectoryWithTrailingSeparator() + "errors";
    }

    /**
     * Checks if any screenshot comparisons failures occurred during the test
     * and combines all comparison errors into one exception
     *
     * @throws IOException
     *             If there were failures during the test
     */
    @After
    public void checkCompareFailures() throws IOException {
        if (screenshotFailures != null && !screenshotFailures.isEmpty()) {
            throw new IOException(
                    "The following screenshots did not match the reference: "
                            + screenshotFailures.toString());
        }

    }

    /**
     * @return the name of a "failure" image which is stored in the folder
     *         defined by {@link #getScreenshotErrorDirectory()} when the test
     *         fails
     */
    private String getScreenshotFailureName() {
        return getScreenshotBaseName() + "_" + getUniqueIdentifier(null)
                + "-failure.png";
    }

    /**
     * @return the base name used for screenshots. This is the first part of the
     *         screenshot file name, typically created as "testclass-testmethod"
     */
    public String getScreenshotBaseName() {
        return screenshotBaseName;
    }

    /**
     * Returns the name of the reference file based on the given parameters.
     *
     * @param testName
     * @param capabilities
     * @param identifier
     * @return the full path of the reference
     */
    private String getScreenshotReferenceName(String identifier) {
        return getScreenshotReferenceName(identifier, null);
    }

    /**
     * Returns the name of the reference file based on the given parameters. The
     * version given in {@literal capabilities} is used unless it is overridden
     * by the {@literal versionOverride} parameter.
     *
     * @param testName
     * @param capabilities
     * @param identifier
     * @return the full path of the reference
     */
    private String getScreenshotReferenceName(String identifier,
            Integer versionOverride) {
        return getScreenshotReferenceDirectory() + File.separator
                + getScreenshotBaseName() + "_"
                + getUniqueIdentifier(versionOverride) + "_" + identifier
                + ".png";
    }

    private String getUniqueIdentifier(Integer versionOverride) {
        String testNameAndParameters = testName.getMethodName();
        // runTest-wildfly9-nginx[Windows_Firefox_24][/buffering/demo][valo]

        String parameters = testNameAndParameters.substring(
                testNameAndParameters.indexOf("[") + 1,
                testNameAndParameters.length() - 1);
        // Windows_Firefox_24][/buffering/demo][valo

        parameters = parameters.replace("][", "_");
        // Windows_Firefox_24_/buffering/demo_valo

        parameters = parameters.replace("/", "");
        // Windows_Firefox_24_bufferingdemo_valo

        if (versionOverride != null) {
            // Windows_Firefox_17_bufferingdemo_valo
            parameters = parameters.replaceFirst(
                    "_" + getDesiredCapabilities().getVersion(),
                    "_" + versionOverride);
        }
        return parameters;
    }

    /**
     * Returns the base name of the screenshot in the error directory. This is a
     * name so that all files matching {@link #getScreenshotErrorBaseName()}*
     * are owned by this test instance (taking into account
     * {@link #getDesiredCapabilities()}) and can safely be removed before
     * running this test.
     */
    private String getScreenshotErrorBaseName() {
        return getScreenshotReferenceName("dummy", null)
                .replace(getScreenshotReferenceDirectory(),
                        getScreenshotErrorDirectory())
                .replace("_dummy.png", "");
    }

    /**
     * Removes any old screenshots related to this test from the errors
     * directory before running the test
     */
    @Before
    public void cleanErrorDirectory() {
        // Remove any screenshots for this test from the error directory
        // before running it. Leave unrelated files as-is
        File errorDirectory = new File(getScreenshotErrorDirectory());

        // Create errors directory if it does not exist
        if (!errorDirectory.exists()) {
            errorDirectory.mkdirs();
        }

        final String errorBase = getScreenshotErrorBaseName();
        File[] files = errorDirectory.listFiles(new FileFilter() {

            @Override
            public boolean accept(File pathname) {
                String thisFile = pathname.getAbsolutePath();
                if (thisFile.startsWith(errorBase)) {
                    return true;
                }
                return false;
            }
        });
        for (File f : files) {
            f.delete();
        }
    }
}