Replace the screenshot usage in framework test (#11547)

- Replace the GridDisabledTest screenshot usage
- Replace the screenshot tests in BasicCrudGridEditorRowTest
- Replace the screenshot tests in ComboboxVaadinIcons
- Replace the screenshot tests in ValoThemeUITest::accordions
- Added screenshotcomparetolerance to some screenshot test to make tests stable

uitest/src/main/java/com/vaadin/tests/components/combobox/ComboBoxVaadinIcons.java

@@ -7,6 +7,7 @@ import com.vaadin.icons.VaadinIcons;
 import com.vaadin.server.VaadinRequest;
 import com.vaadin.tests.components.AbstractTestUI;
 import com.vaadin.ui.ComboBox;
+import com.vaadin.ui.Label;
 
 @Widgetset("com.vaadin.DefaultWidgetSet")
 public class ComboBoxVaadinIcons extends AbstractTestUI {
@@ -19,7 +20,13 @@ public class ComboBoxVaadinIcons extends AbstractTestUI {
         ComboBox<String> comboBox = new ComboBox<>();
         comboBox.setItems(Arrays.stream(icons).map(VaadinIcons::name));
         comboBox.setItemIconGenerator(VaadinIcons::valueOf);
+
+        Label valueLabel = new Label();
+        valueLabel.setValue("Test");
+        valueLabel.setId("value-label");
+        comboBox.addValueChangeListener(event -> valueLabel.setValue(comboBox.getValue()));
         addComponent(comboBox);
+        addComponent(valueLabel);
     }
 
 }

uitest/src/main/java/com/vaadin/tests/themes/valo/GridDisabled.java

@@ -13,6 +13,7 @@ public class GridDisabled extends AbstractTestUI {
         grid.addColumn("foo", String.class);
         grid.addRow("Foo");
         grid.select(grid.addRow("Bar"));
+        grid.setId("disabled-grid");
 
         addComponent(grid);
 

uitest/src/test/java/com/vaadin/tests/components/combobox/ComboBoxVaadinIconsTest.java

@@ -2,7 +2,9 @@ package com.vaadin.tests.components.combobox;
 
 import java.io.IOException;
 
+import org.junit.Assert;
 import org.junit.Test;
+import org.openqa.selenium.By;
 import org.openqa.selenium.Keys;
 
 import com.vaadin.testbench.elements.ComboBoxElement;
@@ -14,10 +16,15 @@ public class ComboBoxVaadinIconsTest extends MultiBrowserTest {
     public void testComboBoxIconRendering() throws IOException {
         openTestURL();
         ComboBoxElement comboBox = $(ComboBoxElement.class).first();
+        waitForElementPresent(By.id("value-label"));
+        Assert.assertEquals(findElement(By.id("value-label")).getText(),
+                "Test");
+
         comboBox.openPopup();
-        compareScreen(comboBox.getSuggestionPopup(), "popup");
         comboBox.sendKeys(Keys.ARROW_DOWN, Keys.ARROW_DOWN, Keys.ENTER);
-        compareScreen(comboBox, "paperplane");
-    }
+        Assert.assertEquals(findElement(By.id("value-label")).getText(),
+                "PAPERPLANE");
 
+
+    }
 }

uitest/src/test/java/com/vaadin/tests/components/grid/GridEditorFrozenColumnsUITest.java

@@ -22,11 +22,11 @@ public class GridEditorFrozenColumnsUITest extends MultiBrowserTest {
         openTestURL();
 
         openEditor(10);
-
+        sleep(500);
         compareScreen("noscroll");
 
         scrollGridHorizontallyTo(100);
-
+        sleep(500);
         compareScreen("scrolled");
     }
 

uitest/src/test/java/com/vaadin/tests/components/table/TableDropIndicatorValoTest.java

@@ -7,6 +7,7 @@ import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.interactions.Actions;
 
+import com.vaadin.testbench.Parameters;
 import com.vaadin.testbench.elements.TableElement;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
@@ -30,6 +31,7 @@ public class TableDropIndicatorValoTest extends MultiBrowserTest {
     public void indicator() throws Exception {
 
         dragRowWithoutDropping(1);
+        Parameters.setScreenshotComparisonTolerance(0.1);
         compareScreen("indicator");
     }
 

uitest/src/test/java/com/vaadin/tests/fieldgroup/BasicCrudGridEditorRowTest.java

@@ -1,9 +1,5 @@
 package com.vaadin.tests.fieldgroup;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertTrue;
-
 import java.util.Locale;
 
 import org.junit.Before;
@@ -21,6 +17,11 @@ import com.vaadin.testbench.elements.GridElement.GridEditorElement;
 import com.vaadin.testbench.parallel.TestCategory;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
 @TestCategory("grid")
 public class BasicCrudGridEditorRowTest extends MultiBrowserTest {
     private GridElement grid;
@@ -29,19 +30,22 @@ public class BasicCrudGridEditorRowTest extends MultiBrowserTest {
     public void openTest() {
         openTestURL();
         grid = $(GridElement.class).first();
-
     }
 
     @Test
     public void lookAndFeel() throws Exception {
         GridCellElement ritaBirthdate = grid.getCell(2, 3);
         waitUntilLoadingIndicatorNotVisible();
-        compareScreen("grid");
+        assertEquals("May 16, 1992",ritaBirthdate.getText());
 
+        // Grid Editor should not present yet
+        waitForElementNotPresent(By.className("v-grid-editor"));
         // Open editor row
         new Actions(getDriver()).doubleClick(ritaBirthdate).perform();
         sleep(200);
-        compareScreen("editorrow");
+        // Compound class name is not allowed to use here,
+        // check the v-grid-editor class only in this case.
+        waitForElementPresent(By.className("v-grid-editor"));
     }
 
     @Test
@@ -80,7 +84,7 @@ public class BasicCrudGridEditorRowTest extends MultiBrowserTest {
         assertNotEquals("Checkbox value did not change", value, cb.getValue());
     }
 
-    @Test
+   @Test
     public void testNoTopStyleSetOnEditorOpenWithFooterOnTop() {
         GridCellElement cell = grid.getCell(2, 3);
         // Open editor row

uitest/src/test/java/com/vaadin/tests/themes/valo/GridDisabledTest.java

@@ -5,17 +5,23 @@ import java.io.IOException;
 import org.junit.Test;
 
 import com.vaadin.testbench.elements.ButtonElement;
+import com.vaadin.testbench.elements.GridElement;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
 public class GridDisabledTest extends MultiBrowserTest {
 
     @Test
     public void disabledGrid() throws IOException {
         openTestURL();
         waitUntilLoadingIndicatorNotVisible();
+        GridElement disabledGrid = $(GridElement.class).id("disabled-grid");
 
+        assertFalse(disabledGrid.getClassNames().toString().contains("v-disabled"));
         $(ButtonElement.class).caption("Disable").first().click();
 
-        compareScreen("disabled");
+        assertTrue(disabledGrid.getClassNames().toString().contains("v-disabled"));
     }
 }

uitest/src/test/java/com/vaadin/tests/themes/valo/ResponsiveStylesTest.java

@@ -9,6 +9,7 @@ import org.junit.Test;
 import org.openqa.selenium.WebElement;
 import org.openqa.selenium.interactions.Actions;
 
+import com.vaadin.testbench.Parameters;
 import com.vaadin.testbench.elements.LabelElement;
 import com.vaadin.tests.tb3.MultiBrowserTest;
 
@@ -83,6 +84,7 @@ public class ResponsiveStylesTest extends MultiBrowserTest {
         actions.perform();
         sleep(200);
 
+        Parameters.setScreenshotComparisonTolerance(0.1);
         compareScreen("expandedMenu");
     }
 }

uitest/src/test/java/com/vaadin/tests/themes/valo/ValoThemeUITest.java

@@ -1,9 +1,11 @@
 package com.vaadin.tests.themes.valo;
 
+import org.junit.Assert;
 import org.junit.Test;
 import org.openqa.selenium.By;
 import org.openqa.selenium.WebElement;
 
+import com.vaadin.testbench.Parameters;
 import com.vaadin.testbench.elements.ButtonElement;
 import com.vaadin.testbench.elements.CheckBoxElement;
 import com.vaadin.testbench.elements.CssLayoutElement;
@@ -13,23 +15,31 @@ import com.vaadin.v7.testbench.elements.TreeElement;
 
 public class ValoThemeUITest extends MultiBrowserTest {
 
+    @Override
+    public void setup() throws Exception {
+        super.setup();
+
+        openTestURL("test");
+        // During the URL open process, we have already verified that the UI
+        // is correct, so that in this case, we can assume everything is fine
+        // as long as the UI is not totally broken
+        Parameters.setScreenshotComparisonTolerance(0.1);
+    }
+
     @Test
     public void labels() throws Exception {
-        openTestURL("test");
         open("Labels");
         compareScreen("labels");
     }
 
     @Test
     public void buttonsLinks() throws Exception {
-        openTestURL("test");
         open("Buttons & Links", "Buttons");
         compareScreen("buttonsLinks_with_disabled");
     }
 
     @Test
     public void textFields() throws Exception {
-        openTestURL("test");
         open("Text Fields <span class=\"valo-menu-badge\">123</span>",
                 "Text Fields");
         compareScreen("textFields");
@@ -37,14 +47,12 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void common() throws Exception {
-        openTestURL("test");
         open("Common UI Elements");
         compareScreen("common");
     }
 
     @Test
     public void datefields() throws Exception {
-        openTestURL("test");
         open("Date Fields");
         // Note that this can look broken in IE9 because of some browser
         // rendering issue... The problem seems to be in the customized
@@ -54,49 +62,42 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void comboboxes() throws Exception {
-        openTestURL("test");
         open("Combo Boxes");
         compareScreen("comboboxes");
     }
 
     @Test
     public void selects() throws Exception {
-        openTestURL("test");
         open("Selects");
         compareScreen("selects");
     }
 
     @Test
     public void checkboxes() throws Exception {
-        openTestURL("test");
         open("Check Boxes & Option Groups", "Check Boxes");
         compareScreen("checkboxes_with_readonly");
     }
 
     @Test
     public void sliders() throws Exception {
-        openTestURL("test");
         open("Sliders & Progress Bars", "Sliders");
         compareScreen("sliders");
     }
 
     @Test
     public void colorpickers() throws Exception {
-        openTestURL("test");
         open("Color Pickers");
         compareScreen("colorpickers");
     }
 
     @Test
     public void menubars() throws Exception {
-        openTestURL("test");
         open("Menu Bars");
         compareScreen("menubars");
     }
 
     @Test
     public void trees() throws Exception {
-        openTestURL("test");
         open("Trees");
         selectTreeNodeByCaption("Quid securi");
         compareScreen("trees");
@@ -111,7 +112,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void tables() throws Exception {
-        openTestURL("test");
         open("Tables");
         check("Components in Cells");
         compareScreen("tables");
@@ -119,7 +119,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void treeTables() throws Exception {
-        openTestURL("test");
         open("Tables");
         check("Hierarchical");
         check("Footer");
@@ -128,28 +127,24 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void dragging() throws Exception {
-        openTestURL("test");
         open("Drag and Drop", "Dragging Components");
         compareScreen("dragging");
     }
 
     @Test
     public void panels() throws Exception {
-        openTestURL("test");
         open("Panels", "Panels & Layout panels");
         compareScreen("panels");
     }
 
     @Test
     public void splitpanels() throws Exception {
-        openTestURL("test");
         open("Split Panels");
         compareScreen("splitpanels");
     }
 
     @Test
     public void tabs() throws Exception {
-        openTestURL("test");
         open("Tabs <span class=\"valo-menu-badge\">123</span>", "Tabs");
         sleep(200);
         compareScreen("tabs");
@@ -158,7 +153,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void tabsClosable() throws Exception {
-        openTestURL("test");
         open("Tabs <span class=\"valo-menu-badge\">123</span>", "Tabs");
         check("Closable");
         check("Disable tabs");
@@ -169,7 +163,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void tabsClosableUnframed() throws Exception {
-        openTestURL("test");
         open("Tabs <span class=\"valo-menu-badge\">123</span>", "Tabs");
         check("Closable");
         // Framed option is checked by default so we are actually unchecking
@@ -181,7 +174,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void tabsAlignRight() throws Exception {
-        openTestURL("test");
         open("Tabs <span class=\"valo-menu-badge\">123</span>", "Tabs");
         check("Right-aligned tabs");
         sleep(200);
@@ -197,7 +189,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void tabsAlignCenter() throws Exception {
-        openTestURL("test");
         open("Tabs <span class=\"valo-menu-badge\">123</span>", "Tabs");
         check("Centered tabs");
         sleep(200);
@@ -206,7 +197,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void tabsIconsOnTop() throws Exception {
-        openTestURL("test");
         open("Tabs <span class=\"valo-menu-badge\">123</span>", "Tabs");
         check("Icons on top");
         sleep(200);
@@ -215,7 +205,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void tabsEqualCompactPadded() throws Exception {
-        openTestURL("test");
         open("Tabs <span class=\"valo-menu-badge\">123</span>", "Tabs");
         check("Equal-width tabs");
         check("Padded tabbar");
@@ -226,14 +215,20 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void accordions() throws Exception {
-        openTestURL("test");
         open("Accordions");
-        compareScreen("accordions");
+
+        // Screenshot test is very unstable here. 
+        // We are testing the label contains the correct text in this case.  
+        CssLayoutElement content = wrap(CssLayoutElement.class,
+                findElement(By.className("valo-content")));
+        LabelElement labelElem = content.$(LabelElement.class).get(1);
+        String text = "Fabio vel iudice vincam, sunt in culpa qui officia. Ut " +
+                "enim ad minim veniam, quis nostrud exercitation.";
+        Assert.assertEquals(text, labelElem.getText());
     }
 
     @Test
     public void popupviews() throws Exception {
-        openTestURL("test");
         open("Popup Views");
         scrollTo(500, 0);
         compareScreen("popupviews");
@@ -241,7 +236,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void calendar() throws Exception {
-        openTestURL("test");
         scrollTo(500, 0);
         open("Calendar");
 
@@ -250,7 +244,6 @@ public class ValoThemeUITest extends MultiBrowserTest {
 
     @Test
     public void forms() throws Exception {
-        openTestURL("test");
         scrollTo(500, 0);
         open("Forms");
         compareScreen("forms");