Disallow RunLocally annotation in framework tests. Fix NPE for

screenshots. 

Change-Id: I72793d85dfaba41aefa5370331c1ae81348607e9

uitest/eclipse-run-selected-test.properties

@@ -22,8 +22,13 @@ com.vaadin.testbench.screenshot.directory=<enter the full path to the screenshot
 ;
 
 ; Simulates @RunLocally with the given value on all test classes without a @RunLocally annotation.
+; Use simple browser name (phantomjs, chrome, firefox, ie8, ie9, ie10, ie11)
 ; com.vaadin.testbench.runLocally=firefox
 
+; By default using @RunLocally annotation in Framework tests is not allowed. 
+; Running tests locally can be done with com.vaadin.testbench.runLocally parameter above. 
+; Uncomment the following line if you want to be able to use @RunLocally annotation
+; com.vaadin.testbench.allowRunLocally=true
 
 ;
 ; For only TestBench 2

uitest/src/com/vaadin/tests/tb3/AbstractTB3Test.java

@@ -142,7 +142,6 @@ public abstract class AbstractTB3Test extends ParallelTest {
         } catch (UnsupportedOperationException e) {
             // Opera does not support this...
         }
-
     }
 
     /**
@@ -244,7 +243,7 @@ public abstract class AbstractTB3Test extends ParallelTest {
     private void openTestURL(Class<?> uiClass, Set<String> parameters) {
         String url = getTestURL(uiClass);
 
-        if(isDebug()) {
+        if (isDebug()) {
             parameters.add("debug");
         }
 

uitest/src/com/vaadin/tests/tb3/PrivateTB3Configuration.java

@@ -25,9 +25,11 @@ import java.net.SocketException;
 import java.util.Enumeration;
 import java.util.Properties;
 
+import org.junit.Assert;
 import org.openqa.selenium.remote.DesiredCapabilities;
 
 import com.vaadin.testbench.annotations.BrowserFactory;
+import com.vaadin.testbench.annotations.RunLocally;
 import com.vaadin.testbench.annotations.RunOnHub;
 import com.vaadin.testbench.parallel.Browser;
 
@@ -45,8 +47,9 @@ public abstract class PrivateTB3Configuration extends ScreenshotTB3Test {
      * 
      */
     public static final String SCREENSHOT_DIRECTORY = "com.vaadin.testbench.screenshot.directory";
-    private static final String RUN_LOCALLY_PROPERTY = "com.vaadin.testbench.runLocally";
     private static final String HOSTNAME_PROPERTY = "com.vaadin.testbench.deployment.hostname";
+    private static final String RUN_LOCALLY_PROPERTY = "com.vaadin.testbench.runLocally";
+    private static final String ALLOW_RUN_LOCALLY_PROPERTY = "com.vaadin.testbench.allowRunLocally";
     private static final String PORT_PROPERTY = "com.vaadin.testbench.deployment.port";
     private static final String DEPLOYMENT_PROPERTY = "com.vaadin.testbench.deployment.url";
     private static final String HUB_URL = "com.vaadin.testbench.hub.url";
@@ -77,6 +80,18 @@ public abstract class PrivateTB3Configuration extends ScreenshotTB3Test {
         }
     }
 
+    @Override
+    public void setup() throws Exception {
+        String allowRunLocally = getProperty(ALLOW_RUN_LOCALLY_PROPERTY);
+        if ((allowRunLocally == null || !allowRunLocally.equals("" + true))
+                && getClass().getAnnotation(RunLocally.class) != null) {
+            Assert.fail("@RunLocally annotation is not allowed by default in framework tests. "
+                    + "See file uitest/eclipse-run-selected-test.properties for more information.");
+        }
+
+        super.setup();
+    }
+
     private static DesiredCapabilities getRunLocallyCapabilities() {
         Browser localBrowser;
         try {

uitest/src/com/vaadin/tests/tb3/ScreenshotTB3Test.java

@@ -340,7 +340,7 @@ public abstract class ScreenshotTB3Test extends AbstractTB3Test {
      */
     @After
     public void checkCompareFailures() throws IOException {
-        if (!screenshotFailures.isEmpty()) {
+        if (screenshotFailures != null && !screenshotFailures.isEmpty()) {
             throw new IOException(
                     "The following screenshots did not match the reference: "
                             + screenshotFailures.toString());