SONAR-8314 Add medium tests for generic test execution report

9 files changed, 246 insertions, 4 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/genericcoverage/GenericTestExecutionReportParser.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/genericcoverage/GenericTestExecutionReportParser.java
index 2762ec37506..0a30a5c8a85 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/genericcoverage/GenericTestExecutionReportParser.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/genericcoverage/GenericTestExecutionReportParser.java
@@ -135,12 +135,12 @@ public class GenericTestExecutionReportParser {
       } else if (ERROR.equals(elementName)) {
         status = TestCase.Status.ERROR;
       }
-      testCase.setStatus(status);
       if (TestCase.Status.OK != status) {
         testCase.setMessage(mandatoryAttribute(child, MESSAGE_ATTR));
         testCase.setStackTrace(child.collectDescendantText());
       }
     }
+    testCase.setStatus(status);
 
   }
 
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/TaskResult.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/TaskResult.java
index e15e7030e1b..879e4a016fd 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/TaskResult.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/mediumtest/TaskResult.java
@@ -259,7 +259,7 @@ public class TaskResult implements org.sonar.scanner.mediumtest.ScanTaskObserver
     return null;
   }
 
-  public ScannerReport.Test testExecutionFor(InputFile testFile, String testName) {
+  public ScannerReport.Test firstTestExecutionForName(InputFile testFile, String testName) {
     int ref = reportComponents.get(((DefaultInputFile) testFile).key()).getRef();
     try (InputStream inputStream = FileUtils.openInputStream(getReportReader().readTests(ref))) {
       ScannerReport.Test test = ScannerReport.Test.parser().parseDelimitedFrom(inputStream);
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/tests/GenericTestExecutionMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/tests/GenericTestExecutionMediumTest.java
new file mode 100644
index 00000000000..4fbc46d350e
--- /dev/null
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/tests/GenericTestExecutionMediumTest.java
@@ -0,0 +1,195 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.mediumtest.tests;
+
+import com.google.common.collect.ImmutableMap;
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.scanner.mediumtest.BatchMediumTester;
+import org.sonar.scanner.mediumtest.TaskResult;
+import org.sonar.scanner.protocol.output.ScannerReport;
+import org.sonar.scanner.protocol.output.ScannerReport.Test.TestStatus;
+import org.sonar.xoo.XooPlugin;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
+
+public class GenericTestExecutionMediumTest {
+
+  @org.junit.Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  public BatchMediumTester tester = BatchMediumTester.builder()
+    .registerPlugin("xoo", new XooPlugin())
+    .addDefaultQProfile("xoo", "Sonar Way")
+    .build();
+
+  @Before
+  public void prepare() {
+    tester.start();
+  }
+
+  @After
+  public void stop() {
+    tester.stop();
+  }
+
+  @Test
+  public void unitTests() throws IOException {
+
+    File baseDir = temp.getRoot();
+    File srcDir = new File(baseDir, "src");
+    srcDir.mkdir();
+    File testDir = new File(baseDir, "test");
+    testDir.mkdir();
+
+    File xooFile = new File(srcDir, "sample.xoo");
+    FileUtils.write(xooFile, "foo");
+
+    File xooTestFile = new File(testDir, "sampleTest.xoo");
+    FileUtils.write(xooTestFile, "failure\nerror\nok\nskipped");
+
+    File xooTestExecutionFile = new File(testDir, "sampleTest.xoo.test");
+    FileUtils.write(xooTestExecutionFile, "skipped::::SKIPPED:UNIT\n" +
+      "failure:2:Failure::FAILURE:UNIT\n" +
+      "error:2:Error:The stack:ERROR:UNIT\n" +
+      "success:4:::OK:INTEGRATION");
+
+    TaskResult result = tester.newTask()
+      .properties(ImmutableMap.<String, String>builder()
+        .put("sonar.task", "scan")
+        .put("sonar.projectBaseDir", baseDir.getAbsolutePath())
+        .put("sonar.projectKey", "com.foo.project")
+        .put("sonar.projectName", "Foo Project")
+        .put("sonar.projectVersion", "1.0-SNAPSHOT")
+        .put("sonar.projectDescription", "Description of Foo Project")
+        .put("sonar.sources", "src")
+        .put("sonar.tests", "test")
+        .build())
+      .start();
+
+    InputFile file = result.inputFile("test/sampleTest.xoo");
+    org.sonar.scanner.protocol.output.ScannerReport.Test success = result.firstTestExecutionForName(file, "success");
+    assertThat(success.getDurationInMs()).isEqualTo(4);
+    assertThat(success.getStatus()).isEqualTo(TestStatus.OK);
+
+    org.sonar.scanner.protocol.output.ScannerReport.Test error = result.firstTestExecutionForName(file, "error");
+    assertThat(error.getDurationInMs()).isEqualTo(2);
+    assertThat(error.getStatus()).isEqualTo(TestStatus.ERROR);
+    assertThat(error.getMsg()).isEqualTo("Error");
+    assertThat(error.getStacktrace()).isEqualTo("The stack");
+  }
+
+  @Test
+  public void singleReport() throws IOException {
+
+    File projectDir = new File("src/test/resources/mediumtest/xoo/sample-generic-test-exec");
+
+    TaskResult result = tester
+      .newScanTask(new File(projectDir, "sonar-project.properties"))
+      .property("sonar.testExecutionReportPaths", "unittest.xml")
+      .start();
+
+    InputFile testFile = result.inputFile("testx/ClassOneTest.xoo");
+    ScannerReport.Test success = result.firstTestExecutionForName(testFile, "test1");
+    assertThat(success.getDurationInMs()).isEqualTo(5);
+    assertThat(success.getStatus()).isEqualTo(TestStatus.OK);
+
+    ScannerReport.Test skipped = result.firstTestExecutionForName(testFile, "test2");
+    assertThat(skipped.getDurationInMs()).isEqualTo(500);
+    assertThat(skipped.getStatus()).isEqualTo(TestStatus.SKIPPED);
+    assertThat(skipped.getMsg()).isEqualTo("short message");
+    assertThat(skipped.getStacktrace()).isEqualTo("other");
+
+    ScannerReport.Test failed = result.firstTestExecutionForName(testFile, "test3");
+    assertThat(failed.getDurationInMs()).isEqualTo(100);
+    assertThat(failed.getStatus()).isEqualTo(TestStatus.FAILURE);
+    assertThat(failed.getMsg()).isEqualTo("short");
+    assertThat(failed.getStacktrace()).isEqualTo("stacktrace");
+
+    ScannerReport.Test error = result.firstTestExecutionForName(testFile, "test4");
+    assertThat(error.getDurationInMs()).isEqualTo(500);
+    assertThat(error.getStatus()).isEqualTo(TestStatus.ERROR);
+    assertThat(error.getMsg()).isEqualTo("short");
+    assertThat(error.getStacktrace()).isEqualTo("stacktrace");
+
+    assertThat(result.allMeasures().get(testFile.key())).extracting("metricKey", "intValue.value", "longValue.value")
+      .containsOnly(
+        tuple(CoreMetrics.TESTS_KEY, 3, 0L),
+        tuple(CoreMetrics.SKIPPED_TESTS_KEY, 1, 0L),
+        tuple(CoreMetrics.TEST_ERRORS_KEY, 1, 0L),
+        tuple(CoreMetrics.TEST_EXECUTION_TIME_KEY, 0, 1105L),
+        tuple(CoreMetrics.TEST_FAILURES_KEY, 1, 0L));
+  }
+
+  @Test
+  public void twoReports() throws IOException {
+
+    File projectDir = new File("src/test/resources/mediumtest/xoo/sample-generic-test-exec");
+
+    TaskResult result = tester
+      .newScanTask(new File(projectDir, "sonar-project.properties"))
+      .property("sonar.testExecutionReportPaths", "unittest.xml,unittest2.xml")
+      .start();
+
+    InputFile testFile = result.inputFile("testx/ClassOneTest.xoo");
+    ScannerReport.Test success = result.firstTestExecutionForName(testFile, "test1");
+    assertThat(success.getDurationInMs()).isEqualTo(5);
+    assertThat(success.getStatus()).isEqualTo(TestStatus.OK);
+
+    ScannerReport.Test success2 = result.firstTestExecutionForName(testFile, "test1b");
+    assertThat(success2.getDurationInMs()).isEqualTo(5);
+    assertThat(success2.getStatus()).isEqualTo(TestStatus.OK);
+
+    ScannerReport.Test skipped = result.firstTestExecutionForName(testFile, "test2");
+    assertThat(skipped.getDurationInMs()).isEqualTo(500);
+    assertThat(skipped.getStatus()).isEqualTo(TestStatus.SKIPPED);
+    assertThat(skipped.getMsg()).isEqualTo("short message");
+    assertThat(skipped.getStacktrace()).isEqualTo("other");
+
+    ScannerReport.Test failed = result.firstTestExecutionForName(testFile, "test3");
+    assertThat(failed.getDurationInMs()).isEqualTo(100);
+    assertThat(failed.getStatus()).isEqualTo(TestStatus.FAILURE);
+    assertThat(failed.getMsg()).isEqualTo("short");
+    assertThat(failed.getStacktrace()).isEqualTo("stacktrace");
+
+    ScannerReport.Test error = result.firstTestExecutionForName(testFile, "test4");
+    assertThat(error.getDurationInMs()).isEqualTo(500);
+    assertThat(error.getStatus()).isEqualTo(TestStatus.ERROR);
+    assertThat(error.getMsg()).isEqualTo("short");
+    assertThat(error.getStacktrace()).isEqualTo("stacktrace");
+
+    assertThat(result.allMeasures().get(testFile.key())).extracting("metricKey", "intValue.value", "longValue.value")
+      .containsOnly(
+        tuple(CoreMetrics.TESTS_KEY, 4, 0L),
+        tuple(CoreMetrics.SKIPPED_TESTS_KEY, 2, 0L),
+        tuple(CoreMetrics.TEST_ERRORS_KEY, 1, 0L),
+        tuple(CoreMetrics.TEST_EXECUTION_TIME_KEY, 0, 1610L),
+        tuple(CoreMetrics.TEST_FAILURES_KEY, 1, 0L));
+  }
+
+}
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/tests/TestExecutionMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/tests/TestExecutionMediumTest.java
index 22d858e8671..afd1f77ed06 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/tests/TestExecutionMediumTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/tests/TestExecutionMediumTest.java
@@ -90,11 +90,11 @@ public class TestExecutionMediumTest {
       .start();
 
     InputFile file = result.inputFile("test/sampleTest.xoo");
-    org.sonar.scanner.protocol.output.ScannerReport.Test success = result.testExecutionFor(file, "success");
+    org.sonar.scanner.protocol.output.ScannerReport.Test success = result.firstTestExecutionForName(file, "success");
     assertThat(success.getDurationInMs()).isEqualTo(4);
     assertThat(success.getStatus()).isEqualTo(TestStatus.OK);
 
-    org.sonar.scanner.protocol.output.ScannerReport.Test error = result.testExecutionFor(file, "error");
+    org.sonar.scanner.protocol.output.ScannerReport.Test error = result.firstTestExecutionForName(file, "error");
     assertThat(error.getDurationInMs()).isEqualTo(2);
     assertThat(error.getStatus()).isEqualTo(TestStatus.ERROR);
     assertThat(error.getMsg()).isEqualTo("Error");
diff --git a/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/sonar-project.properties b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/sonar-project.properties
new file mode 100644
index 00000000000..8810e376701
--- /dev/null
+++ b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/sonar-project.properties
@@ -0,0 +1,6 @@
+sonar.projectKey=sample
+sonar.projectName=Sample
+sonar.projectVersion=0.1-SNAPSHOT
+sonar.sources=xources
+sonar.tests=testx
+sonar.language=xoo
diff --git a/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/testx/ClassOneTest.xoo b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/testx/ClassOneTest.xoo
new file mode 100644
index 00000000000..8c0967e496f
--- /dev/null
+++ b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/testx/ClassOneTest.xoo
@@ -0,0 +1,11 @@
+package org.sonar.tests;
+
+import org.junit.Test;
+
+public class ClassOneTest  {
+
+  @Test
+  public void nothing() {
+
+  }
+}
diff --git a/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/unittest.xml b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/unittest.xml
new file mode 100644
index 00000000000..65df6d58c8b
--- /dev/null
+++ b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/unittest.xml
@@ -0,0 +1,14 @@
+<unitTest version="1">
+  <file path="testx/ClassOneTest.xoo">
+    <testCase name="test1" duration="5"/>
+    <testCase name="test2" duration="500">
+      <skipped message="short message">other</skipped>
+    </testCase>
+    <testCase name="test3" duration="100">
+      <failure message="short">stacktrace</failure>
+    </testCase>
+    <testCase name="test4" duration="500">
+      <error message="short">stacktrace</error>
+    </testCase>
+  </file>
+</unitTest>
diff --git a/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/unittest2.xml b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/unittest2.xml
new file mode 100644
index 00000000000..fb5c28d8b80
--- /dev/null
+++ b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/unittest2.xml
@@ -0,0 +1,8 @@
+<unitTest version="1">
+  <file path="testx/ClassOneTest.xoo">
+    <testCase name="test1b" duration="5"/>
+    <testCase name="test2" duration="500">
+      <skipped message="short message">other</skipped>
+    </testCase>
+  </file>
+</unitTest>
diff --git a/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/xources/hello/HelloJava.xoo b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/xources/hello/HelloJava.xoo
new file mode 100644
index 00000000000..1d9c60d56b7
--- /dev/null
+++ b/sonar-scanner-engine/src/test/resources/mediumtest/xoo/sample-generic-test-exec/xources/hello/HelloJava.xoo
@@ -0,0 +1,8 @@
+package hello;
+
+public class HelloJava {
+
+  public static void main(String[] args) {
+    System.out.println("Hello");
+  }
+}
\ No newline at end of file