SONAR-20268 Add new module for test failure monitoring

build.gradle

@@ -65,6 +65,7 @@ allprojects {
   ext {
     release = project.hasProperty('release') && project.getProperty('release')
     official = project.hasProperty('official') && project.getProperty('official')
+    withTestMonitoring = project.hasProperty('withTestMonitoring')
   }
 
   ext.enableBom = enableBom
@@ -637,12 +638,15 @@ subprojects {
   tasks.withType(Test) {
     configurations {
       utMonitoring
+      testMonitoring
     }
 
     dependencies {
       testImplementation project(":ut-monitoring")
+      testImplementation project(":test-monitoring")
 
       utMonitoring 'org.aspectj:aspectjweaver:1.9.19'
+      testMonitoring 'org.aspectj:aspectjweaver:1.9.19'
     }
   }
 
@@ -678,6 +682,17 @@ subprojects {
     }
   }
 
+  if (ext.withTestMonitoring) {
+    tasks.withType(Test) {
+      doFirst {
+        ext {
+          aspectJWeaver = configurations.testMonitoring.resolvedConfiguration.resolvedArtifacts.find { it.name == 'aspectjweaver' }
+        }
+        jvmArgs "-javaagent:${aspectJWeaver.file}"
+      }
+    }
+  }
+
 
   signing {
     def signingKeyId = findProperty("signingKeyId")

settings.gradle

@@ -59,6 +59,7 @@ include 'sonar-testing-harness'
 include 'sonar-testing-ldap'
 include 'sonar-ws'
 include 'sonar-ws-generator'
+include 'test-monitoring'
 include 'ut-monitoring'
 
 ext.isCiServer = System.getenv().containsKey("CIRRUS_CI")

test-monitoring/build.gradle

@@ -0,0 +1,26 @@
+sonar {
+    properties {
+        property 'sonar.projectName', "${projectTitle} :: Java Tests Monitoring"
+    }
+}
+
+dependencies {
+    // please keep the list grouped by configuration and ordered by name
+
+    api 'com.google.code.gson:gson'
+    api 'junit:junit'
+
+    compileOnlyApi 'org.aspectj:aspectjtools'
+    compileOnlyApi 'com.google.code.findbugs:jsr305'
+
+    testImplementation 'org.assertj:assertj-core'
+    testImplementation 'org.mockito:mockito-core'
+}
+
+tasks.withType(JavaCompile) {
+    options.release = 11
+}
+
+sonar {
+    skipProject = true
+}

test-monitoring/src/main/java/org/sonarqube/monitoring/test/Measure.java

@@ -0,0 +1,198 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.monitoring.test;
+
+public class Measure {
+  private String timestamp;
+  private String branchName;
+  private String commit;
+  private String build;
+  private String category;
+  private String testClass;
+  private String testMethod;
+  private String exceptionClass;
+  private String exceptionMessage;
+  private String exceptionLogs;
+
+  private Measure() {
+  }
+
+  public String getTimestamp() {
+    return timestamp;
+  }
+
+  public void setTimestamp(String timestamp) {
+    this.timestamp = timestamp;
+  }
+
+  public String getBranchName() {
+    return branchName;
+  }
+
+  public void setBranchName(String branchName) {
+    this.branchName = branchName;
+  }
+
+  public String getCommit() {
+    return commit;
+  }
+
+  public void setCommit(String commit) {
+    this.commit = commit;
+  }
+
+  public String getBuild() {
+    return build;
+  }
+
+  public void setBuild(String build) {
+    this.build = build;
+  }
+
+  public String getCategory() {
+    return category;
+  }
+
+  public void setCategory(String category) {
+    this.category = category;
+  }
+
+  public String getTestClass() {
+    return testClass;
+  }
+
+  public void setTestClass(String testClass) {
+    this.testClass = testClass;
+  }
+
+  public String getTestMethod() {
+    return testMethod;
+  }
+
+  public void setTestMethod(String testMethod) {
+    this.testMethod = testMethod;
+  }
+
+  public String getExceptionClass() {
+    return exceptionClass;
+  }
+
+  public void setExceptionClass(String exceptionClass) {
+    this.exceptionClass = exceptionClass;
+  }
+
+  public String getExceptionMessage() {
+    return exceptionMessage;
+  }
+
+  public void setExceptionMessage(String exceptionMessage) {
+    this.exceptionMessage = exceptionMessage;
+  }
+
+  public String getExceptionLogs() {
+    return exceptionLogs;
+  }
+
+  public void setExceptionLogs(String exceptionLogs) {
+    this.exceptionLogs = exceptionLogs;
+  }
+
+  @Override
+  public String toString() {
+    return "Measure{" +
+      "timestamp='" + timestamp + '\'' +
+      ", branchName='" + branchName + '\'' +
+      ", commit='" + commit + '\'' +
+      ", build='" + build + '\'' +
+      ", category='" + category + '\'' +
+      ", testClass='" + testClass + '\'' +
+      ", testMethod='" + testMethod + '\'' +
+      ", exceptionClass='" + exceptionClass + '\'' +
+      ", exceptionMessage='" + exceptionMessage + '\'' +
+      ", exceptionLogs='" + exceptionLogs + '\'' +
+      '}';
+  }
+
+  public static final class MeasureBuilder {
+    private Measure measure;
+
+    private MeasureBuilder() {
+      measure = new Measure();
+    }
+
+    public static MeasureBuilder newMeasureBuilder() {
+      return new MeasureBuilder();
+    }
+
+    public MeasureBuilder setTimestamp(String timestamp) {
+      measure.setTimestamp(timestamp);
+      return this;
+    }
+
+    public MeasureBuilder setBranchName(String branchName) {
+      measure.setBranchName(branchName);
+      return this;
+    }
+
+    public MeasureBuilder setCommit(String commit) {
+      measure.setCommit(commit);
+      return this;
+    }
+
+    public MeasureBuilder setBuild(String build) {
+      measure.setBuild(build);
+      return this;
+    }
+
+    public MeasureBuilder setCategory(String category) {
+      measure.setCategory(category);
+      return this;
+    }
+
+    public MeasureBuilder setTestClass(String testClass) {
+      measure.setTestClass(testClass);
+      return this;
+    }
+
+    public MeasureBuilder setTestMethod(String testMethod) {
+      measure.setTestMethod(testMethod);
+      return this;
+    }
+
+    public MeasureBuilder setExceptionClass(String exceptionClass) {
+      measure.setExceptionClass(exceptionClass);
+      return this;
+    }
+
+    public MeasureBuilder setExceptionMessage(String exceptionMessage) {
+      measure.setExceptionMessage(exceptionMessage);
+      return this;
+    }
+
+    public MeasureBuilder setExceptionLogs(String exceptionLogs) {
+      measure.setExceptionLogs(exceptionLogs);
+      return this;
+    }
+
+    public Measure build() {
+      return measure;
+    }
+  }
+}

test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/TestFailureAspect.java

@@ -0,0 +1,101 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.monitoring.test.aspect;
+
+import com.google.gson.Gson;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.annotation.After;
+import org.aspectj.lang.annotation.Aspect;
+import org.junit.runner.Description;
+import org.junit.runner.notification.Failure;
+import org.sonarqube.monitoring.test.Measure;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.sonarqube.monitoring.test.Measure.MeasureBuilder.newMeasureBuilder;
+
+@Aspect
+public class TestFailureAspect {
+
+  public static final String BRANCH_NAME = System.getenv("GITHUB_BRANCH");
+  public static final String COMMIT_HASH = System.getenv("GIT_SHA1");
+  public static final String BUILD_NUMBER = System.getenv("BUILD_NUMBER");
+  public static final String QA_CATEGORY = System.getenv("QA_CATEGORY");
+
+  private static final Path PATH = Paths.get("/tmp/test-monitoring.log");
+  private static final Gson GSON = new Gson();
+
+  static {
+    try {
+      if (!Files.exists(PATH, LinkOption.NOFOLLOW_LINKS)) {
+        Files.createFile(PATH);
+      }
+      Files.write(PATH, "".getBytes(UTF_8));
+    } catch (IOException e) {
+      // Ignore
+    }
+  }
+
+  @After("execution(public * org.junit.runner.notification.RunNotifier+.fireTestFailure(..))")
+  public void afterFireTestFailure(JoinPoint joinPoint) {
+    Object[] args = joinPoint.getArgs();
+    if (args.length == 1) {
+      Object arg = args[0];
+      if (arg instanceof Failure) {
+        Failure failure = (Failure) arg;
+        persistMeasure(buildMeasure(failure));
+      }
+    }
+  }
+
+  private static Measure buildMeasure(Failure failure) {
+    Throwable throwable = failure.getException();
+    Description description = failure.getDescription();
+    return newMeasureBuilder()
+      .setTimestamp(LocalDateTime.now(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_LOCAL_DATE_TIME))
+      .setBranchName(BRANCH_NAME)
+      .setCommit(COMMIT_HASH)
+      .setBuild(BUILD_NUMBER)
+      .setCategory(QA_CATEGORY)
+      .setTestClass(description.getClassName())
+      .setTestMethod(description.getMethodName())
+      .setExceptionClass(throwable.getClass().getName())
+      .setExceptionMessage(failure.getMessage())
+      .setExceptionLogs(failure.getTrimmedTrace())
+      .build();
+  }
+
+  public static void persistMeasure(Measure measure) {
+    try {
+      Files.write(PATH, GSON.toJson(measure).getBytes(UTF_8), StandardOpenOption.APPEND);
+      Files.write(PATH, "\n".getBytes(UTF_8), StandardOpenOption.APPEND);
+    } catch (IOException e) {
+      // Ignore
+    }
+  }
+}

test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonarqube.monitoring.test.aspect;
+
+import javax.annotation.ParametersAreNonnullByDefault;

test-monitoring/src/main/java/org/sonarqube/monitoring/test/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonarqube.monitoring.test;
+
+import javax.annotation.ParametersAreNonnullByDefault;

test-monitoring/src/main/resources/META-INF/aop.xml

@@ -0,0 +1,12 @@
+<aspectj>
+    <aspects>
+        <aspect name="org.sonarqube.monitoring.test.aspect.TestFailureAspect"/>
+    </aspects>
+    <weaver options="-warn:none -Xlint:ignore">
+        <include within="com.sonarsource..*"/>
+        <include within="org.sonarqube..*"/>
+        <include within="org.sonar..*"/>
+        <include within="com.sonar..*"/>
+        <include within="org.junit..*"/>
+    </weaver>
+</aspectj>

test-monitoring/src/test/java/org/sonarqube/monitoring/test/aspect/TestFailureAspectTest.java

@@ -0,0 +1,89 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.monitoring.test.aspect;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.aspectj.lang.JoinPoint;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.notification.Failure;
+
+import static java.nio.file.Files.createDirectory;
+import static java.nio.file.Files.exists;
+import static java.nio.file.Files.readAllBytes;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+import static org.junit.runner.Description.createTestDescription;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestFailureAspectTest {
+
+  private TestFailureAspect testFailureAspect;
+
+  private static final Path TMP_PATH = Path.of("/tmp");
+
+  @BeforeClass
+  public static void createTmpFolder() throws IOException {
+    if (!exists(TMP_PATH)) {
+      createDirectory(TMP_PATH);
+    }
+  }
+
+  @Before
+  public void setup() {
+    testFailureAspect = new TestFailureAspect();
+  }
+
+  @Test
+  public void afterFireTestFailure_shouldPersistMeasure() {
+    JoinPoint joinPoint = mock(JoinPoint.class);
+    Failure failure = new Failure(
+      createTestDescription("testClass", "testMethod"),
+      new IllegalStateException("some exception"));
+    when(joinPoint.getArgs()).thenReturn(new Object[]{failure});
+
+    testFailureAspect.afterFireTestFailure(joinPoint);
+
+    String fileContent = getFileContent(Paths.get("/tmp/test-monitoring.log"));
+    assertThat(fileContent)
+      .contains("\"timestamp\":\"" )
+      .contains("\"testClass\":\"testClass\"")
+      .contains("\"testMethod\":\"testMethod\"")
+      .contains("\"exceptionClass\":\"java.lang.IllegalStateException\"")
+      .contains("\"exceptionMessage\":\"some exception\"")
+      .contains("\"exceptionLogs\":\"java.lang.IllegalStateException: some exception");
+  }
+
+  private String getFileContent(Path path) {
+    try {
+      byte[] bytes = readAllBytes(path);
+      return new String(bytes, StandardCharsets.UTF_8);
+    } catch (IOException e) {
+      fail("Unable to read file " + path, e);
+    }
+    return null;
+  }
+
+}

wss-unified-agent.config

@@ -9,7 +9,7 @@ gradle.aggregateModules=True
 gradle.preferredEnvironment=wrapper
 gradle.innerModulesAsDependencies=True
 gradle.ignoredConfigurations=.*test.* .*bbt.*
-gradle.excludeModules=.*it-.* .*ut-monitoring.* .*sonar-ws-generator.* .*sonar-testing-ldap.* .*sonar-testing-harness.*
+gradle.excludeModules=.*it-.* .*ut-monitoring.* .*test-monitoring.* .*sonar-ws-generator.* .*sonar-testing-ldap.* .*sonar-testing-harness.*
 
 npm.includeDevDependencies=False
 npm.resolveDependencies=True