From c389c03b751ae12b2999dd792b9be92f57e39ad1 Mon Sep 17 00:00:00 2001 From: Steve Marion Date: Wed, 30 Aug 2023 15:27:35 +0200 Subject: [PATCH] SONAR-20268 Add new module for test failure monitoring --- build.gradle | 15 ++ settings.gradle | 1 + test-monitoring/build.gradle | 26 +++ .../sonarqube/monitoring/test/Measure.java | 198 ++++++++++++++++++ .../test/aspect/TestFailureAspect.java | 101 +++++++++ .../monitoring/test/aspect/package-info.java | 23 ++ .../monitoring/test/package-info.java | 23 ++ .../src/main/resources/META-INF/aop.xml | 12 ++ .../test/aspect/TestFailureAspectTest.java | 89 ++++++++ wss-unified-agent.config | 2 +- 10 files changed, 489 insertions(+), 1 deletion(-) create mode 100644 test-monitoring/build.gradle create mode 100644 test-monitoring/src/main/java/org/sonarqube/monitoring/test/Measure.java create mode 100644 test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/TestFailureAspect.java create mode 100644 test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/package-info.java create mode 100644 test-monitoring/src/main/java/org/sonarqube/monitoring/test/package-info.java create mode 100644 test-monitoring/src/main/resources/META-INF/aop.xml create mode 100644 test-monitoring/src/test/java/org/sonarqube/monitoring/test/aspect/TestFailureAspectTest.java diff --git a/build.gradle b/build.gradle index 9b7c4027944..90234880d1a 100644 --- a/build.gradle +++ b/build.gradle @@ -65,6 +65,7 @@ allprojects { ext { release = project.hasProperty('release') && project.getProperty('release') official = project.hasProperty('official') && project.getProperty('official') + withTestMonitoring = project.hasProperty('withTestMonitoring') } ext.enableBom = enableBom @@ -637,12 +638,15 @@ subprojects { tasks.withType(Test) { configurations { utMonitoring + testMonitoring } dependencies { testImplementation project(":ut-monitoring") + testImplementation project(":test-monitoring") utMonitoring 'org.aspectj:aspectjweaver:1.9.19' + testMonitoring 'org.aspectj:aspectjweaver:1.9.19' } } @@ -678,6 +682,17 @@ subprojects { } } + if (ext.withTestMonitoring) { + tasks.withType(Test) { + doFirst { + ext { + aspectJWeaver = configurations.testMonitoring.resolvedConfiguration.resolvedArtifacts.find { it.name == 'aspectjweaver' } + } + jvmArgs "-javaagent:${aspectJWeaver.file}" + } + } + } + signing { def signingKeyId = findProperty("signingKeyId") diff --git a/settings.gradle b/settings.gradle index 912592cff2b..895b5c8c677 100644 --- a/settings.gradle +++ b/settings.gradle @@ -59,6 +59,7 @@ include 'sonar-testing-harness' include 'sonar-testing-ldap' include 'sonar-ws' include 'sonar-ws-generator' +include 'test-monitoring' include 'ut-monitoring' ext.isCiServer = System.getenv().containsKey("CIRRUS_CI") diff --git a/test-monitoring/build.gradle b/test-monitoring/build.gradle new file mode 100644 index 00000000000..236f8a34266 --- /dev/null +++ b/test-monitoring/build.gradle @@ -0,0 +1,26 @@ +sonar { + properties { + property 'sonar.projectName', "${projectTitle} :: Java Tests Monitoring" + } +} + +dependencies { + // please keep the list grouped by configuration and ordered by name + + api 'com.google.code.gson:gson' + api 'junit:junit' + + compileOnlyApi 'org.aspectj:aspectjtools' + compileOnlyApi 'com.google.code.findbugs:jsr305' + + testImplementation 'org.assertj:assertj-core' + testImplementation 'org.mockito:mockito-core' +} + +tasks.withType(JavaCompile) { + options.release = 11 +} + +sonar { + skipProject = true +} diff --git a/test-monitoring/src/main/java/org/sonarqube/monitoring/test/Measure.java b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/Measure.java new file mode 100644 index 00000000000..efefa0fcccf --- /dev/null +++ b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/Measure.java @@ -0,0 +1,198 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.monitoring.test; + +public class Measure { + private String timestamp; + private String branchName; + private String commit; + private String build; + private String category; + private String testClass; + private String testMethod; + private String exceptionClass; + private String exceptionMessage; + private String exceptionLogs; + + private Measure() { + } + + public String getTimestamp() { + return timestamp; + } + + public void setTimestamp(String timestamp) { + this.timestamp = timestamp; + } + + public String getBranchName() { + return branchName; + } + + public void setBranchName(String branchName) { + this.branchName = branchName; + } + + public String getCommit() { + return commit; + } + + public void setCommit(String commit) { + this.commit = commit; + } + + public String getBuild() { + return build; + } + + public void setBuild(String build) { + this.build = build; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getTestClass() { + return testClass; + } + + public void setTestClass(String testClass) { + this.testClass = testClass; + } + + public String getTestMethod() { + return testMethod; + } + + public void setTestMethod(String testMethod) { + this.testMethod = testMethod; + } + + public String getExceptionClass() { + return exceptionClass; + } + + public void setExceptionClass(String exceptionClass) { + this.exceptionClass = exceptionClass; + } + + public String getExceptionMessage() { + return exceptionMessage; + } + + public void setExceptionMessage(String exceptionMessage) { + this.exceptionMessage = exceptionMessage; + } + + public String getExceptionLogs() { + return exceptionLogs; + } + + public void setExceptionLogs(String exceptionLogs) { + this.exceptionLogs = exceptionLogs; + } + + @Override + public String toString() { + return "Measure{" + + "timestamp='" + timestamp + '\'' + + ", branchName='" + branchName + '\'' + + ", commit='" + commit + '\'' + + ", build='" + build + '\'' + + ", category='" + category + '\'' + + ", testClass='" + testClass + '\'' + + ", testMethod='" + testMethod + '\'' + + ", exceptionClass='" + exceptionClass + '\'' + + ", exceptionMessage='" + exceptionMessage + '\'' + + ", exceptionLogs='" + exceptionLogs + '\'' + + '}'; + } + + public static final class MeasureBuilder { + private Measure measure; + + private MeasureBuilder() { + measure = new Measure(); + } + + public static MeasureBuilder newMeasureBuilder() { + return new MeasureBuilder(); + } + + public MeasureBuilder setTimestamp(String timestamp) { + measure.setTimestamp(timestamp); + return this; + } + + public MeasureBuilder setBranchName(String branchName) { + measure.setBranchName(branchName); + return this; + } + + public MeasureBuilder setCommit(String commit) { + measure.setCommit(commit); + return this; + } + + public MeasureBuilder setBuild(String build) { + measure.setBuild(build); + return this; + } + + public MeasureBuilder setCategory(String category) { + measure.setCategory(category); + return this; + } + + public MeasureBuilder setTestClass(String testClass) { + measure.setTestClass(testClass); + return this; + } + + public MeasureBuilder setTestMethod(String testMethod) { + measure.setTestMethod(testMethod); + return this; + } + + public MeasureBuilder setExceptionClass(String exceptionClass) { + measure.setExceptionClass(exceptionClass); + return this; + } + + public MeasureBuilder setExceptionMessage(String exceptionMessage) { + measure.setExceptionMessage(exceptionMessage); + return this; + } + + public MeasureBuilder setExceptionLogs(String exceptionLogs) { + measure.setExceptionLogs(exceptionLogs); + return this; + } + + public Measure build() { + return measure; + } + } +} diff --git a/test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/TestFailureAspect.java b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/TestFailureAspect.java new file mode 100644 index 00000000000..2cb05c78b16 --- /dev/null +++ b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/TestFailureAspect.java @@ -0,0 +1,101 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.monitoring.test.aspect; + +import com.google.gson.Gson; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.After; +import org.aspectj.lang.annotation.Aspect; +import org.junit.runner.Description; +import org.junit.runner.notification.Failure; +import org.sonarqube.monitoring.test.Measure; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.sonarqube.monitoring.test.Measure.MeasureBuilder.newMeasureBuilder; + +@Aspect +public class TestFailureAspect { + + public static final String BRANCH_NAME = System.getenv("GITHUB_BRANCH"); + public static final String COMMIT_HASH = System.getenv("GIT_SHA1"); + public static final String BUILD_NUMBER = System.getenv("BUILD_NUMBER"); + public static final String QA_CATEGORY = System.getenv("QA_CATEGORY"); + + private static final Path PATH = Paths.get("/tmp/test-monitoring.log"); + private static final Gson GSON = new Gson(); + + static { + try { + if (!Files.exists(PATH, LinkOption.NOFOLLOW_LINKS)) { + Files.createFile(PATH); + } + Files.write(PATH, "".getBytes(UTF_8)); + } catch (IOException e) { + // Ignore + } + } + + @After("execution(public * org.junit.runner.notification.RunNotifier+.fireTestFailure(..))") + public void afterFireTestFailure(JoinPoint joinPoint) { + Object[] args = joinPoint.getArgs(); + if (args.length == 1) { + Object arg = args[0]; + if (arg instanceof Failure) { + Failure failure = (Failure) arg; + persistMeasure(buildMeasure(failure)); + } + } + } + + private static Measure buildMeasure(Failure failure) { + Throwable throwable = failure.getException(); + Description description = failure.getDescription(); + return newMeasureBuilder() + .setTimestamp(LocalDateTime.now(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)) + .setBranchName(BRANCH_NAME) + .setCommit(COMMIT_HASH) + .setBuild(BUILD_NUMBER) + .setCategory(QA_CATEGORY) + .setTestClass(description.getClassName()) + .setTestMethod(description.getMethodName()) + .setExceptionClass(throwable.getClass().getName()) + .setExceptionMessage(failure.getMessage()) + .setExceptionLogs(failure.getTrimmedTrace()) + .build(); + } + + public static void persistMeasure(Measure measure) { + try { + Files.write(PATH, GSON.toJson(measure).getBytes(UTF_8), StandardOpenOption.APPEND); + Files.write(PATH, "\n".getBytes(UTF_8), StandardOpenOption.APPEND); + } catch (IOException e) { + // Ignore + } + } +} diff --git a/test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/package-info.java b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/package-info.java new file mode 100644 index 00000000000..16dd6017c74 --- /dev/null +++ b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/aspect/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonarqube.monitoring.test.aspect; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/test-monitoring/src/main/java/org/sonarqube/monitoring/test/package-info.java b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/package-info.java new file mode 100644 index 00000000000..40402c708e0 --- /dev/null +++ b/test-monitoring/src/main/java/org/sonarqube/monitoring/test/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonarqube.monitoring.test; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/test-monitoring/src/main/resources/META-INF/aop.xml b/test-monitoring/src/main/resources/META-INF/aop.xml new file mode 100644 index 00000000000..32aa78ec8ef --- /dev/null +++ b/test-monitoring/src/main/resources/META-INF/aop.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test-monitoring/src/test/java/org/sonarqube/monitoring/test/aspect/TestFailureAspectTest.java b/test-monitoring/src/test/java/org/sonarqube/monitoring/test/aspect/TestFailureAspectTest.java new file mode 100644 index 00000000000..1b01be0e095 --- /dev/null +++ b/test-monitoring/src/test/java/org/sonarqube/monitoring/test/aspect/TestFailureAspectTest.java @@ -0,0 +1,89 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.monitoring.test.aspect; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.aspectj.lang.JoinPoint; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.notification.Failure; + +import static java.nio.file.Files.createDirectory; +import static java.nio.file.Files.exists; +import static java.nio.file.Files.readAllBytes; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.junit.runner.Description.createTestDescription; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TestFailureAspectTest { + + private TestFailureAspect testFailureAspect; + + private static final Path TMP_PATH = Path.of("/tmp"); + + @BeforeClass + public static void createTmpFolder() throws IOException { + if (!exists(TMP_PATH)) { + createDirectory(TMP_PATH); + } + } + + @Before + public void setup() { + testFailureAspect = new TestFailureAspect(); + } + + @Test + public void afterFireTestFailure_shouldPersistMeasure() { + JoinPoint joinPoint = mock(JoinPoint.class); + Failure failure = new Failure( + createTestDescription("testClass", "testMethod"), + new IllegalStateException("some exception")); + when(joinPoint.getArgs()).thenReturn(new Object[]{failure}); + + testFailureAspect.afterFireTestFailure(joinPoint); + + String fileContent = getFileContent(Paths.get("/tmp/test-monitoring.log")); + assertThat(fileContent) + .contains("\"timestamp\":\"" ) + .contains("\"testClass\":\"testClass\"") + .contains("\"testMethod\":\"testMethod\"") + .contains("\"exceptionClass\":\"java.lang.IllegalStateException\"") + .contains("\"exceptionMessage\":\"some exception\"") + .contains("\"exceptionLogs\":\"java.lang.IllegalStateException: some exception"); + } + + private String getFileContent(Path path) { + try { + byte[] bytes = readAllBytes(path); + return new String(bytes, StandardCharsets.UTF_8); + } catch (IOException e) { + fail("Unable to read file " + path, e); + } + return null; + } + +} diff --git a/wss-unified-agent.config b/wss-unified-agent.config index db13f826b7b..3f4254e63c3 100644 --- a/wss-unified-agent.config +++ b/wss-unified-agent.config @@ -9,7 +9,7 @@ gradle.aggregateModules=True gradle.preferredEnvironment=wrapper gradle.innerModulesAsDependencies=True gradle.ignoredConfigurations=.*test.* .*bbt.* -gradle.excludeModules=.*it-.* .*ut-monitoring.* .*sonar-ws-generator.* .*sonar-testing-ldap.* .*sonar-testing-harness.* +gradle.excludeModules=.*it-.* .*ut-monitoring.* .*test-monitoring.* .*sonar-ws-generator.* .*sonar-testing-ldap.* .*sonar-testing-harness.* npm.includeDevDependencies=False npm.resolveDependencies=True -- 2.39.5