From: lukasz-jarocki-sonarsource Date: Wed, 23 Oct 2024 07:04:30 +0000 (+0200) Subject: NO-JIRA improved performance of tests in sonar-scanner-engine module X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=24bd8856af66bf2e1148c42279739225b254e84e;p=sonarqube.git NO-JIRA improved performance of tests in sonar-scanner-engine module --- diff --git a/sonar-scanner-engine/build.gradle b/sonar-scanner-engine/build.gradle index b8f1cee9c07..de0f9f12d19 100644 --- a/sonar-scanner-engine/build.gradle +++ b/sonar-scanner-engine/build.gradle @@ -73,6 +73,7 @@ dependencies { testImplementation project(':plugins:sonar-xoo-plugin') testImplementation 'org.wiremock:wiremock-standalone' testImplementation 'org.junit-pioneer:junit-pioneer' + testImplementation 'org.awaitility:awaitility' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' testRuntimeOnly 'org.junit.vintage:junit-vintage-engine' diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java index 384e8a8a164..efae70e8f09 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumIT.java @@ -337,9 +337,10 @@ public class FileSystemMediumIT { int nbFiles = 100; int ruleCount = 100000; + String fileContent = StringUtils.repeat(StringUtils.repeat("a", 100) + "\n", ruleCount / 1000); for (int nb = 1; nb <= nbFiles; nb++) { File xooFile = new File(srcDir, "sample" + nb + ".xoo"); - FileUtils.write(xooFile, StringUtils.repeat(StringUtils.repeat("a", 100) + "\n", ruleCount / 1000)); + FileUtils.write(xooFile, fileContent); } AnalysisResult result = tester.newAnalysis() diff --git a/sonar-scanner-engine/src/it/java/org/sonar/scm/svn/SvnBlameCommandIT.java b/sonar-scanner-engine/src/it/java/org/sonar/scm/svn/SvnBlameCommandIT.java index c459e48523f..808b98537e5 100644 --- a/sonar-scanner-engine/src/it/java/org/sonar/scm/svn/SvnBlameCommandIT.java +++ b/sonar-scanner-engine/src/it/java/org/sonar/scm/svn/SvnBlameCommandIT.java @@ -101,7 +101,8 @@ public class SvnBlameCommandIT { @Parameters(name = "SVN server version {0}, WC version {1}") public static Iterable data() { - return Arrays.asList(new Object[][] {{"1.6", 10}, {"1.7", 29}, {"1.8", 31}, {"1.9", 31}}); + //TODO add tests for 1.14 + return Arrays.asList(new Object[][] {{"1.6", 10}, {"1.9", 31}}); } public SvnBlameCommandIT(String serverVersion, int wcVersion) { diff --git a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.7/repo-svn-with-merge.zip b/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.7/repo-svn-with-merge.zip deleted file mode 100644 index d3c3ee5fa1e..00000000000 Binary files a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.7/repo-svn-with-merge.zip and /dev/null differ diff --git a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.7/repo-svn.zip b/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.7/repo-svn.zip deleted file mode 100644 index 291d4fb86f7..00000000000 Binary files a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.7/repo-svn.zip and /dev/null differ diff --git a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.8/repo-svn-with-merge.zip b/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.8/repo-svn-with-merge.zip deleted file mode 100644 index 852a05e4574..00000000000 Binary files a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.8/repo-svn-with-merge.zip and /dev/null differ diff --git a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.8/repo-svn.zip b/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.8/repo-svn.zip deleted file mode 100644 index f31352c5e03..00000000000 Binary files a/sonar-scanner-engine/src/it/resources/org/sonar/scm/svn/test-repos/1.8/repo-svn.zip and /dev/null differ diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java index 06147b4715d..251a12cc8f0 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploder.java @@ -19,6 +19,7 @@ */ package org.sonar.scanner.bootstrap; +import com.google.common.annotations.VisibleForTesting; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -36,6 +37,7 @@ import static org.sonar.core.util.FileUtils.deleteQuietly; public class ScannerPluginJarExploder extends PluginJarExploder { + private static long lockRetrySleep = 200L; private final PluginFiles pluginFiles; public ScannerPluginJarExploder(PluginFiles pluginFiles) { @@ -76,6 +78,11 @@ public class ScannerPluginJarExploder extends PluginJarExploder { return destDir; } + @VisibleForTesting + static void setLockRetrySleep(long sleep) { + lockRetrySleep = sleep; + } + private static FileLock createLockWithRetries(FileChannel channel) throws IOException { int tryCount = 0; while (tryCount++ < 10) { @@ -85,7 +92,7 @@ public class ScannerPluginJarExploder extends PluginJarExploder { // ignore overlapping file exception } try { - Thread.sleep(200L * tryCount); + Thread.sleep(lockRetrySleep * tryCount); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/qualitygate/QualityGateCheck.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/qualitygate/QualityGateCheck.java index aa4c30d352b..31a8394dbf7 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/qualitygate/QualityGateCheck.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/qualitygate/QualityGateCheck.java @@ -19,6 +19,7 @@ */ package org.sonar.scanner.qualitygate; +import com.google.common.annotations.VisibleForTesting; import java.io.InputStream; import java.time.Duration; import java.time.temporal.ChronoUnit; @@ -44,13 +45,13 @@ public class QualityGateCheck implements Startable { private static final Logger LOG = LoggerFactory.getLogger(QualityGateCheck.class); private static final EnumSet TASK_TERMINAL_STATUSES = EnumSet.of(TaskStatus.SUCCESS, TaskStatus.FAILED, TaskStatus.CANCELED); - private static final int POLLING_INTERVAL_IN_MS = 5000; private final DefaultScannerWsClient wsClient; private final GlobalAnalysisMode analysisMode; private final CeTaskReportDataHolder ceTaskReportDataHolder; private final ScanProperties properties; + private int pollingIntervalInSeconds = 5000; private long qualityGateTimeoutInMs; private boolean enabled; @@ -102,6 +103,11 @@ public class QualityGateCheck implements Startable { } } + @VisibleForTesting + void setPollingIntervalInSeconds(int pollingIntervalInSeconds) { + this.pollingIntervalInSeconds = pollingIntervalInSeconds; + } + private Ce.Task waitForCeTaskToFinish(String taskId) { GetRequest getTaskResultReq = new GetRequest("api/ce/task") .setMediaType(MediaTypes.PROTOBUF) @@ -116,8 +122,8 @@ public class QualityGateCheck implements Startable { return task; } - Thread.sleep(POLLING_INTERVAL_IN_MS); - currentTime += POLLING_INTERVAL_IN_MS; + Thread.sleep(pollingIntervalInSeconds); + currentTime += pollingIntervalInSeconds; } catch (HttpException e) { throw MessageException.of(String.format("Failed to get CE Task status - %s", DefaultScannerWsClient.createErrorMessage(e))); } catch (InterruptedException e) { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java index d300d025570..4ded7fb0901 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/bootstrap/ScannerPluginJarExploderTest.java @@ -50,6 +50,7 @@ public class ScannerPluginJarExploderTest { PluginFiles pluginFiles = mock(PluginFiles.class); when(pluginFiles.createTempDir()).thenReturn(tempDir); underTest = new ScannerPluginJarExploder(pluginFiles); + underTest.setLockRetrySleep(5L); } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/qualitygate/QualityGateCheckTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/qualitygate/QualityGateCheckTest.java index 0e7e67cb5c5..b31249ff815 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/qualitygate/QualityGateCheckTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/qualitygate/QualityGateCheckTest.java @@ -71,6 +71,7 @@ class QualityGateCheckTest { @BeforeEach void before() { underTest = new QualityGateCheck(wsClient, analysisMode, reportMetadataHolder, properties); + underTest.setPollingIntervalInSeconds(200); logTester.setLevel(Level.DEBUG); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/util/ProgressReportTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/util/ProgressReportTest.java index 4d04a83fd22..71a3fd644a1 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/util/ProgressReportTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/util/ProgressReportTest.java @@ -20,7 +20,10 @@ package org.sonar.scanner.util; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.DisableOnDebug; @@ -57,18 +60,25 @@ public class ProgressReportTest { } @Test - public void do_log() throws InterruptedException { + public void do_log() { logTester.setLevel(Level.DEBUG); underTest.start("start"); underTest.message("Some message"); boolean logged = false; - Thread.sleep(2000); while (!logged) { - logged = logTester.logs().contains("Some message"); + logged = containsWithRetry("Some message"); } underTest.stop("stop"); - Thread.sleep(1000); - assertThat(logTester.logs().stream().anyMatch(s -> Pattern.matches("stop", s))).isTrue(); + assertThat(containsWithRetry("stop")).isTrue(); + } + + private boolean containsWithRetry(String message) { + try { + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> logTester.logs().contains(message)); + } catch (ConditionTimeoutException e) { + return false; + } + return true; } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java index 419a205fd52..c95966d6d18 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/git/CompositeBlameCommandIT.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.io.CleanupMode; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -57,19 +58,25 @@ import static org.mockito.Mockito.when; class CompositeBlameCommandIT { + private static final ProcessWrapperFactory processWrapperFactory = new ProcessWrapperFactory(); + private static final NativeGitBlameCommand nativeGitBlameCommand = new NativeGitBlameCommand(System2.INSTANCE, processWrapperFactory); + private final AnalysisWarnings analysisWarnings = mock(AnalysisWarnings.class); private final BlameCommand.BlameInput input = mock(BlameCommand.BlameInput.class); private final JGitBlameCommand jGitBlameCommand = new JGitBlameCommand(); - private final ProcessWrapperFactory processWrapperFactory = new ProcessWrapperFactory(); - private final NativeGitBlameCommand nativeGitBlameCommand = new NativeGitBlameCommand(System2.INSTANCE, processWrapperFactory); - // In JUnit4, if the cleanup cannot be performed, the test would not fail. This has changed with JUnit5 // As we cannot find the cause of failure during cleanup, we disable it for now @TempDir (cleanup = NEVER) private File temp; + @BeforeAll + public static void setUp() { + // locating the git executable is a costly operation that can be done just once for all tests in this class + nativeGitBlameCommand.checkIfEnabled(); + } + @ParameterizedTest @MethodSource("namesOfTheTestRepositoriesWithBlameAlgorithm") void testThatBlameAlgorithmOutputsTheSameDataAsGitNativeBlame(String folder, BlameAlgorithmEnum blameAlgorithm) throws Exception { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scm/svn/SvnScmProviderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scm/svn/SvnScmProviderTest.java index 70bbf25bf22..f753e31bd8f 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scm/svn/SvnScmProviderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scm/svn/SvnScmProviderTest.java @@ -86,10 +86,6 @@ public class SvnScmProviderTest { @Before public void before() throws IOException, SVNException { svnTester = new SvnTester(temp.newFolder().toPath()); - - Path worktree = temp.newFolder().toPath(); - svnTester.checkout(worktree, "trunk"); - createAndCommitFile(worktree, "file-in-first-commit.xoo"); } @Test @@ -119,33 +115,36 @@ public class SvnScmProviderTest { } @Test - public void branchChangedFiles_and_lines_from_diverged() throws IOException, SVNException { + public void branchChangedFiles_and_lines_from_diverged() throws IOException, SVNException, InterruptedException { Path trunk = temp.newFolder().toPath(); svnTester.checkout(trunk, "trunk"); - createAndCommitFile(trunk, "file-m1.xoo"); - createAndCommitFile(trunk, "file-m2.xoo"); - createAndCommitFile(trunk, "file-m3.xoo"); - createAndCommitFile(trunk, "lao.txt", CONTENT_LAO); + createFile(trunk, "file-m1.xoo"); + createFile(trunk, "file-m2.xoo"); + createFile(trunk, "file-m3.xoo"); + createFile(trunk, "lao.txt", CONTENT_LAO); + svnTester.commit(trunk); // create branch from trunk svnTester.createBranch("b1"); // still on trunk - appendToAndCommitFile(trunk, "file-m3.xoo"); - createAndCommitFile(trunk, "file-m4.xoo"); + svnTester.appendToFile(trunk, "file-m3.xoo"); + createFile(trunk, "file-m4.xoo"); + svnTester.commit(trunk); Path b1 = temp.newFolder().toPath(); svnTester.checkout(b1, "branches/b1"); Files.createDirectories(b1.resolve("sub")); - createAndCommitFile(b1, "sub/file-b1.xoo"); - appendToAndCommitFile(b1, "file-m1.xoo"); - deleteAndCommitFile(b1, "file-m2.xoo"); + createFile(b1, "sub/file-b1.xoo"); + svnTester.appendToFile(b1, "file-m1.xoo"); + deleteFile(b1, "file-m2.xoo"); createAndCommitFile(b1, "file-m5.xoo"); - deleteAndCommitFile(b1, "file-m5.xoo"); + deleteFile(b1, "file-m5.xoo"); svnCopyAndCommitFile(b1, "file-m1.xoo", "file-m1-copy.xoo"); - appendToAndCommitFile(b1, "file-m1.xoo"); + svnTester.appendToFile(b1, "file-m1.xoo"); + svnTester.commit(b1); // modify file without committing it -> should not be included (think generated files) svnTester.appendToFile(b1, "file-m3.xoo"); @@ -268,8 +267,7 @@ public class SvnScmProviderTest { } private void createAndCommitFile(Path worktree, String filename, String content) throws IOException, SVNException { - svnTester.createFile(worktree, filename, content); - svnTester.add(worktree, filename); + createFile(worktree, filename, content); svnTester.commit(worktree); } @@ -277,14 +275,18 @@ public class SvnScmProviderTest { createAndCommitFile(worktree, filename, filename + "\n"); } - private void appendToAndCommitFile(Path worktree, String filename) throws IOException, SVNException { - svnTester.appendToFile(worktree, filename); - svnTester.commit(worktree); + private void createFile(Path worktree, String filename) throws IOException, SVNException { + svnTester.createFile(worktree, filename, filename + "\n"); + svnTester.add(worktree, filename); + } + + private void createFile(Path worktree, String filename, String content) throws IOException, SVNException { + svnTester.createFile(worktree, filename, content); + svnTester.add(worktree, filename); } - private void deleteAndCommitFile(Path worktree, String filename) throws SVNException { + private void deleteFile(Path worktree, String filename) throws SVNException { svnTester.deleteFile(worktree, filename); - svnTester.commit(worktree); } private void svnCopyAndCommitFile(Path worktree, String src, String dst) throws SVNException {