SONAR-22649 enable parallel generation of test data.

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java
index 0bde18cd641214edbb4d11f928a2ec179be396cf..60401c9a4ff157b836ca83ab74ece7527f5364fb 100644 (file)
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java
@@ -39,7 +39,7 @@ class MyBatisConfBuilder {
   MyBatisConfBuilder(Database database) {
     this.conf = new Configuration();
     this.conf.setEnvironment(new Environment("production", createTransactionFactory(), database.getDataSource()));
-    this.conf.setUseGeneratedKeys(true);
+    this.conf.setUseGeneratedKeys(false);
     this.conf.setLazyLoadingEnabled(false);
     this.conf.setJdbcTypeForNull(JdbcType.NULL);
     Dialect dialect = database.getDialect();

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java
index a7263c547d1aea22e53782ba7759253d1a7602e1..1fe5463af84853c3ddb542f2d0f6d957c10cbd88 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java
@@ -20,14 +20,13 @@
 package org.sonar.db;
 
 import org.apache.ibatis.session.Configuration;
-import org.hamcrest.core.Is;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.sonar.db.rule.RuleMapper;
 
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.junit.Assert.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
+
 
 public class MyBatisTest {
   private static SQDatabase database;
@@ -50,9 +49,10 @@ public class MyBatisTest {
     underTest.start();
 
     Configuration conf = underTest.getSessionFactory().getConfiguration();
-    assertThat(conf.isUseGeneratedKeys(), Is.is(true));
-    assertThat(conf.hasMapper(RuleMapper.class), Is.is(true));
-    assertThat(conf.isLazyLoadingEnabled(), Is.is(false));
+
+    assertThat(conf.isUseGeneratedKeys()).isFalse();
+    assertThat(conf.hasMapper(RuleMapper.class)).isTrue();
+    assertThat(conf.isLazyLoadingEnabled()).isFalse();
   }
 
   @Test
@@ -60,8 +60,8 @@ public class MyBatisTest {
     underTest.start();
 
     try (DbSession session = underTest.openSession(false)) {
-      assertThat(session.getConnection(), notNullValue());
-      assertThat(session.getMapper(RuleMapper.class), notNullValue());
+      assertThat(session.getConnection()).isNotNull();
+      assertThat(session.getMapper(RuleMapper.class)).isNotNull();
     }
   }
 }

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java
index 934c0cc815737b0237d97578b7ca789a7b2bf16a..7acdb362c8f51e9ac0bb606e3784072c303a6144 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java
@@ -76,7 +76,6 @@ public class CeTaskInputDaoTest {
   @Test
   public void selectData_returns_absent_if_uuid_exists_but_data_is_null() {
     insertData(A_UUID);
-    dbTester.commit();
 
     Optional<DbInputStream> result = underTest.selectData(dbTester.getSession(), A_UUID);
     assertThat(result).isNotPresent();
@@ -105,6 +104,5 @@ public class CeTaskInputDaoTest {
 
   private void insertData(String uuid) {
     dbTester.executeInsert(TABLE_NAME, "task_uuid", uuid, "created_at", NOW, "updated_at", NOW);
-    dbTester.commit();
   }
 }

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java b/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java
index cbc7a348a12b42a2abe2410f9b5e570bf2cfa376..654877b0ff695484679b2871050d58c367e6090d 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java
@@ -26,8 +26,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import org.jetbrains.annotations.NotNull;
 import org.sonar.api.measures.CoreMetrics;
@@ -61,26 +65,42 @@ public class PopulateDb {
 
   private static final long NUMBER_OF_USERS = 100_000L;
 
-  public static void main(String[] args) {
+  public static void main(String[] args) throws InterruptedException {
+    final DbTester dbTester = createDbTester();
+
     // read base data
     final Map<String, MetricDto> metricDtosByKey;
     final List<ProjectDto> allProjects;
     final List<PortfolioDto> allPortfolios;
     final Set<RuleDto> enabledRules;
 
-    DbSession dbSession = dbTester.getSession();
-    metricDtosByKey = dbTester.getDbClient().metricDao().selectAll(dbSession).stream().collect(
-      Collectors.toMap(MetricDto::getKey, Function.identity()));
-    allProjects = dbTester.getDbClient().projectDao().selectProjects(dbSession);
-    allPortfolios = dbTester.getDbClient().portfolioDao().selectAll(dbSession);
+    DbSession initSession = dbTester.getSession();
+    metricDtosByKey = dbTester.getDbClient().metricDao().selectAll(initSession).stream().collect(
+      Collectors.toMap(MetricDto::getKey, Function.identity())
+    );
+    allProjects = dbTester.getDbClient().projectDao().selectProjects(initSession);
+    allPortfolios = dbTester.getDbClient().portfolioDao().selectAll(initSession);
     enabledRules = new HashSet<>(dbTester.getDbClient().ruleDao().selectEnabled(dbTester.getSession()));
 
-    ProjectDto projectDto = generateProject(new SqContext(enabledRules, metricDtosByKey, dbTester),
-      new ProjectStructure("project " + (allProjects.size() + 1), 10, 10, 10, 2, 5));
-    allProjects.add(projectDto);
+    ExecutorService executorService = Executors.newFixedThreadPool(1);
+    SqContext sqContext = new SqContext(enabledRules, metricDtosByKey, dbTester);
 
-    createUsers(allProjects);
+    IntStream.rangeClosed(1, 1)
+      .map(i -> i + allProjects.size())
+      .mapToObj(i -> new ProjectStructure("project " + i, 10, 100, 10, 2, 5))
+        .forEach(projectStructure -> {
+          executorService.submit(() -> {
+            sqContext.dbTester.getSession(true);
+            generateProject(
+              sqContext, projectStructure
+            );
+          });
+        });
 
+    executorService.shutdown();
+    executorService.awaitTermination(100, TimeUnit.DAYS);
+
+    createUsers(allProjects);
     allPortfolios.addAll(createPortfolios(new PortfolioGenerationSettings(allPortfolios.size(), 100, 10), allProjects));
 
     // close database connection
@@ -185,6 +205,7 @@ public class PopulateDb {
 
   private static ProjectDto generateProject(SqContext sqContext, ProjectStructure pj) {
     ComponentDto projectCompoDto = sqContext.dbTester.components().insertPublicProject(p -> p.setName(pj.projectName));
+    sqContext.dbTester.forceCommit();
     ProjectDto projectDto = sqContext.dbTester.components().getProjectDto(projectCompoDto);
 
     Streams.concat(
@@ -242,6 +263,7 @@ public class PopulateDb {
         SnapshotDto lastSnapshotDto = snapshots.get(0);
         lastSnapshotDto.setLast(true);
         sqContext.dbTester.components().insertSnapshots(snapshots.toArray(new SnapshotDto[0]));
+        sqContext.dbTester.forceCommit();
       });
 
     return projectDto;

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java
index 07bd20d7fe1013b579f9dd38b66357192c773e25..ae37a612f37d36f09d6ce00e29e04bc8a729b4d6 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java
@@ -74,7 +74,6 @@ public class NotificationQueueDaoTest {
       .mapToObj(i -> toNotificationQueueDto(new Notification("foo_" + i)))
       .collect(toList());
     dao.insert(notifs);
-    db.commit();
 
     List<String> uuids = selectAllUuid();
 
@@ -99,7 +98,6 @@ public class NotificationQueueDaoTest {
       .mapToObj(i -> toNotificationQueueDto(new Notification("foo_" + i)))
       .collect(toList());
     dao.insert(notifs);
-    db.commit();
 
     assertThat(dao.selectOldest(3))
       .extracting(NotificationQueueDto::getData)

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java
index 07222f7e4e16ffab65fd8d8f4c6c7b5839ec1624..755ba05a2747763243787896cfec2ca717de1090 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java
@@ -109,6 +109,5 @@ public class PluginDaoTest {
       "removed", removed,
       "created_at", createdAt,
       "updated_at", updatedAt);
-    db.commit();
   }
 }

diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java
index 8e34db0f454dbba971c3282bcff23df2cb968792..0a593e2ec8fd33545f559159294c4a3cc77e143d 100644 (file)
--- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java
+++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java
@@ -28,6 +28,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
+import org.jetbrains.annotations.NotNull;
 import org.sonar.api.utils.System2;
 import org.sonar.core.util.UuidFactory;
 import org.sonar.core.util.UuidFactoryFast;
@@ -66,7 +67,7 @@ public class DbTester extends AbstractDbTester<TestDbImpl> {
   private final System2 system2;
   private final AuditPersister auditPersister;
   private DbClient client;
-  ThreadLocal<DbSession> session = new ThreadLocal<>();
+  ThreadLocal<DbSessionContext> session = new ThreadLocal<>();
   private final UserDbTester userTester;
   private final ComponentDbTester componentTester;
   private final ProjectLinkDbTester componentLinkTester;
@@ -251,22 +252,32 @@ public class DbTester extends AbstractDbTester<TestDbImpl> {
   @Override
   protected void after() {
     if (session.get() != null) {
-      session.get().rollback();
-      session.get().close();
+      session.get().dbSession.rollback();
+      session.get().dbSession.close();
       session.remove();
     }
     db.stop();
   }
 
   public DbSession getSession() {
+    return getSession(false);
+  }
+
+  public DbSession getSession(boolean batched) {
     if (session.get() == null) {
-      session.set(db.getMyBatis().openSession(false));
+      session.set(new DbSessionContext(db.getMyBatis().openSession(batched), batched));
     }
-    return session.get();
+    return session.get().dbSession;
+  }
+
+  public void forceCommit() {
+    getSession().commit(true);
   }
 
   public void commit() {
-    getSession().commit();
+    if(session.get() != null && !session.get().isBatched) {
+      getSession().commit();
+    }
   }
 
   public DbClient getDbClient() {
@@ -312,6 +323,8 @@ public class DbTester extends AbstractDbTester<TestDbImpl> {
     return ((HikariDataSource) db.getDatabase().getDataSource()).getJdbcUrl();
   }
 
+  private record DbSessionContext(@NotNull DbSession dbSession, boolean isBatched){}
+
   private static class DbSessionConnectionSupplier implements ConnectionSupplier {
     private final DbSession dbSession;
 
@@ -336,8 +349,8 @@ public class DbTester extends AbstractDbTester<TestDbImpl> {
 
     public DbTesterMyBatisConfExtension(Class<?> firstMapperClass, Class<?>... otherMapperClasses) {
       this.mapperClasses = Stream.concat(
-        Stream.of(firstMapperClass),
-        Arrays.stream(otherMapperClasses))
+          Stream.of(firstMapperClass),
+          Arrays.stream(otherMapperClasses))
         .sorted(Comparator.comparing(Class::getName))
         .toArray(Class<?>[]::new);
     }