From 7fb5fa728b6da22b35a1e8062325ec1fbca03c00 Mon Sep 17 00:00:00 2001 From: Steve Marion Date: Wed, 31 Jul 2024 17:27:04 +0200 Subject: [PATCH] SONAR-22649 enable parallel generation of test data. --- .../java/org/sonar/db/MyBatisConfBuilder.java | 2 +- .../test/java/org/sonar/db/MyBatisTest.java | 16 +++---- .../org/sonar/db/ce/CeTaskInputDaoTest.java | 2 - .../org/sonar/db/createdb/PopulateDb.java | 42 ++++++++++++++----- .../NotificationQueueDaoTest.java | 2 - .../org/sonar/db/plugin/PluginDaoTest.java | 1 - .../java/org/sonar/db/DbTester.java | 29 +++++++++---- 7 files changed, 62 insertions(+), 32 deletions(-) diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java index 0bde18cd641..60401c9a4ff 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatisConfBuilder.java @@ -39,7 +39,7 @@ class MyBatisConfBuilder { MyBatisConfBuilder(Database database) { this.conf = new Configuration(); this.conf.setEnvironment(new Environment("production", createTransactionFactory(), database.getDataSource())); - this.conf.setUseGeneratedKeys(true); + this.conf.setUseGeneratedKeys(false); this.conf.setLazyLoadingEnabled(false); this.conf.setJdbcTypeForNull(JdbcType.NULL); Dialect dialect = database.getDialect(); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java index a7263c547d1..1fe5463af84 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/MyBatisTest.java @@ -20,14 +20,13 @@ package org.sonar.db; import org.apache.ibatis.session.Configuration; -import org.hamcrest.core.Is; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.sonar.db.rule.RuleMapper; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.junit.Assert.assertThat; +import static org.assertj.core.api.Assertions.assertThat; + public class MyBatisTest { private static SQDatabase database; @@ -50,9 +49,10 @@ public class MyBatisTest { underTest.start(); Configuration conf = underTest.getSessionFactory().getConfiguration(); - assertThat(conf.isUseGeneratedKeys(), Is.is(true)); - assertThat(conf.hasMapper(RuleMapper.class), Is.is(true)); - assertThat(conf.isLazyLoadingEnabled(), Is.is(false)); + + assertThat(conf.isUseGeneratedKeys()).isFalse(); + assertThat(conf.hasMapper(RuleMapper.class)).isTrue(); + assertThat(conf.isLazyLoadingEnabled()).isFalse(); } @Test @@ -60,8 +60,8 @@ public class MyBatisTest { underTest.start(); try (DbSession session = underTest.openSession(false)) { - assertThat(session.getConnection(), notNullValue()); - assertThat(session.getMapper(RuleMapper.class), notNullValue()); + assertThat(session.getConnection()).isNotNull(); + assertThat(session.getMapper(RuleMapper.class)).isNotNull(); } } } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java index 934c0cc8157..7acdb362c8f 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/ce/CeTaskInputDaoTest.java @@ -76,7 +76,6 @@ public class CeTaskInputDaoTest { @Test public void selectData_returns_absent_if_uuid_exists_but_data_is_null() { insertData(A_UUID); - dbTester.commit(); Optional result = underTest.selectData(dbTester.getSession(), A_UUID); assertThat(result).isNotPresent(); @@ -105,6 +104,5 @@ public class CeTaskInputDaoTest { private void insertData(String uuid) { dbTester.executeInsert(TABLE_NAME, "task_uuid", uuid, "created_at", NOW, "updated_at", NOW); - dbTester.commit(); } } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java b/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java index cbc7a348a12..654877b0ff6 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java @@ -26,8 +26,12 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import org.jetbrains.annotations.NotNull; import org.sonar.api.measures.CoreMetrics; @@ -61,26 +65,42 @@ public class PopulateDb { private static final long NUMBER_OF_USERS = 100_000L; - public static void main(String[] args) { + public static void main(String[] args) throws InterruptedException { + final DbTester dbTester = createDbTester(); + // read base data final Map metricDtosByKey; final List allProjects; final List allPortfolios; final Set enabledRules; - DbSession dbSession = dbTester.getSession(); - metricDtosByKey = dbTester.getDbClient().metricDao().selectAll(dbSession).stream().collect( - Collectors.toMap(MetricDto::getKey, Function.identity())); - allProjects = dbTester.getDbClient().projectDao().selectProjects(dbSession); - allPortfolios = dbTester.getDbClient().portfolioDao().selectAll(dbSession); + DbSession initSession = dbTester.getSession(); + metricDtosByKey = dbTester.getDbClient().metricDao().selectAll(initSession).stream().collect( + Collectors.toMap(MetricDto::getKey, Function.identity()) + ); + allProjects = dbTester.getDbClient().projectDao().selectProjects(initSession); + allPortfolios = dbTester.getDbClient().portfolioDao().selectAll(initSession); enabledRules = new HashSet<>(dbTester.getDbClient().ruleDao().selectEnabled(dbTester.getSession())); - ProjectDto projectDto = generateProject(new SqContext(enabledRules, metricDtosByKey, dbTester), - new ProjectStructure("project " + (allProjects.size() + 1), 10, 10, 10, 2, 5)); - allProjects.add(projectDto); + ExecutorService executorService = Executors.newFixedThreadPool(1); + SqContext sqContext = new SqContext(enabledRules, metricDtosByKey, dbTester); - createUsers(allProjects); + IntStream.rangeClosed(1, 1) + .map(i -> i + allProjects.size()) + .mapToObj(i -> new ProjectStructure("project " + i, 10, 100, 10, 2, 5)) + .forEach(projectStructure -> { + executorService.submit(() -> { + sqContext.dbTester.getSession(true); + generateProject( + sqContext, projectStructure + ); + }); + }); + executorService.shutdown(); + executorService.awaitTermination(100, TimeUnit.DAYS); + + createUsers(allProjects); allPortfolios.addAll(createPortfolios(new PortfolioGenerationSettings(allPortfolios.size(), 100, 10), allProjects)); // close database connection @@ -185,6 +205,7 @@ public class PopulateDb { private static ProjectDto generateProject(SqContext sqContext, ProjectStructure pj) { ComponentDto projectCompoDto = sqContext.dbTester.components().insertPublicProject(p -> p.setName(pj.projectName)); + sqContext.dbTester.forceCommit(); ProjectDto projectDto = sqContext.dbTester.components().getProjectDto(projectCompoDto); Streams.concat( @@ -242,6 +263,7 @@ public class PopulateDb { SnapshotDto lastSnapshotDto = snapshots.get(0); lastSnapshotDto.setLast(true); sqContext.dbTester.components().insertSnapshots(snapshots.toArray(new SnapshotDto[0])); + sqContext.dbTester.forceCommit(); }); return projectDto; diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java index 07bd20d7fe1..ae37a612f37 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/notification/NotificationQueueDaoTest.java @@ -74,7 +74,6 @@ public class NotificationQueueDaoTest { .mapToObj(i -> toNotificationQueueDto(new Notification("foo_" + i))) .collect(toList()); dao.insert(notifs); - db.commit(); List uuids = selectAllUuid(); @@ -99,7 +98,6 @@ public class NotificationQueueDaoTest { .mapToObj(i -> toNotificationQueueDto(new Notification("foo_" + i))) .collect(toList()); dao.insert(notifs); - db.commit(); assertThat(dao.selectOldest(3)) .extracting(NotificationQueueDto::getData) diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java index 07222f7e4e1..755ba05a274 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/plugin/PluginDaoTest.java @@ -109,6 +109,5 @@ public class PluginDaoTest { "removed", removed, "created_at", createdAt, "updated_at", updatedAt); - db.commit(); } } diff --git a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java index 8e34db0f454..0a593e2ec8f 100644 --- a/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java +++ b/server/sonar-db-dao/src/testFixtures/java/org/sonar/db/DbTester.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; import javax.annotation.Nullable; +import org.jetbrains.annotations.NotNull; import org.sonar.api.utils.System2; import org.sonar.core.util.UuidFactory; import org.sonar.core.util.UuidFactoryFast; @@ -66,7 +67,7 @@ public class DbTester extends AbstractDbTester { private final System2 system2; private final AuditPersister auditPersister; private DbClient client; - ThreadLocal session = new ThreadLocal<>(); + ThreadLocal session = new ThreadLocal<>(); private final UserDbTester userTester; private final ComponentDbTester componentTester; private final ProjectLinkDbTester componentLinkTester; @@ -251,22 +252,32 @@ public class DbTester extends AbstractDbTester { @Override protected void after() { if (session.get() != null) { - session.get().rollback(); - session.get().close(); + session.get().dbSession.rollback(); + session.get().dbSession.close(); session.remove(); } db.stop(); } public DbSession getSession() { + return getSession(false); + } + + public DbSession getSession(boolean batched) { if (session.get() == null) { - session.set(db.getMyBatis().openSession(false)); + session.set(new DbSessionContext(db.getMyBatis().openSession(batched), batched)); } - return session.get(); + return session.get().dbSession; + } + + public void forceCommit() { + getSession().commit(true); } public void commit() { - getSession().commit(); + if(session.get() != null && !session.get().isBatched) { + getSession().commit(); + } } public DbClient getDbClient() { @@ -312,6 +323,8 @@ public class DbTester extends AbstractDbTester { return ((HikariDataSource) db.getDatabase().getDataSource()).getJdbcUrl(); } + private record DbSessionContext(@NotNull DbSession dbSession, boolean isBatched){} + private static class DbSessionConnectionSupplier implements ConnectionSupplier { private final DbSession dbSession; @@ -336,8 +349,8 @@ public class DbTester extends AbstractDbTester { public DbTesterMyBatisConfExtension(Class firstMapperClass, Class... otherMapperClasses) { this.mapperClasses = Stream.concat( - Stream.of(firstMapperClass), - Arrays.stream(otherMapperClasses)) + Stream.of(firstMapperClass), + Arrays.stream(otherMapperClasses)) .sorted(Comparator.comparing(Class::getName)) .toArray(Class[]::new); } -- 2.39.5