SONAR-22891 add gradle task to run populator. add logs to execution and errors.

diff --git a/server/sonar-db-dao/build.gradle b/server/sonar-db-dao/build.gradle
index d777d3c04e3a37e7a37540238604e6bb32cc9fb0..955b0a1cebd582045eac628a3eda42fe16fac8fc 100644 (file)
--- a/server/sonar-db-dao/build.gradle
+++ b/server/sonar-db-dao/build.gradle
@@ -1,61 +1,61 @@
 sonar {
-  properties {
-    property 'sonar.projectName', "${projectTitle} :: DAO"
-  }
+    properties {
+        property 'sonar.projectName', "${projectTitle} :: DAO"
+    }
 }
 
 dependencies {
-  // please keep the list grouped by configuration and ordered by name
+    // please keep the list grouped by configuration and ordered by name
 
-  api 'com.google.guava:guava'
-  api 'com.google.protobuf:protobuf-java'
-  api 'commons-io:commons-io'
-  api 'commons-lang:commons-lang'
-  api 'net.jpountz.lz4:lz4'
-  api 'org.mybatis:mybatis'
-  api 'org.sonarsource.api.plugin:sonar-plugin-api'
+    api 'com.google.guava:guava'
+    api 'com.google.protobuf:protobuf-java'
+    api 'commons-io:commons-io'
+    api 'commons-lang:commons-lang'
+    api 'net.jpountz.lz4:lz4'
+    api 'org.mybatis:mybatis'
+    api 'org.sonarsource.api.plugin:sonar-plugin-api'
 
-  api project(':server:sonar-db-core')
-  api project(':server:sonar-db-migration')
-  api project(':sonar-core')
+    api project(':server:sonar-db-core')
+    api project(':server:sonar-db-migration')
+    api project(':sonar-core')
 
-  compileOnlyApi 'com.google.code.findbugs:jsr305'
+    compileOnlyApi 'com.google.code.findbugs:jsr305'
 
-  testImplementation 'com.tngtech.java:junit-dataprovider'
-  testImplementation 'commons-dbutils:commons-dbutils'
-  testImplementation 'junit:junit'
-  testImplementation 'org.assertj:assertj-core'
-  testImplementation 'org.assertj:assertj-guava'
-  testImplementation 'org.mockito:mockito-core'
-  testImplementation 'org.sonarsource.orchestrator:sonar-orchestrator-junit4'
-  testImplementation project(':sonar-testing-harness')
-  testImplementation project(':sonar-plugin-api-impl')
+    testImplementation 'com.tngtech.java:junit-dataprovider'
+    testImplementation 'commons-dbutils:commons-dbutils'
+    testImplementation 'junit:junit'
+    testImplementation 'org.assertj:assertj-core'
+    testImplementation 'org.assertj:assertj-guava'
+    testImplementation 'org.mockito:mockito-core'
+    testImplementation 'org.sonarsource.orchestrator:sonar-orchestrator-junit4'
+    testImplementation project(':sonar-testing-harness')
+    testImplementation project(':sonar-plugin-api-impl')
 
-  testCompileOnly 'com.google.code.findbugs:jsr305'
+    testCompileOnly 'com.google.code.findbugs:jsr305'
 
-  testRuntimeOnly 'com.h2database:h2'
-  testRuntimeOnly 'com.microsoft.sqlserver:mssql-jdbc'
-  testRuntimeOnly 'com.oracle.database.jdbc:ojdbc11'
-  testRuntimeOnly 'org.postgresql:postgresql'
+    testRuntimeOnly 'com.h2database:h2'
+    testRuntimeOnly 'com.microsoft.sqlserver:mssql-jdbc'
+    testRuntimeOnly 'com.oracle.database.jdbc:ojdbc11'
+    testRuntimeOnly 'org.postgresql:postgresql'
 
-  testFixturesApi testFixtures(project(':server:sonar-db-core'))
+    testFixturesApi testFixtures(project(':server:sonar-db-core'))
 
-  testFixturesImplementation 'com.h2database:h2'
+    testFixturesImplementation 'com.h2database:h2'
 
-  testFixturesCompileOnly 'com.google.code.findbugs:jsr305'
+    testFixturesCompileOnly 'com.google.code.findbugs:jsr305'
 }
 
 test {
-  systemProperty 'orchestrator.configUrl', System.getProperty('orchestrator.configUrl')
+    systemProperty 'orchestrator.configUrl', System.getProperty('orchestrator.configUrl')
 }
 
-task dumpSchema(type:JavaExec) {
+task dumpSchema(type: JavaExec) {
     mainClass = 'org.sonar.db.dump.DumpSQSchema'
     classpath = sourceSets.test.runtimeClasspath
 }
 tasks.check.dependsOn dumpSchema
 
-task createDB(type:JavaExec) {
+task createDB(type: JavaExec) {
     mainClass = 'org.sonar.db.createdb.CreateDb'
     classpath = sourceSets.test.runtimeClasspath
     systemProperty 'orchestrator.configUrl', System.getProperty('orchestrator.configUrl')
@@ -64,17 +64,22 @@ task createDB(type:JavaExec) {
     }
 }
 
+task populateDB(type: JavaExec) {
+    mainClass = 'org.sonar.db.createdb.PopulateDb'
+    classpath = sourceSets.test.runtimeClasspath
+}
+
 task testJar(type: Jar) {
-  archiveClassifier = 'tests'
-  from sourceSets.test.output
+    archiveClassifier = 'tests'
+    from sourceSets.test.output
 }
 
 configurations {
-  tests
+    tests
 }
 
 artifacts {
-   tests testJar
+    tests testJar
 }
 
 jar {

diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java b/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java
index 8a21671c33c54543f013191e6082359ef6409545..1a69b3ba18c3858b7ad89307e7e1b9bfdbbe949c 100644 (file)
--- a/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java
+++ b/server/sonar-db-dao/src/test/java/org/sonar/db/createdb/PopulateDb.java
@@ -30,6 +30,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -38,6 +39,8 @@ import org.jetbrains.annotations.NotNull;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.rules.RuleType;
 import org.sonar.api.utils.System2;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.core.util.Uuids;
 import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
@@ -62,9 +65,31 @@ import static org.sonar.db.component.BranchType.BRANCH;
 
 public class PopulateDb {
 
+  private static final Logger LOG = Loggers.get(PopulateDb.class);
+
+  public static final int NB_PROJECT_WISHED = 4;
+  public static final int NB_WORKER = 2;
+  public static final int BRANCH_PER_PROJECT = 7;
+  public static final int FILE_PER_BRANCH = 377;
+  public static final int ISSUE_PER_FILE = 3;
+  public static final int SNAPSHOT_PER_BRANCH = 13;
+  public static final int WEBHOOK_DELIVERIES_PER_COMPONENT = 1;
+  public static final int NB_USER = 100;
+  public static final int NUMBER_OF_PORTOLIOS = 100;
+  public static final int MAX_PROJECT_PER_PORTFOLIO = 10;
+
   public static void main(String[] args) throws InterruptedException {
+    LOG.info("Population procedure starting");
+
+    System.setProperty("sonar.jdbc.url", "jdbc:postgresql://localhost:5432/sonarqube");
+    System.setProperty("sonar.jdbc.username", "sonarqube");
+    System.setProperty("sonar.jdbc.password", "sonarqube");
+    System.setProperty("sonar.jdbc.dialect", "postgresql");
+    System.setProperty("sonar.jdbc.maximumPoolSize", "" + (NB_WORKER + 1));
     final DbTester dbTester = createDbTester();
 
+    LOG.info("Database infrastructure is set up");
+
     // read base data
     final Map<String, MetricDto> metricDtosByKey;
     final List<ProjectDto> allProjects;
@@ -81,28 +106,40 @@ public class PopulateDb {
     adminGroupDto = dbTester.getDbClient().groupDao().selectByName(dbTester.getSession(), "sonar-administrators")
       .orElseThrow(() -> new IllegalStateException("group with name \"sonar-administrators\" is expected to exist"));
     SqContext sqContext = new SqContext(allProjects, enabledRules, metricDtosByKey, adminGroupDto, dbTester);
+    LOG.info("Existing data has been collected");
 
-    ExecutorService executorService = Executors.newFixedThreadPool(1);
 
-    IntStream.rangeClosed(1, 1)
+    ExecutorService executorService = Executors.newFixedThreadPool(NB_WORKER);
+    final AtomicInteger nbProjectsGenerated = new AtomicInteger(0);
+    LOG.info("Starting generation of {} projects", NB_PROJECT_WISHED);
+    IntStream.rangeClosed(1, NB_PROJECT_WISHED)
       .map(i -> i + allProjects.size())
-      .mapToObj(i -> new ProjectStructure("project " + i, 10, 100, 10, 2, 5, 1))
+      .mapToObj(i -> new ProjectStructure("project " + i, BRANCH_PER_PROJECT, FILE_PER_BRANCH, ISSUE_PER_FILE, ISSUE_PER_FILE, SNAPSHOT_PER_BRANCH, WEBHOOK_DELIVERIES_PER_COMPONENT))
       .forEach(projectStructure -> {
         executorService.submit(() -> {
-          sqContext.dbTester.getSession(true);
-          allProjects.add(generateProject(
-            sqContext, projectStructure
-          ));
+          LOG.info("Worker-{}: Starting generation of project: {}", Thread.currentThread().getName(), projectStructure.toString());
+          try {
+            sqContext.dbTester.getSession(true);
+            allProjects.add(generateProject(
+              sqContext, projectStructure
+            ));
+          } catch (Exception e) {
+            LOG.error("Worker-" + Thread.currentThread().getName() + ": Error while generating project", e);
+            return;
+          }
+          nbProjectsGenerated.incrementAndGet();
+          LOG.info("Worker-{}: Project generation completed: {}", Thread.currentThread().getName(), projectStructure.projectName);
         });
       });
 
     executorService.shutdown();
     executorService.awaitTermination(100, TimeUnit.DAYS);
+    LOG.info("Ending generation of {}/{} projects", nbProjectsGenerated.get(), NB_PROJECT_WISHED);
 
-    createUsers(sqContext, 100);
+    createUsers(sqContext, NB_USER);
 
     allPortfolios = new ArrayList<>(dbTester.getDbClient().portfolioDao().selectAll(initSession));
-    allPortfolios.addAll(createPortfolios(sqContext, new PortfolioGenerationSettings(allPortfolios.size(), 100, 10)));
+    allPortfolios.addAll(createPortfolios(sqContext, new PortfolioGenerationSettings(allPortfolios.size(), NUMBER_OF_PORTOLIOS, MAX_PROJECT_PER_PORTFOLIO)));
 
     // close database connection
     dbTester.getDbClient().getDatabase().stop();
@@ -181,11 +218,6 @@ public class PopulateDb {
   }
 
   private static @NotNull DbTester createDbTester() {
-    System.setProperty("sonar.jdbc.url", "jdbc:postgresql://localhost:5432/sonarqube");
-    System.setProperty("sonar.jdbc.username", "sonarqube");
-    System.setProperty("sonar.jdbc.password", "sonarqube");
-    System.setProperty("sonar.jdbc.dialect", "postgresql");
-
     return DbTester.create(System2.INSTANCE);
   }
 
@@ -267,9 +299,9 @@ public class PopulateDb {
           for (SnapshotDto snapshotDto : snapshots) {
             for (int whdNum = 0; whdNum < pj.webhookDeliveriesPerComponent; whdNum++) {
               sqContext.dbTester.webhookDelivery().insert(whd -> whd
-                  .setAnalysisUuid(snapshotDto.getUuid())
-                  .setComponentUuid(fileComponentDto.uuid())
-                  .setWebhookUuid(projectWebHook.getUuid()));
+                .setAnalysisUuid(snapshotDto.getUuid())
+                .setComponentUuid(fileComponentDto.uuid())
+                .setWebhookUuid(projectWebHook.getUuid()));
             }
           }
         }