SONAR-5564 - Added server-data-tests for ES ingestion and synchronizing

server/sonar-data-test/pom.xml

@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.codehaus.sonar</groupId>
+    <artifactId>sonar</artifactId>
+    <version>5.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>sonar-data-test</artifactId>
+  <packaging>jar</packaging>
+  <name>SonarQube :: Server :: Data Tests</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.codehaus.sonar</groupId>
+      <artifactId>sonar-server</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.codehaus.sonar</groupId>
+      <artifactId>sonar-server</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.codehaus.sonar</groupId>
+      <artifactId>sonar-search</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.easytesting</groupId>
+      <artifactId>fest-assert</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <skipTests>${skipServerTests}</skipTests>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java

@@ -0,0 +1,135 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.data.issues;
+
+import com.google.common.collect.Iterables;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.ClassRule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.core.component.ComponentDto;
+import org.sonar.core.issue.db.IssueDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.core.rule.RuleDto;
+import org.sonar.server.db.DbClient;
+import org.sonar.server.rule.RuleTesting;
+import org.sonar.server.search.IndexClient;
+import org.sonar.server.tester.ServerTester;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+
+public class IssueData {
+
+  public final static int MAX_NUMBER_RULES = 2500;
+  public final static int MAX_NUMBER_PROJECTS = 500;
+  public final static int MAX_NUMBER_RESOURCES_PER_PROJECT = 10000;
+
+  public final static int ISSUE_COUNT = 1000000;
+
+  protected static final Logger LOGGER = LoggerFactory.getLogger(IssueData.class);
+
+  @ClassRule
+  public static ServerTester tester = new ServerTester();
+
+  @After
+  public void tearDown() throws Exception {
+    tester.clearDbAndIndexes();
+    if (session != null) {
+      session.close();
+    }
+  }
+
+  @AfterClass
+  public static void reset() throws Exception {
+    tester = new ServerTester();
+  }
+
+  protected Random generator = new Random(System.currentTimeMillis());
+  protected DbClient db = tester.get(DbClient.class);
+  protected IndexClient index = tester.get(IndexClient.class);
+  protected DbSession session = tester.get(DbClient.class).openSession(true);
+
+  protected List<RuleDto> rules = new ArrayList<RuleDto>();
+  protected Map<Long, List<Long>> projects = new HashMap<Long, List<Long>>();
+
+  protected IssueDto getIssue(int id) {
+    RuleDto rule = rules.get(generator.nextInt(rules.size()));
+    Long projectId = Iterables.get(projects.keySet(), generator.nextInt(projects.size()));
+    Long resourceId = projects.get(projectId).get(generator.nextInt(projects.get(projectId).size()));
+    return new IssueDto().setId(new Long(id))
+      .setRootComponentId(projectId)
+      .setRootComponentKey(projectId + "_key")
+      .setComponentId(resourceId)
+      .setComponentKey(resourceId + "_key")
+      .setRule(rule)
+      .setMessage("Lorem ipsum loertium bortim tata toto tutu 14 failures in this issue")
+      .setAssignee("assignee_")
+      .setSeverity("BLOCKER")
+      .setReporter("Luc besson")
+      .setAuthorLogin("Pinpin")
+      .setStatus("OPEN").setResolution("OPEN")
+      .setKee(UUID.randomUUID().toString());
+  }
+
+  protected void generateRules(DbSession dbSession) {
+    // Generate Rules
+    for (int i = 0; i < MAX_NUMBER_RULES; i++) {
+      rules.add(RuleTesting.newDto(RuleKey.of("data_repo", "S" + i)));
+    }
+    DbSession setupSession = db.openSession(false);
+    db.ruleDao().insert(setupSession, rules);
+    setupSession.commit();
+  }
+
+  protected void generateProjects(DbSession setupSession) {
+    // Generate projects & resources
+    for(long p = 1; p<=MAX_NUMBER_PROJECTS; p++) {
+      ComponentDto project = new ComponentDto()
+        .setId(p)
+        .setKey("MyProject")
+        .setProjectId_unit_test_only(p);
+      db.componentDao().insert(setupSession, project);
+      projects.put(project.projectId(), new ArrayList<Long>());
+      List<ComponentDto> resources = new ArrayList<ComponentDto>();
+      for(int i = 0; i<generator.nextInt(MAX_NUMBER_RESOURCES_PER_PROJECT); i++) {
+        ComponentDto resource = new ComponentDto()
+          .setKey("MyComponent_"+(p*MAX_NUMBER_PROJECTS+i))
+          .setProjectId_unit_test_only(project.getId());
+        db.componentDao().insert(setupSession, resource);
+        resources.add(resource);
+      }
+      setupSession.commit();
+      for (ComponentDto resource : resources) {
+        projects.get(project.projectId()).add(resource.getId());
+      }
+    }
+  }
+
+  protected int documentPerSecond(long time) {
+    return (int)Math.round(ISSUE_COUNT/(time/1000.0));
+  }
+}

server/sonar-data-test/src/test/java/org/sonar/data/issues/MassIndexingTest.java

@@ -0,0 +1,63 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.data.issues;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.core.issue.db.IssueDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.server.issue.index.IssueIndex;
+import org.sonar.server.search.IndexDefinition;
+import org.sonar.server.search.action.InsertDto;
+import org.sonar.server.search.action.RefreshIndex;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class MassIndexingTest extends IssueData {
+
+  @Before
+  public void setUp() throws Exception {
+    DbSession setupSession = db.openSession(false);
+    generateRules(setupSession);
+    generateProjects(setupSession);
+    setupSession.commit();
+    setupSession.close();
+  }
+
+  @Test
+  public void enqueue_issues() throws Exception {
+
+    long start = System.currentTimeMillis();
+    int issueInsertCount = ISSUE_COUNT;
+    for (int i = 0; i < issueInsertCount; i++) {
+      session.enqueue(new InsertDto<IssueDto>(IndexDefinition.ISSUES.getIndexType(), getIssue(i), false));
+    }
+    session.enqueue(new RefreshIndex(IndexDefinition.ISSUES.getIndexType()));
+    session.commit();
+    long stop = System.currentTimeMillis();
+
+    //TODO add performance assertions here
+    assertThat(index.get(IssueIndex.class).countAll()).isEqualTo(issueInsertCount);
+
+    long time = stop-start;
+    LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time));
+
+  }
+}

server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java

@@ -0,0 +1,96 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.data.issues;
+
+import org.apache.ibatis.session.ResultContext;
+import org.junit.Before;
+import org.junit.Test;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.server.issue.db.IssueDao;
+import org.sonar.server.search.DbSynchronizationHandler;
+
+import java.util.Date;
+import java.util.Map;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class MassSynchronizingTest extends IssueData {
+
+  MyIssueDao myIssueDao;
+
+  @Before
+  public void setUp() throws Exception {
+
+    myIssueDao = new MyIssueDao();
+
+    DbSession setupSession = db.openSession(false);
+    generateRules(setupSession);
+    generateProjects(setupSession);
+
+
+    // Inserting Issues now (finally)
+    for (int i = 0; i < ISSUE_COUNT; i++) {
+      myIssueDao.insert(setupSession, getIssue(i));
+      if (i % 100 == 0) {
+        setupSession.commit();
+      }
+    }
+    setupSession.commit();
+    setupSession.close();
+  }
+
+  @Test
+  public void synchronize_issues() throws Exception {
+    long start = System.currentTimeMillis();
+    int issueInsertCount = ISSUE_COUNT;
+    myIssueDao.synchronizeAfter(session, new Date(0));
+    long stop = System.currentTimeMillis();
+
+    // TODO add performance assertions here
+    assertThat(myIssueDao.synchronizedIssues).isEqualTo(issueInsertCount);
+
+    long time = stop-start;
+    LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time));
+  }
+
+  class MyIssueDao extends IssueDao {
+    public Integer synchronizedIssues = 0;
+
+    @Override
+    protected boolean hasIndex() {
+      return false;
+    }
+
+    @Override
+    protected DbSynchronizationHandler getSynchronizationResultHandler(DbSession session, Map<String, String> params) {
+      return new DbSynchronizationHandler(session, params) {
+
+        @Override
+        public void handleResult(ResultContext context) {
+          synchronizedIssues++;
+        }
+
+        @Override
+        public void enqueueCollected() {
+        }
+      };
+    }
+  }
+}

server/sonar-data-test/src/test/resources/logback-test.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!--
+  Configuration for default logger. Only used while embedded server is starting,
+  before proper logging configuration is loaded.
+
+  See http://logback.qos.ch/manual/configuration.html
+-->
+<configuration debug="false">
+  <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"/>
+
+  <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>INFO</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+      <pattern>
+        %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n
+      </pattern>
+    </encoder>
+  </appender>
+
+  <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+      <pattern>
+        %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n
+      </pattern>
+    </encoder>
+  </appender>
+
+  <root>
+    <level value="INFO"/>
+    <appender-ref ref="CONSOLE"/>
+  </root>
+
+</configuration>

server/sonar-server/pom.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.codehaus.sonar</groupId>
     <artifactId>server</artifactId>
@@ -251,6 +252,17 @@
     </resources>
 
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>

server/sonar-server/src/main/java/org/sonar/server/db/BaseDao.java

@@ -130,7 +130,7 @@ public abstract class BaseDao<MAPPER, DTO extends Dto<KEY>, KEY extends Serializ
   private Class<MAPPER> mapperClass;
   private System2 system2;
 
-  private boolean hasIndex() {
+  protected boolean hasIndex() {
     return indexDefinition != null;
   }