<artifactId>sonar-search</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.sonar</groupId>
+ <artifactId>sonar-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.sonar</groupId>
+ <artifactId>sonar-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
<dependency>
<groupId>org.easytesting</groupId>
<artifactId>fest-assert</artifactId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.dbunit</groupId>
+ <artifactId>dbunit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
*/
package org.sonar.data.issues;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Iterables;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.ClassRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.sonar.api.issue.Issue;
import org.sonar.api.rule.RuleKey;
+import org.sonar.api.rule.Severity;
import org.sonar.core.component.ComponentDto;
import org.sonar.core.issue.db.IssueDto;
import org.sonar.core.persistence.DbSession;
import org.sonar.core.rule.RuleDto;
+import org.sonar.server.component.ComponentTesting;
import org.sonar.server.db.DbClient;
+import org.sonar.server.issue.IssueTesting;
import org.sonar.server.rule.RuleTesting;
import org.sonar.server.search.IndexClient;
import org.sonar.server.tester.ServerTester;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Random;
-import java.util.UUID;
public class IssueData {
public final static int MAX_NUMBER_PROJECTS = 500;
public final static int MAX_NUMBER_RESOURCES_PER_PROJECT = 10000;
- public final static int ISSUE_COUNT = 1000000;
+ public final static int ISSUE_COUNT = 100000;
protected static final Logger LOGGER = LoggerFactory.getLogger(IssueData.class);
@ClassRule
public static ServerTester tester = new ServerTester();
+ protected Random generator = new Random(System.currentTimeMillis());
+ protected DbClient db = tester.get(DbClient.class);
+ protected IndexClient index = tester.get(IndexClient.class);
+ protected DbSession session = tester.get(DbClient.class).openSession(true);
+
+ protected List<RuleDto> rules = new ArrayList<RuleDto>();
+ protected ArrayListMultimap<ComponentDto, ComponentDto> componentsByProjectId = ArrayListMultimap.create();
+
@After
public void tearDown() throws Exception {
tester.clearDbAndIndexes();
tester = new ServerTester();
}
- protected Random generator = new Random(System.currentTimeMillis());
- protected DbClient db = tester.get(DbClient.class);
- protected IndexClient index = tester.get(IndexClient.class);
- protected DbSession session = tester.get(DbClient.class).openSession(true);
-
- protected List<RuleDto> rules = new ArrayList<RuleDto>();
- protected Map<Long, List<Long>> projects = new HashMap<Long, List<Long>>();
-
protected IssueDto getIssue(int id) {
RuleDto rule = rules.get(generator.nextInt(rules.size()));
- Long projectId = Iterables.get(projects.keySet(), generator.nextInt(projects.size()));
- Long resourceId = projects.get(projectId).get(generator.nextInt(projects.get(projectId).size()));
- return new IssueDto().setId(new Long(id))
- .setRootComponentId(projectId)
- .setRootComponentKey(projectId + "_key")
- .setComponentId(resourceId)
- .setComponentKey(resourceId + "_key")
- .setRule(rule)
+ ComponentDto project = Iterables.get(componentsByProjectId.keySet(), generator.nextInt(componentsByProjectId.keySet().size()));
+ ComponentDto file = componentsByProjectId.get(project).get(generator.nextInt(componentsByProjectId.get(project).size()));
+
+ return IssueTesting.newDto(rule, file, project)
.setMessage("Lorem ipsum loertium bortim tata toto tutu 14 failures in this issue")
.setAssignee("assignee_")
- .setSeverity("BLOCKER")
+ .setSeverity(Severity.BLOCKER)
.setReporter("Luc besson")
.setAuthorLogin("Pinpin")
- .setStatus("OPEN").setResolution("OPEN")
- .setKee(UUID.randomUUID().toString());
+ .setStatus(Issue.STATUS_RESOLVED)
+ .setResolution(Issue.RESOLUTION_FIXED);
}
protected void generateRules(DbSession dbSession) {
// Generate Rules
for (int i = 0; i < MAX_NUMBER_RULES; i++) {
- rules.add(RuleTesting.newDto(RuleKey.of("data_repo", "S" + i)));
+ rules.add(RuleTesting.newDto(RuleKey.of("rule-repo", "rule-key-" + i)));
}
DbSession setupSession = db.openSession(false);
db.ruleDao().insert(setupSession, rules);
protected void generateProjects(DbSession setupSession) {
// Generate projects & resources
- for(long p = 1; p<=MAX_NUMBER_PROJECTS; p++) {
- ComponentDto project = new ComponentDto()
- .setId(p)
- .setKey("MyProject")
- .setProjectId_unit_test_only(p);
+ for (long p = 1; p <= MAX_NUMBER_PROJECTS; p++) {
+ ComponentDto project = ComponentTesting.newProjectDto()
+ .setKey("project-" + p)
+ .setName("Project " + p)
+ .setLongName("Project " + p);
db.componentDao().insert(setupSession, project);
- projects.put(project.projectId(), new ArrayList<Long>());
- List<ComponentDto> resources = new ArrayList<ComponentDto>();
- for(int i = 0; i<generator.nextInt(MAX_NUMBER_RESOURCES_PER_PROJECT); i++) {
- ComponentDto resource = new ComponentDto()
- .setKey("MyComponent_"+(p*MAX_NUMBER_PROJECTS+i))
- .setProjectId_unit_test_only(project.getId());
- db.componentDao().insert(setupSession, resource);
- resources.add(resource);
+
+ for (int i = 0; i < generator.nextInt(MAX_NUMBER_RESOURCES_PER_PROJECT); i++) {
+ ComponentDto file = ComponentTesting.newFileDto(project)
+ .setKey("file-" + (p * MAX_NUMBER_PROJECTS + i))
+ .setName("File " + (p * MAX_NUMBER_PROJECTS + i))
+ .setLongName("File " + (p * MAX_NUMBER_PROJECTS + i));
+ db.componentDao().insert(setupSession, file);
+ componentsByProjectId.put(project, file);
}
setupSession.commit();
- for (ComponentDto resource : resources) {
- projects.get(project.projectId()).add(resource.getId());
- }
}
}
protected int documentPerSecond(long time) {
- return (int)Math.round(ISSUE_COUNT/(time/1000.0));
+ return (int) Math.round(ISSUE_COUNT / (time / 1000.0));
}
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.data.issues;
+
+import com.google.common.collect.Iterables;
+import org.apache.ibatis.session.ResultContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.issue.Issue;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.api.rule.Severity;
+import org.sonar.api.utils.System2;
+import org.sonar.core.component.ComponentDto;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.core.persistence.TestDatabase;
+import org.sonar.core.rule.RuleDto;
+import org.sonar.server.component.ComponentTesting;
+import org.sonar.server.component.db.ComponentDao;
+import org.sonar.server.issue.IssueTesting;
+import org.sonar.server.issue.db.IssueDao;
+import org.sonar.server.rule.RuleTesting;
+import org.sonar.server.rule.db.RuleDao;
+import org.sonar.server.search.DbSynchronizationHandler;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.fest.assertions.Assertions.assertThat;
+
+public class IssuesDbExtractionTest {
+
+ static final Logger LOGGER = LoggerFactory.getLogger(IssuesDbExtractionTest.class);
+
+ final static int RULES_NUMBER = 25;
+ final static int USERS_NUMBER = 100;
+
+ final static int PROJECTS_NUMBER = 10;
+ final static int NUMBER_FILES_PER_PROJECT = 100;
+
+ final static int ISSUE_COUNT = PROJECTS_NUMBER * NUMBER_FILES_PER_PROJECT;
+
+ @Rule
+ public TestDatabase db = new TestDatabase();
+
+ DbSession session;
+
+ Iterator<RuleDto> rules;
+ Iterator<String> users;
+
+ ProxyIssueDao issueDao;
+ RuleDao ruleDao;
+ ComponentDao componentDao;
+
+ @Before
+ public void setUp() throws Exception {
+ issueDao = new ProxyIssueDao();
+ ruleDao = new RuleDao();
+ componentDao = new ComponentDao(System2.INSTANCE);
+
+ session = db.myBatis().openSession(false);
+
+ rules = Iterables.cycle(generateRules(session)).iterator();
+ users = Iterables.cycle(generateUsers()).iterator();
+
+ generateUsers();
+ session.commit();
+ }
+
+ @After
+ public void closeSession() throws Exception {
+ session.close();
+ }
+
+ @Test
+ public void extract_issues() throws Exception {
+ for (long p = 1; p <= PROJECTS_NUMBER; p++) {
+ ComponentDto project = ComponentTesting.newProjectDto()
+ .setKey("project-" + p)
+ .setName("Project " + p)
+ .setLongName("Project " + p);
+ componentDao.insert(session, project);
+
+ for (int i = 0; i < NUMBER_FILES_PER_PROJECT; i++) {
+ String index = p * PROJECTS_NUMBER + i + "";
+
+ ComponentDto file = ComponentTesting.newFileDto(project)
+ .setKey("file-" + index)
+ .setName("File " + index)
+ .setLongName("File " + index);
+ componentDao.insert(session, file);
+
+ issueDao.insert(session, IssueTesting.newDto(rules.next(), file, project)
+ .setMessage("Message on " + index)
+ .setAssignee(users.next())
+ .setReporter(users.next())
+ .setAuthorLogin(users.next())
+ // Change Severity
+ .setSeverity(Severity.BLOCKER)
+ // Change status & resolution
+ .setStatus(Issue.STATUS_RESOLVED)
+ .setResolution(Issue.RESOLUTION_FIXED));
+ }
+ session.commit();
+ }
+
+ long start = System.currentTimeMillis();
+ int issueInsertCount = ISSUE_COUNT;
+ issueDao.synchronizeAfter(session);
+ long stop = System.currentTimeMillis();
+
+ // TODO add performance assertions here
+ assertThat(issueDao.synchronizedIssues).isEqualTo(issueInsertCount);
+
+ long time = stop - start;
+ LOGGER.info("Processed {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, time, documentPerSecond(time));
+ }
+
+ protected List<RuleDto> generateRules(DbSession session) {
+ List<RuleDto> rules = newArrayList();
+ for (int i = 0; i < RULES_NUMBER; i++) {
+ rules.add(RuleTesting.newDto(RuleKey.of("rule-repo", "rule-key-" + i)));
+ }
+ ruleDao.insert(this.session, rules);
+ return rules;
+ }
+
+ protected List<String> generateUsers() {
+ List<String> users = newArrayList();
+ for (int i = 0; i < USERS_NUMBER; i++) {
+ users.add("user-" + i);
+ }
+ return users;
+ }
+
+ private int documentPerSecond(long time) {
+ return (int) Math.round(ISSUE_COUNT / (time / 1000.0));
+ }
+
+ class ProxyIssueDao extends IssueDao {
+ public Integer synchronizedIssues = 0;
+
+ @Override
+ protected boolean hasIndex() {
+ return false;
+ }
+
+ @Override
+ protected DbSynchronizationHandler getSynchronizationResultHandler(DbSession session, Map<String, String> params) {
+ return new DbSynchronizationHandler(session, params) {
+
+ @Override
+ public void handleResult(ResultContext context) {
+ synchronizedIssues++;
+ }
+
+ @Override
+ public void enqueueCollected() {
+ }
+ };
+ }
+ }
+
+}
package org.sonar.data.issues;
import org.apache.ibatis.session.ResultContext;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.sonar.core.persistence.DbSession;
import org.sonar.server.issue.db.IssueDao;
import org.sonar.server.search.DbSynchronizationHandler;
-import java.util.Date;
import java.util.Map;
import static org.fest.assertions.Assertions.assertThat;
public class MassSynchronizingTest extends IssueData {
- MyIssueDao myIssueDao;
+ IssueIndexLessDao myIssueDao;
+
+ DbSession setupSession;
@Before
public void setUp() throws Exception {
+ myIssueDao = new IssueIndexLessDao();
- myIssueDao = new MyIssueDao();
-
- DbSession setupSession = db.openSession(false);
+ setupSession = db.openSession(false);
generateRules(setupSession);
generateProjects(setupSession);
-
- // Inserting Issues now (finally)
for (int i = 0; i < ISSUE_COUNT; i++) {
myIssueDao.insert(setupSession, getIssue(i));
if (i % 100 == 0) {
}
}
setupSession.commit();
+ }
+
+ @After
+ public void closeSession() throws Exception {
setupSession.close();
}
public void synchronize_issues() throws Exception {
long start = System.currentTimeMillis();
int issueInsertCount = ISSUE_COUNT;
- myIssueDao.synchronizeAfter(session, new Date(0));
+ myIssueDao.synchronizeAfter(session);
long stop = System.currentTimeMillis();
// TODO add performance assertions here
assertThat(myIssueDao.synchronizedIssues).isEqualTo(issueInsertCount);
long time = stop-start;
- LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time));
+ LOGGER.info("Processed {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time));
}
- class MyIssueDao extends IssueDao {
+ class IssueIndexLessDao extends IssueDao {
public Integer synchronizedIssues = 0;
@Override
<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Configuration for default logger. Only used while embedded server is starting,
- before proper logging configuration is loaded.
-
- See http://logback.qos.ch/manual/configuration.html
--->
-<configuration debug="false">
- <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"/>
-
- <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>INFO</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
- </filter>
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
- <pattern>
- %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n
- </pattern>
- </encoder>
- </appender>
+<configuration>
- <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
- <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <appender name="STDOUT"
+ class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
<pattern>
- %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n
+ %d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n
</pattern>
</encoder>
</appender>
+ <logger name="org.hibernate">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.dbunit">
+ <level value="WARN"/>
+ </logger>
+
+ <!-- set to level DEBUG to log SQL requests executed by MyBatis -->
+ <logger name="java.sql">
+ <level value="WARN"/>
+ </logger>
+
+ <!-- required for DryRunDatabaseFactoryTest -->
+ <logger name="org.sonar.core.persistence.DryRunDatabaseFactory">
+ <level value="DEBUG"/>
+ </logger>
+
+ <!-- required for DryRunDatabaseFactoryTest -->
+ <logger name="org.elasticsearch">
+ <level value="WARN"/>
+ </logger>
+
<root>
<level value="INFO"/>
- <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="STDOUT"/>
</root>
</configuration>
projects / sonarqube.git / commitdiff