From 4224b99d072e4160d1f601eb63643a831c79762e Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Thu, 6 Nov 2014 16:43:42 +0100 Subject: [PATCH] Add new perf test on issues extraction --- server/sonar-data-test/pom.xml | 21 ++ .../java/org/sonar/data/issues/IssueData.java | 77 ++++---- .../data/issues/IssuesDbExtractionTest.java | 186 ++++++++++++++++++ .../data/issues/MassSynchronizingTest.java | 23 ++- .../src/test/resources/logback-test.xml | 54 ++--- 5 files changed, 285 insertions(+), 76 deletions(-) create mode 100644 server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java diff --git a/server/sonar-data-test/pom.xml b/server/sonar-data-test/pom.xml index 57c7f82e6a8..32eec2d9f75 100644 --- a/server/sonar-data-test/pom.xml +++ b/server/sonar-data-test/pom.xml @@ -29,6 +29,22 @@ sonar-search ${project.version} + + org.codehaus.sonar + sonar-core + ${project.version} + + + org.codehaus.sonar + sonar-core + test-jar + test + ${project.version} + + + com.google.guava + guava + org.easytesting fest-assert @@ -39,6 +55,11 @@ junit test + + org.dbunit + dbunit + test + diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java index a7e8dcc0dd5..b8bdc3560e6 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java @@ -19,28 +19,30 @@ */ package org.sonar.data.issues; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Iterables; import org.junit.After; import org.junit.AfterClass; import org.junit.ClassRule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.api.issue.Issue; import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.Severity; import org.sonar.core.component.ComponentDto; import org.sonar.core.issue.db.IssueDto; import org.sonar.core.persistence.DbSession; import org.sonar.core.rule.RuleDto; +import org.sonar.server.component.ComponentTesting; import org.sonar.server.db.DbClient; +import org.sonar.server.issue.IssueTesting; import org.sonar.server.rule.RuleTesting; import org.sonar.server.search.IndexClient; import org.sonar.server.tester.ServerTester; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Random; -import java.util.UUID; public class IssueData { @@ -48,13 +50,21 @@ public class IssueData { public final static int MAX_NUMBER_PROJECTS = 500; public final static int MAX_NUMBER_RESOURCES_PER_PROJECT = 10000; - public final static int ISSUE_COUNT = 1000000; + public final static int ISSUE_COUNT = 100000; protected static final Logger LOGGER = LoggerFactory.getLogger(IssueData.class); @ClassRule public static ServerTester tester = new ServerTester(); + protected Random generator = new Random(System.currentTimeMillis()); + protected DbClient db = tester.get(DbClient.class); + protected IndexClient index = tester.get(IndexClient.class); + protected DbSession session = tester.get(DbClient.class).openSession(true); + + protected List rules = new ArrayList(); + protected ArrayListMultimap componentsByProjectId = ArrayListMultimap.create(); + @After public void tearDown() throws Exception { tester.clearDbAndIndexes(); @@ -68,37 +78,25 @@ public class IssueData { tester = new ServerTester(); } - protected Random generator = new Random(System.currentTimeMillis()); - protected DbClient db = tester.get(DbClient.class); - protected IndexClient index = tester.get(IndexClient.class); - protected DbSession session = tester.get(DbClient.class).openSession(true); - - protected List rules = new ArrayList(); - protected Map> projects = new HashMap>(); - protected IssueDto getIssue(int id) { RuleDto rule = rules.get(generator.nextInt(rules.size())); - Long projectId = Iterables.get(projects.keySet(), generator.nextInt(projects.size())); - Long resourceId = projects.get(projectId).get(generator.nextInt(projects.get(projectId).size())); - return new IssueDto().setId(new Long(id)) - .setRootComponentId(projectId) - .setRootComponentKey(projectId + "_key") - .setComponentId(resourceId) - .setComponentKey(resourceId + "_key") - .setRule(rule) + ComponentDto project = Iterables.get(componentsByProjectId.keySet(), generator.nextInt(componentsByProjectId.keySet().size())); + ComponentDto file = componentsByProjectId.get(project).get(generator.nextInt(componentsByProjectId.get(project).size())); + + return IssueTesting.newDto(rule, file, project) .setMessage("Lorem ipsum loertium bortim tata toto tutu 14 failures in this issue") .setAssignee("assignee_") - .setSeverity("BLOCKER") + .setSeverity(Severity.BLOCKER) .setReporter("Luc besson") .setAuthorLogin("Pinpin") - .setStatus("OPEN").setResolution("OPEN") - .setKee(UUID.randomUUID().toString()); + .setStatus(Issue.STATUS_RESOLVED) + .setResolution(Issue.RESOLUTION_FIXED); } protected void generateRules(DbSession dbSession) { // Generate Rules for (int i = 0; i < MAX_NUMBER_RULES; i++) { - rules.add(RuleTesting.newDto(RuleKey.of("data_repo", "S" + i))); + rules.add(RuleTesting.newDto(RuleKey.of("rule-repo", "rule-key-" + i))); } DbSession setupSession = db.openSession(false); db.ruleDao().insert(setupSession, rules); @@ -107,29 +105,26 @@ public class IssueData { protected void generateProjects(DbSession setupSession) { // Generate projects & resources - for(long p = 1; p<=MAX_NUMBER_PROJECTS; p++) { - ComponentDto project = new ComponentDto() - .setId(p) - .setKey("MyProject") - .setProjectId_unit_test_only(p); + for (long p = 1; p <= MAX_NUMBER_PROJECTS; p++) { + ComponentDto project = ComponentTesting.newProjectDto() + .setKey("project-" + p) + .setName("Project " + p) + .setLongName("Project " + p); db.componentDao().insert(setupSession, project); - projects.put(project.projectId(), new ArrayList()); - List resources = new ArrayList(); - for(int i = 0; i rules; + Iterator users; + + ProxyIssueDao issueDao; + RuleDao ruleDao; + ComponentDao componentDao; + + @Before + public void setUp() throws Exception { + issueDao = new ProxyIssueDao(); + ruleDao = new RuleDao(); + componentDao = new ComponentDao(System2.INSTANCE); + + session = db.myBatis().openSession(false); + + rules = Iterables.cycle(generateRules(session)).iterator(); + users = Iterables.cycle(generateUsers()).iterator(); + + generateUsers(); + session.commit(); + } + + @After + public void closeSession() throws Exception { + session.close(); + } + + @Test + public void extract_issues() throws Exception { + for (long p = 1; p <= PROJECTS_NUMBER; p++) { + ComponentDto project = ComponentTesting.newProjectDto() + .setKey("project-" + p) + .setName("Project " + p) + .setLongName("Project " + p); + componentDao.insert(session, project); + + for (int i = 0; i < NUMBER_FILES_PER_PROJECT; i++) { + String index = p * PROJECTS_NUMBER + i + ""; + + ComponentDto file = ComponentTesting.newFileDto(project) + .setKey("file-" + index) + .setName("File " + index) + .setLongName("File " + index); + componentDao.insert(session, file); + + issueDao.insert(session, IssueTesting.newDto(rules.next(), file, project) + .setMessage("Message on " + index) + .setAssignee(users.next()) + .setReporter(users.next()) + .setAuthorLogin(users.next()) + // Change Severity + .setSeverity(Severity.BLOCKER) + // Change status & resolution + .setStatus(Issue.STATUS_RESOLVED) + .setResolution(Issue.RESOLUTION_FIXED)); + } + session.commit(); + } + + long start = System.currentTimeMillis(); + int issueInsertCount = ISSUE_COUNT; + issueDao.synchronizeAfter(session); + long stop = System.currentTimeMillis(); + + // TODO add performance assertions here + assertThat(issueDao.synchronizedIssues).isEqualTo(issueInsertCount); + + long time = stop - start; + LOGGER.info("Processed {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, time, documentPerSecond(time)); + } + + protected List generateRules(DbSession session) { + List rules = newArrayList(); + for (int i = 0; i < RULES_NUMBER; i++) { + rules.add(RuleTesting.newDto(RuleKey.of("rule-repo", "rule-key-" + i))); + } + ruleDao.insert(this.session, rules); + return rules; + } + + protected List generateUsers() { + List users = newArrayList(); + for (int i = 0; i < USERS_NUMBER; i++) { + users.add("user-" + i); + } + return users; + } + + private int documentPerSecond(long time) { + return (int) Math.round(ISSUE_COUNT / (time / 1000.0)); + } + + class ProxyIssueDao extends IssueDao { + public Integer synchronizedIssues = 0; + + @Override + protected boolean hasIndex() { + return false; + } + + @Override + protected DbSynchronizationHandler getSynchronizationResultHandler(DbSession session, Map params) { + return new DbSynchronizationHandler(session, params) { + + @Override + public void handleResult(ResultContext context) { + synchronizedIssues++; + } + + @Override + public void enqueueCollected() { + } + }; + } + } + +} diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java index 1ed6da0d4c2..f124d7d280c 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java @@ -20,32 +20,31 @@ package org.sonar.data.issues; import org.apache.ibatis.session.ResultContext; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.sonar.core.persistence.DbSession; import org.sonar.server.issue.db.IssueDao; import org.sonar.server.search.DbSynchronizationHandler; -import java.util.Date; import java.util.Map; import static org.fest.assertions.Assertions.assertThat; public class MassSynchronizingTest extends IssueData { - MyIssueDao myIssueDao; + IssueIndexLessDao myIssueDao; + + DbSession setupSession; @Before public void setUp() throws Exception { + myIssueDao = new IssueIndexLessDao(); - myIssueDao = new MyIssueDao(); - - DbSession setupSession = db.openSession(false); + setupSession = db.openSession(false); generateRules(setupSession); generateProjects(setupSession); - - // Inserting Issues now (finally) for (int i = 0; i < ISSUE_COUNT; i++) { myIssueDao.insert(setupSession, getIssue(i)); if (i % 100 == 0) { @@ -53,6 +52,10 @@ public class MassSynchronizingTest extends IssueData { } } setupSession.commit(); + } + + @After + public void closeSession() throws Exception { setupSession.close(); } @@ -60,17 +63,17 @@ public class MassSynchronizingTest extends IssueData { public void synchronize_issues() throws Exception { long start = System.currentTimeMillis(); int issueInsertCount = ISSUE_COUNT; - myIssueDao.synchronizeAfter(session, new Date(0)); + myIssueDao.synchronizeAfter(session); long stop = System.currentTimeMillis(); // TODO add performance assertions here assertThat(myIssueDao.synchronizedIssues).isEqualTo(issueInsertCount); long time = stop-start; - LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time)); + LOGGER.info("Processed {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time)); } - class MyIssueDao extends IssueDao { + class IssueIndexLessDao extends IssueDao { public Integer synchronizedIssues = 0; @Override diff --git a/server/sonar-data-test/src/test/resources/logback-test.xml b/server/sonar-data-test/src/test/resources/logback-test.xml index df0bb26d407..e595060acc0 100644 --- a/server/sonar-data-test/src/test/resources/logback-test.xml +++ b/server/sonar-data-test/src/test/resources/logback-test.xml @@ -1,38 +1,42 @@ - - - - - - - INFO - ACCEPT - DENY - - - - %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n - - - + - - + + - %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n + %d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + + + + + + + + + + - + -- 2.39.5