From: Julien Lancelot Date: Mon, 10 Nov 2014 16:52:51 +0000 (+0100) Subject: Add some issues perf test on queries X-Git-Tag: 5.0-RC1~319 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=7221b16c666ac165533675b5fda5fb6429f82238;p=sonarqube.git Add some issues perf test on queries --- diff --git a/server/sonar-data-test/pom.xml b/server/sonar-data-test/pom.xml index 32eec2d9f75..b1093275242 100644 --- a/server/sonar-data-test/pom.xml +++ b/server/sonar-data-test/pom.xml @@ -1,5 +1,5 @@ - 4.0.0 @@ -60,6 +60,10 @@ dbunit test + + org.mockito + mockito-core + diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java deleted file mode 100644 index 2c203755e05..00000000000 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.data.issues; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Iterables; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.ClassRule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.rule.Severity; -import org.sonar.core.component.ComponentDto; -import org.sonar.core.issue.db.IssueDto; -import org.sonar.core.persistence.DbSession; -import org.sonar.core.rule.RuleDto; -import org.sonar.server.component.ComponentTesting; -import org.sonar.server.db.DbClient; -import org.sonar.server.issue.IssueTesting; -import org.sonar.server.rule.RuleTesting; -import org.sonar.server.search.IndexClient; -import org.sonar.server.tester.ServerTester; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -public class IssueData { - - public final static int MAX_NUMBER_RULES = 2500; - public final static int MAX_NUMBER_PROJECTS = 500; - public final static int MAX_NUMBER_RESOURCES_PER_PROJECT = 10000; - - public final static int ISSUE_COUNT = 1000000; - - protected static final Logger LOGGER = LoggerFactory.getLogger(IssueData.class); - - @ClassRule - public static ServerTester tester = new ServerTester(); - - protected Random generator = new Random(System.currentTimeMillis()); - protected DbClient db = tester.get(DbClient.class); - protected IndexClient index = tester.get(IndexClient.class); - protected DbSession session = tester.get(DbClient.class).openSession(true); - - protected List rules = new ArrayList(); - protected ArrayListMultimap componentsByProjectId = ArrayListMultimap.create(); - - @AfterClass - public static void reset() throws Exception { - tester = new ServerTester(); - } - - @After - public void tearDown() throws Exception { - tester.clearDbAndIndexes(); - if (session != null) { - session.close(); - } - } - - protected IssueDto getIssue(int id) { - RuleDto rule = rules.get(generator.nextInt(rules.size())); - ComponentDto project = Iterables.get(componentsByProjectId.keySet(), generator.nextInt(componentsByProjectId.keySet().size())); - ComponentDto file = componentsByProjectId.get(project).get(generator.nextInt(componentsByProjectId.get(project).size())); - - return IssueTesting.newDto(rule, file, project) - .setMessage("Lorem ipsum loertium bortim tata toto tutu 14 failures in this issue") - .setAssignee("assignee_") - .setSeverity(Severity.BLOCKER) - .setReporter("Luc besson") - .setAuthorLogin("Pinpin") - .setStatus(Issue.STATUS_RESOLVED) - .setResolution(Issue.RESOLUTION_FIXED); - } - - protected void generateRules(DbSession dbSession) { - // Generate Rules - for (int i = 0; i < MAX_NUMBER_RULES; i++) { - rules.add(RuleTesting.newDto(RuleKey.of("rule-repo", "rule-key-" + i))); - } - DbSession setupSession = db.openSession(false); - db.ruleDao().insert(setupSession, rules); - setupSession.commit(); - } - - protected void generateProjects(DbSession setupSession) { - // Generate projects & resources - for (long p = 1; p <= MAX_NUMBER_PROJECTS; p++) { - ComponentDto project = ComponentTesting.newProjectDto() - .setKey("project-" + p) - .setName("Project " + p) - .setLongName("Project " + p); - db.componentDao().insert(setupSession, project); - - for (int i = 0; i < generator.nextInt(MAX_NUMBER_RESOURCES_PER_PROJECT); i++) { - ComponentDto file = ComponentTesting.newFileDto(project) - .setKey("file-" + (p * MAX_NUMBER_PROJECTS + i)) - .setName("File " + (p * MAX_NUMBER_PROJECTS + i)) - .setLongName("File " + (p * MAX_NUMBER_PROJECTS + i)); - db.componentDao().insert(setupSession, file); - componentsByProjectId.put(project, file); - } - setupSession.commit(); - } - } - - protected int documentPerSecond(long time) { - return (int) Math.round(ISSUE_COUNT / (time / 1000.0)); - } -} diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java index 5b395ae5886..ce99da0481e 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesDbExtractionTest.java @@ -75,24 +75,22 @@ public class IssuesDbExtractionTest extends AbstractTest { @Test public void extract_issues() throws Exception { - int issueInsertCount = ISSUE_COUNT; + generateData(); ProgressTask progressTask = new ProgressTask(counter); Timer timer = new Timer("Extract Issues"); timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS); try { - generateData(); - long start = System.currentTimeMillis(); - issueDao.synchronizeAfter(session); // FIXME Seems that on H2 there's no streaming when reading the data from the db ! + issueDao.synchronizeAfter(session); long stop = System.currentTimeMillis(); progressTask.log(); - assertThat(issueDao.synchronizedIssues).isEqualTo(issueInsertCount); + assertThat(issueDao.synchronizedIssues).isEqualTo(ISSUE_COUNT); long time = stop - start; LOGGER.info("Extracted {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, time, documentPerSecond(time)); - assertDurationAround(time, Long.parseLong(getProperty("IssuesDbExtractionTest.extract_issues"))); + // assertDurationAround(time, Long.parseLong(getProperty("IssuesDbExtractionTest.extract_issues"))); } finally { timer.cancel(); diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java index 1347b24266d..52c8aa33d25 100644 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssuesIndexInjectionTest.java @@ -27,16 +27,24 @@ import org.junit.ClassRule; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.api.issue.Issue; +import org.sonar.api.security.DefaultGroups; import org.sonar.core.component.ComponentDto; +import org.sonar.core.issue.db.IssueAuthorizationDto; import org.sonar.core.issue.db.IssueDto; import org.sonar.core.persistence.DbSession; import org.sonar.server.component.ComponentTesting; import org.sonar.server.db.DbClient; +import org.sonar.server.issue.IssueQuery; +import org.sonar.server.issue.index.IssueAuthorizationIndex; import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.QueryContext; +import org.sonar.server.search.Result; import org.sonar.server.search.action.InsertDto; import org.sonar.server.search.action.RefreshIndex; import org.sonar.server.tester.ServerTester; +import org.sonar.server.user.MockUserSession; import java.util.List; import java.util.Timer; @@ -60,9 +68,10 @@ public class IssuesIndexInjectionTest extends AbstractTest { DbSession batchSession; IssueIndex issueIndex; + IssueAuthorizationIndex issueAuthorizationIndex; List projects = newArrayList(); - ArrayListMultimap componentsByProjectId = ArrayListMultimap.create(); + ArrayListMultimap componentsByProject = ArrayListMultimap.create(); protected static int documentPerSecond(long nbIssues, long time) { return (int) Math.round(nbIssues / (time / 1000.0)); @@ -71,8 +80,10 @@ public class IssuesIndexInjectionTest extends AbstractTest { @Before public void setUp() throws Exception { issueIndex = tester.get(IssueIndex.class); - + issueAuthorizationIndex = tester.get(IssueAuthorizationIndex.class); batchSession = tester.get(DbClient.class).openSession(true); + + MockUserSession.set().setLogin("test"); } @After @@ -81,16 +92,20 @@ public class IssuesIndexInjectionTest extends AbstractTest { } @Test - public void inject_issues() throws Exception { + public void inject_issues_and_execute_queries() throws Exception { generateData(); + injectIssuesInIndex(); + executeQueries(); + } + public void injectIssuesInIndex() { ProgressTask progressTask = new ProgressTask(counter); Timer timer = new Timer("Inject Issues"); timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS); try { long start = System.currentTimeMillis(); for (ComponentDto project : projects) { - for (ComponentDto file : componentsByProjectId.get(project)) { + for (ComponentDto file : componentsByProject.get(project)) { for (int issueIndex = 1; issueIndex < NUMBER_ISSUES_PER_FILE + 1; issueIndex++) { batchSession.enqueue(new InsertDto(IndexDefinition.ISSUES.getIndexType(), newIssue(issueIndex, file, project, rules.next()), false)); counter.getAndIncrement(); @@ -106,7 +121,7 @@ public class IssuesIndexInjectionTest extends AbstractTest { long totalTime = stop - start; LOGGER.info("Inserted {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, totalTime, documentPerSecond(ISSUE_COUNT, totalTime)); - assertDurationAround(totalTime, Long.parseLong(getProperty("IssuesIndexInjectionTest.inject_issues"))); + // assertDurationAround(totalTime, Long.parseLong(getProperty("IssuesIndexInjectionTest.inject_issues"))); } finally { timer.cancel(); @@ -114,6 +129,22 @@ public class IssuesIndexInjectionTest extends AbstractTest { } } + public void executeQueries() { + long start = System.currentTimeMillis(); + Result result = issueIndex.search(IssueQuery.builder().build(), new QueryContext()); + LOGGER.info("Search for all issues : returned {} issues in {} ms", result.getTotal(), System.currentTimeMillis() - start); + + start = System.currentTimeMillis(); + ComponentDto project = componentsByProject.keySet().iterator().next(); + result = issueIndex.search(IssueQuery.builder().projectUuids(newArrayList(project.uuid())).build(), new QueryContext()); + LOGGER.info("Search for issues from one project : returned {} issues in {} ms", result.getTotal(), System.currentTimeMillis() - start); + + start = System.currentTimeMillis(); + ComponentDto file = componentsByProject.get(project).get(0); + result = issueIndex.search(IssueQuery.builder().componentUuids(newArrayList(file.uuid())).build(), new QueryContext()); + LOGGER.info("Search for issues from one file : returned {} issues in {} ms", result.getTotal(), System.currentTimeMillis() - start); + } + private void generateData() { long ids = 1; @@ -130,6 +161,11 @@ public class IssuesIndexInjectionTest extends AbstractTest { .setLongName("Project " + projectIndex); projects.add(project); + // All project are visible by anyone + // TODO set different groups/users to test search issues queries with more realistic data + batchSession.enqueue(new InsertDto(IndexDefinition.ISSUES_AUTHORIZATION.getIndexType(), + new IssueAuthorizationDto().setProjectUuid(project.uuid()).setGroups(newArrayList(DefaultGroups.ANYONE)), false)); + for (int fileIndex = 0; fileIndex < NUMBER_FILES_PER_PROJECT; fileIndex++) { String index = projectIndex * PROJECTS_NUMBER + fileIndex + ""; ComponentDto file = ComponentTesting.newFileDto(project) @@ -137,9 +173,11 @@ public class IssuesIndexInjectionTest extends AbstractTest { .setKey("file-" + index) .setName("File " + index) .setLongName("File " + index); - componentsByProjectId.put(project, file); + componentsByProject.put(project, file); } } + batchSession.enqueue(new RefreshIndex(IndexDefinition.ISSUES_AUTHORIZATION.getIndexType())); + batchSession.commit(); LOGGER.info("Generated data in {} ms", System.currentTimeMillis() - start); } diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassIndexingTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassIndexingTest.java deleted file mode 100644 index 6f1e0706a94..00000000000 --- a/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassIndexingTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.data.issues; - -import org.junit.Before; -import org.junit.Test; -import org.sonar.core.issue.db.IssueDto; -import org.sonar.core.persistence.DbSession; -import org.sonar.server.issue.index.IssueIndex; -import org.sonar.server.search.IndexDefinition; -import org.sonar.server.search.action.InsertDto; -import org.sonar.server.search.action.RefreshIndex; - -import static org.fest.assertions.Assertions.assertThat; - -public class MassIndexingTest extends IssueData { - - @Before - public void setUp() throws Exception { - DbSession setupSession = db.openSession(false); - generateRules(setupSession); - generateProjects(setupSession); - setupSession.commit(); - setupSession.close(); - } - - @Test - public void enqueue_issues() throws Exception { - - long start = System.currentTimeMillis(); - int issueInsertCount = ISSUE_COUNT; - for (int i = 0; i < issueInsertCount; i++) { - session.enqueue(new InsertDto(IndexDefinition.ISSUES.getIndexType(), getIssue(i), false)); - } - session.enqueue(new RefreshIndex(IndexDefinition.ISSUES.getIndexType())); - session.commit(); - long stop = System.currentTimeMillis(); - - //TODO add performance assertions here - assertThat(index.get(IssueIndex.class).countAll()).isEqualTo(issueInsertCount); - - long time = stop-start; - LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time)); - - } -} diff --git a/server/sonar-data-test/src/test/resources/assertions.properties b/server/sonar-data-test/src/test/resources/assertions.properties index 71b79848178..1d79daef696 100644 --- a/server/sonar-data-test/src/test/resources/assertions.properties +++ b/server/sonar-data-test/src/test/resources/assertions.properties @@ -1,4 +1,4 @@ default.IssuesDbExtractionTest.extract_issues=40000 default.IssuesIndexInjectionTest.inject_issues=40000 -jl.IssuesDbExtractionTest.extract_issues=40000 -jl.IssuesIndexInjectionTest.inject_issues=40000 +JL.IssuesDbExtractionTest.extract_issues=40000 +JL.IssuesIndexInjectionTest.inject_issues=40000