From f36ed4b971fc6a4b585a6c8f306b53a3cc3a4714 Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Mon, 13 Oct 2014 16:05:35 +0200 Subject: SONAR-5564 - Added server-data-tests for ES ingestion and synchronizing --- server/sonar-data-test/pom.xml | 56 +++++++++ .../test/java/org/sonar/data/issues/IssueData.java | 135 +++++++++++++++++++++ .../org/sonar/data/issues/MassIndexingTest.java | 63 ++++++++++ .../sonar/data/issues/MassSynchronizingTest.java | 96 +++++++++++++++ .../src/test/resources/logback-test.xml | 38 ++++++ 5 files changed, 388 insertions(+) create mode 100644 server/sonar-data-test/pom.xml create mode 100644 server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java create mode 100644 server/sonar-data-test/src/test/java/org/sonar/data/issues/MassIndexingTest.java create mode 100644 server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java create mode 100644 server/sonar-data-test/src/test/resources/logback-test.xml (limited to 'server/sonar-data-test') diff --git a/server/sonar-data-test/pom.xml b/server/sonar-data-test/pom.xml new file mode 100644 index 00000000000..57c7f82e6a8 --- /dev/null +++ b/server/sonar-data-test/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + + org.codehaus.sonar + sonar + 5.0-SNAPSHOT + + sonar-data-test + jar + SonarQube :: Server :: Data Tests + + + + org.codehaus.sonar + sonar-server + test-jar + test + ${project.version} + + + org.codehaus.sonar + sonar-server + ${project.version} + + + org.codehaus.sonar + sonar-search + ${project.version} + + + org.easytesting + fest-assert + test + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipServerTests} + + + + + + diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java new file mode 100644 index 00000000000..a7e8dcc0dd5 --- /dev/null +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/IssueData.java @@ -0,0 +1,135 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.data.issues; + +import com.google.common.collect.Iterables; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.ClassRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.rule.RuleKey; +import org.sonar.core.component.ComponentDto; +import org.sonar.core.issue.db.IssueDto; +import org.sonar.core.persistence.DbSession; +import org.sonar.core.rule.RuleDto; +import org.sonar.server.db.DbClient; +import org.sonar.server.rule.RuleTesting; +import org.sonar.server.search.IndexClient; +import org.sonar.server.tester.ServerTester; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +public class IssueData { + + public final static int MAX_NUMBER_RULES = 2500; + public final static int MAX_NUMBER_PROJECTS = 500; + public final static int MAX_NUMBER_RESOURCES_PER_PROJECT = 10000; + + public final static int ISSUE_COUNT = 1000000; + + protected static final Logger LOGGER = LoggerFactory.getLogger(IssueData.class); + + @ClassRule + public static ServerTester tester = new ServerTester(); + + @After + public void tearDown() throws Exception { + tester.clearDbAndIndexes(); + if (session != null) { + session.close(); + } + } + + @AfterClass + public static void reset() throws Exception { + tester = new ServerTester(); + } + + protected Random generator = new Random(System.currentTimeMillis()); + protected DbClient db = tester.get(DbClient.class); + protected IndexClient index = tester.get(IndexClient.class); + protected DbSession session = tester.get(DbClient.class).openSession(true); + + protected List rules = new ArrayList(); + protected Map> projects = new HashMap>(); + + protected IssueDto getIssue(int id) { + RuleDto rule = rules.get(generator.nextInt(rules.size())); + Long projectId = Iterables.get(projects.keySet(), generator.nextInt(projects.size())); + Long resourceId = projects.get(projectId).get(generator.nextInt(projects.get(projectId).size())); + return new IssueDto().setId(new Long(id)) + .setRootComponentId(projectId) + .setRootComponentKey(projectId + "_key") + .setComponentId(resourceId) + .setComponentKey(resourceId + "_key") + .setRule(rule) + .setMessage("Lorem ipsum loertium bortim tata toto tutu 14 failures in this issue") + .setAssignee("assignee_") + .setSeverity("BLOCKER") + .setReporter("Luc besson") + .setAuthorLogin("Pinpin") + .setStatus("OPEN").setResolution("OPEN") + .setKee(UUID.randomUUID().toString()); + } + + protected void generateRules(DbSession dbSession) { + // Generate Rules + for (int i = 0; i < MAX_NUMBER_RULES; i++) { + rules.add(RuleTesting.newDto(RuleKey.of("data_repo", "S" + i))); + } + DbSession setupSession = db.openSession(false); + db.ruleDao().insert(setupSession, rules); + setupSession.commit(); + } + + protected void generateProjects(DbSession setupSession) { + // Generate projects & resources + for(long p = 1; p<=MAX_NUMBER_PROJECTS; p++) { + ComponentDto project = new ComponentDto() + .setId(p) + .setKey("MyProject") + .setProjectId_unit_test_only(p); + db.componentDao().insert(setupSession, project); + projects.put(project.projectId(), new ArrayList()); + List resources = new ArrayList(); + for(int i = 0; i(IndexDefinition.ISSUES.getIndexType(), getIssue(i), false)); + } + session.enqueue(new RefreshIndex(IndexDefinition.ISSUES.getIndexType())); + session.commit(); + long stop = System.currentTimeMillis(); + + //TODO add performance assertions here + assertThat(index.get(IssueIndex.class).countAll()).isEqualTo(issueInsertCount); + + long time = stop-start; + LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time)); + + } +} diff --git a/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java b/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java new file mode 100644 index 00000000000..1ed6da0d4c2 --- /dev/null +++ b/server/sonar-data-test/src/test/java/org/sonar/data/issues/MassSynchronizingTest.java @@ -0,0 +1,96 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.data.issues; + +import org.apache.ibatis.session.ResultContext; +import org.junit.Before; +import org.junit.Test; +import org.sonar.core.persistence.DbSession; +import org.sonar.server.issue.db.IssueDao; +import org.sonar.server.search.DbSynchronizationHandler; + +import java.util.Date; +import java.util.Map; + +import static org.fest.assertions.Assertions.assertThat; + +public class MassSynchronizingTest extends IssueData { + + MyIssueDao myIssueDao; + + @Before + public void setUp() throws Exception { + + myIssueDao = new MyIssueDao(); + + DbSession setupSession = db.openSession(false); + generateRules(setupSession); + generateProjects(setupSession); + + + // Inserting Issues now (finally) + for (int i = 0; i < ISSUE_COUNT; i++) { + myIssueDao.insert(setupSession, getIssue(i)); + if (i % 100 == 0) { + setupSession.commit(); + } + } + setupSession.commit(); + setupSession.close(); + } + + @Test + public void synchronize_issues() throws Exception { + long start = System.currentTimeMillis(); + int issueInsertCount = ISSUE_COUNT; + myIssueDao.synchronizeAfter(session, new Date(0)); + long stop = System.currentTimeMillis(); + + // TODO add performance assertions here + assertThat(myIssueDao.synchronizedIssues).isEqualTo(issueInsertCount); + + long time = stop-start; + LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time)); + } + + class MyIssueDao extends IssueDao { + public Integer synchronizedIssues = 0; + + @Override + protected boolean hasIndex() { + return false; + } + + @Override + protected DbSynchronizationHandler getSynchronizationResultHandler(DbSession session, Map params) { + return new DbSynchronizationHandler(session, params) { + + @Override + public void handleResult(ResultContext context) { + synchronizedIssues++; + } + + @Override + public void enqueueCollected() { + } + }; + } + } +} diff --git a/server/sonar-data-test/src/test/resources/logback-test.xml b/server/sonar-data-test/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..df0bb26d407 --- /dev/null +++ b/server/sonar-data-test/src/test/resources/logback-test.xml @@ -0,0 +1,38 @@ + + + + + + + + + INFO + ACCEPT + DENY + + + + %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n + + + + + + + + %d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n + + + + + + + + + + -- cgit v1.2.3