<?xml version="1.0" encoding="UTF-8"?> | |||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | |||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | |||||
<modelVersion>4.0.0</modelVersion> | |||||
<parent> | |||||
<groupId>org.codehaus.sonar</groupId> | |||||
<artifactId>sonar</artifactId> | |||||
<version>5.0-SNAPSHOT</version> | |||||
</parent> | |||||
<artifactId>sonar-data-test</artifactId> | |||||
<packaging>jar</packaging> | |||||
<name>SonarQube :: Server :: Data Tests</name> | |||||
<dependencies> | |||||
<dependency> | |||||
<groupId>org.codehaus.sonar</groupId> | |||||
<artifactId>sonar-server</artifactId> | |||||
<type>test-jar</type> | |||||
<scope>test</scope> | |||||
<version>${project.version}</version> | |||||
</dependency> | |||||
<dependency> | |||||
<groupId>org.codehaus.sonar</groupId> | |||||
<artifactId>sonar-server</artifactId> | |||||
<version>${project.version}</version> | |||||
</dependency> | |||||
<dependency> | |||||
<groupId>org.codehaus.sonar</groupId> | |||||
<artifactId>sonar-search</artifactId> | |||||
<version>${project.version}</version> | |||||
</dependency> | |||||
<dependency> | |||||
<groupId>org.easytesting</groupId> | |||||
<artifactId>fest-assert</artifactId> | |||||
<scope>test</scope> | |||||
</dependency> | |||||
<dependency> | |||||
<groupId>junit</groupId> | |||||
<artifactId>junit</artifactId> | |||||
<scope>test</scope> | |||||
</dependency> | |||||
</dependencies> | |||||
<build> | |||||
<plugins> | |||||
<plugin> | |||||
<groupId>org.apache.maven.plugins</groupId> | |||||
<artifactId>maven-surefire-plugin</artifactId> | |||||
<configuration> | |||||
<skipTests>${skipServerTests}</skipTests> | |||||
</configuration> | |||||
</plugin> | |||||
</plugins> | |||||
</build> | |||||
</project> |
/* | |||||
* SonarQube, open source software quality management tool. | |||||
* Copyright (C) 2008-2014 SonarSource | |||||
* mailto:contact AT sonarsource DOT com | |||||
* | |||||
* SonarQube is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* SonarQube is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.data.issues; | |||||
import com.google.common.collect.Iterables; | |||||
import org.junit.After; | |||||
import org.junit.AfterClass; | |||||
import org.junit.ClassRule; | |||||
import org.slf4j.Logger; | |||||
import org.slf4j.LoggerFactory; | |||||
import org.sonar.api.rule.RuleKey; | |||||
import org.sonar.core.component.ComponentDto; | |||||
import org.sonar.core.issue.db.IssueDto; | |||||
import org.sonar.core.persistence.DbSession; | |||||
import org.sonar.core.rule.RuleDto; | |||||
import org.sonar.server.db.DbClient; | |||||
import org.sonar.server.rule.RuleTesting; | |||||
import org.sonar.server.search.IndexClient; | |||||
import org.sonar.server.tester.ServerTester; | |||||
import java.util.ArrayList; | |||||
import java.util.HashMap; | |||||
import java.util.List; | |||||
import java.util.Map; | |||||
import java.util.Random; | |||||
import java.util.UUID; | |||||
public class IssueData { | |||||
public final static int MAX_NUMBER_RULES = 2500; | |||||
public final static int MAX_NUMBER_PROJECTS = 500; | |||||
public final static int MAX_NUMBER_RESOURCES_PER_PROJECT = 10000; | |||||
public final static int ISSUE_COUNT = 1000000; | |||||
protected static final Logger LOGGER = LoggerFactory.getLogger(IssueData.class); | |||||
@ClassRule | |||||
public static ServerTester tester = new ServerTester(); | |||||
@After | |||||
public void tearDown() throws Exception { | |||||
tester.clearDbAndIndexes(); | |||||
if (session != null) { | |||||
session.close(); | |||||
} | |||||
} | |||||
@AfterClass | |||||
public static void reset() throws Exception { | |||||
tester = new ServerTester(); | |||||
} | |||||
protected Random generator = new Random(System.currentTimeMillis()); | |||||
protected DbClient db = tester.get(DbClient.class); | |||||
protected IndexClient index = tester.get(IndexClient.class); | |||||
protected DbSession session = tester.get(DbClient.class).openSession(true); | |||||
protected List<RuleDto> rules = new ArrayList<RuleDto>(); | |||||
protected Map<Long, List<Long>> projects = new HashMap<Long, List<Long>>(); | |||||
protected IssueDto getIssue(int id) { | |||||
RuleDto rule = rules.get(generator.nextInt(rules.size())); | |||||
Long projectId = Iterables.get(projects.keySet(), generator.nextInt(projects.size())); | |||||
Long resourceId = projects.get(projectId).get(generator.nextInt(projects.get(projectId).size())); | |||||
return new IssueDto().setId(new Long(id)) | |||||
.setRootComponentId(projectId) | |||||
.setRootComponentKey(projectId + "_key") | |||||
.setComponentId(resourceId) | |||||
.setComponentKey(resourceId + "_key") | |||||
.setRule(rule) | |||||
.setMessage("Lorem ipsum loertium bortim tata toto tutu 14 failures in this issue") | |||||
.setAssignee("assignee_") | |||||
.setSeverity("BLOCKER") | |||||
.setReporter("Luc besson") | |||||
.setAuthorLogin("Pinpin") | |||||
.setStatus("OPEN").setResolution("OPEN") | |||||
.setKee(UUID.randomUUID().toString()); | |||||
} | |||||
protected void generateRules(DbSession dbSession) { | |||||
// Generate Rules | |||||
for (int i = 0; i < MAX_NUMBER_RULES; i++) { | |||||
rules.add(RuleTesting.newDto(RuleKey.of("data_repo", "S" + i))); | |||||
} | |||||
DbSession setupSession = db.openSession(false); | |||||
db.ruleDao().insert(setupSession, rules); | |||||
setupSession.commit(); | |||||
} | |||||
protected void generateProjects(DbSession setupSession) { | |||||
// Generate projects & resources | |||||
for(long p = 1; p<=MAX_NUMBER_PROJECTS; p++) { | |||||
ComponentDto project = new ComponentDto() | |||||
.setId(p) | |||||
.setKey("MyProject") | |||||
.setProjectId_unit_test_only(p); | |||||
db.componentDao().insert(setupSession, project); | |||||
projects.put(project.projectId(), new ArrayList<Long>()); | |||||
List<ComponentDto> resources = new ArrayList<ComponentDto>(); | |||||
for(int i = 0; i<generator.nextInt(MAX_NUMBER_RESOURCES_PER_PROJECT); i++) { | |||||
ComponentDto resource = new ComponentDto() | |||||
.setKey("MyComponent_"+(p*MAX_NUMBER_PROJECTS+i)) | |||||
.setProjectId_unit_test_only(project.getId()); | |||||
db.componentDao().insert(setupSession, resource); | |||||
resources.add(resource); | |||||
} | |||||
setupSession.commit(); | |||||
for (ComponentDto resource : resources) { | |||||
projects.get(project.projectId()).add(resource.getId()); | |||||
} | |||||
} | |||||
} | |||||
protected int documentPerSecond(long time) { | |||||
return (int)Math.round(ISSUE_COUNT/(time/1000.0)); | |||||
} | |||||
} |
/* | |||||
* SonarQube, open source software quality management tool. | |||||
* Copyright (C) 2008-2014 SonarSource | |||||
* mailto:contact AT sonarsource DOT com | |||||
* | |||||
* SonarQube is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* SonarQube is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.data.issues; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
import org.sonar.core.issue.db.IssueDto; | |||||
import org.sonar.core.persistence.DbSession; | |||||
import org.sonar.server.issue.index.IssueIndex; | |||||
import org.sonar.server.search.IndexDefinition; | |||||
import org.sonar.server.search.action.InsertDto; | |||||
import org.sonar.server.search.action.RefreshIndex; | |||||
import static org.fest.assertions.Assertions.assertThat; | |||||
public class MassIndexingTest extends IssueData { | |||||
@Before | |||||
public void setUp() throws Exception { | |||||
DbSession setupSession = db.openSession(false); | |||||
generateRules(setupSession); | |||||
generateProjects(setupSession); | |||||
setupSession.commit(); | |||||
setupSession.close(); | |||||
} | |||||
@Test | |||||
public void enqueue_issues() throws Exception { | |||||
long start = System.currentTimeMillis(); | |||||
int issueInsertCount = ISSUE_COUNT; | |||||
for (int i = 0; i < issueInsertCount; i++) { | |||||
session.enqueue(new InsertDto<IssueDto>(IndexDefinition.ISSUES.getIndexType(), getIssue(i), false)); | |||||
} | |||||
session.enqueue(new RefreshIndex(IndexDefinition.ISSUES.getIndexType())); | |||||
session.commit(); | |||||
long stop = System.currentTimeMillis(); | |||||
//TODO add performance assertions here | |||||
assertThat(index.get(IssueIndex.class).countAll()).isEqualTo(issueInsertCount); | |||||
long time = stop-start; | |||||
LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time)); | |||||
} | |||||
} |
/* | |||||
* SonarQube, open source software quality management tool. | |||||
* Copyright (C) 2008-2014 SonarSource | |||||
* mailto:contact AT sonarsource DOT com | |||||
* | |||||
* SonarQube is free software; you can redistribute it and/or | |||||
* modify it under the terms of the GNU Lesser General Public | |||||
* License as published by the Free Software Foundation; either | |||||
* version 3 of the License, or (at your option) any later version. | |||||
* | |||||
* SonarQube is distributed in the hope that it will be useful, | |||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||||
* Lesser General Public License for more details. | |||||
* | |||||
* You should have received a copy of the GNU Lesser General Public License | |||||
* along with this program; if not, write to the Free Software Foundation, | |||||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||||
*/ | |||||
package org.sonar.data.issues; | |||||
import org.apache.ibatis.session.ResultContext; | |||||
import org.junit.Before; | |||||
import org.junit.Test; | |||||
import org.sonar.core.persistence.DbSession; | |||||
import org.sonar.server.issue.db.IssueDao; | |||||
import org.sonar.server.search.DbSynchronizationHandler; | |||||
import java.util.Date; | |||||
import java.util.Map; | |||||
import static org.fest.assertions.Assertions.assertThat; | |||||
public class MassSynchronizingTest extends IssueData { | |||||
MyIssueDao myIssueDao; | |||||
@Before | |||||
public void setUp() throws Exception { | |||||
myIssueDao = new MyIssueDao(); | |||||
DbSession setupSession = db.openSession(false); | |||||
generateRules(setupSession); | |||||
generateProjects(setupSession); | |||||
// Inserting Issues now (finally) | |||||
for (int i = 0; i < ISSUE_COUNT; i++) { | |||||
myIssueDao.insert(setupSession, getIssue(i)); | |||||
if (i % 100 == 0) { | |||||
setupSession.commit(); | |||||
} | |||||
} | |||||
setupSession.commit(); | |||||
setupSession.close(); | |||||
} | |||||
@Test | |||||
public void synchronize_issues() throws Exception { | |||||
long start = System.currentTimeMillis(); | |||||
int issueInsertCount = ISSUE_COUNT; | |||||
myIssueDao.synchronizeAfter(session, new Date(0)); | |||||
long stop = System.currentTimeMillis(); | |||||
// TODO add performance assertions here | |||||
assertThat(myIssueDao.synchronizedIssues).isEqualTo(issueInsertCount); | |||||
long time = stop-start; | |||||
LOGGER.info("processed {} Issues in {}ms with avg {} Issue/second", ISSUE_COUNT, time, this.documentPerSecond(time)); | |||||
} | |||||
class MyIssueDao extends IssueDao { | |||||
public Integer synchronizedIssues = 0; | |||||
@Override | |||||
protected boolean hasIndex() { | |||||
return false; | |||||
} | |||||
@Override | |||||
protected DbSynchronizationHandler getSynchronizationResultHandler(DbSession session, Map<String, String> params) { | |||||
return new DbSynchronizationHandler(session, params) { | |||||
@Override | |||||
public void handleResult(ResultContext context) { | |||||
synchronizedIssues++; | |||||
} | |||||
@Override | |||||
public void enqueueCollected() { | |||||
} | |||||
}; | |||||
} | |||||
} | |||||
} |
<?xml version="1.0" encoding="UTF-8" ?> | |||||
<!-- | |||||
Configuration for default logger. Only used while embedded server is starting, | |||||
before proper logging configuration is loaded. | |||||
See http://logback.qos.ch/manual/configuration.html | |||||
--> | |||||
<configuration debug="false"> | |||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"/> | |||||
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> | |||||
<filter class="ch.qos.logback.classic.filter.LevelFilter"> | |||||
<level>INFO</level> | |||||
<onMatch>ACCEPT</onMatch> | |||||
<onMismatch>DENY</onMismatch> | |||||
</filter> | |||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> | |||||
<pattern> | |||||
%d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n | |||||
</pattern> | |||||
</encoder> | |||||
</appender> | |||||
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> | |||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> | |||||
<pattern> | |||||
%d{yyyy.MM.dd HH:mm:ss} %-5level %msg%n | |||||
</pattern> | |||||
</encoder> | |||||
</appender> | |||||
<root> | |||||
<level value="INFO"/> | |||||
<appender-ref ref="CONSOLE"/> | |||||
</root> | |||||
</configuration> |
<?xml version="1.0" encoding="UTF-8"?> | <?xml version="1.0" encoding="UTF-8"?> | ||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | |||||
<modelVersion>4.0.0</modelVersion> | |||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | |||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | |||||
<modelVersion>4.0.0</modelVersion> | |||||
<parent> | <parent> | ||||
<groupId>org.codehaus.sonar</groupId> | <groupId>org.codehaus.sonar</groupId> | ||||
<artifactId>server</artifactId> | <artifactId>server</artifactId> | ||||
</resources> | </resources> | ||||
<plugins> | <plugins> | ||||
<plugin> | |||||
<groupId>org.apache.maven.plugins</groupId> | |||||
<artifactId>maven-jar-plugin</artifactId> | |||||
<executions> | |||||
<execution> | |||||
<goals> | |||||
<goal>test-jar</goal> | |||||
</goals> | |||||
</execution> | |||||
</executions> | |||||
</plugin> | |||||
<plugin> | <plugin> | ||||
<groupId>org.apache.maven.plugins</groupId> | <groupId>org.apache.maven.plugins</groupId> | ||||
<artifactId>maven-jar-plugin</artifactId> | <artifactId>maven-jar-plugin</artifactId> |
private Class<MAPPER> mapperClass; | private Class<MAPPER> mapperClass; | ||||
private System2 system2; | private System2 system2; | ||||
private boolean hasIndex() { | |||||
protected boolean hasIndex() { | |||||
return indexDefinition != null; | return indexDefinition != null; | ||||
} | } | ||||