<module>sonar-server</module>
<module>sonar-web</module>
<module>sonar-ws-client</module>
- <!--<module>sonar-server-benchmark</module>-->
</modules>
</project>
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar</artifactId>
+ <artifactId>server</artifactId>
<version>5.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
</parent>
<artifactId>sonar-data-test</artifactId>
- <packaging>jar</packaging>
- <name>SonarQube :: Server :: Data Tests</name>
+ <name>SonarQube :: Server :: Benchmark</name>
<dependencies>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-server</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-server</artifactId>
+ <type>test-jar</type>
<version>${project.version}</version>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>org.codehaus.sonar</groupId>
<artifactId>sonar-search</artifactId>
<version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-core</artifactId>
- <type>test-jar</type>
<scope>test</scope>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>org.easytesting</groupId>
- <artifactId>fest-assert</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.dbunit</groupId>
- <artifactId>dbunit</artifactId>
+ <groupId>org.codehaus.sonar</groupId>
+ <artifactId>sonar-testing-harness</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- </dependency>
</dependencies>
<build>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <skipTests>${skipServerTests}</skipTests>
- <argLine>-Xmx256m</argLine>
+ <argLine>-Xmx1G -Xms256m -server -Djava.awt.headless=true</argLine>
</configuration>
</plugin>
</plugins>
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.data.issues;
-
-import com.google.common.collect.Iterables;
-import org.apache.commons.lang.StringUtils;
-import org.apache.ibatis.io.Resources;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-import org.sonar.api.issue.Issue;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.api.rule.Severity;
-import org.sonar.core.component.ComponentDto;
-import org.sonar.core.issue.db.IssueDto;
-import org.sonar.core.rule.RuleDto;
-import org.sonar.server.issue.IssueTesting;
-import org.sonar.server.rule.RuleTesting;
-
-import java.io.File;
-import java.io.FileReader;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Properties;
-
-import static com.google.common.collect.Lists.newArrayList;
-import static org.fest.assertions.Assertions.assertThat;
-
-public class AbstractTest {
-
- final static int RULES_NUMBER = 25;
- final static int USERS_NUMBER = 100;
-
- private static final double ACCEPTED_DURATION_VARIATION_IN_PERCENTS = 10.0;
-
- static Iterator<RuleDto> rules;
- static Iterator<String> users;
- static Iterator<String> severities;
- static Iterator<String> statuses;
- static Iterator<String> closedResolutions;
- static Iterator<String> resolvedResolutions;
-
- static Properties properties = new Properties();
- @Rule
- public TestName testName = new TestName();
-
- @BeforeClass
- public static void loadProperties() {
- try {
- properties = new Properties();
- File propertiesFile = Resources.getResourceAsFile("assertions.properties");
- FileReader reader = new FileReader(propertiesFile);
- properties.load(reader);
- properties.putAll(System.getProperties());
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
-
- severities = Iterables.cycle(Severity.ALL).iterator();
- statuses = Iterables.cycle(Issue.STATUS_OPEN, Issue.STATUS_CONFIRMED, Issue.STATUS_REOPENED, Issue.STATUS_RESOLVED, Issue.STATUS_CLOSED).iterator();
- closedResolutions = Iterables.cycle(Issue.RESOLUTION_FALSE_POSITIVE, Issue.RESOLUTION_FIXED, Issue.RESOLUTION_REMOVED).iterator();
- resolvedResolutions = Iterables.cycle(Issue.RESOLUTION_FALSE_POSITIVE, Issue.RESOLUTION_FIXED).iterator();
- users = Iterables.cycle(generateUsers()).iterator();
- rules = Iterables.cycle(generateRules()).iterator();
- }
-
- protected static List<RuleDto> generateRules() {
- List<RuleDto> rules = newArrayList();
- for (int i = 0; i < RULES_NUMBER; i++) {
- rules.add(RuleTesting.newDto(RuleKey.of("rule-repo", "rule-key-" + i)));
- }
- return rules;
- }
-
- protected static List<String> generateUsers() {
- List<String> users = newArrayList();
- for (int i = 0; i < USERS_NUMBER; i++) {
- users.add("user-" + i);
- }
- return users;
- }
-
- protected String getProperty(String test) {
- String currentUser = StringUtils.defaultString(properties.getProperty("user"), "default");
- String property = currentUser + "." + test;
- String value = properties.getProperty(property);
- if (value == null) {
- throw new IllegalArgumentException(String.format("Property '%s' hasn't been found", property));
- }
- return value;
- }
-
- protected void assertDurationAround(long duration, long expectedDuration) {
- double variation = 100.0 * (0.0 + duration - expectedDuration) / expectedDuration;
- System.out.printf("Test %s : executed in %d ms (%.2f %% from target)\n", testName.getMethodName(), duration, variation);
- assertThat(Math.abs(variation)).as(String.format("Expected %d ms, got %d ms", expectedDuration, duration)).isLessThan(ACCEPTED_DURATION_VARIATION_IN_PERCENTS);
- }
-
- protected IssueDto newIssue(int index, ComponentDto file, ComponentDto project, RuleDto rule) {
- String status = statuses.next();
- String resolution = null;
- if (status.equals(Issue.STATUS_CLOSED)) {
- resolution = closedResolutions.next();
- } else if (status.equals(Issue.STATUS_RESOLVED)) {
- resolution = resolvedResolutions.next();
- }
- return IssueTesting.newDto(rule, file, project)
- .setMessage("Message from rule " + rule.getKey().toString() + " on line " + index)
- .setLine(index)
- .setAssignee(users.next())
- .setReporter(users.next())
- .setAuthorLogin(users.next())
- .setSeverity(severities.next())
- .setStatus(status)
- .setResolution(resolution);
- }
-
-}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.data.issues;
-
-import org.apache.ibatis.session.ResultContext;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.utils.System2;
-import org.sonar.core.component.ComponentDto;
-import org.sonar.core.persistence.DbSession;
-import org.sonar.core.persistence.TestDatabase;
-import org.sonar.server.component.ComponentTesting;
-import org.sonar.server.component.db.ComponentDao;
-import org.sonar.server.issue.db.IssueDao;
-import org.sonar.server.rule.db.RuleDao;
-import org.sonar.server.search.DbSynchronizationHandler;
-
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.fest.assertions.Assertions.assertThat;
-
-@Ignore("Waiting for usage of IssueIndexer")
-public class IssuesDbExtractionTest extends AbstractTest {
-
- static final Logger LOGGER = LoggerFactory.getLogger(IssuesDbExtractionTest.class);
- final static int PROJECTS_NUMBER = 100;
- final static int NUMBER_FILES_PER_PROJECT = 100;
- final static int NUMBER_ISSUES_PER_FILE = 100;
- final static int ISSUE_COUNT = PROJECTS_NUMBER * NUMBER_FILES_PER_PROJECT * NUMBER_ISSUES_PER_FILE;
- @Rule
- public TestDatabase db = new TestDatabase();
- AtomicLong counter = new AtomicLong(0L);
- DbSession session;
-
- ProxyIssueDao issueDao;
- RuleDao ruleDao;
- ComponentDao componentDao;
-
- @Before
- public void setUp() throws Exception {
- issueDao = new ProxyIssueDao();
- ruleDao = new RuleDao();
- componentDao = new ComponentDao(System2.INSTANCE);
-
- session = db.myBatis().openSession(false);
- }
-
- @After
- public void closeSession() throws Exception {
- session.close();
- }
-
- @Test
- public void extract_issues() throws Exception {
- insertReferentials();
-
- ProgressTask progressTask = new ProgressTask(counter);
- Timer timer = new Timer("Extract Issues");
- timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
- try {
- long start = System.currentTimeMillis();
- issueDao.synchronizeAfter(session);
- long stop = System.currentTimeMillis();
- progressTask.log();
-
- assertThat(issueDao.synchronizedIssues).isEqualTo(ISSUE_COUNT);
-
- long time = stop - start;
- LOGGER.info("Iterated over {} issues in {} ms with avg of {} issues/second", ISSUE_COUNT, time, rowsPerSecond(time));
- // assertDurationAround(time, Long.parseLong(getProperty("IssuesDbExtractionTest.extract_issues")));
-
- } finally {
- timer.cancel();
- timer.purge();
- }
- }
-
- private void insertReferentials() {
- long start = System.currentTimeMillis();
-
- for (int i = 0; i < RULES_NUMBER; i++) {
- ruleDao.insert(session, rules.next());
- }
- session.commit();
-
- for (long projectIndex = 0; projectIndex < PROJECTS_NUMBER; projectIndex++) {
- ComponentDto project = ComponentTesting.newProjectDto()
- .setKey("project-" + projectIndex)
- .setName("Project " + projectIndex)
- .setLongName("Project " + projectIndex);
- componentDao.insert(session, project);
-
- for (int fileIndex = 0; fileIndex < NUMBER_FILES_PER_PROJECT; fileIndex++) {
- String index = projectIndex * PROJECTS_NUMBER + fileIndex + "";
- ComponentDto file = ComponentTesting.newFileDto(project)
- .setKey("file-" + index)
- .setName("File " + index)
- .setLongName("File " + index);
- componentDao.insert(session, file);
-
- for (int issueIndex = 1; issueIndex < NUMBER_ISSUES_PER_FILE + 1; issueIndex++) {
- issueDao.insert(session, newIssue(issueIndex, file, project, rules.next()));
- }
- session.commit();
- }
- }
- LOGGER.info("Referentials inserted in {} ms", System.currentTimeMillis() - start);
- }
-
- protected int rowsPerSecond(long time) {
- return (int) Math.round(ISSUE_COUNT / (time / 1000.0));
- }
-
- protected static class ProgressTask extends TimerTask {
-
- public static final long PERIOD_MS = 60000L;
- private static final Logger LOGGER = LoggerFactory.getLogger("PerformanceTests");
- private final AtomicLong counter;
-
- public ProgressTask(AtomicLong counter) {
- this.counter = counter;
- }
-
- @Override
- public void run() {
- log();
- }
-
- public void log() {
- LOGGER.info(String.format("%d issues processed", counter.get()));
- }
- }
-
- class ProxyIssueDao extends IssueDao {
- public Integer synchronizedIssues = 0;
-
- @Override
- protected boolean hasIndex() {
- return false;
- }
-
- @Override
- protected DbSynchronizationHandler getSynchronizationResultHandler(DbSession session, Map<String, String> params) {
- return new DbSynchronizationHandler(session, params) {
-
- @Override
- public void handleResult(ResultContext context) {
- synchronizedIssues++;
- counter.getAndIncrement();
- }
-
- @Override
- public void enqueueCollected() {
-
- }
- };
- }
- }
-
-}
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.data.issues;
-
-import com.google.common.collect.ArrayListMultimap;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.issue.Issue;
-import org.sonar.api.security.DefaultGroups;
-import org.sonar.core.component.ComponentDto;
-import org.sonar.core.issue.db.IssueAuthorizationDto;
-import org.sonar.core.issue.db.IssueDto;
-import org.sonar.core.persistence.DbSession;
-import org.sonar.server.component.ComponentTesting;
-import org.sonar.server.db.DbClient;
-import org.sonar.server.issue.IssueQuery;
-import org.sonar.server.issue.index.IssueAuthorizationIndex;
-import org.sonar.server.issue.index.IssueIndex;
-import org.sonar.server.search.IndexDefinition;
-import org.sonar.server.search.QueryContext;
-import org.sonar.server.search.Result;
-import org.sonar.server.search.action.InsertDto;
-import org.sonar.server.search.action.RefreshIndex;
-import org.sonar.server.tester.ServerTester;
-import org.sonar.server.user.MockUserSession;
-
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static com.google.common.collect.Lists.newArrayList;
-import static org.fest.assertions.Assertions.assertThat;
-
-public class IssuesIndexInjectionTest extends AbstractTest {
-
- static final Logger LOGGER = LoggerFactory.getLogger(IssuesIndexInjectionTest.class);
- final static int PROJECTS = 100;
- final static int FILES_PER_PROJECT = 100;
- final static int ISSUES_PER_FILE = 100;
- final static int ISSUE_COUNT = PROJECTS * FILES_PER_PROJECT * ISSUES_PER_FILE;
-
- @ClassRule
- public static ServerTester tester = new ServerTester();
- AtomicLong counter = new AtomicLong(0L);
- DbSession batchSession;
-
- IssueIndex issueIndex;
- IssueAuthorizationIndex issueAuthorizationIndex;
-
- List<ComponentDto> projects = newArrayList();
- ArrayListMultimap<ComponentDto, ComponentDto> componentsByProject = ArrayListMultimap.create();
-
- protected static int documentPerSecond(long nbIssues, long time) {
- return (int) Math.round(nbIssues / (time / 1000.0));
- }
-
- @Before
- public void setUp() throws Exception {
- issueIndex = tester.get(IssueIndex.class);
- issueAuthorizationIndex = tester.get(IssueAuthorizationIndex.class);
- batchSession = tester.get(DbClient.class).openSession(true);
-
- MockUserSession.set().setLogin("test");
- }
-
- @After
- public void after() throws Exception {
- batchSession.close();
- }
-
- @Test
- public void inject_issues_and_execute_queries() throws Exception {
- generateData();
- injectIssuesInIndex();
- executeQueries();
- }
-
- public void injectIssuesInIndex() {
- ProgressTask progressTask = new ProgressTask(counter);
- Timer timer = new Timer("Inject Issues");
- timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
- try {
- long start = System.currentTimeMillis();
- for (ComponentDto project : projects) {
- for (ComponentDto file : componentsByProject.get(project)) {
- for (int issueIndex = 1; issueIndex < ISSUES_PER_FILE + 1; issueIndex++) {
- batchSession.enqueue(new InsertDto<IssueDto>(IndexDefinition.ISSUES.getIndexType(), newIssue(issueIndex, file, project, rules.next()), false));
- counter.getAndIncrement();
- }
- }
- }
- batchSession.enqueue(new RefreshIndex(IndexDefinition.ISSUES.getIndexType()));
- batchSession.commit();
- long stop = System.currentTimeMillis();
- progressTask.log();
-
- assertThat(issueIndex.countAll()).isEqualTo(ISSUE_COUNT);
-
- long totalTime = stop - start;
- LOGGER.info("Inserted {} Issues in {} ms with avg {} Issue/second", ISSUE_COUNT, totalTime, documentPerSecond(ISSUE_COUNT, totalTime));
- // assertDurationAround(totalTime, Long.parseLong(getProperty("IssuesIndexInjectionTest.inject_issues")));
-
- } finally {
- timer.cancel();
- timer.purge();
- }
- }
-
- public void executeQueries() {
- long start = System.currentTimeMillis();
- Result<Issue> result = issueIndex.search(IssueQuery.builder().build(), new QueryContext());
- LOGGER.info("Search for all issues : returned {} issues in {} ms", result.getTotal(), System.currentTimeMillis() - start);
-
- start = System.currentTimeMillis();
- ComponentDto project = componentsByProject.keySet().iterator().next();
- result = issueIndex.search(IssueQuery.builder().projectUuids(newArrayList(project.uuid())).build(), new QueryContext());
- LOGGER.info("Search for issues from one project : returned {} issues in {} ms", result.getTotal(), System.currentTimeMillis() - start);
-
- start = System.currentTimeMillis();
- ComponentDto file = componentsByProject.get(project).get(0);
- result = issueIndex.search(IssueQuery.builder().componentUuids(newArrayList(file.uuid())).build(), new QueryContext());
- LOGGER.info("Search for issues from one file : returned {} issues in {} ms", result.getTotal(), System.currentTimeMillis() - start);
- }
-
- private void generateData() {
- long ids = 1;
-
- for (int i = 0; i < RULES_NUMBER; i++) {
- rules.next().setId((int) ids++);
- }
-
- long start = System.currentTimeMillis();
- for (long projectIndex = 0; projectIndex < PROJECTS; projectIndex++) {
- ComponentDto project = ComponentTesting.newProjectDto()
- .setId(ids++)
- .setKey("project-" + projectIndex)
- .setName("Project " + projectIndex)
- .setLongName("Project " + projectIndex);
- projects.add(project);
-
- // All project are visible by anyone
- // TODO set different groups/users to test search issues queries with more realistic data
- batchSession.enqueue(new InsertDto<IssueAuthorizationDto>(IndexDefinition.ISSUES_AUTHORIZATION.getIndexType(),
- new IssueAuthorizationDto().setProjectUuid(project.uuid()).setGroups(newArrayList(DefaultGroups.ANYONE)), false));
-
- for (int fileIndex = 0; fileIndex < FILES_PER_PROJECT; fileIndex++) {
- String index = projectIndex * PROJECTS + fileIndex + "";
- ComponentDto file = ComponentTesting.newFileDto(project)
- .setId(ids++)
- .setKey("file-" + index)
- .setName("File " + index)
- .setLongName("File " + index);
- componentsByProject.put(project, file);
- }
- }
- batchSession.enqueue(new RefreshIndex(IndexDefinition.ISSUES_AUTHORIZATION.getIndexType()));
- batchSession.commit();
- LOGGER.info("Generated data in {} ms", System.currentTimeMillis() - start);
- }
-
- protected static class ProgressTask extends TimerTask {
-
- public static final long PERIOD_MS = 60000L;
- private static final Logger LOGGER = LoggerFactory.getLogger("PerformanceTests");
- private final AtomicLong counter;
- private long currentStart;
- private long previousTotal = 0;
-
- public ProgressTask(AtomicLong counter) {
- this.counter = counter;
- this.currentStart = System.currentTimeMillis();
- }
-
- @Override
- public void run() {
- log();
- }
-
- public void log() {
- long currentNumberOfIssues = counter.get() - this.previousTotal;
- LOGGER.info("{} issues inserted with avg {} issue/second", currentNumberOfIssues, documentPerSecond(currentNumberOfIssues, System.currentTimeMillis() - this.currentStart));
- this.previousTotal = counter.get();
- this.currentStart = System.currentTimeMillis();
- }
- }
-
-}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.benchmark;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.commons.lang.math.RandomUtils;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.issue.Issue;
+import org.sonar.api.rule.Severity;
+import org.sonar.api.utils.internal.Uuids;
+import org.sonar.server.issue.IssueQuery;
+import org.sonar.server.issue.index.IssueAuthorizationDao;
+import org.sonar.server.issue.index.IssueAuthorizationIndexer;
+import org.sonar.server.issue.index.IssueDoc;
+import org.sonar.server.issue.index.IssueIndex;
+import org.sonar.server.issue.index.IssueIndexer;
+import org.sonar.server.search.QueryContext;
+import org.sonar.server.search.Result;
+import org.sonar.server.tester.ServerTester;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class IssueIndexBenchmarkTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger("benchmarkIssues");
+
+ final static int PROJECTS = 100;
+ final static int FILES_PER_PROJECT = 100;
+ final static int ISSUES_PER_FILE = 100;
+
+ @ClassRule
+ public static ServerTester tester = new ServerTester();
+
+ @Test
+ public void benchmark() throws Exception {
+ // initialization - feed issues/issueAuthorization with projects and hardcoded users
+ indexAuthorizations();
+
+ // index issues
+ benchmarkIssueIndexing();
+
+ // execute some queries
+ benchmarkQueries();
+ }
+
+ private void indexAuthorizations() {
+ IssueAuthorizationIndexer indexer = tester.get(IssueAuthorizationIndexer.class);
+ List<IssueAuthorizationDao.Dto> authorizations = Lists.newArrayList();
+ for (int i = 0; i < PROJECTS; i++) {
+ IssueAuthorizationDao.Dto authorization = new IssueAuthorizationDao.Dto("PROJECT" + i, System.currentTimeMillis());
+ authorization.addGroup("sonar-users");
+ authorization.addUser("admin");
+ authorizations.add(authorization);
+ }
+ indexer.index(authorizations);
+ }
+
+ private void benchmarkIssueIndexing() {
+ LOGGER.info("Indexing issues");
+ IssueIterator issues = new IssueIterator(PROJECTS, FILES_PER_PROJECT, ISSUES_PER_FILE);
+ ProgressTask progressTask = new ProgressTask("issues", issues.count());
+ Timer timer = new Timer("IssuesIndex");
+ timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
+
+ long start = System.currentTimeMillis();
+ tester.get(IssueIndexer.class).index(issues);
+ long end = System.currentTimeMillis();
+
+ timer.cancel();
+ long period = end - start;
+ LOGGER.info(String.format("%d issues indexed in %d ms (%d/second)", issues.count.get(), period, 1000 * issues.count.get() / period));
+ }
+
+ private void benchmarkQueries() {
+ benchmarkQuery("all issues", IssueQuery.builder().build());
+ benchmarkQuery("project issues", IssueQuery.builder().projectUuids(Arrays.asList("PROJECT33")).build());
+ benchmarkQuery("file issues", IssueQuery.builder().componentUuids(Arrays.asList("FILE333")).build());
+ benchmarkQuery("various", IssueQuery.builder()
+ .resolutions(Arrays.asList(Issue.RESOLUTION_FIXED))
+ .assigned(true)
+ .build());
+ // TODO test facets
+ }
+
+ private void benchmarkQuery(String label, IssueQuery query) {
+ IssueIndex index = tester.get(IssueIndex.class);
+ for (int i = 0; i < 10; i++) {
+ long start = System.currentTimeMillis();
+ Result<Issue> result = index.search(query, new QueryContext());
+ long end = System.currentTimeMillis();
+ LOGGER.info("Request (" + label + "): {} docs in {} ms", result.getTotal(), end - start);
+ }
+ }
+
+ private static class IssueIterator implements Iterator<IssueDoc> {
+ private final int nbProjects, nbFilesPerProject, nbIssuesPerFile;
+ private int currentProject = 0, currentFile = 0;
+ private AtomicLong count = new AtomicLong(0L);
+ private final Iterator<String> users = cycleIterator("guy", 200);
+ private Iterator<String> rules = cycleIterator("squid:rule", 1000);
+ private final Iterator<String> severities = Iterables.cycle(Severity.ALL).iterator();
+ private final Iterator<String> statuses = Iterables.cycle(Issue.STATUSES).iterator();
+ private final Iterator<String> resolutions = Iterables.cycle(Issue.RESOLUTIONS).iterator();
+
+ IssueIterator(int nbProjects, int nbFilesPerProject, int nbIssuesPerFile) {
+ this.nbProjects = nbProjects;
+ this.nbFilesPerProject = nbFilesPerProject;
+ this.nbIssuesPerFile = nbIssuesPerFile;
+ }
+
+ public AtomicLong count() {
+ return count;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return count.get() < nbProjects * nbFilesPerProject * nbIssuesPerFile;
+ }
+
+ @Override
+ public IssueDoc next() {
+ IssueDoc issue = new IssueDoc(Maps.<String, Object>newHashMap());
+ issue.setKey(Uuids.create());
+ issue.setFilePath("src/main/java/Foo" + currentFile);
+ issue.setComponentUuid("FILE" + currentFile);
+ issue.setProjectUuid("PROJECT" + currentProject);
+ issue.setActionPlanKey("PLAN" + currentProject);
+ issue.setAssignee(users.next());
+ issue.setAuthorLogin(users.next());
+ issue.setLine(RandomUtils.nextInt());
+ issue.setCreationDate(new Date());
+ issue.setUpdateDate(new Date());
+ issue.setFuncUpdateDate(new Date());
+ issue.setFuncCreationDate(new Date());
+ issue.setFuncCloseDate(null);
+ issue.setAttributes(null);
+ issue.setDebt(1000L);
+ issue.setEffortToFix(3.14);
+ issue.setLanguage("php");
+ issue.setReporter(users.next());
+ issue.setRuleKey(rules.next());
+ issue.setResolution(resolutions.next());
+ issue.setStatus(statuses.next());
+ issue.setSeverity(severities.next());
+ issue.setMessage(RandomUtils.nextLong() + "this is the message. Not too short.");
+ count.incrementAndGet();
+ if (count.get() % nbIssuesPerFile == 0) {
+ currentFile++;
+ }
+ if (count.get() % (nbFilesPerProject * nbIssuesPerFile) == 0) {
+ currentProject++;
+ }
+
+ return issue;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static Iterator<String> cycleIterator(String prefix, int size) {
+ List<String> values = Lists.newArrayList();
+ for (int i = 0; i < size; i++) {
+ values.add(String.format("%s%d", prefix, i));
+ }
+ return Iterators.cycle(values);
+ }
+
+ static class ProgressTask extends TimerTask {
+ public static final long PERIOD_MS = 60000L;
+
+ private final String label;
+ private final AtomicLong counter;
+ private long previousCount = 0L;
+ private long previousTime = 0L;
+
+ public ProgressTask(String label, AtomicLong counter) {
+ this.label = label;
+ this.counter = counter;
+ this.previousTime = System.currentTimeMillis();
+ }
+
+ @Override
+ public void run() {
+ long currentCount = counter.get();
+ long now = System.currentTimeMillis();
+ LOGGER.info("{} {} indexed ({} {}/second)",
+ currentCount, label, documentsPerSecond(currentCount - previousCount, now - previousTime), label);
+ this.previousCount = currentCount;
+ this.previousTime = now;
+ }
+ }
+
+ static int documentsPerSecond(long nbIssues, long time) {
+ return (int) Math.round(nbIssues / (time / 1000.0));
+ }
+}
+++ /dev/null
-default.IssuesDbExtractionTest.extract_issues=40000
-default.IssuesIndexInjectionTest.inject_issues=40000
-JL.IssuesDbExtractionTest.extract_issues=40000
-JL.IssuesIndexInjectionTest.inject_issues=40000
<level value="WARN"/>
</logger>
- <!-- required for DryRunDatabaseFactoryTest -->
- <logger name="org.sonar.core.persistence.DryRunDatabaseFactory">
- <level value="DEBUG"/>
- </logger>
-
- <!-- required for DryRunDatabaseFactoryTest -->
- <logger name="org.elasticsearch">
- <level value="WARN"/>
- </logger>
-
<root>
<level value="INFO"/>
<appender-ref ref="STDOUT"/>
+++ /dev/null
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>server</artifactId>
- <version>5.0-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
- <artifactId>sonar-server-benchmark</artifactId>
- <name>SonarQube :: Server :: Benchmark</name>
-
- <dependencies>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-server</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-server</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-search</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-core</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.sonar</groupId>
- <artifactId>sonar-testing-harness</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skipTests>${skipServerTests}</skipTests>
- <argLine>-Xmx256m</argLine>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
+++ /dev/null
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.server.benchmark;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import org.apache.commons.lang.math.RandomUtils;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.issue.Issue;
-import org.sonar.api.rule.Severity;
-import org.sonar.api.utils.internal.Uuids;
-import org.sonar.server.issue.IssueQuery;
-import org.sonar.server.issue.index.IssueAuthorizationDao;
-import org.sonar.server.issue.index.IssueAuthorizationIndexer;
-import org.sonar.server.issue.index.IssueDoc;
-import org.sonar.server.issue.index.IssueIndex;
-import org.sonar.server.issue.index.IssueIndexer;
-import org.sonar.server.search.QueryContext;
-import org.sonar.server.search.Result;
-import org.sonar.server.tester.ServerTester;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class IssueIndexBenchmarkTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger("benchmarkIssues");
-
- final static int PROJECTS = 100;
- final static int FILES_PER_PROJECT = 100;
- final static int ISSUES_PER_FILE = 100;
-
- @ClassRule
- public static ServerTester tester = new ServerTester();
-
- @Test
- public void benchmark() throws Exception {
- // initialization - feed issues/issueAuthorization with projects and hardcoded users
- indexAuthorizations();
-
- // index issues
- benchmarkIssueIndexing();
-
- // execute some queries
- benchmarkQueries();
- }
-
- private void indexAuthorizations() {
- IssueAuthorizationIndexer indexer = tester.get(IssueAuthorizationIndexer.class);
- List<IssueAuthorizationDao.Dto> authorizations = Lists.newArrayList();
- for (int i = 0; i < PROJECTS; i++) {
- IssueAuthorizationDao.Dto authorization = new IssueAuthorizationDao.Dto("PROJECT" + i, System.currentTimeMillis());
- authorization.addGroup("sonar-users");
- authorization.addUser("admin");
- authorizations.add(authorization);
- }
- indexer.index(authorizations);
- }
-
- private void benchmarkIssueIndexing() {
- LOGGER.info("Indexing issues");
- IssueIterator issues = new IssueIterator(PROJECTS, FILES_PER_PROJECT, ISSUES_PER_FILE);
- ProgressTask progressTask = new ProgressTask("issues", issues.count());
- Timer timer = new Timer("IssuesIndex");
- timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
-
- long start = System.currentTimeMillis();
- tester.get(IssueIndexer.class).index(issues);
- long end = System.currentTimeMillis();
-
- timer.cancel();
- long period = end - start;
- LOGGER.info(String.format("%d issues indexed in %d ms (%d/second)", issues.count.get(), period, 1000 * issues.count.get() / period));
- }
-
- private void benchmarkQueries() {
- benchmarkQuery("all issues", IssueQuery.builder().build());
- benchmarkQuery("project issues", IssueQuery.builder().projectUuids(Arrays.asList("PROJECT33")).build());
- benchmarkQuery("file issues", IssueQuery.builder().componentUuids(Arrays.asList("FILE333")).build());
- benchmarkQuery("various", IssueQuery.builder()
- .resolutions(Arrays.asList(Issue.RESOLUTION_FIXED))
- .assigned(true)
- .build());
- // TODO test facets
- }
-
- private void benchmarkQuery(String label, IssueQuery query) {
- IssueIndex index = tester.get(IssueIndex.class);
- for (int i = 0; i < 10; i++) {
- long start = System.currentTimeMillis();
- Result<Issue> result = index.search(query, new QueryContext());
- long end = System.currentTimeMillis();
- LOGGER.info("Request (" + label + "): {} docs in {} ms", result.getTotal(), end - start);
- }
- }
-
- private static class IssueIterator implements Iterator<IssueDoc> {
- private final int nbProjects, nbFilesPerProject, nbIssuesPerFile;
- private int currentProject = 0, currentFile = 0;
- private AtomicLong count = new AtomicLong(0L);
- private final Iterator<String> users = cycleIterator("guy", 200);
- private Iterator<String> rules = cycleIterator("squid:rule", 1000);
- private final Iterator<String> severities = Iterables.cycle(Severity.ALL).iterator();
- private final Iterator<String> statuses = Iterables.cycle(Issue.STATUSES).iterator();
- private final Iterator<String> resolutions = Iterables.cycle(Issue.RESOLUTIONS).iterator();
-
- IssueIterator(int nbProjects, int nbFilesPerProject, int nbIssuesPerFile) {
- this.nbProjects = nbProjects;
- this.nbFilesPerProject = nbFilesPerProject;
- this.nbIssuesPerFile = nbIssuesPerFile;
- }
-
- public AtomicLong count() {
- return count;
- }
-
- @Override
- public boolean hasNext() {
- return count.get() < nbProjects * nbFilesPerProject * nbIssuesPerFile;
- }
-
- @Override
- public IssueDoc next() {
- IssueDoc issue = new IssueDoc(Maps.<String, Object>newHashMap());
- issue.setKey(Uuids.create());
- issue.setFilePath("src/main/java/Foo" + currentFile);
- issue.setComponentUuid("FILE" + currentFile);
- issue.setProjectUuid("PROJECT" + currentProject);
- issue.setActionPlanKey("PLAN" + currentProject);
- issue.setAssignee(users.next());
- issue.setAuthorLogin(users.next());
- issue.setLine(RandomUtils.nextInt());
- issue.setCreationDate(new Date());
- issue.setUpdateDate(new Date());
- issue.setFuncUpdateDate(new Date());
- issue.setFuncCreationDate(new Date());
- issue.setFuncCloseDate(null);
- issue.setAttributes(null);
- issue.setDebt(1000L);
- issue.setEffortToFix(3.14);
- issue.setLanguage("php");
- issue.setReporter(users.next());
- issue.setRuleKey(rules.next());
- issue.setResolution(resolutions.next());
- issue.setStatus(statuses.next());
- issue.setSeverity(severities.next());
- issue.setMessage(RandomUtils.nextLong() + "this is the message. Not too short.");
- count.incrementAndGet();
- if (count.get() % nbIssuesPerFile == 0) {
- currentFile++;
- }
- if (count.get() % (nbFilesPerProject * nbIssuesPerFile) == 0) {
- currentProject++;
- }
-
- return issue;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
- private static Iterator<String> cycleIterator(String prefix, int size) {
- List<String> values = Lists.newArrayList();
- for (int i = 0; i < size; i++) {
- values.add(String.format("%s%d", prefix, i));
- }
- return Iterators.cycle(values);
- }
-
- static class ProgressTask extends TimerTask {
- public static final long PERIOD_MS = 60000L;
-
- private final String label;
- private final AtomicLong counter;
- private long previousCount = 0L;
- private long previousTime = 0L;
-
- public ProgressTask(String label, AtomicLong counter) {
- this.label = label;
- this.counter = counter;
- this.previousTime = System.currentTimeMillis();
- }
-
- @Override
- public void run() {
- long currentCount = counter.get();
- long now = System.currentTimeMillis();
- LOGGER.info("{} {} indexed ({} {}/second)",
- currentCount, label, documentsPerSecond(currentCount - previousCount, now - previousTime), label);
- this.previousCount = currentCount;
- this.previousTime = now;
- }
- }
-
- static int documentsPerSecond(long nbIssues, long time) {
- return (int) Math.round(nbIssues / (time / 1000.0));
- }
-}
+++ /dev/null
-<?xml version="1.0" encoding="UTF-8" ?>
-
-<configuration>
-
- <appender name="STDOUT"
- class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- %d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n
- </pattern>
- </encoder>
- </appender>
-
- <logger name="org.hibernate">
- <level value="WARN"/>
- </logger>
-
- <logger name="org.dbunit">
- <level value="WARN"/>
- </logger>
-
- <!-- set to level DEBUG to log SQL requests executed by MyBatis -->
- <logger name="java.sql">
- <level value="WARN"/>
- </logger>
-
- <!-- required for DryRunDatabaseFactoryTest -->
- <logger name="org.sonar.core.persistence.DryRunDatabaseFactory">
- <level value="DEBUG"/>
- </logger>
-
- <!-- required for DryRunDatabaseFactoryTest -->
- <logger name="org.elasticsearch">
- <level value="WARN"/>
- </logger>
-
- <root>
- <level value="INFO"/>
- <appender-ref ref="STDOUT"/>
- </root>
-
-</configuration>
projects / sonarqube.git / commitdiff