import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.math.RandomUtils;
import org.junit.ClassRule;
import org.junit.Test;
import org.sonar.server.search.QueryContext;
import org.sonar.server.search.Result;
import org.sonar.server.tester.ServerTester;
+import org.sonar.server.user.MockUserSession;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Timer;
-import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicLong;
public class IssueIndexBenchmarkTest {
private void benchmarkIssueIndexing() {
LOGGER.info("Indexing issues");
IssueIterator issues = new IssueIterator(PROJECTS, FILES_PER_PROJECT, ISSUES_PER_FILE);
- ProgressTask progressTask = new ProgressTask("issues", issues.count());
+ ProgressTask progressTask = new ProgressTask(LOGGER, "issues", issues.count());
Timer timer = new Timer("IssuesIndex");
timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
timer.cancel();
long period = end - start;
- LOGGER.info(String.format("%d issues indexed in %d ms (%d/second)", issues.count.get(), period, 1000 * issues.count.get() / period));
+ LOGGER.info(String.format("%d issues indexed in %d ms (%d docs/second)", issues.count.get(), period, 1000 * issues.count.get() / period));
+ LOGGER.info(String.format("Index disk: " + FileUtils.byteCountToDisplaySize(FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir()))));
}
private void benchmarkQueries() {
+ MockUserSession.set().setUserGroups("sonar-users");
benchmarkQuery("all issues", IssueQuery.builder().build());
benchmarkQuery("project issues", IssueQuery.builder().projectUuids(Arrays.asList("PROJECT33")).build());
benchmarkQuery("file issues", IssueQuery.builder().componentUuids(Arrays.asList("FILE333")).build());
}
return Iterators.cycle(values);
}
-
- static class ProgressTask extends TimerTask {
- public static final long PERIOD_MS = 60000L;
-
- private final String label;
- private final AtomicLong counter;
- private long previousCount = 0L;
- private long previousTime = 0L;
-
- public ProgressTask(String label, AtomicLong counter) {
- this.label = label;
- this.counter = counter;
- this.previousTime = System.currentTimeMillis();
- }
-
- @Override
- public void run() {
- long currentCount = counter.get();
- long now = System.currentTimeMillis();
- LOGGER.info("{} {} indexed ({} {}/second)",
- currentCount, label, documentsPerSecond(currentCount - previousCount, now - previousTime), label);
- this.previousCount = currentCount;
- this.previousTime = now;
- }
- }
-
- static int documentsPerSecond(long nbIssues, long time) {
- return (int) Math.round(nbIssues / (time / 1000.0));
- }
}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.benchmark;
+
+import org.slf4j.Logger;
+
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicLong;
+
+class ProgressTask extends TimerTask {
+
+ public static final long PERIOD_MS = 60000L;
+
+ private final Logger logger;
+ private final String label;
+ private final AtomicLong counter;
+ private long previousCount = 0L;
+ private long previousTime = 0L;
+
+ public ProgressTask(Logger logger, String label, AtomicLong counter) {
+ this.logger = logger;
+ this.label = label;
+ this.counter = counter;
+ this.previousTime = System.currentTimeMillis();
+ }
+
+ @Override
+ public void run() {
+ long currentCount = counter.get();
+ long now = System.currentTimeMillis();
+ logger.info("{} {} indexed ({} docs/second)",
+ currentCount, label, documentsPerSecond(currentCount - previousCount, now - previousTime), label);
+ this.previousCount = currentCount;
+ this.previousTime = now;
+ }
+
+ private int documentsPerSecond(long nbDocs, long time) {
+ return (int) Math.round(nbDocs / (time / 1000.0));
+ }
+}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.benchmark;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.server.source.index.SourceLineDoc;
+import org.sonar.server.source.index.SourceLineIndex;
+import org.sonar.server.source.index.SourceLineIndexer;
+import org.sonar.server.source.index.SourceLineResultSetIterator;
+import org.sonar.server.tester.ServerTester;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Timer;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class SourceIndexBenchmarkTest {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger("benchmarkSources");
+
+ final static long FILES = 10000L;
+ static final int LINES_PER_FILE = 200;
+
+ @ClassRule
+ public static ServerTester tester = new ServerTester();
+
+ @Test
+ public void benchmark() throws Exception {
+ // index source lines
+ benchmarkIndexing();
+
+ // execute some queries
+ benchmarkQueries();
+ }
+
+ private void benchmarkIndexing() {
+ LOGGER.info("Indexing source lines");
+
+ SourceIterator files = new SourceIterator(FILES, LINES_PER_FILE);
+ ProgressTask progressTask = new ProgressTask(LOGGER, "files of " + LINES_PER_FILE + " lines", files.count());
+ Timer timer = new Timer("SourceIndexer");
+ timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
+
+ long start = System.currentTimeMillis();
+ tester.get(SourceLineIndexer.class).index(files);
+ long end = System.currentTimeMillis();
+
+ timer.cancel();
+ long period = end - start;
+ long nbDocs = files.count.get() * LINES_PER_FILE;
+ LOGGER.info(String.format("%d files indexed in %d ms (%d docs/second)", nbDocs, period, 1000 * nbDocs / period));
+ LOGGER.info(String.format("Index disk: " + FileUtils.byteCountToDisplaySize(FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir()))));
+
+ }
+
+ private void benchmarkQueries() {
+ SourceLineIndex index = tester.get(SourceLineIndex.class);
+ for (int i = 1; i <= 100; i++) {
+ long start = System.currentTimeMillis();
+ List<SourceLineDoc> result = index.getLines("FILE" + i, 20, 150);
+ long end = System.currentTimeMillis();
+ assertThat(result).hasSize(131);
+ LOGGER.info("Request: {} docs in {} ms", result.size(), end - start);
+ }
+ }
+
+ private static class SourceIterator implements Iterator<SourceLineResultSetIterator.SourceFile> {
+ private final long nbFiles;
+ private final int nbLinesPerFile;
+ private int currentProject = 0;
+ private AtomicLong count = new AtomicLong(0L);
+
+ SourceIterator(long nbFiles, int nbLinesPerFile) {
+ this.nbFiles = nbFiles;
+ this.nbLinesPerFile = nbLinesPerFile;
+ }
+
+ public AtomicLong count() {
+ return count;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return count.get() < nbFiles;
+ }
+
+ @Override
+ public SourceLineResultSetIterator.SourceFile next() {
+ String fileUuid = "FILE" + count.get();
+ SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile(fileUuid, System.currentTimeMillis());
+ for (int indexLine = 1; indexLine <= nbLinesPerFile; indexLine++) {
+ SourceLineDoc line = new SourceLineDoc(Maps.<String, Object>newHashMap());
+ line.setFileUuid(fileUuid);
+ line.setLine(indexLine);
+ line.setHighlighting(StringUtils.repeat("HIGHLIGHTING", 5));
+ line.setItConditions(4);
+ line.setItCoveredConditions(2);
+ line.setItLineHits(2);
+ line.setOverallConditions(8);
+ line.setOverallCoveredConditions(2);
+ line.setOverallLineHits(2);
+ line.setUtConditions(8);
+ line.setUtCoveredConditions(2);
+ line.setUtLineHits(2);
+ line.setProjectUuid("PROJECT" + currentProject);
+ line.setScmAuthor("a_guy");
+ line.setScmRevision("ABCDEFGHIJKL");
+ line.setSource(StringUtils.repeat("SOURCE", 10));
+ file.addLine(line);
+ }
+ count.incrementAndGet();
+ if (count.get() % 500 == 0) {
+ currentProject++;
+ }
+ return file;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+}
private static EsServerHolder HOLDER = null;
private final String clusterName, nodeName;
private final int port;
+ private final File homeDir;
private final SearchServer server;
- private EsServerHolder(SearchServer server, String clusterName, String nodeName, int port) {
+ private EsServerHolder(SearchServer server, String clusterName, String nodeName, int port, File homeDir) {
this.server = server;
this.clusterName = clusterName;
this.nodeName = nodeName;
this.port = port;
+ this.homeDir = homeDir;
}
public String getClusterName() {
return server;
}
+ public File getHomeDir() {
+ return homeDir;
+ }
+
private void reset() {
TransportClient client = new TransportClient(ImmutableSettings.settingsBuilder()
.put("node.name", nodeName)
properties.setProperty(ProcessConstants.PATH_HOME, homeDir.getAbsolutePath());
SearchServer server = new SearchServer(new Props(properties));
server.start();
- HOLDER = new EsServerHolder(server, clusterName, nodeName, port);
+ HOLDER = new EsServerHolder(server, clusterName, nodeName, port, homeDir);
}
HOLDER.reset();
return HOLDER;
private static final String PROP_PREFIX = "mediumTests.";
private Platform platform;
+ private EsServerHolder esServerHolder;
private final File homeDir = TestUtils.newTempDir("tmp-sq-");
private final List components = Lists.newArrayList(WsTester.class);
private final Properties initialProps = new Properties();
try {
Properties properties = new Properties();
properties.putAll(initialProps);
- EsServerHolder esServerHolder = EsServerHolder.get();
-
+ esServerHolder = EsServerHolder.get();
properties.setProperty(ProcessConstants.CLUSTER_NAME, esServerHolder.getClusterName());
properties.setProperty(ProcessConstants.CLUSTER_NODE_NAME, esServerHolder.getNodeName());
properties.setProperty(ProcessConstants.SEARCH_PORT, String.valueOf(esServerHolder.getPort()));
} catch (Exception e) {
LOG.error("Fail to stop web server", e);
}
+ esServerHolder = null;
FileUtils.deleteQuietly(homeDir);
}
return get(WsTester.class);
}
+ public EsServerHolder getEsServerHolder() {
+ return esServerHolder;
+ }
+
private void checkStarted() {
if (platform == null || !platform.isStarted()) {
throw new IllegalStateException("Not started");
projects / sonarqube.git / commitdiff