source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube, open source software quality management tool.
   3  * Copyright (C) 2008-2014 SonarSource
   4  * mailto:contact AT sonarsource DOT com
   5  *
   6  * SonarQube is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * SonarQube is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20
  21 package org.sonar.server.benchmark;
  22
  23 import com.google.common.collect.Iterables;
  24 import com.google.common.collect.Iterators;
  25 import com.google.common.collect.Lists;
  26 import com.google.common.collect.Maps;
  27 import org.apache.commons.io.FileUtils;
  28 import org.apache.commons.lang.math.RandomUtils;
  29 import org.junit.ClassRule;
  30 import org.junit.Test;
  31 import org.slf4j.Logger;
  32 import org.slf4j.LoggerFactory;
  33 import org.sonar.api.issue.Issue;
  34 import org.sonar.api.rule.Severity;
  35 import org.sonar.api.utils.internal.Uuids;
  36 import org.sonar.server.issue.IssueQuery;
  37 import org.sonar.server.issue.index.IssueAuthorizationDao;
  38 import org.sonar.server.issue.index.IssueAuthorizationIndexer;
  39 import org.sonar.server.issue.index.IssueDoc;
  40 import org.sonar.server.issue.index.IssueIndex;
  41 import org.sonar.server.issue.index.IssueIndexer;
  42 import org.sonar.server.search.QueryContext;
  43 import org.sonar.server.search.Result;
  44 import org.sonar.server.tester.ServerTester;
  45 import org.sonar.server.user.MockUserSession;
  46
  47 import java.util.Arrays;
  48 import java.util.Date;
  49 import java.util.Iterator;
  50 import java.util.List;
  51 import java.util.Timer;
  52 import java.util.concurrent.atomic.AtomicLong;
  53
  54 public class IssueIndexBenchmarkTest {
  55
  56   private static final Logger LOGGER = LoggerFactory.getLogger("benchmarkIssues");
  57
  58   final static int PROJECTS = 100;
  59   final static int FILES_PER_PROJECT = 100;
  60   final static int ISSUES_PER_FILE = 100;
  61
  62   @ClassRule
  63   public static ServerTester tester = new ServerTester();
  64
  65   @Test
  66   public void benchmark() throws Exception {
  67     // initialization - feed issues/issueAuthorization with projects and hardcoded users
  68     indexAuthorizations();
  69
  70     // index issues
  71     benchmarkIssueIndexing();
  72
  73     // execute some queries
  74     benchmarkQueries();
  75   }
  76
  77   private void indexAuthorizations() {
  78     LOGGER.info("Indexing issue authorizations");
  79     IssueAuthorizationIndexer indexer = tester.get(IssueAuthorizationIndexer.class);
  80     List<IssueAuthorizationDao.Dto> authorizations = Lists.newArrayList();
  81     for (int i = 0; i < PROJECTS; i++) {
  82       IssueAuthorizationDao.Dto authorization = new IssueAuthorizationDao.Dto("PROJECT" + i, System.currentTimeMillis());
  83       authorization.addGroup("sonar-users");
  84       authorization.addUser("admin");
  85       authorizations.add(authorization);
  86     }
  87     long start = System.currentTimeMillis();
  88     indexer.index(authorizations);
  89     long period = System.currentTimeMillis() - start;
  90     LOGGER.info(String.format("%d authorizations indexed in %d ms (%d docs/second)", PROJECTS, period, 1000 * PROJECTS / period));
  91   }
  92
  93   private void benchmarkIssueIndexing() {
  94     LOGGER.info("Indexing issues");
  95     IssueIterator issues = new IssueIterator(PROJECTS, FILES_PER_PROJECT, ISSUES_PER_FILE);
  96     ProgressTask progressTask = new ProgressTask(LOGGER, "issues", issues.count());
  97     Timer timer = new Timer("IssuesIndex");
  98     timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
  99
 100     long start = System.currentTimeMillis();
 101     tester.get(IssueIndexer.class).index(issues);
 102
 103     timer.cancel();
 104     long period = System.currentTimeMillis() - start;
 105     LOGGER.info(String.format("%d issues indexed in %d ms (%d docs/second)", issues.count.get(), period, 1000 * issues.count.get() / period));
 106     LOGGER.info(String.format("Index disk: " + FileUtils.byteCountToDisplaySize(FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir()))));
 107   }
 108
 109   private void benchmarkQueries() {
 110     MockUserSession.set().setUserGroups("sonar-users");
 111     benchmarkQuery("all issues", IssueQuery.builder().build());
 112     benchmarkQuery("project issues", IssueQuery.builder().projectUuids(Arrays.asList("PROJECT33")).build());
 113     benchmarkQuery("file issues", IssueQuery.builder().componentUuids(Arrays.asList("FILE333")).build());
 114     benchmarkQuery("various", IssueQuery.builder()
 115       .resolutions(Arrays.asList(Issue.RESOLUTION_FIXED))
 116       .assigned(true)
 117       .build());
 118     // TODO test facets
 119   }
 120
 121   private void benchmarkQuery(String label, IssueQuery query) {
 122     IssueIndex index = tester.get(IssueIndex.class);
 123     for (int i = 0; i < 10; i++) {
 124       long start = System.currentTimeMillis();
 125       Result<Issue> result = index.search(query, new QueryContext());
 126       long end = System.currentTimeMillis();
 127       LOGGER.info("Request (" + label + "): {} docs in {} ms", result.getTotal(), end - start);
 128     }
 129   }
 130
 131   private static class IssueIterator implements Iterator<IssueDoc> {
 132     private final int nbProjects, nbFilesPerProject, nbIssuesPerFile;
 133     private int currentProject = 0, currentFile = 0;
 134     private AtomicLong count = new AtomicLong(0L);
 135     private final Iterator<String> users = cycleIterator("guy", 200);
 136     private Iterator<String> rules = cycleIterator("squid:rule", 1000);
 137     private final Iterator<String> severities = Iterables.cycle(Severity.ALL).iterator();
 138     private final Iterator<String> statuses = Iterables.cycle(Issue.STATUSES).iterator();
 139     private final Iterator<String> resolutions = Iterables.cycle(Issue.RESOLUTIONS).iterator();
 140
 141     IssueIterator(int nbProjects, int nbFilesPerProject, int nbIssuesPerFile) {
 142       this.nbProjects = nbProjects;
 143       this.nbFilesPerProject = nbFilesPerProject;
 144       this.nbIssuesPerFile = nbIssuesPerFile;
 145     }
 146
 147     public AtomicLong count() {
 148       return count;
 149     }
 150
 151     @Override
 152     public boolean hasNext() {
 153       return count.get() < nbProjects * nbFilesPerProject * nbIssuesPerFile;
 154     }
 155
 156     @Override
 157     public IssueDoc next() {
 158       IssueDoc issue = new IssueDoc(Maps.<String, Object>newHashMap());
 159       issue.setKey(Uuids.create());
 160       issue.setFilePath("src/main/java/Foo" + currentFile);
 161       issue.setComponentUuid("FILE" + currentFile);
 162       issue.setProjectUuid("PROJECT" + currentProject);
 163       issue.setActionPlanKey("PLAN" + currentProject);
 164       issue.setAssignee(users.next());
 165       issue.setAuthorLogin(users.next());
 166       issue.setLine(RandomUtils.nextInt());
 167       issue.setTechnicalCreationDate(new Date());
 168       issue.setTechnicalUpdateDate(new Date());
 169       issue.setFuncUpdateDate(new Date());
 170       issue.setFuncCreationDate(new Date());
 171       issue.setFuncCloseDate(null);
 172       issue.setAttributes(null);
 173       issue.setDebt(1000L);
 174       issue.setEffortToFix(3.14);
 175       issue.setLanguage("php");
 176       issue.setReporter(users.next());
 177       issue.setRuleKey(rules.next());
 178       issue.setResolution(resolutions.next());
 179       issue.setStatus(statuses.next());
 180       issue.setSeverity(severities.next());
 181       issue.setMessage(RandomUtils.nextLong() + "this is the message. Not too short.");
 182       count.incrementAndGet();
 183       if (count.get() % nbIssuesPerFile == 0) {
 184         currentFile++;
 185       }
 186       if (count.get() % (nbFilesPerProject * nbIssuesPerFile) == 0) {
 187         currentProject++;
 188       }
 189
 190       return issue;
 191     }
 192
 193     @Override
 194     public void remove() {
 195       throw new UnsupportedOperationException();
 196     }
 197   }
 198
 199   private static Iterator<String> cycleIterator(String prefix, int size) {
 200     List<String> values = Lists.newArrayList();
 201     for (int i = 0; i < size; i++) {
 202       values.add(String.format("%s%d", prefix, i));
 203     }
 204     return Iterators.cycle(values);
 205   }
 206 }