SONAR-6813 Drop ES index sourceLines

2 files changed, 0 insertions, 313 deletions

diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java
deleted file mode 100644
index 49193d73ac3..00000000000
--- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceDbBenchmarkTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.benchmark;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Timer;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.io.IOUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.api.utils.System2;
-import org.sonar.core.util.Uuids;
-import org.sonar.db.DbTester;
-import org.sonar.db.protobuf.DbFileSources;
-import org.sonar.db.source.FileSourceDao;
-import org.sonar.db.source.FileSourceDto;
-import org.sonar.server.source.index.FileSourcesUpdaterHelper;
-import org.sonar.server.source.index.SourceLineResultSetIterator;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class SourceDbBenchmarkTest {
-
-  public static final Logger LOGGER = LoggerFactory.getLogger("benchmarkSourceDbScrolling");
-
-  public static final int NUMBER_OF_FILES = 1000;
-  public static final int NUMBER_OF_LINES = 3220;
-  public static final String PROJECT_UUID = Uuids.create();
-
-  @Rule
-  public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
-  @Rule
-  public Benchmark benchmark = new Benchmark();
-
-  @Test
-  public void benchmark() throws Exception {
-    prepareTable();
-    scrollRows();
-  }
-
-  private void scrollRows() {
-    LOGGER.info("Scroll table FILE_SOURCES");
-    AtomicLong counter = new AtomicLong();
-    ProgressTask progress = new ProgressTask(LOGGER, "source file", counter);
-    Timer timer = new Timer("SourceDbScroll");
-    timer.schedule(progress, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
-
-    try {
-      long start = System.currentTimeMillis();
-      SourceLineResultSetIterator it = SourceLineResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), 0L, null);
-      while (it.hasNext()) {
-        FileSourcesUpdaterHelper.Row row = it.next();
-        assertThat(row.getUpdateRequests().size()).isEqualTo(NUMBER_OF_LINES);
-        assertThat(row.getFileUuid()).isNotEmpty();
-        counter.incrementAndGet();
-      }
-      long end = System.currentTimeMillis();
-      long period = end - start;
-      long throughputPerSecond = 1000L * counter.get() / period;
-      LOGGER.info(String.format("%d FILE_SOURCES rows scrolled in %d ms (%d rows/second)", counter.get(), period, throughputPerSecond));
-      benchmark.expectBetween("Throughput to scroll FILE_SOURCES", throughputPerSecond, 10, 14);
-
-    } finally {
-      timer.cancel();
-    }
-  }
-
-  private void prepareTable() throws IOException {
-    LOGGER.info("Populate table FILE_SOURCES");
-    FileSourceDao dao = new FileSourceDao(dbTester.myBatis());
-    for (int i = 0; i < NUMBER_OF_FILES; i++) {
-      dao.insert(generateDto());
-    }
-  }
-
-  private FileSourceDto generateDto() throws IOException {
-    long now = System.currentTimeMillis();
-    byte[] data = generateData();
-    FileSourceDto dto = new FileSourceDto();
-    dto.setCreatedAt(now);
-    dto.setUpdatedAt(now);
-    dto.setBinaryData(data);
-    dto.setDataHash("49d7230271f2bd24c759e54bcd66547d");
-    dto.setProjectUuid(PROJECT_UUID);
-    dto.setFileUuid(Uuids.create());
-    dto.setLineHashes(IOUtils.toString(getClass().getResourceAsStream("SourceDbBenchmarkTest/line_hashes.txt")));
-    dto.setDataType(FileSourceDto.Type.SOURCE);
-    return dto;
-  }
-
-  private byte[] generateData() {
-    DbFileSources.Data.Builder dataBuilder = DbFileSources.Data.newBuilder();
-    DbFileSources.Line.Builder lineBuilder = DbFileSources.Line.newBuilder();
-    for (int i = 1; i <= NUMBER_OF_LINES; i++) {
-      lineBuilder.clear();
-      dataBuilder.addLines(lineBuilder
-        .setLine(i)
-        .setScmRevision("REVISION_" + i)
-        .setScmAuthor("a_guy")
-        .setSource("this is not java code " + i)
-        .setUtLineHits(i)
-        .setUtConditions(i + 1)
-        .setUtCoveredConditions(i)
-        .setItLineHits(i)
-        .setItConditions(i + 1)
-        .setItCoveredConditions(i)
-        .setOverallLineHits(i)
-        .setOverallConditions(i + 1)
-        .setOverallCoveredConditions(i)
-        .setScmDate(1_500_000_000_000L)
-        .setHighlighting("2,9,k;9,18,k")
-        .addAllDuplication(Arrays.asList(19, 33, 141))
-        .build());
-    }
-    return FileSourceDto.encodeSourceData(dataBuilder.build());
-  }
-}
diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java
deleted file mode 100644
index a45f6851771..00000000000
--- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * SonarQube, open source software quality management tool.
- * Copyright (C) 2008-2014 SonarSource
- * mailto:contact AT sonarsource DOT com
- *
- * SonarQube is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * SonarQube is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-package org.sonar.server.benchmark;
-
-import java.util.Arrays;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Timer;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.io.FileUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.sonar.db.protobuf.DbFileSources;
-import org.sonar.server.es.EsClient;
-import org.sonar.server.source.index.FileSourcesUpdaterHelper;
-import org.sonar.server.source.index.SourceLineDoc;
-import org.sonar.server.source.index.SourceLineIndex;
-import org.sonar.server.source.index.SourceLineIndexDefinition;
-import org.sonar.server.source.index.SourceLineIndexer;
-import org.sonar.server.source.index.SourceLineResultSetIterator;
-import org.sonar.server.tester.ServerTester;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-/**
- * Performance tests of the Elasticsearch index sourcelines
- * <ul>
- *   <li>throughput of indexing of documents</li>
- *   <li>size of ES data directory</li>
- *   <li>time to request index</li>
- * </ul>
- */
-public class SourceIndexBenchmarkTest {
-
-  private static final Logger LOGGER = LoggerFactory.getLogger("benchmarkSourceIndexing");
-  private static final long FILES = 1000L;
-  private static final int LINES_PER_FILE = 3220;
-
-  @Rule
-  public ServerTester tester = new ServerTester().withEsIndexes();
-
-  @Rule
-  public Benchmark benchmark = new Benchmark();
-
-  @Test
-  public void benchmark() {
-    // index source lines
-    benchmarkIndexing();
-
-    // execute some queries
-    benchmarkQueries();
-  }
-
-  private void benchmarkIndexing() {
-    LOGGER.info("Indexing source lines");
-
-    SourceIterator files = new SourceIterator(FILES, LINES_PER_FILE);
-    ProgressTask progressTask = new ProgressTask(LOGGER, "files of " + LINES_PER_FILE + " lines", files.count());
-    Timer timer = new Timer("SourceIndexer");
-    timer.schedule(progressTask, ProgressTask.PERIOD_MS, ProgressTask.PERIOD_MS);
-
-    long start = System.currentTimeMillis();
-    tester.get(SourceLineIndexer.class).index(files);
-    long end = System.currentTimeMillis();
-
-    timer.cancel();
-    long period = end - start;
-    long nbLines = files.count.get() * LINES_PER_FILE;
-    long throughputPerSecond = 1000L * nbLines / period;
-    LOGGER.info(String.format("%d lines indexed in %d ms (%d docs/second)", nbLines, period, throughputPerSecond));
-    benchmark.expectAround("Throughput to index source lines", throughputPerSecond, 8950, Benchmark.DEFAULT_ERROR_MARGIN_PERCENTS);
-
-    // be sure that physical files do not evolve during estimation of size
-    tester.get(EsClient.class).prepareOptimize(SourceLineIndexDefinition.INDEX).setWaitForMerge(true).get();
-    long dirSize = FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir());
-    LOGGER.info(String.format("ES dir: " + FileUtils.byteCountToDisplaySize(dirSize)));
-    benchmark.expectBetween("ES dir size (b)", dirSize, 172L * FileUtils.ONE_MB, 182L * FileUtils.ONE_MB);
-  }
-
-  private void benchmarkQueries() {
-    SourceLineIndex index = tester.get(SourceLineIndex.class);
-    for (int i = 1; i <= 100; i++) {
-      long start = System.currentTimeMillis();
-      List<SourceLineDoc> result = index.getLines("FILE" + i, 20, 150);
-      long end = System.currentTimeMillis();
-      assertThat(result).hasSize(131);
-      LOGGER.info("Request: {} docs in {} ms", result.size(), end - start);
-    }
-    // TODO assertions
-  }
-
-  private static class SourceIterator implements Iterator<FileSourcesUpdaterHelper.Row> {
-    private final long nbFiles;
-    private final int nbLinesPerFile;
-    private int currentProject = 0;
-    private AtomicLong count = new AtomicLong(0L);
-    private final DbFileSources.Data.Builder dataBuilder = DbFileSources.Data.newBuilder();
-    private final DbFileSources.Line.Builder lineBuilder = DbFileSources.Line.newBuilder();
-
-    SourceIterator(long nbFiles, int nbLinesPerFile) {
-      this.nbFiles = nbFiles;
-      this.nbLinesPerFile = nbLinesPerFile;
-    }
-
-    public AtomicLong count() {
-      return count;
-    }
-
-    @Override
-    public boolean hasNext() {
-      return count.get() < nbFiles;
-    }
-
-    @Override
-    public FileSourcesUpdaterHelper.Row next() {
-      String projectUuid = "P" + currentProject;
-      String fileUuid = "FILE" + count.get();
-      dataBuilder.clear();
-
-      for (int indexLine = 1; indexLine <= nbLinesPerFile; indexLine++) {
-        lineBuilder.clear();
-        dataBuilder.addLines(lineBuilder
-          .setLine(indexLine)
-          .setScmRevision("REVISION_" + indexLine)
-          .setScmAuthor("a_guy")
-          .setSource("this is not java code " + indexLine)
-          .setUtLineHits(2)
-          .setUtConditions(8)
-          .setUtCoveredConditions(2)
-          .setItLineHits(2)
-          .setItConditions(8)
-          .setItCoveredConditions(2)
-          .setOverallLineHits(2)
-          .setOverallConditions(8)
-          .setOverallCoveredConditions(2)
-          .setScmDate(1_500_000_000_000L)
-          .setHighlighting("2,9,k;9,18,k")
-          .addAllDuplication(Arrays.asList(19, 33, 141))
-          .build());
-      }
-      count.incrementAndGet();
-      if (count.get() % 500 == 0) {
-        currentProject++;
-      }
-      return SourceLineResultSetIterator.toRow(projectUuid, fileUuid, new Date(), dataBuilder.build());
-    }
-
-    @Override
-    public void remove() {
-      throw new UnsupportedOperationException();
-    }
-  }
-
-}