Improve stability of sonar-server-benchmarks

Optimize Lucene FS so it does not evolve when estimating size

diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java
index b839f2a4cd79f2d5e7d25df2dc90631f2db27a5e..05ddb828634cfa38e0dd5f03cc85519d0c6ca853 100644 (file)
--- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java
+++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java
@@ -21,6 +21,7 @@ package org.sonar.server.benchmark;
 
 import org.hamcrest.Matchers;
 import org.junit.rules.ErrorCollector;
+import org.slf4j.LoggerFactory;
 
 public class Benchmark extends ErrorCollector {
 
@@ -28,7 +29,7 @@ public class Benchmark extends ErrorCollector {
 
   static {
     if (ENABLED) {
-      System.out.println("Assertions are calibrated for SonarSource dedicated box. " +
+      LoggerFactory.getLogger(Benchmark.class).warn("Assertions are calibrated for SonarSource dedicated box. " +
         "They can be disabled by setting the property -DenableBenchmarkAssertions=false.");
     }
   }

diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java
index 4f4115c5c3030c75373349d545184399a813602b..6ad0d6915a6f500c84bba67cbfb37a3f17df8cf6 100644 (file)
--- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java
+++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java
@@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
 import org.sonar.api.issue.Issue;
 import org.sonar.api.rule.Severity;
 import org.sonar.api.utils.internal.Uuids;
+import org.sonar.server.es.EsClient;
 import org.sonar.server.issue.IssueQuery;
 import org.sonar.server.issue.index.IssueAuthorizationDao;
 import org.sonar.server.issue.index.IssueAuthorizationIndexer;
@@ -55,7 +56,7 @@ public class IssueIndexBenchmarkTest {
 
   private static final Logger LOGGER = LoggerFactory.getLogger("benchmarkIssues");
 
-  final static int PROJECTS = 100;
+  final static int PROJECTS = 1;
   final static int FILES_PER_PROJECT = 100;
   final static int ISSUES_PER_FILE = 100;
 
@@ -111,6 +112,8 @@ public class IssueIndexBenchmarkTest {
     LOGGER.info(String.format("%d issues indexed in %d ms (%d docs/second)", issues.count.get(), period, 1000 * issues.count.get() / period));
     benchmark.expectBetween("Time to index issues", period, 350000L, 430000L);
 
+    // be sure that physical files do not evolve during estimation of size
+    tester.get(EsClient.class).prepareOptimize("issues").get();
     long dirSize = FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir());
     LOGGER.info(String.format("ES dir: " + FileUtils.byteCountToDisplaySize(dirSize)));
     benchmark.expectBetween("ES dir size (b)", dirSize, 385L * FileUtils.ONE_MB, 420L * FileUtils.ONE_MB);

diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java
index c4503b8ff79d3c9eaf93aa853e0ef4c435d5c135..8c55a25c39c76694d2dc82ee0bf83de3fef0f5ca 100644 (file)
--- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java
+++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java
@@ -28,8 +28,10 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sonar.server.es.EsClient;
 import org.sonar.server.source.index.SourceLineDoc;
 import org.sonar.server.source.index.SourceLineIndex;
+import org.sonar.server.source.index.SourceLineIndexDefinition;
 import org.sonar.server.source.index.SourceLineIndexer;
 import org.sonar.server.source.index.SourceLineResultSetIterator;
 import org.sonar.server.tester.ServerTester;
@@ -80,6 +82,8 @@ public class SourceIndexBenchmarkTest {
     long nbLines = files.count.get() * LINES_PER_FILE;
     LOGGER.info(String.format("%d lines indexed in %d ms (%d docs/second)", nbLines, period, nbLines / period));
 
+    // be sure that physical files do not evolve during estimation of size
+    tester.get(EsClient.class).prepareOptimize(SourceLineIndexDefinition.INDEX).get();
     long dirSize = FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir());
     LOGGER.info(String.format("ES dir: " + FileUtils.byteCountToDisplaySize(dirSize)));
     benchmark.expectBetween("ES dir size (b)", dirSize, 80L * FileUtils.ONE_MB, 100L * FileUtils.ONE_MB);

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
index b08af55a94e42bd2aadc94a1dcb40a7ee43e4e9a..a1af20bc681a138d3e31139fd8f86a213a463ea6 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
@@ -149,11 +149,7 @@ public class BulkIndexer implements Startable {
       // optimize lucene segments and revert index settings
       // Optimization must be done before re-applying replicas:
       // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/
-      // TODO do not use nativeClient, else request is not profiled
-      client.nativeClient().admin().indices().prepareOptimize(indexName)
-        .setMaxNumSegments(1)
-        .setWaitForMerge(true)
-        .get();
+      client.prepareOptimize(indexName).get();
 
       updateSettings(largeInitialSettings);
     }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
index 2e737b8c4bea9ac5da753a15b8d5ccb1632679cd..eb658126ed88e7729e197a107d4e6cd9a591face 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
 import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
+import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -185,6 +186,14 @@ public class EsClient implements Startable {
     return new ProxyIndexRequestBuilder(client, profiling).setIndex(index).setType(type);
   }
 
+  public OptimizeRequestBuilder prepareOptimize(String indexName) {
+    // TODO add proxy for profiling
+    return nativeClient().admin().indices().prepareOptimize(indexName)
+      .setMaxNumSegments(1)
+      .setWaitForMerge(true);
+  }
+
+
   public long getLastUpdatedAt(String indexName, String typeName) {
     SearchRequestBuilder request = prepareSearch(indexName)
       .setTypes(typeName)