From: Simon Brandhof Date: Mon, 8 Dec 2014 15:08:05 +0000 (+0100) Subject: Improve stability of sonar-server-benchmarks X-Git-Tag: 5.0-RC1~27 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=93ea25fb4634f91147b4c49b5df2c2ad48177e00;p=sonarqube.git Improve stability of sonar-server-benchmarks Optimize Lucene FS so it does not evolve when estimating size --- diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java index b839f2a4cd7..05ddb828634 100644 --- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java +++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/Benchmark.java @@ -21,6 +21,7 @@ package org.sonar.server.benchmark; import org.hamcrest.Matchers; import org.junit.rules.ErrorCollector; +import org.slf4j.LoggerFactory; public class Benchmark extends ErrorCollector { @@ -28,7 +29,7 @@ public class Benchmark extends ErrorCollector { static { if (ENABLED) { - System.out.println("Assertions are calibrated for SonarSource dedicated box. " + + LoggerFactory.getLogger(Benchmark.class).warn("Assertions are calibrated for SonarSource dedicated box. " + "They can be disabled by setting the property -DenableBenchmarkAssertions=false."); } } diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java index 4f4115c5c30..6ad0d6915a6 100644 --- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java +++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/IssueIndexBenchmarkTest.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import org.sonar.api.issue.Issue; import org.sonar.api.rule.Severity; import org.sonar.api.utils.internal.Uuids; +import org.sonar.server.es.EsClient; import org.sonar.server.issue.IssueQuery; import org.sonar.server.issue.index.IssueAuthorizationDao; import org.sonar.server.issue.index.IssueAuthorizationIndexer; @@ -55,7 +56,7 @@ public class IssueIndexBenchmarkTest { private static final Logger LOGGER = LoggerFactory.getLogger("benchmarkIssues"); - final static int PROJECTS = 100; + final static int PROJECTS = 1; final static int FILES_PER_PROJECT = 100; final static int ISSUES_PER_FILE = 100; @@ -111,6 +112,8 @@ public class IssueIndexBenchmarkTest { LOGGER.info(String.format("%d issues indexed in %d ms (%d docs/second)", issues.count.get(), period, 1000 * issues.count.get() / period)); benchmark.expectBetween("Time to index issues", period, 350000L, 430000L); + // be sure that physical files do not evolve during estimation of size + tester.get(EsClient.class).prepareOptimize("issues").get(); long dirSize = FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir()); LOGGER.info(String.format("ES dir: " + FileUtils.byteCountToDisplaySize(dirSize))); benchmark.expectBetween("ES dir size (b)", dirSize, 385L * FileUtils.ONE_MB, 420L * FileUtils.ONE_MB); diff --git a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java index c4503b8ff79..8c55a25c39c 100644 --- a/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java +++ b/server/sonar-server-benchmarks/src/test/java/org/sonar/server/benchmark/SourceIndexBenchmarkTest.java @@ -28,8 +28,10 @@ import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonar.server.es.EsClient; import org.sonar.server.source.index.SourceLineDoc; import org.sonar.server.source.index.SourceLineIndex; +import org.sonar.server.source.index.SourceLineIndexDefinition; import org.sonar.server.source.index.SourceLineIndexer; import org.sonar.server.source.index.SourceLineResultSetIterator; import org.sonar.server.tester.ServerTester; @@ -80,6 +82,8 @@ public class SourceIndexBenchmarkTest { long nbLines = files.count.get() * LINES_PER_FILE; LOGGER.info(String.format("%d lines indexed in %d ms (%d docs/second)", nbLines, period, nbLines / period)); + // be sure that physical files do not evolve during estimation of size + tester.get(EsClient.class).prepareOptimize(SourceLineIndexDefinition.INDEX).get(); long dirSize = FileUtils.sizeOfDirectory(tester.getEsServerHolder().getHomeDir()); LOGGER.info(String.format("ES dir: " + FileUtils.byteCountToDisplaySize(dirSize))); benchmark.expectBetween("ES dir size (b)", dirSize, 80L * FileUtils.ONE_MB, 100L * FileUtils.ONE_MB); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index b08af55a94e..a1af20bc681 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -149,11 +149,7 @@ public class BulkIndexer implements Startable { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ - // TODO do not use nativeClient, else request is not profiled - client.nativeClient().admin().indices().prepareOptimize(indexName) - .setMaxNumSegments(1) - .setWaitForMerge(true) - .get(); + client.prepareOptimize(indexName).get(); updateSettings(largeInitialSettings); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java index 2e737b8c4be..eb658126ed8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder; +import org.elasticsearch.action.admin.indices.optimize.OptimizeRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -185,6 +186,14 @@ public class EsClient implements Startable { return new ProxyIndexRequestBuilder(client, profiling).setIndex(index).setType(type); } + public OptimizeRequestBuilder prepareOptimize(String indexName) { + // TODO add proxy for profiling + return nativeClient().admin().indices().prepareOptimize(indexName) + .setMaxNumSegments(1) + .setWaitForMerge(true); + } + + public long getLastUpdatedAt(String indexName, String typeName) { SearchRequestBuilder request = prepareSearch(indexName) .setTypes(typeName)