From: Stephane Gamard Date: Mon, 28 Jul 2014 15:00:12 +0000 (+0200) Subject: SONAR-4898 - Centralized profiling X-Git-Tag: 4.5-RC1~343 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=b16c54633a335da1713c05720f0c079205790572;p=sonarqube.git SONAR-4898 - Centralized profiling --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java index 2bb4a22bd97..6d81789168f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java @@ -34,8 +34,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.sonar.core.activity.Activity; import org.sonar.core.activity.db.ActivityDto; import org.sonar.core.cluster.WorkQueue; -import org.sonar.core.profiling.Profiling; -import org.sonar.core.profiling.StopWatch; import org.sonar.server.search.BaseIndex; import org.sonar.server.search.ESNode; import org.sonar.server.search.IndexDefinition; @@ -53,8 +51,8 @@ import java.util.Map; */ public class ActivityIndex extends BaseIndex { - public ActivityIndex(Profiling profiling, ActivityNormalizer normalizer, WorkQueue workQueue, ESNode node) { - super(IndexDefinition.LOG, normalizer, workQueue, node, profiling); + public ActivityIndex(ActivityNormalizer normalizer, WorkQueue workQueue, ESNode node) { + super(IndexDefinition.LOG, normalizer, workQueue, node); } @Override @@ -91,15 +89,11 @@ public class ActivityIndex extends BaseIndex { } public Result findAll() { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) .setQuery(QueryBuilders.matchAllQuery()) .setTypes(this.getIndexType()) .setSize(Integer.MAX_VALUE); - basicProfile.stop(request.toString()); - SearchResponse response = request.get(); - fullProfile.stop(response.toString()); + SearchResponse response = node.execute(request); return new Result(this, response); } @@ -110,9 +104,6 @@ public class ActivityIndex extends BaseIndex { public SearchResponse search(ActivityQuery query, QueryOptions options, @Nullable FilterBuilder domainFilter) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); - // Prepare query SearchRequestBuilder esSearch = getClient() .prepareSearch(this.getIndexName()) @@ -153,9 +144,7 @@ public class ActivityIndex extends BaseIndex { esSearch.setScroll(TimeValue.timeValueMinutes(3)); } - basicProfile.stop(esSearch.toString()); - SearchResponse response = esSearch.get(); - fullProfile.stop(response.toString()); + SearchResponse response = node.execute(esSearch); return response; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index a77c50062b8..a851c038c72 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -34,8 +34,6 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.core.cluster.WorkQueue; -import org.sonar.core.profiling.Profiling; -import org.sonar.core.profiling.StopWatch; import org.sonar.core.qualityprofile.db.ActiveRuleDto; import org.sonar.core.qualityprofile.db.ActiveRuleKey; import org.sonar.server.qualityprofile.ActiveRule; @@ -54,8 +52,8 @@ import java.util.Map; public class ActiveRuleIndex extends BaseIndex { - public ActiveRuleIndex(Profiling profiling, ActiveRuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { - super(IndexDefinition.ACTIVE_RULE, normalizer, workQueue, node, profiling); + public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { + super(IndexDefinition.ACTIVE_RULE, normalizer, workQueue, node); } @Override @@ -120,8 +118,6 @@ public class ActiveRuleIndex extends BaseIndex findByRule(RuleKey key) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) .setQuery(QueryBuilders .hasParentQuery(this.getParentType(), @@ -132,9 +128,7 @@ public class ActiveRuleIndex extends BaseIndex activeRules = new ArrayList(); for (SearchHit hit : response.getHits()) { @@ -144,16 +138,12 @@ public class ActiveRuleIndex extends BaseIndex findByProfile(String key) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); SearchRequestBuilder request = getClient().prepareSearch(getIndexName()) .setQuery(QueryBuilders.termQuery(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), key)) .setRouting(key) // TODO replace by scrolling .setSize(Integer.MAX_VALUE); - basicProfile.stop(request.toString()); - SearchResponse response = request.get(); - fullProfile.stop(response.toString()); + SearchResponse response = node.execute(request); List activeRules = new ArrayList(); for (SearchHit hit : response.getHits()) { activeRules.add(toDoc(hit.getSource())); @@ -184,8 +174,6 @@ public class ActiveRuleIndex extends BaseIndex> getStatsByProfileKeys(List keys) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) .setQuery(QueryBuilders.filteredQuery( QueryBuilders.termsQuery(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field(), keys), @@ -201,9 +189,7 @@ public class ActiveRuleIndex extends BaseIndex> stats = new HashMap>(); Aggregation aggregation = response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field()); for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index df61f911521..6a06ffef27b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -44,8 +44,6 @@ import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.api.server.debt.DebtCharacteristic; import org.sonar.core.cluster.WorkQueue; -import org.sonar.core.profiling.Profiling; -import org.sonar.core.profiling.StopWatch; import org.sonar.core.rule.RuleDto; import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; import org.sonar.server.rule.Rule; @@ -70,8 +68,8 @@ import static com.google.common.collect.Lists.newArrayList; public class RuleIndex extends BaseIndex { - public RuleIndex(Profiling profiling, RuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { - super(IndexDefinition.RULE, normalizer, workQueue, node, profiling); + public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { + super(IndexDefinition.RULE, normalizer, workQueue, node); } protected String getKeyValue(RuleKey key) { @@ -337,9 +335,6 @@ public class RuleIndex extends BaseIndex { } public Result search(RuleQuery query, QueryOptions options) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); - SearchRequestBuilder esSearch = getClient() .prepareSearch(this.getIndexName()) .setTypes(this.getIndexType()) @@ -359,9 +354,7 @@ public class RuleIndex extends BaseIndex { QueryBuilder qb = this.getQuery(query, options); esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); - basicProfile.stop(esSearch.toString()); - SearchResponse esResult = esSearch.get(); - fullProfile.stop(esResult.toString()); + SearchResponse esResult = node.execute(esSearch); return new Result(this, esResult); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index 6db97c0ec31..371ea996c47 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -54,8 +54,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.core.cluster.WorkQueue; import org.sonar.core.persistence.Dto; -import org.sonar.core.profiling.Profiling; -import org.sonar.core.profiling.StopWatch; import javax.annotation.Nullable; import java.io.IOException; @@ -76,17 +74,15 @@ public abstract class BaseIndex, KEY extends Serial private static final Logger LOG = LoggerFactory.getLogger(BaseIndex.class); - private final ESNode node; + protected final ESNode node; private final BaseNormalizer normalizer; private final IndexDefinition indexDefinition; - protected final Profiling profiling; protected BaseIndex(IndexDefinition indexDefinition, BaseNormalizer normalizer, - WorkQueue workQueue, ESNode node, Profiling profiling) { + WorkQueue workQueue, ESNode node) { this.normalizer = normalizer; this.node = node; this.indexDefinition = indexDefinition; - this.profiling = profiling; } @Override @@ -127,7 +123,7 @@ public abstract class BaseIndex, KEY extends Serial try { SearchScrollRequestBuilder esRequest = getClient().prepareSearchScroll(scrollId) .setScroll(TimeValue.timeValueMinutes(3)); - Collections.addAll(hits, esRequest.get().getHits().getHits()); + Collections.addAll(hits, ((SearchResponse) node.execute(esRequest)).getHits().getHits()); } catch (Exception e) { throw new IllegalStateException("Error while filling in the scroll buffer", e); } @@ -203,8 +199,6 @@ public abstract class BaseIndex, KEY extends Serial @Override public Date getLastSynchronization() { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); Date date; SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) @@ -214,10 +208,7 @@ public abstract class BaseIndex, KEY extends Serial .addAggregation(AggregationBuilders.max("latest") .field(BaseNormalizer.UPDATED_AT_FIELD)); - - basicProfile.stop(request.toString()); - SearchResponse response = request.get(); - fullProfile.stop(response.toString()); + SearchResponse response = node.execute(request); Max max = (Max) response.getAggregations().get("latest"); @@ -378,13 +369,12 @@ public abstract class BaseIndex, KEY extends Serial @Override public void refresh() { - getClient() + node.execute(getClient() .admin() .indices() .prepareRefresh(this.getIndexName()) .setForce(false) - .setIndices(this.getIndexName()) - .get(); + .setIndices(this.getIndexName())); } /* Base CRUD methods */ @@ -392,18 +382,13 @@ public abstract class BaseIndex, KEY extends Serial protected abstract DOMAIN toDoc(Map fields); public DOMAIN getByKey(KEY key) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); - GetRequestBuilder request = getClient().prepareGet() .setType(this.getIndexType()) .setIndex(this.getIndexName()) .setId(this.getKeyValue(key)) .setRouting(this.getKeyValue(key)); - basicProfile.stop(request.toString()); - GetResponse response = request.get(); - fullProfile.stop(response.toString()); + GetResponse response = node.execute(request); if (response.isExists()) { return toDoc(response.getSource()); @@ -412,8 +397,6 @@ public abstract class BaseIndex, KEY extends Serial } protected void updateDocument(Collection requests, KEY key) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); LOG.debug("UPDATE _id:{} in index {}", key, this.getIndexName()); BulkRequestBuilder bulkRequest = getClient().prepareBulk(); for (UpdateRequest request : requests) { @@ -430,9 +413,7 @@ public abstract class BaseIndex, KEY extends Serial .type(this.getIndexType())); } } - basicProfile.stop(bulkRequest.toString()); - BulkResponse response = bulkRequest.get(); - fullProfile.stop(response.toString()); + BulkResponse response = node.execute(bulkRequest); } @Override @@ -483,17 +464,13 @@ public abstract class BaseIndex, KEY extends Serial } private void deleteDocument(KEY key) throws ExecutionException, InterruptedException { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); LOG.debug("DELETE _id:{} in index {}", key, this.getIndexName()); DeleteRequestBuilder request = getClient() .prepareDelete() .setIndex(this.getIndexName()) .setType(this.getIndexType()) .setId(this.getKeyValue(key)); - basicProfile.stop(request.toString()); - DeleteResponse response = request.get(); - fullProfile.stop(response.toString()); + DeleteResponse response = node.execute(request); } @Override @@ -561,8 +538,6 @@ public abstract class BaseIndex, KEY extends Serial public Map countByField(IndexField indexField, FilterBuilder filter) { - StopWatch fullProfile = profiling.start("es", Profiling.Level.FULL); - StopWatch basicProfile = profiling.start("es", Profiling.Level.BASIC); Map counts = new HashMap(); SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) @@ -578,9 +553,7 @@ public abstract class BaseIndex, KEY extends Serial .size(Integer.MAX_VALUE) .minDocCount(0)); - basicProfile.stop(request.toString()); - SearchResponse response = request.get(); - fullProfile.stop(response.toString()); + SearchResponse response = node.execute(request); Terms values = response.getAggregations().get(indexField.field()); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/ESNode.java b/server/sonar-server/src/main/java/org/sonar/server/search/ESNode.java index 4a3c1e2749f..4213543b164 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/ESNode.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/ESNode.java @@ -23,6 +23,9 @@ package org.sonar.server.search; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.TransportClient; @@ -37,6 +40,8 @@ import org.picocontainer.Startable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.config.Settings; +import org.sonar.core.profiling.Profiling; +import org.sonar.core.profiling.StopWatch; import org.sonar.server.search.es.ListUpdate; import org.sonar.server.search.es.ListUpdate.UpdateListScriptFactory; @@ -57,6 +62,9 @@ public class ESNode implements Startable { // available only after startup private Client client; private Node node; + protected final Profiling profiling; + private final Profiling.Level profilingLevel; + public ESNode(Settings settings) { this(settings, DEFAULT_HEALTH_TIMEOUT); @@ -66,6 +74,9 @@ public class ESNode implements Startable { ESNode(Settings settings, String healthTimeout) { this.settings = settings; this.healthTimeout = healthTimeout; + this.profiling = new Profiling(settings); + String settingsValue = settings.getString(Profiling.CONFIG_PROFILING_LEVEL); + this.profilingLevel = Profiling.Level.fromConfigString(settingsValue); } @Override @@ -278,4 +289,22 @@ public class ESNode implements Startable { } return client; } + + public K execute(ActionRequestBuilder action) { + StopWatch requestProfile = null; + if (profilingLevel.ordinal() >= Profiling.Level.BASIC.ordinal()) { + requestProfile = profiling.start("search", Profiling.Level.BASIC); + } + ListenableActionFuture acc = action.execute(); + try { + K response = (K) acc.get(); + return response; + } catch (Exception e) { + throw new IllegalStateException("ES error: ", e); + } finally { + if (requestProfile != null) { + requestProfile.stop(action.toString()); + } + } + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java index 7a1ef3335a1..09fbe08fe86 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java @@ -29,7 +29,6 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.api.config.Settings; import org.sonar.core.cluster.NullQueue; -import org.sonar.core.profiling.Profiling; import java.io.File; import java.io.IOException; @@ -80,7 +79,7 @@ public class BaseIndexTest { private BaseIndex getIndex(final ESNode esNode) { BaseIndex index = new BaseIndex( IndexDefinition.TEST, - null, new NullQueue(), esNode, new Profiling(new Settings())) { + null, new NullQueue(), esNode) { @Override protected String getKeyValue(Serializable key) { return null;