From: Teryk Bellahsene Date: Fri, 17 Apr 2015 13:41:52 +0000 (+0200) Subject: refactor ES indexes - SONAR-6255 X-Git-Tag: 5.2-RC1~2196 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=01f3bef8f77526fb5f5f2dbe94390d42be7fa2bc;p=sonarqube.git refactor ES indexes - SONAR-6255 --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java index a67224f95a7..b45817e7c74 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java @@ -21,7 +21,6 @@ package org.sonar.server.activity.index; import com.google.common.collect.ImmutableMap; import org.sonar.api.config.Settings; -import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; @@ -50,8 +49,8 @@ public class ActivityIndexDefinition implements IndexDefinition { public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); index.getSettings().put("analysis.analyzer.default.type", "keyword"); - EsUtils.refreshHandledByIndexer(index); - EsUtils.setShards(index, settings); + index.setShards(settings); + index.refreshHandledByIndexer(); // type "activity" NewIndex.NewIndexType mapping = index.createType(TYPE); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistTestResultsStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistTestResultsStep.java new file mode 100644 index 00000000000..8dd727245fa --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistTestResultsStep.java @@ -0,0 +1,79 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.computation.step; + +import org.sonar.api.i18n.I18n; +import org.sonar.api.resources.Qualifiers; +import org.sonar.batch.protocol.output.BatchReport; +import org.sonar.batch.protocol.output.BatchReportReader; +import org.sonar.core.persistence.DbSession; +import org.sonar.core.persistence.MyBatis; +import org.sonar.core.source.db.FileSourceDto; +import org.sonar.server.computation.ComputationContext; +import org.sonar.server.db.DbClient; + +public class PersistTestResultsStep implements ComputationStep { + + private final DbClient dbClient; + + public PersistTestResultsStep(DbClient dbClient, I18n i18n) { + this.dbClient = dbClient; + } + + @Override + public String[] supportedProjectQualifiers() { + return new String[] {Qualifiers.PROJECT}; + } + + @Override + public void execute(ComputationContext context) { + DbSession session = dbClient.openSession(false); + try { + int rootComponentRef = context.getReportMetadata().getRootComponentRef(); + recursivelyProcessComponent(session, context, rootComponentRef); + session.commit(); + } finally { + MyBatis.closeQuietly(session); + } + } + + private void recursivelyProcessComponent(DbSession session, ComputationContext context, int componentRef) { + BatchReportReader reportReader = context.getReportReader(); + BatchReport.Component component = reportReader.readComponent(componentRef); + if (component.getIsTest() && reportReader.readTestResults(componentRef) != null) { + persistTestResults(session, component); + } + + for (Integer childRef : component.getChildRefList()) { + recursivelyProcessComponent(session, context, childRef); + } + } + + private void persistTestResults(DbSession session, BatchReport.Component component) { + dbClient.fileSourceDao().select(component.getUuid()); + dbClient.fileSourceDao().update(session, new FileSourceDto()); + } + + @Override + public String getDescription() { + return "Persist test results"; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java index 2eda91efb5f..05b17e4a73e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java @@ -21,18 +21,14 @@ package org.sonar.server.es; import com.google.common.base.Function; import com.google.common.collect.Lists; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.joda.time.format.ISODateTimeFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.sonar.api.config.Settings; -import org.sonar.process.ProcessProperties; import org.sonar.server.search.BaseDoc; import javax.annotation.CheckForNull; import javax.annotation.Nullable; - import java.util.*; public class EsUtils { @@ -83,16 +79,4 @@ public class EsUtils { return null; } - public static void setShards(NewIndex index, Settings settings) { - boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE); - if (clusterMode) { - index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4); - index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1); - // else keep defaults (one shard) - } - } - - public static void refreshHandledByIndexer(NewIndex index) { - index.getSettings().put("index.refresh_interval", "-1"); - } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index 611c1c5ec92..ece5f4d0804 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -23,16 +23,33 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; import org.apache.commons.lang.StringUtils; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.ImmutableSettings; +import org.sonar.api.config.Settings; +import org.sonar.process.ProcessProperties; import org.sonar.server.search.IndexField; import javax.annotation.CheckForNull; + import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; public class NewIndex { + public void refreshHandledByIndexer() { + getSettings().put("index.refresh_interval", "-1"); + } + + public void setShards(Settings settings) { + boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE); + if (clusterMode) { + getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4); + getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1); + // else keep defaults (one shard) + } + } + public static class NewIndexType { private final String name; private final Map attributes = new TreeMap(); @@ -100,7 +117,7 @@ public class NewIndex { } public NewIndexType createDynamicNestedField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "nested", "dynamic", "true", "include_in_parent", "true")); + return setProperty(fieldName, ImmutableMap.of("type", "nested", "dynamic", "true")); } public NewIndexType createShortField(String fieldName) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index 571852be309..0157b9539ea 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -21,7 +21,6 @@ package org.sonar.server.issue.index; import com.google.common.collect.ImmutableMap; import org.sonar.api.config.Settings; -import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; @@ -92,8 +91,8 @@ public class IssueIndexDefinition implements IndexDefinition { public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); - EsUtils.refreshHandledByIndexer(index); - EsUtils.setShards(index, settings); + index.refreshHandledByIndexer(); + index.setShards(settings); // type "authorization" NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_AUTHORIZATION); diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java index ecae4389568..f979520e6ae 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java @@ -21,7 +21,6 @@ package org.sonar.server.source.index; import com.google.common.collect.ImmutableMap; import org.sonar.api.config.Settings; -import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; @@ -60,8 +59,8 @@ public class SourceLineIndexDefinition implements IndexDefinition { public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); - EsUtils.refreshHandledByIndexer(index); - EsUtils.setShards(index, settings); + index.refreshHandledByIndexer(); + index.setShards(settings); // type "sourceline" NewIndex.NewIndexType mapping = index.createType(TYPE); diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java index 511bf9ba33d..a6da0650834 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java @@ -93,6 +93,15 @@ public class TestDoc extends BaseDoc { return this; } + public Long durationInMs() { + return getField(FIELD_DURATION_IN_MS); + } + + public TestDoc setDurationInMs(Long durationInMs) { + setField(FIELD_DURATION_IN_MS, durationInMs); + return this; + } + // TODO TBE - it should be a CoverageBlockDoc list public List> coverageBlocks() { return getField(FIELD_COVERAGE_BLOCKS); diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java index c4de32f3bf2..8c0b119c92a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java @@ -21,7 +21,6 @@ package org.sonar.server.test.index; import org.sonar.api.config.Settings; -import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; @@ -31,14 +30,13 @@ public class TestIndexDefinition implements IndexDefinition { public static final String FIELD_UUID = "uuid"; public static final String FIELD_NAME = "name"; public static final String FIELD_STATUS = "status"; + public static final String FIELD_TYPE = "type"; + public static final String FIELD_DURATION_IN_MS = "durationInMs"; public static final String FIELD_MESSAGE = "message"; public static final String FIELD_STACKTRACE = "stacktrace"; - public static final String FIELD_TYPE = "type"; public static final String FIELD_COVERAGE_BLOCKS = "coverageBlocks"; public static final String FIELD_COVERAGE_BLOCK_UUID = "uuid"; - public static final String FIELD_COVERAGE_BLOCK_KEY = "key"; public static final String FIELD_COVERAGE_BLOCK_LINES = "lines"; - public static final String FIELD_COVERAGE_BLOCK_COVERED_LINES = "coveredLines"; private final Settings settings; @@ -50,22 +48,21 @@ public class TestIndexDefinition implements IndexDefinition { public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); - EsUtils.refreshHandledByIndexer(index); - EsUtils.setShards(index, settings); + index.refreshHandledByIndexer(); + index.setShards(settings); NewIndex.NewIndexType nestedMapping = index.createType(TYPE); - nestedMapping.stringFieldBuilder(FIELD_COVERAGE_BLOCK_KEY).build(); nestedMapping.stringFieldBuilder(FIELD_COVERAGE_BLOCK_UUID).build(); nestedMapping.createIntegerField(FIELD_COVERAGE_BLOCK_LINES); - nestedMapping.createIntegerField(FIELD_COVERAGE_BLOCK_COVERED_LINES); NewIndex.NewIndexType mapping = index.createType(TYPE); mapping.stringFieldBuilder(FIELD_UUID).build(); mapping.stringFieldBuilder(FIELD_NAME).build(); mapping.stringFieldBuilder(FIELD_STATUS).disableSearch().build(); + mapping.stringFieldBuilder(FIELD_TYPE).disableSearch().build(); + mapping.createLongField(FIELD_DURATION_IN_MS); mapping.stringFieldBuilder(FIELD_MESSAGE).disableSearch().build(); mapping.stringFieldBuilder(FIELD_STACKTRACE).disableSearch().build(); - mapping.stringFieldBuilder(FIELD_TYPE).disableSearch().build(); mapping.nestedObjectBuilder(FIELD_COVERAGE_BLOCKS, nestedMapping).build(); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java index 5c158a7d1db..bb07d1fc6d7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java @@ -21,7 +21,6 @@ package org.sonar.server.user.index; import com.google.common.collect.ImmutableMap; import org.sonar.api.config.Settings; -import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; @@ -52,7 +51,7 @@ public class UserIndexDefinition implements IndexDefinition { public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); - EsUtils.setShards(index, settings); + index.setShards(settings); // type "user" NewIndex.NewIndexType mapping = index.createType(TYPE_USER); diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java index bb6f746fa92..2b60c6465f2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java @@ -22,7 +22,6 @@ package org.sonar.server.view.index; import com.google.common.collect.ImmutableMap; import org.sonar.api.config.Settings; -import org.sonar.server.es.EsUtils; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.NewIndex; @@ -48,7 +47,7 @@ public class ViewIndexDefinition implements IndexDefinition { public void define(IndexDefinitionContext context) { NewIndex index = context.create(INDEX); - EsUtils.setShards(index, settings); + index.setShards(settings); // type "view" NewIndex.NewIndexType mapping = index.createType(TYPE_VIEW);