refactor ES indexes - SONAR-6255

10 files changed, 120 insertions, 39 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java
index a67224f95a7..b45817e7c74 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndexDefinition.java
@@ -21,7 +21,6 @@ package org.sonar.server.activity.index;
 
 import com.google.common.collect.ImmutableMap;
 import org.sonar.api.config.Settings;
-import org.sonar.server.es.EsUtils;
 import org.sonar.server.es.IndexDefinition;
 import org.sonar.server.es.NewIndex;
 
@@ -50,8 +49,8 @@ public class ActivityIndexDefinition implements IndexDefinition {
   public void define(IndexDefinitionContext context) {
     NewIndex index = context.create(INDEX);
     index.getSettings().put("analysis.analyzer.default.type", "keyword");
-    EsUtils.refreshHandledByIndexer(index);
-    EsUtils.setShards(index, settings);
+    index.setShards(settings);
+    index.refreshHandledByIndexer();
 
     // type "activity"
     NewIndex.NewIndexType mapping = index.createType(TYPE);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistTestResultsStep.java b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistTestResultsStep.java
new file mode 100644
index 00000000000..8dd727245fa
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/computation/step/PersistTestResultsStep.java
@@ -0,0 +1,79 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.computation.step;
+
+import org.sonar.api.i18n.I18n;
+import org.sonar.api.resources.Qualifiers;
+import org.sonar.batch.protocol.output.BatchReport;
+import org.sonar.batch.protocol.output.BatchReportReader;
+import org.sonar.core.persistence.DbSession;
+import org.sonar.core.persistence.MyBatis;
+import org.sonar.core.source.db.FileSourceDto;
+import org.sonar.server.computation.ComputationContext;
+import org.sonar.server.db.DbClient;
+
+public class PersistTestResultsStep implements ComputationStep {
+
+  private final DbClient dbClient;
+
+  public PersistTestResultsStep(DbClient dbClient, I18n i18n) {
+    this.dbClient = dbClient;
+  }
+
+  @Override
+  public String[] supportedProjectQualifiers() {
+    return new String[] {Qualifiers.PROJECT};
+  }
+
+  @Override
+  public void execute(ComputationContext context) {
+    DbSession session = dbClient.openSession(false);
+    try {
+      int rootComponentRef = context.getReportMetadata().getRootComponentRef();
+      recursivelyProcessComponent(session, context, rootComponentRef);
+      session.commit();
+    } finally {
+      MyBatis.closeQuietly(session);
+    }
+  }
+
+  private void recursivelyProcessComponent(DbSession session, ComputationContext context, int componentRef) {
+    BatchReportReader reportReader = context.getReportReader();
+    BatchReport.Component component = reportReader.readComponent(componentRef);
+    if (component.getIsTest() && reportReader.readTestResults(componentRef) != null) {
+      persistTestResults(session, component);
+    }
+
+    for (Integer childRef : component.getChildRefList()) {
+      recursivelyProcessComponent(session, context, childRef);
+    }
+  }
+
+  private void persistTestResults(DbSession session, BatchReport.Component component) {
+    dbClient.fileSourceDao().select(component.getUuid());
+    dbClient.fileSourceDao().update(session, new FileSourceDto());
+  }
+
+  @Override
+  public String getDescription() {
+    return "Persist test results";
+  }
+}
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java
index 2eda91efb5f..05b17e4a73e 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsUtils.java
@@ -21,18 +21,14 @@ package org.sonar.server.es;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Lists;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.joda.time.format.ISODateTimeFormat;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.sonar.api.config.Settings;
-import org.sonar.process.ProcessProperties;
 import org.sonar.server.search.BaseDoc;
 
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
-
 import java.util.*;
 
 public class EsUtils {
@@ -83,16 +79,4 @@ public class EsUtils {
     return null;
   }
 
-  public static void setShards(NewIndex index, Settings settings) {
-    boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE);
-    if (clusterMode) {
-      index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4);
-      index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1);
-      // else keep defaults (one shard)
-    }
-  }
-
-  public static void refreshHandledByIndexer(NewIndex index) {
-    index.getSettings().put("index.refresh_interval", "-1");
-  }
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
index 611c1c5ec92..ece5f4d0804 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
@@ -23,16 +23,33 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedMap;
 import org.apache.commons.lang.StringUtils;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.ImmutableSettings;
+import org.sonar.api.config.Settings;
+import org.sonar.process.ProcessProperties;
 import org.sonar.server.search.IndexField;
 
 import javax.annotation.CheckForNull;
+
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
 public class NewIndex {
 
+  public void refreshHandledByIndexer() {
+    getSettings().put("index.refresh_interval", "-1");
+  }
+
+  public void setShards(Settings settings) {
+    boolean clusterMode = settings.getBoolean(ProcessProperties.CLUSTER_ACTIVATE);
+    if (clusterMode) {
+      getSettings().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4);
+      getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1);
+      // else keep defaults (one shard)
+    }
+  }
+
   public static class NewIndexType {
     private final String name;
     private final Map<String, Object> attributes = new TreeMap<String, Object>();
@@ -100,7 +117,7 @@ public class NewIndex {
     }
 
     public NewIndexType createDynamicNestedField(String fieldName) {
-      return setProperty(fieldName, ImmutableMap.of("type", "nested", "dynamic", "true", "include_in_parent", "true"));
+      return setProperty(fieldName, ImmutableMap.of("type", "nested", "dynamic", "true"));
     }
 
     public NewIndexType createShortField(String fieldName) {
diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
index 571852be309..0157b9539ea 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java
@@ -21,7 +21,6 @@ package org.sonar.server.issue.index;
 
 import com.google.common.collect.ImmutableMap;
 import org.sonar.api.config.Settings;
-import org.sonar.server.es.EsUtils;
 import org.sonar.server.es.IndexDefinition;
 import org.sonar.server.es.NewIndex;
 
@@ -92,8 +91,8 @@ public class IssueIndexDefinition implements IndexDefinition {
   public void define(IndexDefinitionContext context) {
     NewIndex index = context.create(INDEX);
 
-    EsUtils.refreshHandledByIndexer(index);
-    EsUtils.setShards(index, settings);
+    index.refreshHandledByIndexer();
+    index.setShards(settings);
 
     // type "authorization"
     NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_AUTHORIZATION);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
index ecae4389568..f979520e6ae 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
@@ -21,7 +21,6 @@ package org.sonar.server.source.index;
 
 import com.google.common.collect.ImmutableMap;
 import org.sonar.api.config.Settings;
-import org.sonar.server.es.EsUtils;
 import org.sonar.server.es.IndexDefinition;
 import org.sonar.server.es.NewIndex;
 
@@ -60,8 +59,8 @@ public class SourceLineIndexDefinition implements IndexDefinition {
   public void define(IndexDefinitionContext context) {
     NewIndex index = context.create(INDEX);
 
-    EsUtils.refreshHandledByIndexer(index);
-    EsUtils.setShards(index, settings);
+    index.refreshHandledByIndexer();
+    index.setShards(settings);
 
     // type "sourceline"
     NewIndex.NewIndexType mapping = index.createType(TYPE);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java
index 511bf9ba33d..a6da0650834 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestDoc.java
@@ -93,6 +93,15 @@ public class TestDoc extends BaseDoc {
     return this;
   }
 
+  public Long durationInMs() {
+    return getField(FIELD_DURATION_IN_MS);
+  }
+
+  public TestDoc setDurationInMs(Long durationInMs) {
+    setField(FIELD_DURATION_IN_MS, durationInMs);
+    return this;
+  }
+
   // TODO TBE - it should be a CoverageBlockDoc list
   public List<Map<String, Object>> coverageBlocks() {
     return getField(FIELD_COVERAGE_BLOCKS);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java
index c4de32f3bf2..8c0b119c92a 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexDefinition.java
@@ -21,7 +21,6 @@
 package org.sonar.server.test.index;
 
 import org.sonar.api.config.Settings;
-import org.sonar.server.es.EsUtils;
 import org.sonar.server.es.IndexDefinition;
 import org.sonar.server.es.NewIndex;
 
@@ -31,14 +30,13 @@ public class TestIndexDefinition implements IndexDefinition {
   public static final String FIELD_UUID = "uuid";
   public static final String FIELD_NAME = "name";
   public static final String FIELD_STATUS = "status";
+  public static final String FIELD_TYPE = "type";
+  public static final String FIELD_DURATION_IN_MS = "durationInMs";
   public static final String FIELD_MESSAGE = "message";
   public static final String FIELD_STACKTRACE = "stacktrace";
-  public static final String FIELD_TYPE = "type";
   public static final String FIELD_COVERAGE_BLOCKS = "coverageBlocks";
   public static final String FIELD_COVERAGE_BLOCK_UUID = "uuid";
-  public static final String FIELD_COVERAGE_BLOCK_KEY = "key";
   public static final String FIELD_COVERAGE_BLOCK_LINES = "lines";
-  public static final String FIELD_COVERAGE_BLOCK_COVERED_LINES = "coveredLines";
 
   private final Settings settings;
 
@@ -50,22 +48,21 @@ public class TestIndexDefinition implements IndexDefinition {
   public void define(IndexDefinitionContext context) {
     NewIndex index = context.create(INDEX);
 
-    EsUtils.refreshHandledByIndexer(index);
-    EsUtils.setShards(index, settings);
+    index.refreshHandledByIndexer();
+    index.setShards(settings);
 
     NewIndex.NewIndexType nestedMapping = index.createType(TYPE);
-    nestedMapping.stringFieldBuilder(FIELD_COVERAGE_BLOCK_KEY).build();
     nestedMapping.stringFieldBuilder(FIELD_COVERAGE_BLOCK_UUID).build();
     nestedMapping.createIntegerField(FIELD_COVERAGE_BLOCK_LINES);
-    nestedMapping.createIntegerField(FIELD_COVERAGE_BLOCK_COVERED_LINES);
 
     NewIndex.NewIndexType mapping = index.createType(TYPE);
     mapping.stringFieldBuilder(FIELD_UUID).build();
     mapping.stringFieldBuilder(FIELD_NAME).build();
     mapping.stringFieldBuilder(FIELD_STATUS).disableSearch().build();
+    mapping.stringFieldBuilder(FIELD_TYPE).disableSearch().build();
+    mapping.createLongField(FIELD_DURATION_IN_MS);
     mapping.stringFieldBuilder(FIELD_MESSAGE).disableSearch().build();
     mapping.stringFieldBuilder(FIELD_STACKTRACE).disableSearch().build();
-    mapping.stringFieldBuilder(FIELD_TYPE).disableSearch().build();
     mapping.nestedObjectBuilder(FIELD_COVERAGE_BLOCKS, nestedMapping).build();
   }
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java
index 5c158a7d1db..bb07d1fc6d7 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java
@@ -21,7 +21,6 @@ package org.sonar.server.user.index;
 
 import com.google.common.collect.ImmutableMap;
 import org.sonar.api.config.Settings;
-import org.sonar.server.es.EsUtils;
 import org.sonar.server.es.IndexDefinition;
 import org.sonar.server.es.NewIndex;
 
@@ -52,7 +51,7 @@ public class UserIndexDefinition implements IndexDefinition {
   public void define(IndexDefinitionContext context) {
     NewIndex index = context.create(INDEX);
 
-    EsUtils.setShards(index, settings);
+    index.setShards(settings);
 
     // type "user"
     NewIndex.NewIndexType mapping = index.createType(TYPE_USER);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java
index bb6f746fa92..2b60c6465f2 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java
@@ -22,7 +22,6 @@ package org.sonar.server.view.index;
 
 import com.google.common.collect.ImmutableMap;
 import org.sonar.api.config.Settings;
-import org.sonar.server.es.EsUtils;
 import org.sonar.server.es.IndexDefinition;
 import org.sonar.server.es.NewIndex;
 
@@ -48,7 +47,7 @@ public class ViewIndexDefinition implements IndexDefinition {
   public void define(IndexDefinitionContext context) {
     NewIndex index = context.create(INDEX);
 
-    EsUtils.setShards(index, settings);
+    index.setShards(settings);
 
     // type "view"
     NewIndex.NewIndexType mapping = index.createType(TYPE_VIEW);