SONAR-9480 make WS operations on rules resilient to Elasticsearch errors

8 files changed, 408 insertions, 11 deletions

diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java
index feb3148fa57..35c51c7f2e5 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java
@@ -22,12 +22,12 @@ package org.sonar.db.es;
 public final class EsQueueDto {
 
   public enum Type {
-    USER
+    USER, RULE, RULE_EXTENSION
   }
 
   private String uuid;
   private Type docType;
-  private String docUuid;
+  private String docId;
 
   public String getUuid() {
     return uuid;
@@ -47,12 +47,12 @@ public final class EsQueueDto {
     return this;
   }
 
-  public String getDocUuid() {
-    return docUuid;
+  public String getDocId() {
+    return docId;
   }
 
-  private EsQueueDto setDocUuid(String s) {
-    this.docUuid = s;
+  private EsQueueDto setDocId(String s) {
+    this.docId = s;
     return this;
   }
 
@@ -61,12 +61,31 @@ public final class EsQueueDto {
     StringBuilder sb = new StringBuilder("EsQueueDto{");
     sb.append("uuid='").append(uuid).append('\'');
     sb.append(", docType=").append(docType);
-    sb.append(", docUuid='").append(docUuid).append('\'');
+    sb.append(", docId='").append(docId).append('\'');
     sb.append('}');
     return sb.toString();
   }
 
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof EsQueueDto)) {
+      return false;
+    }
+
+    EsQueueDto that = (EsQueueDto) o;
+
+    return uuid.equals(that.uuid);
+  }
+
+  @Override
+  public int hashCode() {
+    return uuid.hashCode();
+  }
+
   public static EsQueueDto create(Type docType, String docUuid) {
-    return new EsQueueDto().setDocType(docType).setDocUuid(docUuid);
+    return new EsQueueDto().setDocType(docType).setDocId(docUuid);
   }
 }
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java
new file mode 100644
index 00000000000..7f821be5889
--- /dev/null
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java
@@ -0,0 +1,88 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.db.es;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Splitter;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.lang.String.format;
+
+public class RuleExtensionId {
+  private final String organizationUuid;
+  private final String repositoryName;
+  private final String ruleKey;
+  private final String id;
+
+  private static final Splitter ID_SPLITTER = Splitter.on(CharMatcher.anyOf(":|"));
+
+  public RuleExtensionId(String organizationUuid, String repositoryName, String ruleKey) {
+    this.organizationUuid = organizationUuid;
+    this.repositoryName = repositoryName;
+    this.ruleKey = ruleKey;
+    this.id = format("%s:%s|%s",repositoryName,ruleKey,organizationUuid);
+  }
+
+  public RuleExtensionId(String ruleExtensionId) {
+    List<String> splittedId = ID_SPLITTER.splitToList(ruleExtensionId);
+    checkArgument(splittedId.size() == 3, "Incorrect Id %s", ruleExtensionId);
+    this.id = ruleExtensionId;
+    this.repositoryName = splittedId.get(0);
+    this.ruleKey = splittedId.get(1);
+    this.organizationUuid = splittedId.get(2);
+  }
+
+  public String getOrganizationUuid() {
+    return organizationUuid;
+  }
+
+  public String getRepositoryName() {
+    return repositoryName;
+  }
+
+  public String getRuleKey() {
+    return ruleKey;
+  }
+
+  public String getId() {
+    return id;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof RuleExtensionId)) {
+      return false;
+    }
+
+    RuleExtensionId that = (RuleExtensionId) o;
+
+    return id.equals(that.id);
+  }
+
+  @Override
+  public int hashCode() {
+    return id.hashCode();
+  }
+}
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java
index e4902888070..94e47de330f 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java
@@ -22,8 +22,10 @@ package org.sonar.db.rule;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.ibatis.session.ResultHandler;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rules.RuleQuery;
 import org.sonar.db.Dao;
@@ -34,6 +36,7 @@ import org.sonar.db.organization.OrganizationDto;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Optional.ofNullable;
 import static org.sonar.db.DatabaseUtils.executeLargeInputs;
+import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput;
 
 public class RuleDao implements Dao {
 
@@ -149,6 +152,24 @@ public class RuleDao implements Dao {
     }
   }
 
+  public void scrollRuleExtensionByRuleKeys(DbSession dbSession, Collection<RuleExtensionId> ruleExtensionIds, Consumer<RuleExtensionForIndexingDto> consumer) {
+    RuleMapper mapper = mapper(dbSession);
+
+    executeLargeInputsWithoutOutput(ruleExtensionIds,
+      pageOfRuleExtensionIds -> mapper
+        .selectRuleExtensionForIndexingByKeys(pageOfRuleExtensionIds)
+        .forEach(consumer));
+  }
+
+  public void scrollRuleByRuleKeys(DbSession dbSession, Collection<RuleKey> ruleKeys, Consumer<RuleForIndexingDto> consumer) {
+    RuleMapper mapper = mapper(dbSession);
+
+    executeLargeInputsWithoutOutput(ruleKeys,
+      pageOfRuleKeys -> mapper
+        .selectRuleForIndexingByKeys(pageOfRuleKeys)
+        .forEach(consumer));
+  }
+
   private static RuleMapper mapper(DbSession session) {
     return session.getMapper(RuleMapper.class);
   }
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java
new file mode 100644
index 00000000000..ef127131794
--- /dev/null
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java
@@ -0,0 +1,81 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.db.rule;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+import org.sonar.api.rule.RuleKey;
+
+public class RuleExtensionForIndexingDto {
+
+  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+
+  private String pluginName;
+  private String pluginRuleKey;
+  private String organizationUuid;
+  private String tags;
+
+  public String getPluginName() {
+    return pluginName;
+  }
+
+  public RuleExtensionForIndexingDto setPluginName(String pluginName) {
+    this.pluginName = pluginName;
+    return this;
+  }
+
+  public String getPluginRuleKey() {
+    return pluginRuleKey;
+  }
+
+  public RuleExtensionForIndexingDto setPluginRuleKey(String pluginRuleKey) {
+    this.pluginRuleKey = pluginRuleKey;
+    return this;
+  }
+
+  public String getOrganizationUuid() {
+    return organizationUuid;
+  }
+
+  public RuleExtensionForIndexingDto setOrganizationUuid(String organizationUuid) {
+    this.organizationUuid = organizationUuid;
+    return this;
+  }
+
+  public String getTags() {
+    return tags;
+  }
+
+  public RuleExtensionForIndexingDto setTags(String tags) {
+    this.tags = tags;
+    return this;
+  }
+
+  public RuleKey getRuleKey() {
+    return RuleKey.of(pluginName, pluginRuleKey);
+  }
+
+  public Set<String> getTagsAsSet() {
+    return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
+  }
+
+}
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java
new file mode 100644
index 00000000000..52459e6338e
--- /dev/null
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java
@@ -0,0 +1,135 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.db.rule;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.api.rule.RuleStatus;
+import org.sonar.api.rules.RuleType;
+
+public class RuleForIndexingDto {
+
+  private Integer id;
+  private String repository;
+  private String pluginRuleKey;
+  private String name;
+  private String description;
+  private RuleDto.Format descriptionFormat;
+  private Integer severity;
+  private RuleStatus status;
+  private boolean isTemplate;
+  private String systemTags;
+  private String templateRuleKey;
+  private String templateName;
+  private String internalKey;
+  private String language;
+  private int type;
+  private long createdAt;
+  private long updatedAt;
+
+  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+
+  public Integer getId() {
+    return id;
+  }
+
+  public String getRepository() {
+    return repository;
+  }
+
+  public String getPluginRuleKey() {
+    return pluginRuleKey;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public RuleDto.Format getDescriptionFormat() {
+    return descriptionFormat;
+  }
+
+  public Integer getSeverity() {
+    return severity;
+  }
+
+  public RuleStatus getStatus() {
+    return status;
+  }
+
+  public boolean isTemplate() {
+    return isTemplate;
+  }
+
+  public String getSystemTags() {
+    return systemTags;
+  }
+
+  public String getTemplateRuleKey() {
+    return templateRuleKey;
+  }
+
+  public String getTemplateName() {
+    return templateName;
+  }
+
+  public String getInternalKey() {
+    return internalKey;
+  }
+
+  public String getLanguage() {
+    return language;
+  }
+
+  public int getType() {
+    return type;
+  }
+
+  public long getCreatedAt() {
+    return createdAt;
+  }
+
+  public long getUpdatedAt() {
+    return updatedAt;
+  }
+
+  public RuleType getTypeAsRuleType() {
+    return RuleType.valueOf(type);
+  }
+
+  public String getSeverityAsString() {
+    return severity != null ? SeverityUtil.getSeverityFromOrdinal(severity) : null;
+  }
+
+  public RuleKey getRuleKey() {
+    return RuleKey.of(repository, pluginRuleKey);
+  }
+
+  public Set<String> getSystemTagsAsSet() {
+    return ImmutableSet.copyOf(TAGS_SPLITTER.split(systemTags == null ? "" : systemTags));
+  }
+}
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java
index 77ab19067d1..d5a98e60200 100644
--- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java
+++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java
@@ -22,6 +22,7 @@ package org.sonar.db.rule;
 import java.util.List;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.session.ResultHandler;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rules.RuleQuery;
 
@@ -51,6 +52,10 @@ public interface RuleMapper {
 
   List<RuleDefinitionDto> selectDefinitionByKeys(@Param("ruleKeys") List<RuleKey> keys);
 
+  List<RuleForIndexingDto> selectRuleForIndexingByKeys(@Param("ruleKeys") List<RuleKey> keys);
+
+  List<RuleExtensionForIndexingDto> selectRuleExtensionForIndexingByKeys(@Param("ruleExtensionIds") List<RuleExtensionId> ruleExtensionIds);
+
   List<RuleDto> selectByQuery(@Param("organizationUuid") String organizationUuid, @Param("query") RuleQuery ruleQuery);
 
   void insertDefinition(RuleDefinitionDto ruleDefinitionDto);
diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml
index 08b3761380c..62ec6e3421b 100644
--- a/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml
+++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml
@@ -6,7 +6,7 @@
   <sql id="esQueueColumns">
     uuid,
     doc_type as docType,
-    doc_uuid as docUuid,
+    doc_id as docId,
     created_at as createdAt
   </sql>
 
@@ -14,12 +14,12 @@
     insert into es_queue (
       uuid,
       doc_type,
-      doc_uuid,
+      doc_id,
       created_at
     ) values (
       #{dto.uuid, jdbcType=VARCHAR},
       #{dto.docType, jdbcType=VARCHAR},
-      #{dto.docUuid, jdbcType=VARCHAR},
+      #{dto.docId, jdbcType=VARCHAR},
       #{now, jdbcType=BIGINT}
     )
   </insert>
diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml
index 25ef4923224..0e29a3e0d94 100644
--- a/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml
+++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml
@@ -139,6 +139,26 @@
       and r.plugin_rule_key=#{rule,jdbcType=VARCHAR}
   </select>
 
+  <select id="selectRuleExtensionForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto">
+    select
+    r.plugin_name as "pluginName",
+    r.plugin_rule_key as "pluginRuleKey",
+    rm.organization_uuid as "organizationUuid",
+    rm.tags as "tags"
+    from
+    rules r
+    inner join
+    rules_metadata rm on rm.rule_id = r.id
+    where
+    rm.tags is not null and
+    rm.tags != '' and
+    <foreach collection="ruleExtensionIds" index="index" item="ruleExtId" open="" separator=" or " close="">
+      ( r.plugin_name=#{ruleExtId.repositoryName,jdbcType=VARCHAR} and
+      r.plugin_rule_key=#{ruleExtId.ruleKey,jdbcType=VARCHAR} and
+      rm.organization_uuid=#{ruleExtId.organizationUuid,jdbcType=VARCHAR} )
+    </foreach>
+  </select>
+
   <select id="selectMetadataByKey" parameterType="map" resultType="org.sonar.db.rule.RuleMetadataDto">
     select
       rm.rule_id as "ruleId",
@@ -185,6 +205,34 @@
       </foreach>
   </select>
 
+  <select id="selectRuleForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleForIndexingDto">
+    select
+      r.id as "id",
+      r.plugin_name as "repository",
+      r.plugin_rule_key as "pluginRuleKey",
+      r.name as "name",
+      r.description as "description",
+      r.description_format as "descriptionFormat",
+      r.priority as "severity",
+      r.status as "status",
+      r.is_template as "isTemplate",
+      r.system_tags as "systemTags",
+      t.plugin_rule_key as "templateRuleKey",
+      t.plugin_name as "templateName",
+      r.plugin_config_key as "internalKey",
+      r.language as "language",
+      r.rule_type as "type",
+      r.created_at as "createdAt",
+      r.updated_at as "updatedAt"
+    from
+      rules r
+    left outer join rules t on t.id = r.template_id
+    where
+      <foreach collection="ruleKeys" index="index" item="ruleKey" open="" separator=" or " close="">
+        (r.plugin_name=#{ruleKey.repository,jdbcType=VARCHAR} and r.plugin_rule_key=#{ruleKey.rule,jdbcType=VARCHAR})
+      </foreach>
+  </select>
+
   <select id="selectByQuery" parameterType="map" resultType="Rule">
     select
       <include refid="selectJoinedTablesColumns"/>