SONAR-9480 make WS operations on rules resilient to Elasticsearch errors

server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl

@@ -662,7 +662,7 @@ CREATE INDEX "CE_TASK_UUID" ON "WEBHOOK_DELIVERIES" ("CE_TASK_UUID");
 CREATE TABLE "ES_QUEUE" (
   "UUID" VARCHAR(40) NOT NULL PRIMARY KEY,
   "DOC_TYPE" VARCHAR(40) NOT NULL,
-  "DOC_UUID" VARCHAR(255) NOT NULL,
+  "DOC_ID" VARCHAR(4000) NOT NULL,
   "CREATED_AT" BIGINT NOT NULL
 );
 CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID");

server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java

@@ -22,12 +22,12 @@ package org.sonar.db.es;
 public final class EsQueueDto {
 
   public enum Type {
-    USER
+    USER, RULE, RULE_EXTENSION
   }
 
   private String uuid;
   private Type docType;
-  private String docUuid;
+  private String docId;
 
   public String getUuid() {
     return uuid;
@@ -47,12 +47,12 @@ public final class EsQueueDto {
     return this;
   }
 
-  public String getDocUuid() {
-    return docUuid;
+  public String getDocId() {
+    return docId;
   }
 
-  private EsQueueDto setDocUuid(String s) {
-    this.docUuid = s;
+  private EsQueueDto setDocId(String s) {
+    this.docId = s;
     return this;
   }
 
@@ -61,12 +61,31 @@ public final class EsQueueDto {
     StringBuilder sb = new StringBuilder("EsQueueDto{");
     sb.append("uuid='").append(uuid).append('\'');
     sb.append(", docType=").append(docType);
-    sb.append(", docUuid='").append(docUuid).append('\'');
+    sb.append(", docId='").append(docId).append('\'');
     sb.append('}');
     return sb.toString();
   }
 
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof EsQueueDto)) {
+      return false;
+    }
+
+    EsQueueDto that = (EsQueueDto) o;
+
+    return uuid.equals(that.uuid);
+  }
+
+  @Override
+  public int hashCode() {
+    return uuid.hashCode();
+  }
+
   public static EsQueueDto create(Type docType, String docUuid) {
-    return new EsQueueDto().setDocType(docType).setDocUuid(docUuid);
+    return new EsQueueDto().setDocType(docType).setDocId(docUuid);
   }
 }

server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java

@@ -0,0 +1,88 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.db.es;
+
+import com.google.common.base.CharMatcher;
+import com.google.common.base.Splitter;
+import java.util.List;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.lang.String.format;
+
+public class RuleExtensionId {
+  private final String organizationUuid;
+  private final String repositoryName;
+  private final String ruleKey;
+  private final String id;
+
+  private static final Splitter ID_SPLITTER = Splitter.on(CharMatcher.anyOf(":|"));
+
+  public RuleExtensionId(String organizationUuid, String repositoryName, String ruleKey) {
+    this.organizationUuid = organizationUuid;
+    this.repositoryName = repositoryName;
+    this.ruleKey = ruleKey;
+    this.id = format("%s:%s|%s",repositoryName,ruleKey,organizationUuid);
+  }
+
+  public RuleExtensionId(String ruleExtensionId) {
+    List<String> splittedId = ID_SPLITTER.splitToList(ruleExtensionId);
+    checkArgument(splittedId.size() == 3, "Incorrect Id %s", ruleExtensionId);
+    this.id = ruleExtensionId;
+    this.repositoryName = splittedId.get(0);
+    this.ruleKey = splittedId.get(1);
+    this.organizationUuid = splittedId.get(2);
+  }
+
+  public String getOrganizationUuid() {
+    return organizationUuid;
+  }
+
+  public String getRepositoryName() {
+    return repositoryName;
+  }
+
+  public String getRuleKey() {
+    return ruleKey;
+  }
+
+  public String getId() {
+    return id;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof RuleExtensionId)) {
+      return false;
+    }
+
+    RuleExtensionId that = (RuleExtensionId) o;
+
+    return id.equals(that.id);
+  }
+
+  @Override
+  public int hashCode() {
+    return id.hashCode();
+  }
+}

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java

@@ -22,8 +22,10 @@ package org.sonar.db.rule;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.ibatis.session.ResultHandler;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rules.RuleQuery;
 import org.sonar.db.Dao;
@@ -34,6 +36,7 @@ import org.sonar.db.organization.OrganizationDto;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Optional.ofNullable;
 import static org.sonar.db.DatabaseUtils.executeLargeInputs;
+import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput;
 
 public class RuleDao implements Dao {
 
@@ -149,6 +152,24 @@ public class RuleDao implements Dao {
     }
   }
 
+  public void scrollRuleExtensionByRuleKeys(DbSession dbSession, Collection<RuleExtensionId> ruleExtensionIds, Consumer<RuleExtensionForIndexingDto> consumer) {
+    RuleMapper mapper = mapper(dbSession);
+
+    executeLargeInputsWithoutOutput(ruleExtensionIds,
+      pageOfRuleExtensionIds -> mapper
+        .selectRuleExtensionForIndexingByKeys(pageOfRuleExtensionIds)
+        .forEach(consumer));
+  }
+
+  public void scrollRuleByRuleKeys(DbSession dbSession, Collection<RuleKey> ruleKeys, Consumer<RuleForIndexingDto> consumer) {
+    RuleMapper mapper = mapper(dbSession);
+
+    executeLargeInputsWithoutOutput(ruleKeys,
+      pageOfRuleKeys -> mapper
+        .selectRuleForIndexingByKeys(pageOfRuleKeys)
+        .forEach(consumer));
+  }
+
   private static RuleMapper mapper(DbSession session) {
     return session.getMapper(RuleMapper.class);
   }

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java

@@ -0,0 +1,81 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.db.rule;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+import org.sonar.api.rule.RuleKey;
+
+public class RuleExtensionForIndexingDto {
+
+  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+
+  private String pluginName;
+  private String pluginRuleKey;
+  private String organizationUuid;
+  private String tags;
+
+  public String getPluginName() {
+    return pluginName;
+  }
+
+  public RuleExtensionForIndexingDto setPluginName(String pluginName) {
+    this.pluginName = pluginName;
+    return this;
+  }
+
+  public String getPluginRuleKey() {
+    return pluginRuleKey;
+  }
+
+  public RuleExtensionForIndexingDto setPluginRuleKey(String pluginRuleKey) {
+    this.pluginRuleKey = pluginRuleKey;
+    return this;
+  }
+
+  public String getOrganizationUuid() {
+    return organizationUuid;
+  }
+
+  public RuleExtensionForIndexingDto setOrganizationUuid(String organizationUuid) {
+    this.organizationUuid = organizationUuid;
+    return this;
+  }
+
+  public String getTags() {
+    return tags;
+  }
+
+  public RuleExtensionForIndexingDto setTags(String tags) {
+    this.tags = tags;
+    return this;
+  }
+
+  public RuleKey getRuleKey() {
+    return RuleKey.of(pluginName, pluginRuleKey);
+  }
+
+  public Set<String> getTagsAsSet() {
+    return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
+  }
+
+}

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java

@@ -0,0 +1,135 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.db.rule;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.api.rule.RuleStatus;
+import org.sonar.api.rules.RuleType;
+
+public class RuleForIndexingDto {
+
+  private Integer id;
+  private String repository;
+  private String pluginRuleKey;
+  private String name;
+  private String description;
+  private RuleDto.Format descriptionFormat;
+  private Integer severity;
+  private RuleStatus status;
+  private boolean isTemplate;
+  private String systemTags;
+  private String templateRuleKey;
+  private String templateName;
+  private String internalKey;
+  private String language;
+  private int type;
+  private long createdAt;
+  private long updatedAt;
+
+  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+
+  public Integer getId() {
+    return id;
+  }
+
+  public String getRepository() {
+    return repository;
+  }
+
+  public String getPluginRuleKey() {
+    return pluginRuleKey;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public RuleDto.Format getDescriptionFormat() {
+    return descriptionFormat;
+  }
+
+  public Integer getSeverity() {
+    return severity;
+  }
+
+  public RuleStatus getStatus() {
+    return status;
+  }
+
+  public boolean isTemplate() {
+    return isTemplate;
+  }
+
+  public String getSystemTags() {
+    return systemTags;
+  }
+
+  public String getTemplateRuleKey() {
+    return templateRuleKey;
+  }
+
+  public String getTemplateName() {
+    return templateName;
+  }
+
+  public String getInternalKey() {
+    return internalKey;
+  }
+
+  public String getLanguage() {
+    return language;
+  }
+
+  public int getType() {
+    return type;
+  }
+
+  public long getCreatedAt() {
+    return createdAt;
+  }
+
+  public long getUpdatedAt() {
+    return updatedAt;
+  }
+
+  public RuleType getTypeAsRuleType() {
+    return RuleType.valueOf(type);
+  }
+
+  public String getSeverityAsString() {
+    return severity != null ? SeverityUtil.getSeverityFromOrdinal(severity) : null;
+  }
+
+  public RuleKey getRuleKey() {
+    return RuleKey.of(repository, pluginRuleKey);
+  }
+
+  public Set<String> getSystemTagsAsSet() {
+    return ImmutableSet.copyOf(TAGS_SPLITTER.split(systemTags == null ? "" : systemTags));
+  }
+}

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java

@@ -22,6 +22,7 @@ package org.sonar.db.rule;
 import java.util.List;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.session.ResultHandler;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rules.RuleQuery;
 
@@ -51,6 +52,10 @@ public interface RuleMapper {
 
   List<RuleDefinitionDto> selectDefinitionByKeys(@Param("ruleKeys") List<RuleKey> keys);
 
+  List<RuleForIndexingDto> selectRuleForIndexingByKeys(@Param("ruleKeys") List<RuleKey> keys);
+
+  List<RuleExtensionForIndexingDto> selectRuleExtensionForIndexingByKeys(@Param("ruleExtensionIds") List<RuleExtensionId> ruleExtensionIds);
+
   List<RuleDto> selectByQuery(@Param("organizationUuid") String organizationUuid, @Param("query") RuleQuery ruleQuery);
 
   void insertDefinition(RuleDefinitionDto ruleDefinitionDto);

server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml

@@ -6,7 +6,7 @@
   <sql id="esQueueColumns">
     uuid,
     doc_type as docType,
-    doc_uuid as docUuid,
+    doc_id as docId,
     created_at as createdAt
   </sql>
 
@@ -14,12 +14,12 @@
     insert into es_queue (
       uuid,
       doc_type,
-      doc_uuid,
+      doc_id,
       created_at
     ) values (
       #{dto.uuid, jdbcType=VARCHAR},
       #{dto.docType, jdbcType=VARCHAR},
-      #{dto.docUuid, jdbcType=VARCHAR},
+      #{dto.docId, jdbcType=VARCHAR},
       #{now, jdbcType=BIGINT}
     )
   </insert>

server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml

@@ -139,6 +139,26 @@
       and r.plugin_rule_key=#{rule,jdbcType=VARCHAR}
   </select>
 
+  <select id="selectRuleExtensionForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto">
+    select
+    r.plugin_name as "pluginName",
+    r.plugin_rule_key as "pluginRuleKey",
+    rm.organization_uuid as "organizationUuid",
+    rm.tags as "tags"
+    from
+    rules r
+    inner join
+    rules_metadata rm on rm.rule_id = r.id
+    where
+    rm.tags is not null and
+    rm.tags != '' and
+    <foreach collection="ruleExtensionIds" index="index" item="ruleExtId" open="" separator=" or " close="">
+      ( r.plugin_name=#{ruleExtId.repositoryName,jdbcType=VARCHAR} and
+      r.plugin_rule_key=#{ruleExtId.ruleKey,jdbcType=VARCHAR} and
+      rm.organization_uuid=#{ruleExtId.organizationUuid,jdbcType=VARCHAR} )
+    </foreach>
+  </select>
+
   <select id="selectMetadataByKey" parameterType="map" resultType="org.sonar.db.rule.RuleMetadataDto">
     select
       rm.rule_id as "ruleId",
@@ -185,6 +205,34 @@
       </foreach>
   </select>
 
+  <select id="selectRuleForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleForIndexingDto">
+    select
+      r.id as "id",
+      r.plugin_name as "repository",
+      r.plugin_rule_key as "pluginRuleKey",
+      r.name as "name",
+      r.description as "description",
+      r.description_format as "descriptionFormat",
+      r.priority as "severity",
+      r.status as "status",
+      r.is_template as "isTemplate",
+      r.system_tags as "systemTags",
+      t.plugin_rule_key as "templateRuleKey",
+      t.plugin_name as "templateName",
+      r.plugin_config_key as "internalKey",
+      r.language as "language",
+      r.rule_type as "type",
+      r.created_at as "createdAt",
+      r.updated_at as "updatedAt"
+    from
+      rules r
+    left outer join rules t on t.id = r.template_id
+    where
+      <foreach collection="ruleKeys" index="index" item="ruleKey" open="" separator=" or " close="">
+        (r.plugin_name=#{ruleKey.repository,jdbcType=VARCHAR} and r.plugin_rule_key=#{ruleKey.rule,jdbcType=VARCHAR})
+      </foreach>
+  </select>
+
   <select id="selectByQuery" parameterType="map" resultType="Rule">
     select
       <include refid="selectJoinedTablesColumns"/>

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java

@@ -49,9 +49,9 @@ public class CreateEsQueueTable extends DdlChange {
         .setLimit(40)
         .build())
       .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
-        .setColumnName("doc_uuid")
+        .setColumnName("doc_id")
         .setIsNullable(false)
-        .setLimit(255)
+        .setLimit(4000)
         .build())
       .addColumn(BigIntegerColumnDef.newBigIntegerColumnDefBuilder()
         .setColumnName("created_at")

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java

@@ -48,7 +48,7 @@ public class CreateEsQueueTableTest {
 
     db.assertColumnDefinition(TABLE, "uuid", Types.VARCHAR, 40, false);
     db.assertColumnDefinition(TABLE, "doc_type", Types.VARCHAR, 40, false);
-    db.assertColumnDefinition(TABLE, "doc_uuid", Types.VARCHAR, 255, false);
+    db.assertColumnDefinition(TABLE, "doc_id", Types.VARCHAR, 4000, false);
     db.assertColumnDefinition(TABLE, "created_at", Types.BIGINT, null, false);
   }
 }

server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java

@@ -77,7 +77,7 @@ public class BulkIndexer {
   private final String indexName;
   private final BulkProcessor bulkProcessor;
   private final AtomicLong counter = new AtomicLong(0L);
-  private final AtomicLong successCounter = new AtomicLong(0L);
+  private final ResilientIndexerResult successCounter = new ResilientIndexerResult();
   private final SizeHandler sizeHandler;
   private final BulkProcessorListener bulkProcessorListener;
   @Nullable
@@ -103,7 +103,7 @@ public class BulkIndexer {
   public void start() {
     sizeHandler.beforeStart(this);
     counter.set(0L);
-    successCounter.set(0L);
+    successCounter.clear();
   }
 
   public void start(DbSession dbSession, DbClient dbClient, Collection<EsQueueDto> esQueueDtos) {
@@ -112,13 +112,13 @@ public class BulkIndexer {
     this.esQueueDtos = esQueueDtos;
     sizeHandler.beforeStart(this);
     counter.set(0L);
-    successCounter.set(0L);
+    successCounter.clear();
   }
 
   /**
    * @return the number of documents successfully indexed
    */
-  public long stop() {
+  public ResilientIndexerResult stop() {
     try {
       bulkProcessor.awaitClose(1, TimeUnit.MINUTES);
     } catch (InterruptedException e) {
@@ -129,7 +129,7 @@ public class BulkIndexer {
     }
     client.prepareRefresh(indexName).get();
     sizeHandler.afterStop(this);
-    return successCounter.get();
+    return successCounter;
   }
 
   public void add(ActionRequest<?> request) {
@@ -204,8 +204,9 @@ public class BulkIndexer {
       for (BulkItemResponse item : response.getItems()) {
         if (item.isFailed()) {
           LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage());
+          successCounter.increaseFailure();
         } else {
-          successCounter.incrementAndGet();
+          successCounter.increaseSuccess();
         }
       }
 
@@ -221,7 +222,7 @@ public class BulkIndexer {
       if (esQueueDtos != null) {
         List<EsQueueDto> itemsToDelete = Arrays.stream(bulkResponse.getItems())
           .filter(b -> !b.isFailed())
-          .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocUuid())).findFirst().orElse(null))
+          .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocId())).findFirst().orElse(null))
           .filter(Objects::nonNull)
           .collect(toList());
 

server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java

@@ -38,6 +38,7 @@ import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.es.EsQueueDto;
+import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.user.index.UserIndexer;
 
 import static java.lang.String.format;
@@ -64,14 +65,16 @@ public class RecoveryIndexer implements Startable {
   private final Settings settings;
   private final DbClient dbClient;
   private final UserIndexer userIndexer;
+  private final RuleIndexer ruleIndexer;
   private final long minAgeInMs;
   private final long loopLimit;
 
-  public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer) {
+  public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer, RuleIndexer ruleIndexer) {
     this.system2 = system2;
     this.settings = settings;
     this.dbClient = dbClient;
     this.userIndexer = userIndexer;
+    this.ruleIndexer = ruleIndexer;
     this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS);
     this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT);
   }
@@ -107,42 +110,43 @@ public class RecoveryIndexer implements Startable {
     try (DbSession dbSession = dbClient.openSession(false)) {
       Profiler profiler = Profiler.create(LOGGER).start();
       long beforeDate = system2.now() - minAgeInMs;
-      long total = 0L;
-      long totalSuccess = 0L;
+      ResilientIndexerResult result = new ResilientIndexerResult();
 
       Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
       while (!items.isEmpty()) {
-        total += items.size();
-        long loopSuccess = 0L;
+        ResilientIndexerResult loopResult = new ResilientIndexerResult();
 
         ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
         for (Map.Entry<EsQueueDto.Type, Collection<EsQueueDto>> entry : itemsByType.asMap().entrySet()) {
-          loopSuccess += doIndex(dbSession, entry.getKey(), entry.getValue());
+          loopResult.add(doIndex(dbSession, entry.getKey(), entry.getValue()));
         }
 
-        totalSuccess += loopSuccess;
-        if (1.0d * (items.size() - loopSuccess) / items.size() >= CIRCUIT_BREAKER_IN_PERCENT) {
-          LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", items.size() - loopSuccess, items.size());
+        result.add(loopResult);
+        if (loopResult.getFailureRatio() >= CIRCUIT_BREAKER_IN_PERCENT) {
+          LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal());
           break;
         }
         items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
       }
-      if (total > 0L) {
-        profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", total, total - totalSuccess));
+      if (result.getTotal() > 0L) {
+        profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures()));
       }
     } catch (Throwable t) {
       LOGGER.error(LOG_PREFIX + "fail to recover documents", t);
     }
   }
 
-  private long doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) {
+  private ResilientIndexerResult doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) {
     LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type);
     switch (type) {
       case USER:
         return userIndexer.index(dbSession, typeItems);
+      case RULE_EXTENSION:
+      case RULE:
+        return ruleIndexer.index(dbSession, typeItems);
       default:
         LOGGER.error(LOG_PREFIX + "ignore {} documents with unsupported type {}", typeItems.size(), type);
-        return 0;
+        return new ResilientIndexerResult();
     }
   }
 

server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexer.java

@@ -0,0 +1,41 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.es;
+
+import java.util.Collection;
+import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
+
+/**
+ * This kind of indexers that are resilient
+ */
+public interface ResilientIndexer {
+
+  /**
+   * Index the items and delete them from es_queue table when the indexation
+   * is done, keep it if there is a failure on the item of the collection
+   *
+   * @param dbSession the db session
+   * @param items     the items to be indexed
+   * @return the number of successful indexation
+   */
+  ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items);
+}

server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexerResult.java

@@ -0,0 +1,77 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonar.server.es;
+
+/**
+ * The type Resilient indexer result.
+ */
+public class ResilientIndexerResult {
+  private long total = 0L;
+  private long failures = 0L;
+
+  public ResilientIndexerResult clear() {
+    total = 0L;
+    failures = 0L;
+    return this;
+  }
+
+  public ResilientIndexerResult increaseFailure() {
+    failures += 1;
+    total += 1;
+    return this;
+  }
+
+  public ResilientIndexerResult increaseSuccess() {
+    total += 1;
+    return this;
+  }
+
+  public long getFailures() {
+    return failures;
+  }
+
+  public long getTotal() {
+    return total;
+  }
+
+  public long getSuccess() {
+    return total - failures;
+  }
+
+  /**
+   * Get the failure ratio,
+   * if the total is 0, we always return 1 in order to break loop
+   * @see {@link RecoveryIndexer#recover()}
+   */
+  public double getFailureRatio() {
+    return total == 0 ? 1 : 1.0d * failures / total;
+  }
+
+  public double getSuccessRatio() {
+    return total == 0 ? 0 : 1 -  1.0d * failures / total;
+  }
+
+  public ResilientIndexerResult add(ResilientIndexerResult other) {
+    this.total += other.getTotal();
+    this.failures += other.getFailures();
+    return this;
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/organization/ws/EnableSupportAction.java

@@ -89,8 +89,7 @@ public class EnableSupportAction implements OrganizationsWsAction {
         createDefaultMembersGroup(dbSession);
         List<RuleKey> disabledTemplateAndCustomRuleKeys = disableTemplateRulesAndCustomRules(dbSession);
         enableFeature(dbSession);
-        dbSession.commit();
-        ruleIndexer.indexRuleDefinitions(disabledTemplateAndCustomRuleKeys);
+        ruleIndexer.commitAndIndex(dbSession, disabledTemplateAndCustomRuleKeys);
       }
     }
     response.noContent();

server/sonar-server/src/main/java/org/sonar/server/rule/RegisterRules.java

@@ -130,7 +130,7 @@ public class RegisterRules implements Startable {
       keysToIndex.addAll(removedRules.stream().map(RuleDefinitionDto::getKey).collect(Collectors.toList()));
 
       persistRepositories(dbSession, context.repositories());
-      ruleIndexer.indexRuleDefinitions(keysToIndex);
+      ruleIndexer.commitAndIndex(dbSession, keysToIndex);
       activeRuleIndexer.indexChanges(dbSession, changes);
       profiler.stopDebug();
 

server/sonar-server/src/main/java/org/sonar/server/rule/RuleCreator.java

@@ -86,8 +86,7 @@ public class RuleCreator {
       .map(existingRule -> updateExistingRule(existingRule, newRule, dbSession))
       .orElseGet(() -> createCustomRule(customRuleKey, newRule, templateRule, dbSession));
 
-    dbSession.commit();
-    ruleIndexer.indexRuleDefinition(customRuleKey);
+    ruleIndexer.commitAndIndex(dbSession, customRuleKey);
     return customRuleKey;
   }
 

server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java

@@ -32,7 +32,6 @@ import java.util.Set;
 import java.util.function.Consumer;
 import javax.annotation.Nonnull;
 import org.apache.commons.lang.builder.EqualsBuilder;
-import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.rule.Severity;
 import org.sonar.api.server.ServerSide;
@@ -82,11 +81,9 @@ public class RuleUpdater {
     apply(update, rule, userSession);
     update(dbSession, rule);
     updateParameters(dbSession, organization, update, rule);
-    dbSession.commit();
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, organization, rule.getKey());
 
-    RuleKey ruleKey = rule.getKey();
-    ruleIndexer.indexRuleDefinition(ruleKey);
-    ruleIndexer.indexRuleExtension(organization, ruleKey);
     return true;
   }
 

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java

@@ -27,6 +27,9 @@ import org.apache.commons.lang.builder.ReflectionToStringBuilder;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.rules.RuleType;
+import org.sonar.db.rule.RuleDto;
+import org.sonar.db.rule.RuleForIndexingDto;
+import org.sonar.markdown.Markdown;
 import org.sonar.server.es.BaseDoc;
 
 /**
@@ -201,4 +204,34 @@ public class RuleDoc extends BaseDoc {
     return ReflectionToStringBuilder.toString(this);
   }
 
+  public static RuleDoc of(RuleForIndexingDto ruleForIndexingDto) {
+    RuleDoc ruleDoc = new RuleDoc()
+      .setKey(ruleForIndexingDto.getRuleKey().toString())
+      .setRepository(ruleForIndexingDto.getRepository())
+      .setInternalKey(ruleForIndexingDto.getInternalKey())
+      .setIsTemplate(ruleForIndexingDto.isTemplate())
+      .setLanguage(ruleForIndexingDto.getLanguage())
+      .setName(ruleForIndexingDto.getName())
+      .setRuleKey(ruleForIndexingDto.getPluginRuleKey())
+      .setSeverity(ruleForIndexingDto.getSeverityAsString())
+      .setStatus(ruleForIndexingDto.getStatus().toString())
+      .setType(ruleForIndexingDto.getTypeAsRuleType())
+      .setCreatedAt(ruleForIndexingDto.getCreatedAt())
+      .setUpdatedAt(ruleForIndexingDto.getUpdatedAt());
+
+    if (ruleForIndexingDto.getPluginRuleKey() != null && ruleForIndexingDto.getRepository() != null) {
+      ruleDoc.setTemplateKey(RuleKey.of(ruleForIndexingDto.getPluginRuleKey(), ruleForIndexingDto.getRepository()).toString());
+    } else {
+      ruleDoc.setTemplateKey(null);
+    }
+
+    if (ruleForIndexingDto.getDescription() != null && ruleForIndexingDto.getDescriptionFormat() != null) {
+      if (RuleDto.Format.HTML == ruleForIndexingDto.getDescriptionFormat()) {
+        ruleDoc.setHtmlDescription(ruleForIndexingDto.getDescription());
+      } else {
+        ruleDoc.setHtmlDescription(Markdown.convertToHtml(ruleForIndexingDto.getDescription()));;
+      }
+    }
+    return ruleDoc;
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java

@@ -24,6 +24,8 @@ import java.util.Map;
 import java.util.Set;
 import org.apache.commons.lang.builder.ReflectionToStringBuilder;
 import org.sonar.api.rule.RuleKey;
+import org.sonar.db.rule.RuleExtensionForIndexingDto;
+import org.sonar.db.rule.RuleForIndexingDto;
 import org.sonar.db.rule.RuleMetadataDto;
 import org.sonar.server.es.BaseDoc;
 
@@ -86,6 +88,20 @@ public class RuleExtensionDoc extends BaseDoc {
       .setTags(ruleExtension.getTags());
   }
 
+  public static RuleExtensionDoc of(RuleForIndexingDto rule) {
+    return new RuleExtensionDoc()
+      .setRuleKey(rule.getRuleKey())
+      .setScope(RuleExtensionScope.system())
+      .setTags(rule.getSystemTagsAsSet());
+  }
+
+  public static RuleExtensionDoc of(RuleExtensionForIndexingDto rule) {
+    return new RuleExtensionDoc()
+      .setRuleKey(rule.getRuleKey())
+      .setScope(RuleExtensionScope.organization(rule.getOrganizationUuid()))
+      .setTags(rule.getTagsAsSet());
+  }
+
   @Override
   public String toString() {
     return ReflectionToStringBuilder.toString(this);

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java

@@ -20,26 +20,38 @@
 package org.sonar.server.rule.index;
 
 import com.google.common.collect.ImmutableSet;
-import java.util.Arrays;
+import com.google.common.collect.ListMultimap;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import org.elasticsearch.action.index.IndexRequest;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.api.rule.RuleKey;
+import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
 import org.sonar.db.organization.OrganizationDto;
+import org.sonar.db.rule.RuleExtensionForIndexingDto;
+import org.sonar.db.rule.RuleForIndexingDto;
 import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.BulkIndexer.Size;
 import org.sonar.server.es.EsClient;
 import org.sonar.server.es.IndexType;
+import org.sonar.server.es.ResilientIndexer;
+import org.sonar.server.es.ResilientIndexerResult;
 import org.sonar.server.es.StartupIndexer;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.Collections.singletonList;
+import static java.util.Objects.requireNonNull;
+import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
+import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
 
-public class RuleIndexer implements StartupIndexer {
+public class RuleIndexer implements StartupIndexer, ResilientIndexer {
 
   private final EsClient esClient;
   private final DbClient dbClient;
@@ -56,7 +68,7 @@ public class RuleIndexer implements StartupIndexer {
 
   @Override
   public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
-    BulkIndexer bulk = new BulkIndexer(esClient, RuleIndexDefinition.INDEX, Size.LARGE);
+    BulkIndexer bulk = new BulkIndexer(esClient, INDEX, Size.LARGE);
     bulk.start();
 
     // index all definitions and system extensions
@@ -76,35 +88,113 @@ public class RuleIndexer implements StartupIndexer {
     bulk.stop();
   }
 
-  public void indexRuleDefinition(RuleKey ruleKey) {
-    indexRuleDefinitions(singletonList(ruleKey));
+  public void commitAndIndex(DbSession dbSession, RuleKey ruleKey) {
+    commitAndIndex(dbSession, singletonList(ruleKey));
   }
 
-  public void indexRuleDefinitions(List<RuleKey> ruleKeys) {
-    BulkIndexer bulk = new BulkIndexer(esClient, RuleIndexDefinition.INDEX, Size.REGULAR);
-    bulk.start();
+  public void commitAndIndex(DbSession dbSession, Collection<RuleKey> ruleDtos) {
+    List<EsQueueDto> items = ruleDtos.stream()
+      .map(key -> EsQueueDto.create(EsQueueDto.Type.RULE, key.toString()))
+      .collect(MoreCollectors.toArrayList());
 
-    try (RuleIterator rules = new RuleIteratorForMultipleChunks(dbClient, ruleKeys)) {
-      doIndexRuleDefinitions(rules, bulk);
-    }
+    dbClient.esQueueDao().insert(dbSession, items);
+    dbSession.commit();
+    postCommit(dbSession, ruleDtos, items);
+  }
 
-    bulk.stop();
+  public void commitAndIndex(DbSession dbSession, OrganizationDto organizationDto, RuleKey ruleKey) {
+    List<EsQueueDto> items = singletonList(EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, ruleKey + "|" + organizationDto.getUuid()));
+
+    dbClient.esQueueDao().insert(dbSession, items);
+    dbSession.commit();
+    postCommit(dbSession, ruleKey, organizationDto, items);
   }
 
-  public void indexRuleExtension(OrganizationDto organization, RuleKey ruleKey) {
-    try (DbSession dbSession = dbClient.openSession(false)) {
-      dbClient.ruleDao()
-        .selectMetadataByKey(dbSession, ruleKey, organization)
-        .map(ruleExtension -> RuleExtensionDoc.of(ruleKey, RuleExtensionScope.organization(organization), ruleExtension))
-        .map(Arrays::asList)
-        .map(List::iterator)
-        .ifPresent(metadata -> {
-          BulkIndexer bulk = new BulkIndexer(esClient, RuleIndexDefinition.INDEX, Size.REGULAR);
-          bulk.start();
-          doIndexRuleExtensions(metadata, bulk);
-          bulk.stop();
-        });
+  /**
+   * Entry point for Byteman tests. See directory tests/resilience.
+   * The parameter "ruleKeys" is used only by the Byteman script.
+   */
+  private void postCommit(DbSession dbSession, Collection<RuleKey> ruleKeys, Collection<EsQueueDto> items) {
+    index(dbSession, items);
+  }
+
+  private void postCommit(DbSession dbSession, RuleKey ruleKeys, OrganizationDto organizationDto, Collection<EsQueueDto> items) {
+    index(dbSession, items);
+  }
+
+  @Override
+  public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+    if (items.isEmpty()) {
+      return new ResilientIndexerResult();
     }
+
+    ResilientIndexerResult result = new ResilientIndexerResult();
+
+    ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
+
+    result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE)));
+    result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION)));
+
+    return result;
+  }
+
+  private ResilientIndexerResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
+    BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR);
+    bulkIndexer.start(dbSession, dbClient, items);
+
+    Set<RuleKey> rules = items
+      .stream()
+      .filter(i -> {
+        requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
+        return i.getDocType() == EsQueueDto.Type.RULE;
+      })
+      .map(i -> RuleKey.parse(i.getDocId()))
+      .collect(toHashSet(items.size()));
+
+    dbClient.ruleDao().scrollRuleByRuleKeys(dbSession, rules,
+      // only index requests, no deletion requests.
+      // Deactivated users are not deleted but updated.
+      r -> {
+        rules.remove(r.getRuleKey());
+        bulkIndexer.add(newRuleDocIndexRequest(r));
+        bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
+      });
+
+    // the remaining items reference rows that don't exist in db. They must
+    // be deleted from index.
+    rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString()));
+    rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.toString()));
+
+    return bulkIndexer.stop();
+  }
+
+  private ResilientIndexerResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) {
+    BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR);
+    bulkIndexer.start(dbSession, dbClient, items);
+
+    Set<RuleExtensionId> docIds = items
+      .stream()
+      .filter(i -> {
+        requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
+        return i.getDocType() == EsQueueDto.Type.RULE_EXTENSION;
+      })
+      .map(RuleIndexer::explodeRuleExtensionDocId)
+      .collect(toHashSet(items.size()));
+
+    dbClient.ruleDao().scrollRuleExtensionByRuleKeys(dbSession, docIds,
+      // only index requests, no deletion requests.
+      // Deactivated users are not deleted but updated.
+      r -> {
+        docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()) );
+        bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
+      });
+
+
+    // the remaining items reference rows that don't exist in db. They must
+    // be deleted from index.
+    docIds.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.getId()));
+
+    return bulkIndexer.stop();
   }
 
   private static void doIndexRuleDefinitions(Iterator<RuleDocWithSystemScope> rules, BulkIndexer bulk) {
@@ -136,4 +226,40 @@ public class RuleIndexer implements StartupIndexer {
       .source(ruleExtension.getFields())
       .parent(ruleExtension.getParent());
   }
+
+  private static IndexRequest newRuleDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
+    RuleDoc ruleDoc = RuleDoc.of(ruleForIndexingDto);
+
+    return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType(), ruleDoc.key().toString())
+      .source(ruleDoc.getFields());
+  }
+
+  private static IndexRequest newRuleExtensionDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
+    RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleForIndexingDto);
+
+    return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId())
+      .source(ruleExtensionDoc.getFields())
+      .parent(ruleExtensionDoc.getParent());
+  }
+
+  private static IndexRequest newRuleExtensionDocIndexRequest(RuleExtensionForIndexingDto ruleExtensionForIndexingDto) {
+    RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleExtensionForIndexingDto);
+
+    return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId())
+      .source(ruleExtensionDoc.getFields())
+      .parent(ruleExtensionDoc.getParent());
+  }
+
+  private BulkIndexer newBulkIndexerForRules(Size bulkSize) {
+    return new BulkIndexer(esClient, INDEX_TYPE_RULE.getIndex(), bulkSize);
+  }
+
+  private static ListMultimap<EsQueueDto.Type, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) {
+    return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType));
+  }
+
+  private static RuleExtensionId explodeRuleExtensionDocId(EsQueueDto esQueueDto) {
+    checkArgument(esQueueDto.getDocType() == EsQueueDto.Type.RULE_EXTENSION);
+    return new RuleExtensionId(esQueueDto.getDocId());
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/rule/ws/DeleteAction.java

@@ -88,9 +88,7 @@ public class DeleteAction implements RulesWsAction {
       rule.setStatus(RuleStatus.REMOVED);
       rule.setUpdatedAt(system2.now());
       dbClient.ruleDao().update(dbSession, rule);
-
-      dbSession.commit();
-      ruleIndexer.indexRuleDefinition(ruleKey);
+      ruleIndexer.commitAndIndex(dbSession, ruleKey);
     }
   }
 }

server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java

@@ -37,6 +37,8 @@ import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.BulkIndexer.Size;
 import org.sonar.server.es.EsClient;
 import org.sonar.server.es.IndexType;
+import org.sonar.server.es.ResilientIndexer;
+import org.sonar.server.es.ResilientIndexerResult;
 import org.sonar.server.es.StartupIndexer;
 
 import static java.util.Collections.singletonList;
@@ -44,7 +46,7 @@ import static java.util.Objects.requireNonNull;
 import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
 import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
 
-public class UserIndexer implements StartupIndexer {
+public class UserIndexer implements StartupIndexer, ResilientIndexer {
 
   private final DbClient dbClient;
   private final EsClient esClient;
@@ -104,17 +106,18 @@ public class UserIndexer implements StartupIndexer {
   /**
    * @return the number of items that have been successfully indexed
    */
-  public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+  @Override
+  public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
     if (items.isEmpty()) {
-      return 0L;
+      return new ResilientIndexerResult();
     }
     Set<String> logins = items
       .stream()
       .filter(i -> {
-        requireNonNull(i.getDocUuid(), () -> "BUG - " + i + " has not been persisted before indexing");
-        return true;
+        requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
+        return i.getDocType() == EsQueueDto.Type.USER;
       })
-      .map(EsQueueDto::getDocUuid)
+      .map(EsQueueDto::getDocId)
       .collect(toHashSet(items.size()));
 
     ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();

server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java

@@ -70,7 +70,7 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> {
       if (esQueueDtos != null) {
         logins = esQueueDtos.stream()
           .filter(i -> i.getDocType() == EsQueueDto.Type.USER)
-          .map(EsQueueDto::getDocUuid).collect(toArrayList());
+          .map(EsQueueDto::getDocId).collect(toArrayList());
         sql += "where (" + repeat("u.login=?", " or ", logins.size()) + ")";
       }
 

server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java

@@ -20,21 +20,15 @@
 package org.sonar.server.es;
 
 import com.google.common.collect.ImmutableMap;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.stream.IntStream;
-import org.apache.commons.lang.math.RandomUtils;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.utils.internal.TestSystem2;
 import org.sonar.db.DbTester;
-import org.sonar.db.es.EsQueueDto;
 import org.sonar.server.es.BulkIndexer.Size;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -114,42 +108,6 @@ public class BulkIndexerTest {
     assertThat(count()).isEqualTo(removeFrom);
   }
 
-  @Test
-  @Ignore
-  public void when_index_is_done_EsQueues_must_be_deleted() {
-    BulkIndexer indexer = new BulkIndexer(esTester.client(), INDEX, Size.REGULAR);
-    int nbOfDelete = 10 + RandomUtils.nextInt(10);
-    int nbOfInsert = 10 + RandomUtils.nextInt(10);
-    int nbOfDocumentNotToBeDeleted = 10 + RandomUtils.nextInt(10);
-    Collection<EsQueueDto> esQueueDtos = new ArrayList<>();
-
-    // Those documents must be kept
-    FakeDoc[] docs = new FakeDoc[nbOfDocumentNotToBeDeleted];
-    for (int i = 1; i <= nbOfDocumentNotToBeDeleted; i++) {
-      docs[i] = FakeIndexDefinition.newDoc(-i);
-    }
-    esTester.putDocuments(INDEX_TYPE_FAKE, docs);
-
-    // Create nbOfDelete documents to be deleted
-    docs = new FakeDoc[nbOfDelete];
-    for (int i = 1; i <= nbOfDelete; i++) {
-      docs[i] = FakeIndexDefinition.newDoc(i);
-    }
-    esTester.putDocuments(INDEX_TYPE_FAKE, docs);
-    assertThat(count()).isEqualTo(nbOfDelete + nbOfDocumentNotToBeDeleted);
-
-    indexer.start(dbTester.getSession(), dbTester.getDbClient(), esQueueDtos);
-    // Create nbOfDelete for old Documents
-    IntStream.rangeClosed(1, nbOfDelete).forEach(
-      i -> indexer.addDeletion(INDEX_TYPE_FAKE, "" + i));
-    // Create nbOfInsert for new Documents
-    IntStream.rangeClosed(nbOfDelete + 1, nbOfInsert).forEach(
-      i -> indexer.add(newIndexRequest(i)));
-    indexer.stop();
-
-    assertThat(count()).isEqualTo(nbOfInsert + nbOfDocumentNotToBeDeleted);
-  }
-
   private long count() {
     return esTester.countDocuments("fakes", "fake");
   }

server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java

@@ -30,6 +30,7 @@ import java.util.stream.IntStream;
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
 import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
 import org.sonar.api.config.Settings;
@@ -40,10 +41,14 @@ import org.sonar.api.utils.log.LoggerLevel;
 import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
 import org.sonar.db.es.EsQueueDto;
+import org.sonar.db.rule.RuleDto;
 import org.sonar.db.user.UserDto;
+import org.sonar.server.rule.index.RuleIndexDefinition;
+import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.user.index.UserIndexDefinition;
 import org.sonar.server.user.index.UserIndexer;
 
+import static java.util.stream.IntStream.rangeClosed;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -57,16 +62,20 @@ public class RecoveryIndexerTest {
 
   private static final long PAST = 1_000L;
   private TestSystem2 system2 = new TestSystem2().setNow(PAST);
+  private MapSettings emptySettings = new MapSettings();
 
   @Rule
-  public final EsTester es = new EsTester(new UserIndexDefinition(new MapSettings().asConfig()));
+  public final EsTester es = new EsTester(new UserIndexDefinition(emptySettings.asConfig()), new RuleIndexDefinition(emptySettings.asConfig()));
   @Rule
   public final DbTester db = DbTester.create(system2);
   @Rule
   public final LogTester logTester = new LogTester().setLevel(TRACE);
   @Rule
-  public TestRule safeguard = new Timeout(60, TimeUnit.SECONDS);
+  public TestRule safeguard = new DisableOnDebug(new Timeout(60, TimeUnit.SECONDS));
 
+
+  private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class);
+  private UserIndexer mockedUserIndexer = mock(UserIndexer.class);
   private RecoveryIndexer underTest;
 
   @After
@@ -79,7 +88,8 @@ public class RecoveryIndexerTest {
   @Test
   public void display_default_configuration_at_startup() {
     UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client());
-    underTest = newRecoveryIndexer(userIndexer, new MapSettings());
+    RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
+    underTest = newRecoveryIndexer(userIndexer, ruleIndexer, emptySettings);
 
     underTest.start();
 
@@ -93,7 +103,7 @@ public class RecoveryIndexerTest {
     Settings settings = new MapSettings()
       .setProperty("sonar.search.recovery.initialDelayInMs", "0")
       .setProperty("sonar.search.recovery.delayInMs", "1");
-    underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mock(UserIndexer.class)));
+    underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer));
     AtomicInteger calls = new AtomicInteger(0);
     doAnswer(invocation -> {
       calls.incrementAndGet();
@@ -109,13 +119,33 @@ public class RecoveryIndexerTest {
   }
 
   @Test
-  public void successfully_index_old_records() {
+  public void successfully_index_RULE_records() {
+    EsQueueDto item1 = createUnindexedRule();
+    EsQueueDto item2 = createUnindexedRule();
+
+    ProxyRuleIndexer ruleIndexer = new ProxyRuleIndexer();
+    advanceInTime();
+
+    underTest = newRecoveryIndexer(mockedUserIndexer, ruleIndexer);
+    underTest.recover();
+
+    assertThatQueueHasSize(0);
+    assertThat(ruleIndexer.called)
+      .extracting(EsQueueDto::getUuid)
+      .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid());
+
+    assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 2 RULE");
+    assertThatLogsContain(INFO, "Elasticsearch recovery - 4 documents processed [0 failures]");
+  }
+
+  @Test
+  public void successfully_index_USER_records() {
     EsQueueDto item1 = createUnindexedUser();
     EsQueueDto item2 = createUnindexedUser();
 
     ProxyUserIndexer userIndexer = new ProxyUserIndexer();
     advanceInTime();
-    underTest = newRecoveryIndexer(userIndexer);
+    underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer);
     underTest.recover();
 
     assertThatQueueHasSize(0);
@@ -134,7 +164,7 @@ public class RecoveryIndexerTest {
 
     ProxyUserIndexer userIndexer = new ProxyUserIndexer();
     // do not advance in time
-    underTest = newRecoveryIndexer(userIndexer);
+    underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer);
     underTest.recover();
 
     assertThatQueueHasSize(2);
@@ -161,7 +191,7 @@ public class RecoveryIndexerTest {
     FailingOnceUserIndexer failingOnceUserIndexer = new FailingOnceUserIndexer();
     advanceInTime();
 
-    underTest = newRecoveryIndexer(failingOnceUserIndexer);
+    underTest = newRecoveryIndexer(failingOnceUserIndexer, mockedRuleIndexer);
     underTest.recover();
 
     // No rows treated
@@ -175,7 +205,7 @@ public class RecoveryIndexerTest {
     advanceInTime();
     FailingUserIndexer userIndexer = new FailingUserIndexer();
 
-    underTest = newRecoveryIndexer(userIndexer);
+    underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer);
     underTest.start();
 
     // all runs fail, but they are still scheduled
@@ -191,7 +221,7 @@ public class RecoveryIndexerTest {
     advanceInTime();
     FailingOnceUserIndexer userIndexer = new FailingOnceUserIndexer();
 
-    underTest = newRecoveryIndexer(userIndexer);
+    underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer);
     underTest.start();
 
     // first run fails, second run succeeds
@@ -213,7 +243,7 @@ public class RecoveryIndexerTest {
     PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(1);
     Settings settings = new MapSettings()
       .setProperty("sonar.search.recovery.loopLimit", "3");
-    underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings);
+    underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings);
     underTest.recover();
 
     assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [2/3 documents], waiting for next run");
@@ -233,7 +263,7 @@ public class RecoveryIndexerTest {
     PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(4, 4, 2);
     Settings settings = new MapSettings()
       .setProperty("sonar.search.recovery.loopLimit", "5");
-    underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings);
+    underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings);
     underTest.recover();
 
     assertThatLogsDoNotContain(ERROR, "too many failures");
@@ -249,7 +279,7 @@ public class RecoveryIndexerTest {
     advanceInTime();
 
     FailingAlwaysOnSameElementIndexer indexer = new FailingAlwaysOnSameElementIndexer(buggy);
-    underTest = newRecoveryIndexer(indexer);
+    underTest = newRecoveryIndexer(indexer, mockedRuleIndexer);
     underTest.recover();
 
     assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [1/1 documents], waiting for next run");
@@ -264,7 +294,21 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+      called.addAll(items);
+      return super.index(dbSession, items);
+    }
+  }
+
+  private class ProxyRuleIndexer extends RuleIndexer {
+    private final List<EsQueueDto> called = new ArrayList<>();
+
+    ProxyRuleIndexer() {
+      super(es.client(), db.getDbClient());
+    }
+
+    @Override
+    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       called.addAll(items);
       return super.index(dbSession, items);
     }
@@ -278,7 +322,7 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       called.addAll(items);
       throw new RuntimeException("boom");
     }
@@ -293,7 +337,7 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       try {
         if (counter.getCount() == 2) {
           throw new RuntimeException("boom");
@@ -314,10 +358,15 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+      ResilientIndexerResult result = new ResilientIndexerResult();
       List<EsQueueDto> filteredItems = items.stream().filter(
         i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList());
-      return super.index(dbSession, filteredItems);
+      if (items.contains(failing)) {
+        result.increaseFailure();
+      }
+
+      return result.add(super.index(dbSession, filteredItems));
     }
   }
 
@@ -332,17 +381,21 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public long index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       System.out.println("called with " + items.size());
       called.addAll(items);
       int success = successfulReturns.next();
+      ResilientIndexerResult result = new ResilientIndexerResult();
       items.stream().limit(success).forEach(i -> {
         System.out.println(" + success");
         db.getDbClient().esQueueDao().delete(dbSession, i);
+        result.increaseSuccess();
         indexed.add(i);
       });
+
+      rangeClosed(1, items.size() - success).forEach(i -> result.increaseFailure());
       dbSession.commit();
-      return success;
+      return result;
     }
   }
 
@@ -368,19 +421,20 @@ public class RecoveryIndexerTest {
 
   private RecoveryIndexer newRecoveryIndexer() {
     UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client());
-    return newRecoveryIndexer(userIndexer);
+    RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
+    return newRecoveryIndexer(userIndexer, ruleIndexer);
   }
 
-  private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer) {
+  private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer) {
     Settings settings = new MapSettings()
       .setProperty("sonar.search.recovery.initialDelayInMs", "0")
       .setProperty("sonar.search.recovery.delayInMs", "1")
       .setProperty("sonar.search.recovery.minAgeInMs", "1");
-    return newRecoveryIndexer(userIndexer, settings);
+    return newRecoveryIndexer(userIndexer, ruleIndexer, settings);
   }
 
-  private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, Settings settings) {
-    return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer);
+  private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, Settings settings) {
+    return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer);
   }
 
   private EsQueueDto createUnindexedUser() {
@@ -391,4 +445,13 @@ public class RecoveryIndexerTest {
 
     return item;
   }
+
+  private EsQueueDto createUnindexedRule() {
+    RuleDto rule = db.rules().insertRule();
+    EsQueueDto item = EsQueueDto.create(EsQueueDto.Type.RULE, rule.getKey().toString());
+    db.getDbClient().esQueueDao().insert(db.getSession(), item);
+    db.commit();
+
+    return item;
+  }
 }

server/sonar-server/src/test/java/org/sonar/server/es/ResilientIndexerResultTest.java

@@ -0,0 +1,62 @@
+package org.sonar.server.es;/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+import java.util.stream.IntStream;
+import org.apache.commons.lang.math.RandomUtils;
+import org.assertj.core.data.Offset;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ResilientIndexerResultTest {
+
+  private final ResilientIndexerResult underTest = new ResilientIndexerResult();
+
+  @Before
+  public void clear() {
+    underTest.clear();
+  }
+
+  @Test
+  public void ensure_correctness() {
+    int success = 1 + RandomUtils.nextInt(100);
+    int failures = RandomUtils.nextInt(100);
+    IntStream.rangeClosed(1, success).forEach(i -> underTest.increaseSuccess());
+    IntStream.rangeClosed(1, failures).forEach(i -> underTest.increaseFailure());
+
+    assertThat(underTest.getFailures()).isEqualTo(failures);
+    assertThat(underTest.getSuccess()).isEqualTo(success);
+    assertThat(underTest.getTotal()).isEqualTo(success + failures);
+    assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1);
+    assertThat(underTest.getFailureRatio()).isEqualTo(1.0d * failures / (success + failures), Offset.offset(0.000001d));
+    assertThat(underTest.getSuccessRatio()).isEqualTo(1.0d * success / (success + failures), Offset.offset(0.000001d));
+  }
+
+  @Test
+  public void correctness_even_with_no_data() {
+    assertThat(underTest.getFailures()).isEqualTo(0);
+    assertThat(underTest.getSuccess()).isEqualTo(0);
+    assertThat(underTest.getTotal()).isEqualTo(0);
+    assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1);
+    assertThat(underTest.getFailureRatio()).isEqualTo(1);
+    assertThat(underTest.getSuccessRatio()).isEqualTo(0);
+  }
+}

server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java

@@ -141,7 +141,7 @@ public class IssueServiceMediumTest {
       ruleDao.insertOrUpdate(session, rule.getMetadata().setRuleId(rule.getId()));
     }
     session.commit();
-    ruleIndexer.indexRuleDefinition(rule.getDefinition().getKey());
+    //FIXME ruleIndexer.commitAndIndex(dbSession, rule.getDefinition().getKey());
     return rule;
   }
 

server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java

@@ -81,25 +81,25 @@ import static org.sonar.server.issue.IssueDocTesting.newDoc;
 public class IssueIndexTest {
 
   private System2 system2 = mock(System2.class);
-
+  private MapSettings settings = new MapSettings();
   @Rule
-  public EsTester tester = new EsTester(
-    new IssueIndexDefinition(new MapSettings().asConfig()),
-    new ViewIndexDefinition(new MapSettings().asConfig()),
-    new RuleIndexDefinition(new MapSettings().asConfig()));
+  public EsTester esTester = new EsTester(
+    new IssueIndexDefinition(settings.asConfig()),
+    new ViewIndexDefinition(settings.asConfig()),
+    new RuleIndexDefinition(settings.asConfig());
   @Rule
-  public DbTester db = DbTester.create(system2);
+  public DbTester dbTester = DbTester.create(system2);
   @Rule
   public UserSessionRule userSessionRule = UserSessionRule.standalone();
   @Rule
   public ExpectedException expectedException = ExpectedException.none();
 
-  private IssueIndexer issueIndexer = new IssueIndexer(tester.client(), new IssueIteratorFactory(null));
-  private ViewIndexer viewIndexer = new ViewIndexer(null, tester.client());
-  private RuleIndexer ruleIndexer = new RuleIndexer(tester.client(), db.getDbClient());
-  private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(tester, issueIndexer);
+  private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(null));
+  private ViewIndexer viewIndexer = new ViewIndexer(null, esTester.client());
+  private RuleIndexer ruleIndexer = new RuleIndexer(esTester.client(), dbTester.getDbClient());
+  private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(esTester, issueIndexer);
 
-  private IssueIndex underTest = new IssueIndex(tester.client(), system2, userSessionRule, new AuthorizationTypeSupport(userSessionRule));
+  private IssueIndex underTest = new IssueIndex(esTester.client(), system2, userSessionRule, new AuthorizationTypeSupport(userSessionRule));
 
   @Before
   public void setUp() {
@@ -1329,13 +1329,11 @@ public class IssueIndexTest {
 
   @Test
   public void list_tags() {
-    RuleDefinitionDto r1 = db.rules().insert();
-    ruleIndexer.indexRuleDefinition(r1.getKey());
-
-    RuleDefinitionDto r2 = db.rules().insert();
-    ruleIndexer.indexRuleDefinition(r2.getKey());
+    RuleDefinitionDto r1 = dbTester.rules().insert();
+    RuleDefinitionDto r2 = dbTester.rules().insert();
+    ruleIndexer.commitAndIndex(dbTester.getSession(), asList(r1.getKey(), r2.getKey()));
 
-    OrganizationDto org = db.organizations().insert();
+    OrganizationDto org = dbTester.organizations().insert();
     ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto());
     ComponentDto file = newFileDto(project, null);
     indexIssues(

server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java

@@ -56,25 +56,26 @@ import static org.sonar.test.JsonAssert.assertJson;
 
 public class TagsActionTest {
 
+  private MapSettings settings = new MapSettings();
   @Rule
   public UserSessionRule userSession = UserSessionRule.standalone();
   @Rule
-  public DbTester db = DbTester.create();
+  public DbTester dbTester = DbTester.create();
   @Rule
-  public EsTester es = new EsTester(new IssueIndexDefinition(new MapSettings().asConfig()), new RuleIndexDefinition(new MapSettings().asConfig()));
+  public EsTester esTester = new EsTester(new IssueIndexDefinition(settings.asConfig()), new RuleIndexDefinition(settings.asConfig()));
 
-  private IssueIndexer issueIndexer = new IssueIndexer(es.client(), new IssueIteratorFactory(db.getDbClient()));
-  private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
-  private PermissionIndexerTester permissionIndexerTester = new PermissionIndexerTester(es, issueIndexer);
-  private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new AuthorizationTypeSupport(userSession));
-  private RuleIndex ruleIndex = new RuleIndex(es.client());
+  private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbTester.getDbClient()));
+  private RuleIndexer ruleIndexer = new RuleIndexer(esTester.client(), dbTester.getDbClient());
+  private PermissionIndexerTester permissionIndexerTester = new PermissionIndexerTester(esTester, issueIndexer);
+  private IssueIndex issueIndex = new IssueIndex(esTester.client(), System2.INSTANCE, userSession, new AuthorizationTypeSupport(userSession));
+  private RuleIndex ruleIndex = new RuleIndex(esTester.client());
 
-  private WsActionTester tester = new WsActionTester(new TagsAction(issueIndex, ruleIndex, db.getDbClient(), TestDefaultOrganizationProvider.from(db)));
+  private WsActionTester tester = new WsActionTester(new TagsAction(issueIndex, ruleIndex, dbTester.getDbClient(), TestDefaultOrganizationProvider.from(dbTester)));
   private OrganizationDto organization;
 
   @Before
   public void before() {
-    organization = db.organizations().insert();
+    organization = dbTester.organizations().insert();
   }
 
   @Test
@@ -92,15 +93,15 @@ public class TagsActionTest {
   @Test
   public void return_tags_from_rules() throws Exception {
     userSession.logIn();
-    RuleDefinitionDto r = db.rules().insert(setSystemTags("tag1"));
-    ruleIndexer.indexRuleDefinition(r.getKey());
-    db.rules().insertOrUpdateMetadata(r, organization, setTags("tag2"));
-    ruleIndexer.indexRuleExtension(organization, r.getKey());
+    RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag1"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
+    dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag2"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
 
-    RuleDefinitionDto r2 = db.rules().insert(setSystemTags("tag3"));
-    ruleIndexer.indexRuleDefinition(r2.getKey());
-    db.rules().insertOrUpdateMetadata(r2, organization, setTags("tag4", "tag5"));
-    ruleIndexer.indexRuleExtension(organization, r2.getKey());
+    RuleDefinitionDto r2 = dbTester.rules().insert(setSystemTags("tag3"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), r2.getKey());
+    dbTester.rules().insertOrUpdateMetadata(r2, organization, setTags("tag4", "tag5"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r2.getKey());
 
     String result = tester.newRequest()
       .setParam("organization", organization.getKey())
@@ -114,10 +115,10 @@ public class TagsActionTest {
     insertIssueWithBrowsePermission(insertRuleWithoutTags(), "tag1", "tag2");
     insertIssueWithBrowsePermission(insertRuleWithoutTags(), "tag3", "tag4", "tag5");
 
-    RuleDefinitionDto r = db.rules().insert(setSystemTags("tag6"));
-    ruleIndexer.indexRuleDefinition(r.getKey());
-    db.rules().insertOrUpdateMetadata(r, organization, setTags("tag7"));
-    ruleIndexer.indexRuleExtension(organization, r.getKey());
+    RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag6"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
+    dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag7"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
 
     String result = tester.newRequest()
       .setParam("organization", organization.getKey())
@@ -176,10 +177,10 @@ public class TagsActionTest {
     userSession.logIn();
     insertIssueWithBrowsePermission(insertRuleWithoutTags(), "convention");
 
-    RuleDefinitionDto r = db.rules().insert(setSystemTags("cwe"));
-    ruleIndexer.indexRuleDefinition(r.getKey());
-    db.rules().insertOrUpdateMetadata(r, organization, setTags("security"));
-    ruleIndexer.indexRuleExtension(organization, r.getKey());
+    RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("cwe"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
+    dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("security"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
 
     String result = tester.newRequest()
       .setParam("organization", organization.getKey())
@@ -220,7 +221,7 @@ public class TagsActionTest {
   }
 
   private RuleDefinitionDto insertRuleWithoutTags() {
-    return db.rules().insert(setSystemTags());
+    return dbTester.rules().insert(setSystemTags());
   }
 
   private void insertIssueWithBrowsePermission(RuleDefinitionDto rule, String... tags) {
@@ -229,8 +230,8 @@ public class TagsActionTest {
   }
 
   private IssueDto insertIssueWithoutBrowsePermission(RuleDefinitionDto rule, String... tags) {
-    IssueDto issue = db.issues().insertIssue(organization, i -> i.setRule(rule).setTags(asList(tags)));
-    ComponentDto project = db.getDbClient().componentDao().selectByUuid(db.getSession(), issue.getProjectUuid()).get();
+    IssueDto issue = dbTester.issues().insertIssue(organization, i -> i.setRule(rule).setTags(asList(tags)));
+    ComponentDto project = dbTester.getDbClient().componentDao().selectByUuid(dbTester.getSession(), issue.getProjectUuid()).get();
     userSession.addProjectPermission(USER, project);
     issueIndexer.index(Collections.singletonList(issue.getKey()));
     return issue;

server/sonar-server/src/test/java/org/sonar/server/organization/ws/EnableSupportActionTest.java

@@ -27,6 +27,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.mockito.ArgumentCaptor;
+import org.sonar.api.config.MapSettings;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.server.ws.WebService;
@@ -39,12 +40,14 @@ import org.sonar.db.permission.template.PermissionTemplateGroupDto;
 import org.sonar.db.rule.RuleDefinitionDto;
 import org.sonar.db.user.GroupDto;
 import org.sonar.db.user.UserDto;
+import org.sonar.server.es.EsTester;
 import org.sonar.server.exceptions.ForbiddenException;
 import org.sonar.server.exceptions.UnauthorizedException;
 import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.organization.OrganizationFlags;
 import org.sonar.server.organization.OrganizationFlagsImpl;
 import org.sonar.server.organization.TestDefaultOrganizationProvider;
+import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.tester.UserSessionRule;
 import org.sonar.server.usergroups.DefaultGroupCreatorImpl;
@@ -54,7 +57,8 @@ import org.sonar.server.ws.WsActionTester;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.tuple;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.sonar.db.permission.OrganizationPermission.ADMINISTER;
 
@@ -65,20 +69,23 @@ public class EnableSupportActionTest {
   @Rule
   public UserSessionRule userSession = UserSessionRule.standalone();
   @Rule
-  public DbTester db = DbTester.create();
+  public DbTester dbTester = DbTester.create();
+  @Rule
+  public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings()));
+
 
-  private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
-  private OrganizationFlags organizationFlags = new OrganizationFlagsImpl(db.getDbClient());
-  private RuleIndexer ruleIndexer = mock(RuleIndexer.class);
-  private EnableSupportAction underTest = new EnableSupportAction(userSession, db.getDbClient(), defaultOrganizationProvider, organizationFlags,
-    new DefaultGroupCreatorImpl(db.getDbClient()), new DefaultGroupFinder(db.getDbClient()), ruleIndexer);
+  private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
+  private OrganizationFlags organizationFlags = new OrganizationFlagsImpl(dbTester.getDbClient());
+  private RuleIndexer ruleIndexer = spy(new RuleIndexer(esTester.client(), dbTester.getDbClient()));
+  private EnableSupportAction underTest = new EnableSupportAction(userSession, dbTester.getDbClient(), defaultOrganizationProvider, organizationFlags,
+    new DefaultGroupCreatorImpl(dbTester.getDbClient()), new DefaultGroupFinder(dbTester.getDbClient()), ruleIndexer);
   private WsActionTester tester = new WsActionTester(underTest);
 
   @Test
   public void enabling_support_saves_internal_property_and_flags_caller_as_root() {
-    UserDto user = db.users().insertUser();
-    UserDto otherUser = db.users().insertUser();
-    db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users");
+    UserDto user = dbTester.users().insertUser();
+    UserDto otherUser = dbTester.users().insertUser();
+    dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users");
     verifyFeatureEnabled(false);
     verifyRoot(user, false);
     verifyRoot(otherUser, false);
@@ -93,48 +100,48 @@ public class EnableSupportActionTest {
 
   @Test
   public void enabling_support_creates_default_members_group_and_associate_org_members() throws Exception {
-    OrganizationDto defaultOrganization = db.getDefaultOrganization();
-    OrganizationDto anotherOrganization = db.organizations().insert();
-    UserDto user1 = db.users().insertUser();
-    UserDto user2 = db.users().insertUser();
-    UserDto userInAnotherOrganization = db.users().insertUser();
-    db.organizations().addMember(defaultOrganization, user1);
-    db.organizations().addMember(defaultOrganization, user2);
-    db.organizations().addMember(anotherOrganization, userInAnotherOrganization);
-    db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users");
+    OrganizationDto defaultOrganization = dbTester.getDefaultOrganization();
+    OrganizationDto anotherOrganization = dbTester.organizations().insert();
+    UserDto user1 = dbTester.users().insertUser();
+    UserDto user2 = dbTester.users().insertUser();
+    UserDto userInAnotherOrganization = dbTester.users().insertUser();
+    dbTester.organizations().addMember(defaultOrganization, user1);
+    dbTester.organizations().addMember(defaultOrganization, user2);
+    dbTester.organizations().addMember(anotherOrganization, userInAnotherOrganization);
+    dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users");
     logInAsSystemAdministrator(user1.getLogin());
 
     call();
 
-    Optional<Integer> defaultGroupId = db.getDbClient().organizationDao().getDefaultGroupId(db.getSession(), defaultOrganization.getUuid());
+    Optional<Integer> defaultGroupId = dbTester.getDbClient().organizationDao().getDefaultGroupId(dbTester.getSession(), defaultOrganization.getUuid());
     assertThat(defaultGroupId).isPresent();
-    GroupDto membersGroup = db.getDbClient().groupDao().selectById(db.getSession(), defaultGroupId.get());
+    GroupDto membersGroup = dbTester.getDbClient().groupDao().selectById(dbTester.getSession(), defaultGroupId.get());
     assertThat(membersGroup).isNotNull();
     assertThat(membersGroup.getName()).isEqualTo("Members");
-    assertThat(db.getDbClient().groupMembershipDao().selectGroupIdsByUserId(db.getSession(), user1.getId())).containsOnly(defaultGroupId.get());
-    assertThat(db.getDbClient().groupMembershipDao().selectGroupIdsByUserId(db.getSession(), user2.getId())).containsOnly(defaultGroupId.get());
-    assertThat(db.getDbClient().groupMembershipDao().selectGroupIdsByUserId(db.getSession(), userInAnotherOrganization.getId())).isEmpty();
+    assertThat(dbTester.getDbClient().groupMembershipDao().selectGroupIdsByUserId(dbTester.getSession(), user1.getId())).containsOnly(defaultGroupId.get());
+    assertThat(dbTester.getDbClient().groupMembershipDao().selectGroupIdsByUserId(dbTester.getSession(), user2.getId())).containsOnly(defaultGroupId.get());
+    assertThat(dbTester.getDbClient().groupMembershipDao().selectGroupIdsByUserId(dbTester.getSession(), userInAnotherOrganization.getId())).isEmpty();
   }
 
   @Test
   public void enabling_support_copy_sonar_users_permissions_to_members_group() throws Exception {
-    OrganizationDto defaultOrganization = db.getDefaultOrganization();
-    UserDto user = db.users().insertUser();
-    GroupDto sonarUsersGroup = db.users().insertDefaultGroup(defaultOrganization, "sonar-users");
-    ComponentDto project = db.components().insertPrivateProject(defaultOrganization);
-    db.users().insertPermissionOnGroup(sonarUsersGroup, "user");
-    db.users().insertProjectPermissionOnGroup(sonarUsersGroup, "codeviewer", project);
+    OrganizationDto defaultOrganization = dbTester.getDefaultOrganization();
+    UserDto user = dbTester.users().insertUser();
+    GroupDto sonarUsersGroup = dbTester.users().insertDefaultGroup(defaultOrganization, "sonar-users");
+    ComponentDto project = dbTester.components().insertPrivateProject(defaultOrganization);
+    dbTester.users().insertPermissionOnGroup(sonarUsersGroup, "user");
+    dbTester.users().insertProjectPermissionOnGroup(sonarUsersGroup, "codeviewer", project);
     // Should be ignored
-    GroupDto anotherGroup = db.users().insertGroup();
-    db.users().insertPermissionOnGroup(anotherGroup, "admin");
+    GroupDto anotherGroup = dbTester.users().insertGroup();
+    dbTester.users().insertPermissionOnGroup(anotherGroup, "admin");
     logInAsSystemAdministrator(user.getLogin());
 
     call();
 
-    int defaultGroupId = db.getDbClient().organizationDao().getDefaultGroupId(db.getSession(), defaultOrganization.getUuid()).get();
+    int defaultGroupId = dbTester.getDbClient().organizationDao().getDefaultGroupId(dbTester.getSession(), defaultOrganization.getUuid()).get();
     assertThat(defaultGroupId).isNotEqualTo(sonarUsersGroup.getId());
     List<GroupPermissionDto> result = new ArrayList<>();
-    db.getDbClient().groupPermissionDao().selectAllPermissionsByGroupId(db.getSession(), defaultOrganization.getUuid(), defaultGroupId,
+    dbTester.getDbClient().groupPermissionDao().selectAllPermissionsByGroupId(dbTester.getSession(), defaultOrganization.getUuid(), defaultGroupId,
       context -> result.add((GroupPermissionDto) context.getResultObject()));
     assertThat(result).extracting(GroupPermissionDto::getResourceId, GroupPermissionDto::getRole).containsOnly(
       tuple(null, "user"), tuple(project.getId(), "codeviewer"));
@@ -142,36 +149,36 @@ public class EnableSupportActionTest {
 
   @Test
   public void enabling_support_copy_sonar_users_permission_templates_to_members_group() throws Exception {
-    OrganizationDto defaultOrganization = db.getDefaultOrganization();
-    UserDto user = db.users().insertUser();
-    GroupDto sonarUsersGroup = db.users().insertDefaultGroup(defaultOrganization, "sonar-users");
-    PermissionTemplateDto permissionTemplate = db.permissionTemplates().insertTemplate(db.getDefaultOrganization());
-    db.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "user");
-    db.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "admin");
+    OrganizationDto defaultOrganization = dbTester.getDefaultOrganization();
+    UserDto user = dbTester.users().insertUser();
+    GroupDto sonarUsersGroup = dbTester.users().insertDefaultGroup(defaultOrganization, "sonar-users");
+    PermissionTemplateDto permissionTemplate = dbTester.permissionTemplates().insertTemplate(dbTester.getDefaultOrganization());
+    dbTester.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "user");
+    dbTester.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "admin");
     // Should be ignored
-    GroupDto otherGroup = db.users().insertGroup();
-    db.permissionTemplates().addGroupToTemplate(permissionTemplate, otherGroup, "user");
+    GroupDto otherGroup = dbTester.users().insertGroup();
+    dbTester.permissionTemplates().addGroupToTemplate(permissionTemplate, otherGroup, "user");
     logInAsSystemAdministrator(user.getLogin());
 
     call();
 
-    int defaultGroupId = db.getDbClient().organizationDao().getDefaultGroupId(db.getSession(), defaultOrganization.getUuid()).get();
-    assertThat(db.getDbClient().permissionTemplateDao().selectAllGroupPermissionTemplatesByGroupId(db.getSession(), defaultGroupId))
+    int defaultGroupId = dbTester.getDbClient().organizationDao().getDefaultGroupId(dbTester.getSession(), defaultOrganization.getUuid()).get();
+    assertThat(dbTester.getDbClient().permissionTemplateDao().selectAllGroupPermissionTemplatesByGroupId(dbTester.getSession(), defaultGroupId))
       .extracting(PermissionTemplateGroupDto::getGroupId, PermissionTemplateGroupDto::getPermission)
       .containsOnly(tuple(defaultGroupId, "user"), tuple(defaultGroupId, "admin"));
   }
 
   @Test
   public void enabling_organizations_should_remove_template_rule_and_custom_rule() {
-    RuleDefinitionDto normal = db.rules().insert();
-    RuleDefinitionDto template = db.rules().insert(r -> r.setIsTemplate(true));
-    RuleDefinitionDto custom = db.rules().insert(r -> r.setTemplateId(template.getId()));
+    RuleDefinitionDto normal = dbTester.rules().insert();
+    RuleDefinitionDto template = dbTester.rules().insert(r -> r.setIsTemplate(true));
+    RuleDefinitionDto custom = dbTester.rules().insert(r -> r.setTemplateId(template.getId()));
 
-    UserDto user = db.users().insertUser();
-    db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users");
+    UserDto user = dbTester.users().insertUser();
+    dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users");
     logInAsSystemAdministrator(user.getLogin());
 
-    assertThat(db.getDbClient().ruleDao().selectAllDefinitions(db.getSession()))
+    assertThat(dbTester.getDbClient().ruleDao().selectAllDefinitions(dbTester.getSession()))
       .extracting(RuleDefinitionDto::getKey, RuleDefinitionDto::getStatus)
       .containsExactlyInAnyOrder(
         tuple(normal.getKey(), RuleStatus.READY),
@@ -181,7 +188,7 @@ public class EnableSupportActionTest {
 
     call();
 
-    assertThat(db.getDbClient().ruleDao().selectAllDefinitions(db.getSession()))
+    assertThat(dbTester.getDbClient().ruleDao().selectAllDefinitions(dbTester.getSession()))
       .extracting(RuleDefinitionDto::getKey, RuleDefinitionDto::getStatus)
       .containsExactlyInAnyOrder(
         tuple(normal.getKey(), RuleStatus.READY),
@@ -192,15 +199,15 @@ public class EnableSupportActionTest {
     @SuppressWarnings("unchecked")
     Class<ArrayList<RuleKey>> listClass = (Class<ArrayList<RuleKey>>)(Class)ArrayList.class;
     ArgumentCaptor<ArrayList<RuleKey>> indexedRuleKeys = ArgumentCaptor.forClass(listClass);
-    verify(ruleIndexer).indexRuleDefinitions(indexedRuleKeys.capture());
+    verify(ruleIndexer).commitAndIndex(any(), indexedRuleKeys.capture());
     assertThat(indexedRuleKeys.getValue()).containsExactlyInAnyOrder(template.getKey(), custom.getKey());
   }
 
   @Test
   public void throw_IAE_when_members_group_already_exists() throws Exception {
-    UserDto user = db.users().insertUser();
-    db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users");
-    db.users().insertGroup(db.getDefaultOrganization(), "Members");
+    UserDto user = dbTester.users().insertUser();
+    dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users");
+    dbTester.users().insertGroup(dbTester.getDefaultOrganization(), "Members");
     logInAsSystemAdministrator(user.getLogin());
 
     expectedException.expect(IllegalArgumentException.class);
@@ -231,7 +238,7 @@ public class EnableSupportActionTest {
 
   @Test
   public void throw_ISE_when_default_organization_has_not_default_group() {
-    UserDto user = db.users().insertUser();
+    UserDto user = dbTester.users().insertUser();
     logInAsSystemAdministrator(user.getLogin());
 
     expectedException.expect(IllegalStateException.class);
@@ -242,7 +249,7 @@ public class EnableSupportActionTest {
 
   @Test
   public void do_nothing_if_support_is_already_enabled() {
-    db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users");
+    dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users");
     logInAsSystemAdministrator("foo");
 
     call();
@@ -265,7 +272,7 @@ public class EnableSupportActionTest {
   }
 
   private void logInAsSystemAdministrator(String login) {
-    userSession.logIn(login).addPermission(ADMINISTER, db.getDefaultOrganization());
+    userSession.logIn(login).addPermission(ADMINISTER, dbTester.getDefaultOrganization());
   }
 
   private void call() {
@@ -274,10 +281,10 @@ public class EnableSupportActionTest {
   }
 
   private void verifyFeatureEnabled(boolean enabled) {
-    assertThat(organizationFlags.isEnabled(db.getSession())).isEqualTo(enabled);
+    assertThat(organizationFlags.isEnabled(dbTester.getSession())).isEqualTo(enabled);
   }
 
   private void verifyRoot(UserDto user, boolean root) {
-    db.rootFlag().verify(user.getLogin(), root);
+    dbTester.rootFlag().verify(user.getLogin(), root);
   }
 }

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ChangeParentActionTest.java

@@ -21,7 +21,6 @@ package org.sonar.server.qualityprofile.ws;
 
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Stream;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -34,7 +33,6 @@ import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.rule.Severity;
 import org.sonar.api.server.ws.WebService;
 import org.sonar.api.utils.System2;
-import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
@@ -65,6 +63,7 @@ import org.sonar.server.util.TypeValidations;
 import org.sonar.server.ws.TestRequest;
 import org.sonar.server.ws.WsActionTester;
 
+import static java.util.Arrays.asList;
 import static java.util.Collections.emptySet;
 import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -158,7 +157,7 @@ public class ChangeParentActionTest {
 
     RuleDefinitionDto rule1 = createRule();
     createActiveRule(rule1, parent1);
-    ruleIndexer.indexRuleDefinition(rule1.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule1.getKey());
     activeRuleIndexer.indexOnStartup(emptySet());
 
     assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty();
@@ -188,7 +187,7 @@ public class ChangeParentActionTest {
     RuleDefinitionDto rule2 = createRule();
     createActiveRule(rule1, parent1);
     createActiveRule(rule2, parent2);
-    ruleIndexer.indexRuleDefinitions(Stream.of(rule1, rule2).map(RuleDefinitionDto::getKey).collect(MoreCollectors.toList()));
+    ruleIndexer.commitAndIndex(dbSession, asList(rule1.getKey(), rule2.getKey()));
     activeRuleIndexer.indexOnStartup(emptySet());
 
     // Set parent 1
@@ -216,7 +215,7 @@ public class ChangeParentActionTest {
 
     RuleDefinitionDto rule1 = createRule();
     createActiveRule(rule1, parent);
-    ruleIndexer.indexRuleDefinition(rule1.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule1.getKey());
     activeRuleIndexer.indexOnStartup(emptySet());
 
     // Set parent
@@ -244,7 +243,7 @@ public class ChangeParentActionTest {
     RuleDefinitionDto rule2 = createRule();
     createActiveRule(rule1, parent1);
     createActiveRule(rule2, parent2);
-    ruleIndexer.indexRuleDefinition(rule1.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule1.getKey());
     activeRuleIndexer.indexOnStartup(emptySet());
 
     assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty();
@@ -303,7 +302,7 @@ public class ChangeParentActionTest {
 
     RuleDefinitionDto rule1 = createRule();
     createActiveRule(rule1, parent);
-    ruleIndexer.indexRuleDefinition(rule1.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule1.getKey());
     activeRuleIndexer.indexOnStartup(emptySet());
 
     assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty();

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/CreateActionTest.java

@@ -252,7 +252,7 @@ public class CreateActionTest {
   private void insertRule(RuleDefinitionDto ruleDto) {
     dbClient.ruleDao().insert(dbSession, ruleDto);
     dbSession.commit();
-    ruleIndexer.indexRuleDefinition(ruleDto.getKey());
+    ruleIndexer.commitAndIndex(dbSession, ruleDto.getKey());
   }
 
   private CreateWsResponse executeRequest(String name, String language) {

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/InheritanceActionTest.java

@@ -59,6 +59,7 @@ import org.sonar.server.util.TypeValidations;
 import org.sonar.server.ws.WsActionTester;
 import org.sonarqube.ws.QualityProfiles.InheritanceWsResponse;
 
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.sonar.server.qualityprofile.QProfileTesting.newQProfileDto;
 import static org.sonar.test.JsonAssert.assertJson;
@@ -68,7 +69,7 @@ import static org.sonarqube.ws.client.qualityprofile.QualityProfileWsParameters.
 public class InheritanceActionTest {
 
   @Rule
-  public DbTester db = DbTester.create();
+  public DbTester dbTester = DbTester.create();
   @Rule
   public EsTester es = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig()));
   @Rule
@@ -87,12 +88,12 @@ public class InheritanceActionTest {
 
   @Before
   public void setUp() {
-    dbClient = db.getDbClient();
-    dbSession = db.getSession();
+    dbClient = dbTester.getDbClient();
+    dbSession = dbTester.getSession();
     esClient = es.client();
     ruleIndexer = new RuleIndexer(esClient, dbClient);
     activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
-    TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
+    TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
     underTest = new InheritanceAction(
       dbClient,
       new QProfileWsSupport(dbClient, userSession, defaultOrganizationProvider),
@@ -106,7 +107,7 @@ public class InheritanceActionTest {
       new TypeValidations(new ArrayList<>()),
       activeRuleIndexer,
       userSession);
-    organization = db.organizations().insert();
+    organization = dbTester.organizations().insert();
   }
 
   @Test
@@ -118,7 +119,7 @@ public class InheritanceActionTest {
     /*
      * sonar way (2) <- companyWide (2) <- buWide (2, 1 overriding) <- (forProject1 (2), forProject2 (2))
      */
-    QProfileDto sonarway = db.qualityProfiles().insert(organization, p -> p.setKee("xoo-sonar-way").setLanguage("xoo").setName("Sonar way").setIsBuiltIn(true));
+    QProfileDto sonarway = dbTester.qualityProfiles().insert(organization, p -> p.setKee("xoo-sonar-way").setLanguage("xoo").setName("Sonar way").setIsBuiltIn(true));
     ActiveRuleDto activeRule1 = createActiveRule(rule1, sonarway);
     ActiveRuleDto activeRule2 = createActiveRule(rule2, sonarway);
 
@@ -153,22 +154,18 @@ public class InheritanceActionTest {
 
   @Test
   public void inheritance_parent_child() throws Exception {
-    RuleDefinitionDto rule1 = db.rules().insert();
-    ruleIndexer.indexRuleDefinition(rule1.getKey());
-
-    RuleDefinitionDto rule2 = db.rules().insert();
-    ruleIndexer.indexRuleDefinition(rule1.getKey());
-
-    RuleDefinitionDto rule3 = db.rules().insert();
-    ruleIndexer.indexRuleDefinition(rule1.getKey());
-
-    QProfileDto parent = db.qualityProfiles().insert(organization);
-    db.qualityProfiles().activateRule(parent, rule1);
-    db.qualityProfiles().activateRule(parent, rule2);
+    RuleDefinitionDto rule1 = dbTester.rules().insert();
+    RuleDefinitionDto rule2 = dbTester.rules().insert();
+    RuleDefinitionDto rule3 = dbTester.rules().insert();
+    ruleIndexer.commitAndIndex(dbTester.getSession(), asList(rule1.getKey(), rule2.getKey(), rule3.getKey()));
+
+    QProfileDto parent = dbTester.qualityProfiles().insert(organization);
+    dbTester.qualityProfiles().activateRule(parent, rule1);
+    dbTester.qualityProfiles().activateRule(parent, rule2);
     long parentRules = 2;
 
-    QProfileDto child = db.qualityProfiles().insert(organization, q -> q.setParentKee(parent.getKee()));
-    db.qualityProfiles().activateRule(child, rule3);
+    QProfileDto child = dbTester.qualityProfiles().insert(organization, q -> q.setParentKee(parent.getKee()));
+    dbTester.qualityProfiles().activateRule(child, rule3);
     long childRules = 1;
 
     activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes());
@@ -191,11 +188,11 @@ public class InheritanceActionTest {
 
   @Test
   public void inheritance_ignores_removed_rules() throws Exception {
-    RuleDefinitionDto rule = db.rules().insert(r -> r.setStatus(RuleStatus.REMOVED));
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    RuleDefinitionDto rule = dbTester.rules().insert(r -> r.setStatus(RuleStatus.REMOVED));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), rule.getKey());
 
-    QProfileDto profile = db.qualityProfiles().insert(organization);
-    db.qualityProfiles().activateRule(profile, rule);
+    QProfileDto profile = dbTester.qualityProfiles().insert(organization);
+    dbTester.qualityProfiles().activateRule(profile, rule);
     long activeRules = 0;
 
     activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes());
@@ -266,8 +263,7 @@ public class InheritanceActionTest {
       .setUpdatedAt(now)
       .setCreatedAt(now);
     dbClient.ruleDao().insert(dbSession, rule);
-    dbSession.commit();
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
     return rule;
   }
 

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/QProfilesWsMediumTest.java

@@ -109,8 +109,7 @@ public class QProfilesWsMediumTest {
     QProfileDto profile = createProfile("java");
     RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto");
     createActiveRule(rule, profile);
-    dbSession.commit();
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
     activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes());
 
     // 0. Assert No Active Rule for profile
@@ -209,8 +208,7 @@ public class QProfilesWsMediumTest {
   public void activate_rule() throws Exception {
     QProfileDto profile = createProfile("java");
     RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto");
-    dbSession.commit();
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
 
     // 0. Assert No Active Rule for profile
     assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, profile.getKee())).isEmpty();
@@ -230,8 +228,7 @@ public class QProfilesWsMediumTest {
   public void activate_rule_diff_languages() throws Exception {
     QProfileDto profile = createProfile("java");
     RuleDefinitionDto rule = createRule("php", "toto");
-    dbSession.commit();
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
 
     // 0. Assert No Active Rule for profile
     assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, profile.getKee())).isEmpty();
@@ -253,8 +250,7 @@ public class QProfilesWsMediumTest {
   public void activate_rule_override_severity() throws Exception {
     QProfileDto profile = createProfile("java");
     RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto");
-    dbSession.commit();
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
 
     // 0. Assert No Active Rule for profile
     assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, profile.getKee())).isEmpty();
@@ -456,8 +452,7 @@ public class QProfilesWsMediumTest {
       .setSeverity(Severity.BLOCKER)
       .setStatus(RuleStatus.READY);
     dbClient.ruleDao().insert(dbSession, rule);
-    dbSession.commit();
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    ruleIndexer.commitAndIndex(dbSession, rule.getKey());
     return rule;
   }
 

server/sonar-server/src/test/java/org/sonar/server/rule/RuleCreatorTest.java

@@ -63,17 +63,17 @@ public class RuleCreatorTest {
   public ExpectedException expectedException = ExpectedException.none();
 
   @Rule
-  public DbTester db = DbTester.create(system2);
+  public DbTester dbTester = DbTester.create(system2);
 
   @Rule
   public EsTester es = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig()));
 
   private RuleIndex ruleIndex = new RuleIndex(es.client());
-  private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
-  private DbSession dbSession = db.getSession();
+  private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), dbTester.getDbClient());
+  private DbSession dbSession = dbTester.getSession();
 
-  private RuleCreator underTest = new RuleCreator(system2, new RuleIndexer(es.client(), db.getDbClient()), db.getDbClient(), newFullTypeValidations(),
-    TestDefaultOrganizationProvider.from(db));
+  private RuleCreator underTest = new RuleCreator(system2, new RuleIndexer(es.client(), dbTester.getDbClient()), dbTester.getDbClient(), newFullTypeValidations(),
+    TestDefaultOrganizationProvider.from(dbTester));
 
   @Test
   public void create_custom_rule() {
@@ -88,7 +88,7 @@ public class RuleCreatorTest {
       .setParameters(ImmutableMap.of("regex", "a.*"));
     RuleKey customRuleKey = underTest.create(dbSession, newRule);
 
-    RuleDto rule = db.getDbClient().ruleDao().selectOrFailByKey(dbSession, db.getDefaultOrganization(), customRuleKey);
+    RuleDto rule = dbTester.getDbClient().ruleDao().selectOrFailByKey(dbSession, dbTester.getDefaultOrganization(), customRuleKey);
     assertThat(rule).isNotNull();
     assertThat(rule.getKey()).isEqualTo(RuleKey.of("java", "CUSTOM_RULE"));
     assertThat(rule.getTemplateId()).isEqualTo(templateRule.getId());
@@ -105,7 +105,7 @@ public class RuleCreatorTest {
     assertThat(rule.getTags()).containsOnly("usertag1", "usertag2");
     assertThat(rule.getSystemTags()).containsOnly("tag1", "tag4");
 
-    List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
+    List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
     assertThat(params).hasSize(1);
 
     RuleParamDto param = params.get(0);
@@ -132,7 +132,7 @@ public class RuleCreatorTest {
 
     RuleKey customRuleKey = underTest.create(dbSession, newRule);
 
-    List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
+    List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
     assertThat(params).hasSize(1);
     RuleParamDto param = params.get(0);
     assertThat(param.getName()).isEqualTo("regex");
@@ -153,7 +153,7 @@ public class RuleCreatorTest {
 
     RuleKey customRuleKey = underTest.create(dbSession, newRule);
 
-    List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
+    List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
     assertThat(params).hasSize(1);
     RuleParamDto param = params.get(0);
     assertThat(param.getName()).isEqualTo("myIntegers");
@@ -175,7 +175,7 @@ public class RuleCreatorTest {
 
     RuleKey customRuleKey = underTest.create(dbSession, newRule);
 
-    List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
+    List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
     assertThat(params).hasSize(1);
     RuleParamDto param = params.get(0);
     assertThat(param.getName()).isEqualTo("myIntegers");
@@ -237,8 +237,8 @@ public class RuleCreatorTest {
       .setDescription("Old description")
       .setDescriptionFormat(Format.MARKDOWN)
       .setSeverity(Severity.INFO);
-    db.rules().insert(rule.getDefinition());
-    db.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*"));
+    dbTester.rules().insert(rule.getDefinition());
+    dbTester.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*"));
     dbSession.commit();
 
     // Create custom rule with same key, but with different values
@@ -250,7 +250,7 @@ public class RuleCreatorTest {
       .setParameters(ImmutableMap.of("regex", "c.*"));
     RuleKey customRuleKey = underTest.create(dbSession, newRule);
 
-    RuleDefinitionDto result = db.getDbClient().ruleDao().selectOrFailDefinitionByKey(dbSession, customRuleKey);
+    RuleDefinitionDto result = dbTester.getDbClient().ruleDao().selectOrFailDefinitionByKey(dbSession, customRuleKey);
     assertThat(result.getKey()).isEqualTo(RuleKey.of("java", key));
     assertThat(result.getStatus()).isEqualTo(RuleStatus.READY);
 
@@ -259,7 +259,7 @@ public class RuleCreatorTest {
     assertThat(result.getDescription()).isEqualTo("Old description");
     assertThat(result.getSeverityString()).isEqualTo(Severity.INFO);
 
-    List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
+    List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey);
     assertThat(params).hasSize(1);
     assertThat(params.get(0).getDefaultValue()).isEqualTo("a.*");
   }
@@ -276,8 +276,8 @@ public class RuleCreatorTest {
       .setName("Old name")
       .setDescription("Old description")
       .setSeverity(Severity.INFO);
-    db.rules().insert(rule.getDefinition());
-    db.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*"));
+    dbTester.rules().insert(rule.getDefinition());
+    dbTester.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*"));
     dbSession.commit();
 
     // Create custom rule with same key, but with different values
@@ -427,7 +427,7 @@ public class RuleCreatorTest {
   public void fail_to_create_custom_rule_when_wrong_rule_template() {
     // insert rule
     RuleDefinitionDto rule = newRule(RuleKey.of("java", "S001")).setIsTemplate(false);
-    db.rules().insert(rule);
+    dbTester.rules().insert(rule);
     dbSession.commit();
 
     expectedException.expect(IllegalArgumentException.class);
@@ -458,7 +458,7 @@ public class RuleCreatorTest {
   }
 
   private RuleDto createTemplateRule() {
-    RuleDto templateRule = RuleTesting.newDto(RuleKey.of("java", "S001"), db.getDefaultOrganization())
+    RuleDto templateRule = RuleTesting.newDto(RuleKey.of("java", "S001"), dbTester.getDefaultOrganization())
       .setIsTemplate(true)
       .setLanguage("java")
       .setConfigKey("S001")
@@ -470,10 +470,10 @@ public class RuleCreatorTest {
       .setSystemTags(Sets.newHashSet("tag1", "tag4"))
       .setCreatedAt(new Date().getTime())
       .setUpdatedAt(new Date().getTime());
-    db.rules().insert(templateRule.getDefinition());
-    db.rules().insertOrUpdateMetadata(templateRule.getMetadata().setRuleId(templateRule.getId()));
-    db.rules().insertRuleParam(templateRule.getDefinition(), param -> param.setName("regex").setType("STRING").setDescription("Reg ex").setDefaultValue(".*"));
-    ruleIndexer.indexRuleDefinition(templateRule.getDefinition().getKey());
+    dbTester.rules().insert(templateRule.getDefinition());
+    dbTester.rules().insertOrUpdateMetadata(templateRule.getMetadata().setRuleId(templateRule.getId()));
+    dbTester.rules().insertRuleParam(templateRule.getDefinition(), param -> param.setName("regex").setType("STRING").setDescription("Reg ex").setDefaultValue(".*"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), templateRule.getDefinition().getKey());
     return templateRule;
   }
 
@@ -488,9 +488,9 @@ public class RuleCreatorTest {
       .setGapDescription("desc")
       .setCreatedAt(new Date().getTime())
       .setUpdatedAt(new Date().getTime());
-    db.rules().insert(templateRule);
-    db.rules().insertRuleParam(templateRule, param -> param.setName("myIntegers").setType("INTEGER,multiple=true,values=1;2;3").setDescription("My Integers").setDefaultValue("1"));
-    ruleIndexer.indexRuleDefinition(templateRule.getKey());
+    dbTester.rules().insert(templateRule);
+    dbTester.rules().insertRuleParam(templateRule, param -> param.setName("myIntegers").setType("INTEGER,multiple=true,values=1;2;3").setDescription("My Integers").setDefaultValue("1"));
+    ruleIndexer.commitAndIndex(dbTester.getSession(), templateRule.getKey());
     return templateRule;
   }
 
@@ -505,9 +505,9 @@ public class RuleCreatorTest {
       .setGapDescription("desc")
       .setCreatedAt(new Date().getTime())
       .setUpdatedAt(new Date().getTime());
-    db.rules().insert(templateRule);
-    db.rules().insertRuleParam(templateRule, param -> param.setName("first").setType("INTEGER").setDescription("First integer").setDefaultValue("0"));
-    db.rules().insertRuleParam(templateRule, param -> param.setName("second").setType("INTEGER").setDescription("Second integer").setDefaultValue("0"));
+    dbTester.rules().insert(templateRule);
+    dbTester.rules().insertRuleParam(templateRule, param -> param.setName("first").setType("INTEGER").setDescription("First integer").setDefaultValue("0"));
+    dbTester.rules().insertRuleParam(templateRule, param -> param.setName("second").setType("INTEGER").setDescription("Second integer").setDefaultValue("0"));
     return templateRule;
   }
 

server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java

@@ -20,6 +20,7 @@
 package org.sonar.server.rule.index;
 
 import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.config.internal.MapSettings;
@@ -68,16 +69,14 @@ public class RuleIndexerTest {
 
   @Test
   public void index_nothing() {
-    // underTest.index(Iterators.emptyIterator());
+    underTest.index(dbSession, Collections.emptyList());
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(0L);
   }
 
   @Test
   public void index() {
     dbClient.ruleDao().insert(dbSession, rule);
-    dbSession.commit();
-
-    underTest.indexRuleDefinition(rule.getKey());
+    underTest.commitAndIndex(dbSession, rule.getKey());
 
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
   }
@@ -87,13 +86,12 @@ public class RuleIndexerTest {
     // Create and Index rule
     dbClient.ruleDao().insert(dbSession, rule.setStatus(RuleStatus.READY));
     dbSession.commit();
-    underTest.indexRuleDefinition(rule.getKey());
+    underTest.commitAndIndex(dbTester.getSession(), rule.getKey());
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
 
     // Remove rule
     dbTester.getDbClient().ruleDao().update(dbTester.getSession(), rule.setStatus(RuleStatus.READY).setUpdatedAt(2000000000000L));
-    dbTester.getSession().commit();
-    underTest.indexRuleDefinition(rule.getKey());
+    underTest.commitAndIndex(dbTester.getSession(), rule.getKey());
 
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
   }
@@ -101,10 +99,10 @@ public class RuleIndexerTest {
   @Test
   public void index_rule_extension_with_long_id() {
     RuleDefinitionDto rule = dbTester.rules().insert(r -> r.setRuleKey(RuleTesting.randomRuleKeyOfMaximumLength()));
-    underTest.indexRuleDefinition(rule.getKey());
+    underTest.commitAndIndex(dbTester.getSession(), rule.getKey());
     OrganizationDto organization = dbTester.organizations().insert();
     dbTester.rules().insertOrUpdateMetadata(rule, organization, m -> m.setTags(ImmutableSet.of("bla")));
-    underTest.indexRuleExtension(organization, rule.getKey());
+    underTest.commitAndIndex(dbTester.getSession(), organization, rule.getKey());
 
     RuleExtensionDoc doc = new RuleExtensionDoc()
       .setRuleKey(rule.getKey())

server/sonar-server/src/test/java/org/sonar/server/rule/ws/DeleteActionTest.java

@@ -19,42 +19,50 @@
  */
 package org.sonar.server.rule.ws;
 
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
-import org.mockito.Mockito;
+import org.sonar.api.config.MapSettings;
 import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
 import org.sonar.db.rule.RuleDefinitionDto;
+import org.sonar.server.es.EsTester;
 import org.sonar.server.exceptions.ForbiddenException;
 import org.sonar.server.exceptions.UnauthorizedException;
 import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.RuleActivator;
+import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.tester.UserSessionRule;
 import org.sonar.server.ws.WsActionTester;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
 import static org.sonar.db.permission.OrganizationPermission.ADMINISTER_QUALITY_PROFILES;
 
 public class DeleteActionTest {
   private static final long PAST = 10000L;
 
-  @org.junit.Rule
+  @Rule
   public UserSessionRule userSession = UserSessionRule.standalone();
-  @org.junit.Rule
+  @Rule
   public DbTester dbTester = DbTester.create();
-  @org.junit.Rule
+  @Rule
+  public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings()));
+  @Rule
   public ExpectedException thrown = ExpectedException.none();
 
   private DbClient dbClient = dbTester.getDbClient();
   private DbSession dbSession = dbTester.getSession();
-  private RuleIndexer ruleIndexer = mock(RuleIndexer.class);
+  private RuleIndexer ruleIndexer = spy(new RuleIndexer(esTester.client(), dbClient));
   private RuleActivator ruleActivator = mock(RuleActivator.class);
   private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.fromUuid("ORG1");
   private RuleWsSupport ruleWsSupport = new RuleWsSupport(mock(DbClient.class), userSession, defaultOrganizationProvider);
@@ -79,8 +87,7 @@ public class DeleteActionTest {
       .setParam("key", customRule.getKey().toString())
       .execute();
 
-    Mockito.verify(ruleIndexer).indexRuleDefinition(eq(customRule.getKey()));
-    Mockito.verifyNoMoreInteractions(ruleIndexer);
+    verify(ruleIndexer).commitAndIndex(any(), eq(customRule.getKey()));
 
     // Verify custom rule has status REMOVED
     RuleDefinitionDto customRuleReloaded = dbClient.ruleDao().selectOrFailDefinitionByKey(dbSession, customRule.getKey());

server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionMediumTest.java

@@ -26,10 +26,10 @@ public class SearchActionMediumTest {
   // @Test
   // public void search_profile_active_rules_with_inheritance() throws Exception {
   // QProfileDto profile = QProfileTesting.newXooP1(defaultOrganizationDto);
-  // tester.get(QualityProfileDao.class).insert(dbSession, profile);
+  // esTester.get(QualityProfileDao.class).insert(dbSession, profile);
   //
   // QProfileDto profile2 = QProfileTesting.newXooP2(defaultOrganizationDto).setParentKee(profile.getKee());
-  // tester.get(QualityProfileDao.class).insert(dbSession, profile2);
+  // esTester.get(QualityProfileDao.class).insert(dbSession, profile2);
   //
   // dbSession.commit();
   //
@@ -37,15 +37,15 @@ public class SearchActionMediumTest {
   // insertRule(rule);
   //
   // ActiveRuleDto activeRule = newActiveRule(profile, rule);
-  // tester.get(ActiveRuleDao.class).insert(dbSession, activeRule);
+  // esTester.get(ActiveRuleDao.class).insert(dbSession, activeRule);
   // ActiveRuleDto activeRule2 = newActiveRule(profile2, rule).setInheritance(ActiveRuleDto.OVERRIDES).setSeverity(Severity.CRITICAL);
-  // tester.get(ActiveRuleDao.class).insert(dbSession, activeRule2);
+  // esTester.get(ActiveRuleDao.class).insert(dbSession, activeRule2);
   //
   // dbSession.commit();
   //
   // activeRuleIndexer.index();
   //
-  // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.TEXT_QUERY, "x1");
   // request.setParam(PARAM_ACTIVATION, "true");
   // request.setParam(PARAM_QPROFILE, profile2.getKee());
@@ -57,7 +57,7 @@ public class SearchActionMediumTest {
   // @Test
   // public void search_all_active_rules_params() throws Exception {
   // QProfileDto profile = QProfileTesting.newXooP1(defaultOrganizationDto);
-  // tester.get(QualityProfileDao.class).insert(dbSession, profile);
+  // esTester.get(QualityProfileDao.class).insert(dbSession, profile);
   // RuleDefinitionDto rule = RuleTesting.newXooX1().getDefinition();
   // insertRule(rule);
   // dbSession.commit();
@@ -77,21 +77,21 @@ public class SearchActionMediumTest {
   // ruleDao.insertRuleParam(dbSession, rule, param2);
   //
   // ActiveRuleDto activeRule = newActiveRule(profile, rule);
-  // tester.get(ActiveRuleDao.class).insert(dbSession, activeRule);
+  // esTester.get(ActiveRuleDao.class).insert(dbSession, activeRule);
   //
   // ActiveRuleParamDto activeRuleParam = ActiveRuleParamDto.createFor(param)
   // .setValue("The VALUE");
-  // tester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam);
+  // esTester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam);
   //
   // ActiveRuleParamDto activeRuleParam2 = ActiveRuleParamDto.createFor(param2)
   // .setValue("The Other Value");
-  // tester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam2);
+  // esTester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam2);
   //
   // dbSession.commit();
   //
   // activeRuleIndexer.index();
   //
-  // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.TEXT_QUERY, "x1");
   // request.setParam(PARAM_ACTIVATION, "true");
   // request.setParam(WebService.Param.FIELDS, "params");
@@ -103,7 +103,7 @@ public class SearchActionMediumTest {
   // @Test
   // public void get_note_as_markdown_and_html() throws Exception {
   // QProfileDto profile = QProfileTesting.newXooP1("org-123");
-  // tester.get(QualityProfileDao.class).insert(dbSession, profile);
+  // esTester.get(QualityProfileDao.class).insert(dbSession, profile);
   // RuleDto rule = RuleTesting.newXooX1(defaultOrganizationDto).setNoteData("this is *bold*");
   // insertRule(rule.getDefinition());
   // ruleDao.insertOrUpdate(dbSession, rule.getMetadata().setRuleId(rule.getId()));
@@ -112,7 +112,7 @@ public class SearchActionMediumTest {
   //
   // activeRuleIndexer.index();
   //
-  // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.FIELDS, "htmlNote, mdNote");
   // WsTester.Result result = request.execute();
   // result.assertJson(this.getClass(), "get_note_as_markdown_and_html.json");
@@ -128,7 +128,7 @@ public class SearchActionMediumTest {
   //
   // activeRuleIndexer.index();
   //
-  // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(PARAM_TAGS, "tag1");
   // request.setParam(WebService.Param.FIELDS, "sysTags, tags");
   // request.setParam(WebService.Param.FACETS, "tags");
@@ -138,7 +138,7 @@ public class SearchActionMediumTest {
   //
   // @Test
   // public void severities_facet_should_have_all_severities() throws Exception {
-  // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.FACETS, "severities");
   // request.execute().assertJson(this.getClass(), "severities_facet.json");
   // }
@@ -160,7 +160,7 @@ public class SearchActionMediumTest {
   // dbSession.commit();
   //
   // // 1. Sort Name Asc
-  // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.FIELDS, "");
   // request.setParam(WebService.Param.SORT, "name");
   // request.setParam(WebService.Param.ASCENDING, "true");
@@ -169,7 +169,7 @@ public class SearchActionMediumTest {
   // result.assertJson("{\"total\":3,\"p\":1,\"ps\":100,\"rules\":[{\"key\":\"xoo:x2\"},{\"key\":\"xoo:x1\"},{\"key\":\"xoo:x3\"}]}");
   //
   // // 2. Sort Name DESC
-  // request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.FIELDS, "");
   // request.setParam(WebService.Param.SORT, RuleIndexDefinition.FIELD_RULE_NAME);
   // request.setParam(WebService.Param.ASCENDING, "false");
@@ -194,7 +194,7 @@ public class SearchActionMediumTest {
   // dbSession.clearCache();
   //
   // // 1. find today's rules
-  // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.FIELDS, "");
   // request.setParam(PARAM_AVAILABLE_SINCE, DateUtils.formatDate(since));
   // request.setParam(WebService.Param.SORT, RuleIndexDefinition.FIELD_RULE_KEY);
@@ -202,7 +202,7 @@ public class SearchActionMediumTest {
   // result.assertJson("{\"total\":2,\"p\":1,\"ps\":100,\"rules\":[{\"key\":\"xoo:x1\"},{\"key\":\"xoo:x2\"}]}");
   //
   // // 2. no rules since tomorrow
-  // request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
+  // request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
   // request.setParam(WebService.Param.FIELDS, "");
   // request.setParam(PARAM_AVAILABLE_SINCE, DateUtils.formatDate(DateUtils.addDays(since, 1)));
   // result = request.execute();
@@ -222,7 +222,7 @@ public class SearchActionMediumTest {
   // ruleDao.insertOrUpdate(dbSession, ruleDto.getMetadata().setRuleId(ruleDto.getId()));
   // dbSession.commit();
   //
-  // WsTester.TestRequest request = tester.wsTester()
+  // WsTester.TestRequest request = esTester.wsTester()
   // .newGetRequest(API_ENDPOINT, API_SEARCH_METHOD)
   // .setParam(WebService.Param.FIELDS, "name,defaultDebtRemFn,debtRemFn,effortToFixDescription,debtOverloaded");
   // WsTester.Result result = request.execute();

server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java

@@ -739,7 +739,7 @@ public class SearchActionTest {
   @SafeVarargs
   private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer<RuleMetadataDto>... populaters) {
     RuleMetadataDto metadata = db.rules().insertOrUpdateMetadata(rule, organization, populaters);
-    ruleIndexer.indexRuleExtension(organization, rule.getKey());
+    ruleIndexer.commitAndIndex(db.getSession(), organization, rule.getKey());
     return metadata;
   }
 

server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionMediumTest.java

@@ -271,7 +271,7 @@ public class ShowActionMediumTest {
       .setUpdatedAt(new Date().getTime());
     ruleDao.insert(session, ruleDto);
     session.commit();
-    ruleIndexer.indexRuleDefinition(ruleDto.getKey());
+    ruleIndexer.commitAndIndex(session, ruleDto.getKey());
     RuleParamDto regexParam = RuleParamDto.createFor(ruleDto).setName("regex").setType("STRING").setDescription("Reg *exp*").setDefaultValue(".*");
     ruleDao.insertRuleParam(session, ruleDto, regexParam);
 

server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java

@@ -230,14 +230,14 @@ public class ShowActionTest {
 
   private RuleDefinitionDto insertRule() {
     RuleDefinitionDto rule = dbTester.rules().insert();
-    ruleIndexer.indexRuleDefinition(rule.getKey());
+    ruleIndexer.commitAndIndex(dbTester.getSession(), rule.getKey());
     return rule;
   }
 
   @SafeVarargs
   private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer<RuleMetadataDto>... populaters) {
     RuleMetadataDto metadata = dbTester.rules().insertOrUpdateMetadata(rule, organization, populaters);
-    ruleIndexer.indexRuleExtension(organization, rule.getKey());
+    ruleIndexer.commitAndIndex(dbTester.getSession(), organization, rule.getKey());
     return metadata;
   }
 }

server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java

@@ -99,7 +99,7 @@ public class TagsActionTest {
   @Test
   public void return_system_tag() throws Exception {
     RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag"));
-    ruleIndexer.indexRuleDefinition(r.getKey());
+    ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
 
     String result = tester.newRequest().execute().getInput();
     assertJson(result).isSimilarTo("{\"tags\":[\"tag\"]}");
@@ -108,9 +108,9 @@ public class TagsActionTest {
   @Test
   public void return_tag() throws Exception {
     RuleDefinitionDto r = dbTester.rules().insert(setSystemTags());
-    ruleIndexer.indexRuleDefinition(r.getKey());
+    ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
     dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag"));
-    ruleIndexer.indexRuleExtension(organization, r.getKey());
+    ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
 
     String result = tester.newRequest().setParam("organization", organization.getKey()).execute().getInput();
     assertJson(result).isSimilarTo("{\"tags\":[\"tag\"]}");

tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java

@@ -92,8 +92,7 @@ public class UserEsResilienceTest {
   public void creation_and_update_of_user_are_resilient_to_indexing_crash() throws Exception {
     String login = "crash";
 
-    // creation of user succeeds but index is not up-to-date (indexing
-    // crashes are not propagated to web services)
+    // creation of user succeeds in db but indexing crashes --> ws fails
     expectHttpError(500, () -> tester.users().generate(u -> u.setLogin(login)));
 
     // user exists in db, it can't be created again.