SONAR-9481 Make operations on QProfiles resilient to ES errors

server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl

@@ -663,6 +663,8 @@ CREATE TABLE "ES_QUEUE" (
   "UUID" VARCHAR(40) NOT NULL PRIMARY KEY,
   "DOC_TYPE" VARCHAR(40) NOT NULL,
   "DOC_ID" VARCHAR(4000) NOT NULL,
+  "DOC_ID_TYPE" VARCHAR(20),
+  "DOC_ROUTING" VARCHAR(4000),
   "CREATED_AT" BIGINT NOT NULL
 );
 CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID");

server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java

@@ -19,15 +19,20 @@
  */
 package org.sonar.db.es;
 
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
 public final class EsQueueDto {
 
   public enum Type {
-    USER, RULE, RULE_EXTENSION
+    USER, RULE, RULE_EXTENSION, ACTIVE_RULE
   }
 
   private String uuid;
   private Type docType;
   private String docId;
+  private String docIdType;
+  private String docRouting;
 
   public String getUuid() {
     return uuid;
@@ -56,12 +61,34 @@ public final class EsQueueDto {
     return this;
   }
 
+  @CheckForNull
+  public String getDocIdType() {
+    return docIdType;
+  }
+
+  private EsQueueDto setDocIdType(@Nullable String s) {
+    this.docIdType = s;
+    return this;
+  }
+
+  @CheckForNull
+  public String getDocRouting() {
+    return docRouting;
+  }
+
+  private EsQueueDto setDocRouting(@Nullable String s) {
+    this.docRouting = s;
+    return this;
+  }
+
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder("EsQueueDto{");
     sb.append("uuid='").append(uuid).append('\'');
     sb.append(", docType=").append(docType);
     sb.append(", docId='").append(docId).append('\'');
+    sb.append(", docIdType='").append(docIdType).append('\'');
+    sb.append(", docRouting='").append(docRouting).append('\'');
     sb.append('}');
     return sb.toString();
   }
@@ -88,4 +115,9 @@ public final class EsQueueDto {
   public static EsQueueDto create(Type docType, String docUuid) {
     return new EsQueueDto().setDocType(docType).setDocId(docUuid);
   }
+
+  public static EsQueueDto create(Type docType, String docId, @Nullable String docIdType, @Nullable String docRouting) {
+    return new EsQueueDto().setDocType(docType)
+      .setDocId(docId).setDocIdType(docIdType).setDocRouting(docRouting);
+  }
 }

server/sonar-db-dao/src/main/java/org/sonar/db/qualityprofile/ActiveRuleDao.java

@@ -24,6 +24,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.Consumer;
 import org.sonar.db.Dao;
 import org.sonar.db.DatabaseUtils;
 import org.sonar.db.DbSession;
@@ -31,6 +32,7 @@ import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.rule.RuleParamDto;
 
 import static org.sonar.db.DatabaseUtils.executeLargeInputs;
+import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput;
 import static org.sonar.db.KeyLongValue.toMap;
 
 public class ActiveRuleDao implements Dao {
@@ -170,6 +172,30 @@ public class ActiveRuleDao implements Dao {
       partition -> mapper(dbSession).countActiveRulesByQuery(query.getOrganization().getUuid(), partition, query.getRuleStatus(), query.getInheritance())));
   }
 
+  public void scrollAllForIndexing(DbSession dbSession, Consumer<IndexedActiveRuleDto> consumer) {
+    mapper(dbSession).scrollAllForIndexing(context -> {
+      IndexedActiveRuleDto dto = (IndexedActiveRuleDto) context.getResultObject();
+      consumer.accept(dto);
+    });
+  }
+
+  public void scrollByIdsForIndexing(DbSession dbSession, Collection<Long> ids, Consumer<IndexedActiveRuleDto> consumer) {
+    ActiveRuleMapper mapper = mapper(dbSession);
+    executeLargeInputsWithoutOutput(ids,
+      pageOfIds -> mapper
+        .scrollByIdsForIndexing(pageOfIds, context -> {
+          IndexedActiveRuleDto dto = (IndexedActiveRuleDto) context.getResultObject();
+          consumer.accept(dto);
+        }));
+  }
+
+  public void scrollByRuleProfileForIndexing(DbSession dbSession, String ruleProfileUuid, Consumer<IndexedActiveRuleDto> consumer) {
+    mapper(dbSession).scrollByRuleProfileUuidForIndexing(ruleProfileUuid, context -> {
+      IndexedActiveRuleDto dto = (IndexedActiveRuleDto) context.getResultObject();
+      consumer.accept(dto);
+    });
+  }
+
   private static ActiveRuleMapper mapper(DbSession dbSession) {
     return dbSession.getMapper(ActiveRuleMapper.class);
   }

server/sonar-db-dao/src/main/java/org/sonar/db/qualityprofile/ActiveRuleMapper.java

@@ -24,6 +24,7 @@ import java.util.List;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
 import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.session.ResultHandler;
 import org.sonar.api.rule.RuleStatus;
 import org.sonar.db.KeyLongValue;
 
@@ -73,4 +74,9 @@ public interface ActiveRuleMapper {
   List<KeyLongValue> countActiveRulesByQuery(@Param("organizationUuid") String organizationUuid, @Param("profileUuids") List<String> profileUuids,
     @Nullable @Param("ruleStatus") RuleStatus ruleStatus, @Param("inheritance") String inheritance);
 
+  void scrollAllForIndexing(ResultHandler handler);
+
+  void scrollByIdsForIndexing(@Param("ids") Collection<Long> ids, ResultHandler handler);
+
+  void scrollByRuleProfileUuidForIndexing(@Param("ruleProfileUuid") String ruleProfileUuid, ResultHandler handler);
 }

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDocWithSystemScope.java → server/sonar-db-dao/src/main/java/org/sonar/db/qualityprofile/IndexedActiveRuleDto.java

@@ -17,24 +17,40 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
+package org.sonar.db.qualityprofile;
 
-package org.sonar.server.rule.index;
+import javax.annotation.CheckForNull;
 
-public class RuleDocWithSystemScope {
+public class IndexedActiveRuleDto {
+  private long id;
+  private int severity;
+  private String inheritance;
+  private String repository;
+  private String key;
+  private String ruleProfileUuid;
 
-  private final RuleDoc ruleDoc;
-  private final RuleExtensionDoc ruleExtensionDoc;
+  public long getId() {
+    return id;
+  }
+
+  public int getSeverity() {
+    return severity;
+  }
+
+  @CheckForNull
+  public String getInheritance() {
+    return inheritance;
+  }
 
-  public RuleDocWithSystemScope(RuleDoc ruleDoc, RuleExtensionDoc ruleExtensionDoc) {
-    this.ruleDoc = ruleDoc;
-    this.ruleExtensionDoc = ruleExtensionDoc;
+  public String getRepository() {
+    return repository;
   }
 
-  public RuleDoc getRuleDoc() {
-    return ruleDoc;
+  public String getKey() {
+    return key;
   }
 
-  public RuleExtensionDoc getRuleExtensionDoc() {
-    return ruleExtensionDoc;
+  public String getRuleProfileUuid() {
+    return ruleProfileUuid;
   }
 }

server/sonar-db-dao/src/main/java/org/sonar/db/qualityprofile/QualityProfileDao.java

@@ -74,6 +74,11 @@ public class QualityProfileDao implements Dao {
     return mapper(dbSession).selectBuiltInRuleProfiles();
   }
 
+  @CheckForNull
+  public RulesProfileDto selectRuleProfile(DbSession dbSession, String ruleProfileUuid) {
+    return mapper(dbSession).selectRuleProfile(ruleProfileUuid);
+  }
+
   public void insert(DbSession dbSession, RulesProfileDto dto) {
     QualityProfileMapper mapper = mapper(dbSession);
     mapper.insertRuleProfile(dto, new Date(system.now()));

server/sonar-db-dao/src/main/java/org/sonar/db/qualityprofile/QualityProfileMapper.java

@@ -42,6 +42,9 @@ public interface QualityProfileMapper {
 
   List<RulesProfileDto> selectBuiltInRuleProfiles();
 
+  @CheckForNull
+  RulesProfileDto selectRuleProfile(@Param("uuid") String ruleProfileUuid);
+
   List<QProfileDto> selectOrderedByOrganizationUuid(@Param("organizationUuid") String organizationUuid);
 
   @CheckForNull

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java

@@ -25,12 +25,12 @@ import java.util.Optional;
 import java.util.function.Consumer;
 import javax.annotation.Nullable;
 import org.apache.ibatis.session.ResultHandler;
-import org.sonar.db.es.RuleExtensionId;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rules.RuleQuery;
 import org.sonar.db.Dao;
 import org.sonar.db.DbSession;
 import org.sonar.db.RowNotFoundException;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.db.organization.OrganizationDto;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -152,24 +152,38 @@ public class RuleDao implements Dao {
     }
   }
 
-  public void scrollRuleExtensionByRuleKeys(DbSession dbSession, Collection<RuleExtensionId> ruleExtensionIds, Consumer<RuleExtensionForIndexingDto> consumer) {
+  public void scrollIndexingRuleExtensionsByIds(DbSession dbSession, Collection<RuleExtensionId> ruleExtensionIds, Consumer<RuleExtensionForIndexingDto> consumer) {
     RuleMapper mapper = mapper(dbSession);
 
     executeLargeInputsWithoutOutput(ruleExtensionIds,
       pageOfRuleExtensionIds -> mapper
-        .selectRuleExtensionForIndexingByKeys(pageOfRuleExtensionIds)
+        .selectIndexingRuleExtensionsByIds(pageOfRuleExtensionIds)
         .forEach(consumer));
   }
 
-  public void scrollRuleByRuleKeys(DbSession dbSession, Collection<RuleKey> ruleKeys, Consumer<RuleForIndexingDto> consumer) {
+  public void scrollIndexingRuleExtensions(DbSession dbSession, Consumer<RuleExtensionForIndexingDto> consumer) {
+    mapper(dbSession).scrollIndexingRuleExtensions(context -> {
+      RuleExtensionForIndexingDto dto = (RuleExtensionForIndexingDto) context.getResultObject();
+      consumer.accept(dto);
+    });
+  }
+
+  public void scrollIndexingRulesByKeys(DbSession dbSession, Collection<RuleKey> ruleKeys, Consumer<RuleForIndexingDto> consumer) {
     RuleMapper mapper = mapper(dbSession);
 
     executeLargeInputsWithoutOutput(ruleKeys,
       pageOfRuleKeys -> mapper
-        .selectRuleForIndexingByKeys(pageOfRuleKeys)
+        .selectIndexingRulesByKeys(pageOfRuleKeys)
         .forEach(consumer));
   }
 
+  public void scrollIndexingRules(DbSession dbSession, Consumer<RuleForIndexingDto> consumer) {
+    mapper(dbSession).scrollIndexingRules(context -> {
+      RuleForIndexingDto dto = (RuleForIndexingDto) context.getResultObject();
+      consumer.accept(dto);
+    });
+  }
+
   private static RuleMapper mapper(DbSession session) {
     return session.getMapper(RuleMapper.class);
   }

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java

@@ -40,7 +40,7 @@ public class RuleForIndexingDto {
   private boolean isTemplate;
   private String systemTags;
   private String templateRuleKey;
-  private String templateName;
+  private String templateRepository;
   private String internalKey;
   private String language;
   private int type;
@@ -93,8 +93,8 @@ public class RuleForIndexingDto {
     return templateRuleKey;
   }
 
-  public String getTemplateName() {
-    return templateName;
+  public String getTemplateRepository() {
+    return templateRepository;
   }
 
   public String getInternalKey() {

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java

@@ -52,9 +52,13 @@ public interface RuleMapper {
 
   List<RuleDefinitionDto> selectDefinitionByKeys(@Param("ruleKeys") List<RuleKey> keys);
 
-  List<RuleForIndexingDto> selectRuleForIndexingByKeys(@Param("ruleKeys") List<RuleKey> keys);
+  void scrollIndexingRules(ResultHandler handler);
 
-  List<RuleExtensionForIndexingDto> selectRuleExtensionForIndexingByKeys(@Param("ruleExtensionIds") List<RuleExtensionId> ruleExtensionIds);
+  List<RuleForIndexingDto> selectIndexingRulesByKeys(@Param("ruleKeys") List<RuleKey> keys);
+
+  void scrollIndexingRuleExtensions(ResultHandler handler);
+
+  List<RuleExtensionForIndexingDto> selectIndexingRuleExtensionsByIds(@Param("ruleExtensionIds") List<RuleExtensionId> ruleExtensionIds);
 
   List<RuleDto> selectByQuery(@Param("organizationUuid") String organizationUuid, @Param("query") RuleQuery ruleQuery);
 

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMetadataDto.java

@@ -135,6 +135,10 @@ public class RuleMetadataDto {
     return tags == null ? new HashSet<>() : new TreeSet<>(Arrays.asList(StringUtils.split(tags, ',')));
   }
 
+  String getTagsAsString() {
+    return tags;
+  }
+
   public RuleMetadataDto setTags(Set<String> tags) {
     String raw = tags.isEmpty() ? null : StringUtils.join(tags, ',');
     checkArgument(raw == null || raw.length() <= 4000, "Rule tags are too long: %s", raw);

server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml

@@ -7,6 +7,8 @@
     uuid,
     doc_type as docType,
     doc_id as docId,
+    doc_id_type as docIdType,
+    doc_routing as docRouting,
     created_at as createdAt
   </sql>
 
@@ -15,11 +17,15 @@
       uuid,
       doc_type,
       doc_id,
+      doc_id_type,
+      doc_routing,
       created_at
     ) values (
       #{dto.uuid, jdbcType=VARCHAR},
       #{dto.docType, jdbcType=VARCHAR},
       #{dto.docId, jdbcType=VARCHAR},
+      #{dto.docIdType, jdbcType=VARCHAR},
+      #{dto.docRouting, jdbcType=VARCHAR},
       #{now, jdbcType=BIGINT}
     )
   </insert>

server/sonar-db-dao/src/main/resources/org/sonar/db/qualityprofile/ActiveRuleMapper.xml

@@ -272,5 +272,32 @@
     group by oqp.uuid
   </select>
 
+  <select id="scrollAllForIndexing" resultType="org.sonar.db.qualityprofile.IndexedActiveRuleDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    <include refid="scrollAllForIndexingSql"/>
+  </select>
+
+  <select id="scrollByIdsForIndexing" parameterType="map" resultType="org.sonar.db.qualityprofile.IndexedActiveRuleDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    <include refid="scrollAllForIndexingSql"/>
+    where ar.id in
+    <foreach collection="ids" open="(" close=")" item="id" separator=",">#{id, jdbcType=BIGINT}</foreach>
+  </select>
+
+  <select id="scrollByRuleProfileUuidForIndexing" parameterType="String" resultType="org.sonar.db.qualityprofile.IndexedActiveRuleDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    <include refid="scrollAllForIndexingSql"/>
+    where rp.kee = #{ruleProfileUuid, jdbcType=VARCHAR}
+  </select>
+
+  <sql id="scrollAllForIndexingSql">
+    select
+    ar.id as "id",
+    ar.failure_level as "severity",
+    ar.inheritance as "inheritance",
+    r.plugin_name as "repository",
+    r.plugin_rule_key as "key",
+    rp.kee as "ruleProfileUuid"
+    from active_rules ar
+    inner join rules_profiles rp on rp.id = ar.profile_id
+    inner join rules r on r.id = ar.rule_id
+  </sql>
 </mapper>
 

server/sonar-db-dao/src/main/resources/org/sonar/db/qualityprofile/QualityProfileMapper.xml

@@ -109,6 +109,12 @@
     where rp.is_built_in = ${_true}
   </select>
 
+  <select id="selectRuleProfile" resultType="org.sonar.db.qualityprofile.RulesProfileDto">
+    select <include refid="ruleProfileColumns"/>
+    from rules_profiles rp
+    where rp.kee = #{uuid, jdbcType=VARCHAR}
+  </select>
+
   <select id="selectOrderedByOrganizationUuid" parameterType="map" resultType="org.sonar.db.qualityprofile.QProfileDto">
     select
     <include refid="qProfileColumns"/>

server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml

@@ -139,26 +139,45 @@
       and r.plugin_rule_key=#{rule,jdbcType=VARCHAR}
   </select>
 
-  <select id="selectRuleExtensionForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto">
-    select
-    r.plugin_name as "pluginName",
-    r.plugin_rule_key as "pluginRuleKey",
-    rm.organization_uuid as "organizationUuid",
-    rm.tags as "tags"
-    from
-    rules r
-    inner join
-    rules_metadata rm on rm.rule_id = r.id
-    where
-    rm.tags is not null and
-    rm.tags != '' and
+  <select id="selectIndexingRuleExtensionsByIds" parameterType="map" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto">
+    <include refid="sqlSelectIndexingRuleExtensions" />
+    and
     <foreach collection="ruleExtensionIds" index="index" item="ruleExtId" open="" separator=" or " close="">
-      ( r.plugin_name=#{ruleExtId.repositoryName,jdbcType=VARCHAR} and
-      r.plugin_rule_key=#{ruleExtId.ruleKey,jdbcType=VARCHAR} and
-      rm.organization_uuid=#{ruleExtId.organizationUuid,jdbcType=VARCHAR} )
+      ( r.plugin_name = #{ruleExtId.repositoryName, jdbcType=VARCHAR} and
+      r.plugin_rule_key = #{ruleExtId.ruleKey, jdbcType=VARCHAR} and
+      rm.organization_uuid = #{ruleExtId.organizationUuid, jdbcType=VARCHAR} )
     </foreach>
   </select>
 
+  <select id="scrollIndexingRuleExtensions" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    <include refid="sqlSelectIndexingRuleExtensions" />
+  </select>
+
+  <sql id="sqlSelectIndexingRuleExtensions">
+    select
+      r.plugin_name as "pluginName",
+      r.plugin_rule_key as "pluginRuleKey",
+      rm.organization_uuid as "organizationUuid",
+      rm.tags as "tags"
+    from rules r
+    inner join rules_metadata rm on rm.rule_id = r.id
+    where
+      rm.tags is not null and
+      rm.tags != ''
+  </sql>
+
+  <sql id="sqlSelectIndexingRuleExtensions" databaseId="oracle">
+    select
+      r.plugin_name as "pluginName",
+      r.plugin_rule_key as "pluginRuleKey",
+      rm.organization_uuid as "organizationUuid",
+      rm.tags as "tags"
+    from rules r
+    inner join rules_metadata rm on rm.rule_id = r.id
+    where
+      rm.tags is not null
+  </sql>
+
   <select id="selectMetadataByKey" parameterType="map" resultType="org.sonar.db.rule.RuleMetadataDto">
     select
       rm.rule_id as "ruleId",
@@ -205,7 +224,19 @@
       </foreach>
   </select>
 
-  <select id="selectRuleForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleForIndexingDto">
+  <select id="selectIndexingRulesByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleForIndexingDto">
+    <include refid="sqlSelectIndexingRules"/>
+    where
+      <foreach collection="ruleKeys" index="index" item="ruleKey" open="" separator=" or " close="">
+        (r.plugin_name=#{ruleKey.repository,jdbcType=VARCHAR} and r.plugin_rule_key=#{ruleKey.rule,jdbcType=VARCHAR})
+      </foreach>
+  </select>
+
+  <select id="scrollIndexingRules" resultType="org.sonar.db.rule.RuleForIndexingDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+    <include refid="sqlSelectIndexingRules"/>
+  </select>
+
+  <sql id="sqlSelectIndexingRules">
     select
       r.id as "id",
       r.plugin_name as "repository",
@@ -218,20 +249,15 @@
       r.is_template as "isTemplate",
       r.system_tags as "systemTags",
       t.plugin_rule_key as "templateRuleKey",
-      t.plugin_name as "templateName",
+      t.plugin_name as "templateRepository",
       r.plugin_config_key as "internalKey",
       r.language as "language",
       r.rule_type as "type",
       r.created_at as "createdAt",
       r.updated_at as "updatedAt"
-    from
-      rules r
+    from rules r
     left outer join rules t on t.id = r.template_id
-    where
-      <foreach collection="ruleKeys" index="index" item="ruleKey" open="" separator=" or " close="">
-        (r.plugin_name=#{ruleKey.repository,jdbcType=VARCHAR} and r.plugin_rule_key=#{ruleKey.rule,jdbcType=VARCHAR})
-      </foreach>
-  </select>
+  </sql>
 
   <select id="selectByQuery" parameterType="map" resultType="Rule">
     select

server/sonar-db-dao/src/test/java/org/sonar/db/qualityprofile/ActiveRuleDaoTest.java

@@ -19,8 +19,10 @@
  */
 package org.sonar.db.qualityprofile;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.function.Consumer;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -578,4 +580,68 @@ public class ActiveRuleDaoTest {
       .containsOnly(entry(profile1.getKee(), 1L));
   }
 
+  @Test
+  public void scrollAllForIndexing_empty_table() {
+    Accumulator accumulator = new Accumulator();
+    underTest.scrollAllForIndexing(dbSession, accumulator);
+    assertThat(accumulator.list).isEmpty();
+  }
+
+  @Test
+  public void scrollAllForIndexing() {
+    ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+    ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+    ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+
+    Accumulator accumulator = new Accumulator();
+    underTest.scrollAllForIndexing(dbSession, accumulator);
+    assertThat(accumulator.list)
+      .extracting(IndexedActiveRuleDto::getId, IndexedActiveRuleDto::getRepository, IndexedActiveRuleDto::getKey, IndexedActiveRuleDto::getRuleProfileUuid,
+        IndexedActiveRuleDto::getSeverity, IndexedActiveRuleDto::getInheritance)
+      .containsExactlyInAnyOrder(
+        tuple((long)ar1.getId(), ar1.getRuleKey().repository(), ar1.getRuleKey().rule(), profile1.getRulesProfileUuid(), ar1.getSeverity(), ar1.getInheritance()),
+        tuple((long)ar2.getId(), ar2.getRuleKey().repository(), ar2.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar2.getSeverity(), ar2.getInheritance()),
+        tuple((long)ar3.getId(), ar3.getRuleKey().repository(), ar3.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar3.getSeverity(), ar3.getInheritance()));
+  }
+
+  @Test
+  public void scrollByIdsForIndexing() {
+    ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+    ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+    ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+
+    Accumulator accumulator = new Accumulator();
+    underTest.scrollByIdsForIndexing(dbSession, asList((long)ar1.getId(), (long)ar2.getId()), accumulator);
+    assertThat(accumulator.list)
+      .extracting(IndexedActiveRuleDto::getId, IndexedActiveRuleDto::getRepository, IndexedActiveRuleDto::getKey, IndexedActiveRuleDto::getRuleProfileUuid,
+        IndexedActiveRuleDto::getSeverity)
+      .containsExactlyInAnyOrder(
+        tuple((long)ar1.getId(), ar1.getRuleKey().repository(), ar1.getRuleKey().rule(), profile1.getRulesProfileUuid(), ar1.getSeverity()),
+        tuple((long)ar2.getId(), ar2.getRuleKey().repository(), ar2.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar2.getSeverity()));
+  }
+
+  @Test
+  public void scrollByRuleProfileForIndexing() {
+    ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+    ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+    ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+
+    Accumulator accumulator = new Accumulator();
+    underTest.scrollByRuleProfileForIndexing(dbSession, profile2.getRulesProfileUuid(), accumulator);
+    assertThat(accumulator.list)
+      .extracting(IndexedActiveRuleDto::getId, IndexedActiveRuleDto::getRepository, IndexedActiveRuleDto::getKey, IndexedActiveRuleDto::getRuleProfileUuid,
+        IndexedActiveRuleDto::getSeverity)
+      .containsExactlyInAnyOrder(
+        tuple((long)ar2.getId(), ar2.getRuleKey().repository(), ar2.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar2.getSeverity()),
+        tuple((long)ar3.getId(), ar3.getRuleKey().repository(), ar3.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar3.getSeverity()));
+  }
+
+  private static class Accumulator implements Consumer<IndexedActiveRuleDto> {
+    private final List<IndexedActiveRuleDto> list = new ArrayList<>();
+
+    @Override
+    public void accept(IndexedActiveRuleDto dto) {
+      list.add(dto);
+    }
+  }
 }

server/sonar-db-dao/src/test/java/org/sonar/db/qualityprofile/QualityProfileDaoTest.java

@@ -147,6 +147,14 @@ public class QualityProfileDaoTest {
     assertThat(reloaded.isBuiltIn()).isEqualTo(update.isBuiltIn());
   }
 
+  @Test
+  public void selectRuleProfile() {
+    RulesProfileDto rp = insertRulesProfile();
+
+    assertThat(underTest.selectRuleProfile(dbSession, rp.getKee()).getId()).isEqualTo(rp.getId());
+    assertThat(underTest.selectRuleProfile(dbSession, "missing")).isNull();
+  }
+
   @Test
   public void deleteRulesProfilesByUuids() {
     RulesProfileDto rp1 = insertRulesProfile();
@@ -245,7 +253,7 @@ public class QualityProfileDaoTest {
   }
 
   @Test
-  public void find_all_is_sorted_by_profile_name() {
+  public void selectOrderedByOrganizationUuid_is_sorted_by_profile_name() {
     QProfileDto dto1 = new QProfileDto()
       .setKee("js_first")
       .setRulesProfileUuid("rp-js_first")
@@ -315,7 +323,7 @@ public class QualityProfileDaoTest {
   }
 
   @Test
-  public void get_by_name_and_language() {
+  public void selectByNameAndLanguage() {
     List<QProfileDto> sharedData = createSharedData();
 
     QProfileDto dto = underTest.selectByNameAndLanguage(dbSession, organization, "Sonar Way", "java");
@@ -328,7 +336,7 @@ public class QualityProfileDaoTest {
   }
 
   @Test
-  public void get_by_name_and_languages() {
+  public void selectByNameAndLanguages() {
     createSharedData();
 
     List<QProfileDto> dtos = underTest.selectByNameAndLanguages(dbSession, organization, "Sonar Way", singletonList("java"));
@@ -362,7 +370,7 @@ public class QualityProfileDaoTest {
   }
 
   @Test
-  public void should_not_find_by_language_in_wrong_organization() {
+  public void should_not_selectByLanguage_in_wrong_organization() {
     QProfileDto profile = QualityProfileTesting.newQualityProfileDto()
       .setOrganizationUuid(organization.getUuid());
     underTest.insert(dbSession, profile);
@@ -372,7 +380,7 @@ public class QualityProfileDaoTest {
   }
 
   @Test
-  public void should_not_find_by_language_with_wrong_language() {
+  public void should_not_selectByLanguage_with_wrong_language() {
     QProfileDto profile = QualityProfileTesting.newQualityProfileDto()
       .setOrganizationUuid(organization.getUuid());
     underTest.insert(dbSession, profile);
@@ -382,7 +390,7 @@ public class QualityProfileDaoTest {
   }
 
   @Test
-  public void find_children() {
+  public void selectChildren() {
     QProfileDto original1 = new QProfileDto()
       .setKee("java_child1")
       .setRulesProfileUuid("rp-java_child1")

server/sonar-db-dao/src/test/java/org/sonar/db/rule/RuleDaoTest.java

@@ -25,6 +25,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Consumer;
 import org.apache.ibatis.session.ResultHandler;
 import org.junit.Before;
 import org.junit.Rule;
@@ -40,6 +41,7 @@ import org.sonar.api.utils.DateUtils;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbTester;
 import org.sonar.db.RowNotFoundException;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.organization.OrganizationTesting;
 
@@ -47,6 +49,7 @@ import static com.google.common.collect.Sets.newHashSet;
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
 
 public class RuleDaoTest {
 
@@ -55,219 +58,219 @@ public class RuleDaoTest {
   @Rule
   public ExpectedException thrown = ExpectedException.none();
   @Rule
-  public DbTester dbTester = DbTester.create(System2.INSTANCE);
+  public DbTester db = DbTester.create(System2.INSTANCE);
 
-  private RuleDao underTest = dbTester.getDbClient().ruleDao();
+  private RuleDao underTest = db.getDbClient().ruleDao();
   private OrganizationDto organization;
 
   @Before
   public void before() {
-    organization = dbTester.organizations().insert(o -> o.setUuid(ORGANIZATION_UUID));
+    organization = db.organizations().insert(o -> o.setUuid(ORGANIZATION_UUID));
   }
 
   @Test
   public void selectByKey() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    assertThat(underTest.selectByKey(dbTester.getSession(), organization, RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
+    assertThat(underTest.selectByKey(db.getSession(), organization, RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
 
-    Optional<RuleDto> rule = underTest.selectByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001"));
+    Optional<RuleDto> rule = underTest.selectByKey(db.getSession(), organization, RuleKey.of("java", "S001"));
     assertThat(rule.isPresent()).isTrue();
     assertThat(rule.get().getId()).isEqualTo(1);
   }
 
   @Test
   public void selectByKey_populates_organizationUuid_even_when_organization_has_no_metadata() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    assertThat(underTest.selectByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001")).get().getOrganizationUuid())
+    assertThat(underTest.selectByKey(db.getSession(), organization, RuleKey.of("java", "S001")).get().getOrganizationUuid())
       .isEqualTo(ORGANIZATION_UUID);
   }
 
   @Test
   public void selectDefinitionByKey() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    assertThat(underTest.selectDefinitionByKey(dbTester.getSession(), RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
+    assertThat(underTest.selectDefinitionByKey(db.getSession(), RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
 
-    Optional<RuleDefinitionDto> rule = underTest.selectDefinitionByKey(dbTester.getSession(), RuleKey.of("java", "S001"));
+    Optional<RuleDefinitionDto> rule = underTest.selectDefinitionByKey(db.getSession(), RuleKey.of("java", "S001"));
     assertThat(rule.isPresent()).isTrue();
     assertThat(rule.get().getId()).isEqualTo(1);
   }
 
   @Test
   public void selectById() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectById(55l, organizationUuid, dbTester.getSession())).isEmpty();
-    Optional<RuleDto> ruleDtoOptional = underTest.selectById(1l, organizationUuid, dbTester.getSession());
+    assertThat(underTest.selectById(55l, organizationUuid, db.getSession())).isEmpty();
+    Optional<RuleDto> ruleDtoOptional = underTest.selectById(1l, organizationUuid, db.getSession());
     assertThat(ruleDtoOptional).isPresent();
     assertThat(ruleDtoOptional.get().getId()).isEqualTo(1);
   }
 
   @Test
   public void selectById_populates_organizationUuid_even_when_organization_has_no_metadata() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectById(1l, organizationUuid, dbTester.getSession()).get().getOrganizationUuid())
+    assertThat(underTest.selectById(1l, organizationUuid, db.getSession()).get().getOrganizationUuid())
       .isEqualTo(organizationUuid);
   }
 
   @Test
   public void selectDefinitionById() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    assertThat(underTest.selectDefinitionById(55l, dbTester.getSession())).isEmpty();
-    Optional<RuleDefinitionDto> ruleDtoOptional = underTest.selectDefinitionById(1l, dbTester.getSession());
+    assertThat(underTest.selectDefinitionById(55l, db.getSession())).isEmpty();
+    Optional<RuleDefinitionDto> ruleDtoOptional = underTest.selectDefinitionById(1l, db.getSession());
     assertThat(ruleDtoOptional).isPresent();
     assertThat(ruleDtoOptional.get().getId()).isEqualTo(1);
   }
 
   @Test
   public void selectByIds() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1))).hasSize(1);
-    assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1, 2))).hasSize(2);
-    assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1, 2, 3))).hasSize(2);
+    assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1))).hasSize(1);
+    assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1, 2))).hasSize(2);
+    assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1, 2, 3))).hasSize(2);
 
-    assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(123))).isEmpty();
+    assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(123))).isEmpty();
   }
 
   @Test
   public void selectByIds_populates_organizationUuid_even_when_organization_has_no_metadata() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1, 2)))
+    assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1, 2)))
       .extracting(RuleDto::getOrganizationUuid)
       .containsExactly(organizationUuid, organizationUuid);
   }
 
   @Test
   public void selectDefinitionByIds() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(1))).hasSize(1);
-    assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(1, 2))).hasSize(2);
-    assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(1, 2, 3))).hasSize(2);
+    assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(1))).hasSize(1);
+    assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(1, 2))).hasSize(2);
+    assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(1, 2, 3))).hasSize(2);
 
-    assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(123))).isEmpty();
+    assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(123))).isEmpty();
   }
 
   @Test
   public void selectOrFailByKey() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
     OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid("org-1");
-    RuleDto rule = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001"));
+    RuleDto rule = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("java", "S001"));
     assertThat(rule.getId()).isEqualTo(1);
   }
 
   @Test
   public void selectOrFailByKey_fails_if_rule_not_found() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
     thrown.expect(RowNotFoundException.class);
     thrown.expectMessage("Rule with key 'NOT:FOUND' does not exist");
 
     OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid("org-1");
-    underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("NOT", "FOUND"));
+    underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("NOT", "FOUND"));
   }
 
   @Test
   public void selectOrFailByKey_populates_organizationUuid_even_when_organization_has_no_metadata() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
     OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid(organizationUuid);
-    assertThat(underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001")).getOrganizationUuid())
+    assertThat(underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("java", "S001")).getOrganizationUuid())
       .isEqualTo(organizationUuid);
   }
 
   @Test
   public void selectOrFailDefinitionByKey_fails_if_rule_not_found() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
     thrown.expect(RowNotFoundException.class);
     thrown.expectMessage("Rule with key 'NOT:FOUND' does not exist");
 
-    underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("NOT", "FOUND"));
+    underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("NOT", "FOUND"));
   }
 
   @Test
   public void selectByKeys() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectByKeys(dbTester.getSession(), organizationUuid, Collections.emptyList())).isEmpty();
-    assertThat(underTest.selectByKeys(dbTester.getSession(), organizationUuid, asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
+    assertThat(underTest.selectByKeys(db.getSession(), organizationUuid, Collections.emptyList())).isEmpty();
+    assertThat(underTest.selectByKeys(db.getSession(), organizationUuid, asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
 
-    List<RuleDto> rules = underTest.selectByKeys(dbTester.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
+    List<RuleDto> rules = underTest.selectByKeys(db.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
     assertThat(rules).hasSize(1);
     assertThat(rules.get(0).getId()).isEqualTo(1);
   }
 
   @Test
   public void selectByKeys_populates_organizationUuid_even_when_organization_has_no_metadata() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectByKeys(dbTester.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER"))))
+    assertThat(underTest.selectByKeys(db.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER"))))
       .extracting(RuleDto::getOrganizationUuid)
       .containsExactly(organizationUuid);
   }
 
   @Test
   public void selectDefinitionByKeys() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    assertThat(underTest.selectDefinitionByKeys(dbTester.getSession(), Collections.emptyList())).isEmpty();
-    assertThat(underTest.selectDefinitionByKeys(dbTester.getSession(), asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
+    assertThat(underTest.selectDefinitionByKeys(db.getSession(), Collections.emptyList())).isEmpty();
+    assertThat(underTest.selectDefinitionByKeys(db.getSession(), asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
 
-    List<RuleDefinitionDto> rules = underTest.selectDefinitionByKeys(dbTester.getSession(), asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
+    List<RuleDefinitionDto> rules = underTest.selectDefinitionByKeys(db.getSession(), asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
     assertThat(rules).hasSize(1);
     assertThat(rules.get(0).getId()).isEqualTo(1);
   }
 
   @Test
   public void selectAll() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    assertThat(underTest.selectAll(dbTester.getSession(), "org-1"))
+    assertThat(underTest.selectAll(db.getSession(), "org-1"))
       .extracting(RuleDto::getId)
       .containsOnly(1, 2, 10);
   }
 
   @Test
   public void selectAll_populates_organizationUuid_even_when_organization_has_no_metadata() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectAll(dbTester.getSession(), organizationUuid))
+    assertThat(underTest.selectAll(db.getSession(), organizationUuid))
       .extracting(RuleDto::getOrganizationUuid)
       .containsExactly(organizationUuid, organizationUuid, organizationUuid);
   }
 
   @Test
   public void selectAllDefinitions() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
-    List<RuleDefinitionDto> ruleDtos = underTest.selectAllDefinitions(dbTester.getSession());
+    List<RuleDefinitionDto> ruleDtos = underTest.selectAllDefinitions(db.getSession());
 
     assertThat(ruleDtos).extracting("id").containsOnly(1, 2, 10);
   }
 
   @Test
   public void selectEnabled_with_ResultHandler() {
-    dbTester.prepareDbUnit(getClass(), "selectEnabled.xml");
+    db.prepareDbUnit(getClass(), "selectEnabled.xml");
 
     final List<RuleDefinitionDto> rules = new ArrayList<>();
     ResultHandler resultHandler = resultContext -> rules.add((RuleDefinitionDto) resultContext.getResultObject());
-    underTest.selectEnabled(dbTester.getSession(), resultHandler);
+    underTest.selectEnabled(db.getSession(), resultHandler);
 
     assertThat(rules.size()).isEqualTo(1);
     RuleDefinitionDto ruleDto = rules.get(0);
@@ -285,23 +288,23 @@ public class RuleDaoTest {
 
   @Test
   public void select_by_query() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
 
     String organizationUuid = "org-1";
-    assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create())).hasSize(2);
-    assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create().withKey("S001"))).hasSize(1);
-    assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create().withConfigKey("S1"))).hasSize(1);
-    assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create().withRepositoryKey("java"))).hasSize(2);
-    assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid,
+    assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create())).hasSize(2);
+    assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create().withKey("S001"))).hasSize(1);
+    assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create().withConfigKey("S1"))).hasSize(1);
+    assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create().withRepositoryKey("java"))).hasSize(2);
+    assertThat(underTest.selectByQuery(db.getSession(), organizationUuid,
       RuleQuery.create().withKey("S001").withConfigKey("S1").withRepositoryKey("java"))).hasSize(1);
   }
 
   @Test
   public void select_by_query_populates_organizationUuid_even_when_organization_has_no_metadata() {
-    dbTester.prepareDbUnit(getClass(), "shared.xml");
+    db.prepareDbUnit(getClass(), "shared.xml");
     String organizationUuid = "org-1";
 
-    assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create()))
+    assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create()))
       .extracting(RuleDto::getOrganizationUuid)
       .containsExactly(organizationUuid, organizationUuid);
   }
@@ -328,10 +331,10 @@ public class RuleDaoTest {
       .setType(RuleType.BUG)
       .setCreatedAt(1_500_000_000_000L)
       .setUpdatedAt(2_000_000_000_000L);
-    underTest.insert(dbTester.getSession(), newRule);
-    dbTester.getSession().commit();
+    underTest.insert(db.getSession(), newRule);
+    db.getSession().commit();
 
-    RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("plugin", "NewRuleKey"));
+    RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("plugin", "NewRuleKey"));
     assertThat(ruleDto.getId()).isNotNull();
     assertThat(ruleDto.getName()).isEqualTo("new name");
     assertThat(ruleDto.getDescription()).isEqualTo("new description");
@@ -356,7 +359,7 @@ public class RuleDaoTest {
 
   @Test
   public void update_RuleDefinitionDto() {
-    dbTester.prepareDbUnit(getClass(), "update.xml");
+    db.prepareDbUnit(getClass(), "update.xml");
 
     RuleDefinitionDto ruleToUpdate = new RuleDefinitionDto()
       .setId(1)
@@ -379,10 +382,10 @@ public class RuleDaoTest {
       .setType(RuleType.BUG)
       .setUpdatedAt(2_000_000_000_000L);
 
-    underTest.update(dbTester.getSession(), ruleToUpdate);
-    dbTester.getSession().commit();
+    underTest.update(db.getSession(), ruleToUpdate);
+    db.getSession().commit();
 
-    RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("plugin", "NewRuleKey"));
+    RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("plugin", "NewRuleKey"));
     assertThat(ruleDto.getName()).isEqualTo("new name");
     assertThat(ruleDto.getDescription()).isEqualTo("new description");
     assertThat(ruleDto.getDescriptionFormat()).isEqualTo(RuleDto.Format.MARKDOWN);
@@ -406,7 +409,7 @@ public class RuleDaoTest {
 
   @Test
   public void update_RuleMetadataDto_inserts_row_in_RULE_METADATA_if_not_exists_yet() {
-    dbTester.prepareDbUnit(getClass(), "update.xml");
+    db.prepareDbUnit(getClass(), "update.xml");
     String organizationUuid = "org-1";
 
     RuleMetadataDto metadataToUpdate = new RuleMetadataDto()
@@ -423,11 +426,11 @@ public class RuleDaoTest {
       .setCreatedAt(3_500_000_000_000L)
       .setUpdatedAt(4_000_000_000_000L);
 
-    underTest.insertOrUpdate(dbTester.getSession(), metadataToUpdate);
-    dbTester.getSession().commit();
+    underTest.insertOrUpdate(db.getSession(), metadataToUpdate);
+    db.getSession().commit();
 
     OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid(organizationUuid);
-    RuleDto ruleDto = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
+    RuleDto ruleDto = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
     assertThat(ruleDto.getName()).isEqualTo("Avoid Null");
     assertThat(ruleDto.getDescription()).isEqualTo("Should avoid NULL");
     assertThat(ruleDto.getDescriptionFormat()).isNull();
@@ -459,7 +462,7 @@ public class RuleDaoTest {
 
   @Test
   public void update_RuleMetadataDto_updates_row_in_RULE_METADATA_if_already_exists() {
-    dbTester.prepareDbUnit(getClass(), "update.xml");
+    db.prepareDbUnit(getClass(), "update.xml");
     String organizationUuid = "org-1";
     OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid(organizationUuid);
     RuleMetadataDto metadataV1 = new RuleMetadataDto()
@@ -481,11 +484,11 @@ public class RuleDaoTest {
       .setCreatedAt(6_500_000_000_000L)
       .setUpdatedAt(7_000_000_000_000L);
 
-    underTest.insertOrUpdate(dbTester.getSession(), metadataV1);
-    dbTester.commit();
+    underTest.insertOrUpdate(db.getSession(), metadataV1);
+    db.commit();
 
-    assertThat(dbTester.countRowsOfTable("RULES_METADATA")).isEqualTo(1);
-    RuleDto ruleDto = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
+    assertThat(db.countRowsOfTable("RULES_METADATA")).isEqualTo(1);
+    RuleDto ruleDto = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
     assertThat(ruleDto.getName()).isEqualTo("Avoid Null");
     assertThat(ruleDto.getDescription()).isEqualTo("Should avoid NULL");
     assertThat(ruleDto.getDescriptionFormat()).isNull();
@@ -514,10 +517,10 @@ public class RuleDaoTest {
     assertThat(ruleDto.getCreatedAt()).isEqualTo(3_500_000_000_000L);
     assertThat(ruleDto.getUpdatedAt()).isEqualTo(4_000_000_000_000L);
 
-    underTest.insertOrUpdate(dbTester.getSession(), metadataV2);
-    dbTester.commit();
+    underTest.insertOrUpdate(db.getSession(), metadataV2);
+    db.commit();
 
-    ruleDto = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
+    ruleDto = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
     assertThat(ruleDto.getName()).isEqualTo("Avoid Null");
     assertThat(ruleDto.getDescription()).isEqualTo("Should avoid NULL");
     assertThat(ruleDto.getDescriptionFormat()).isNull();
@@ -549,8 +552,8 @@ public class RuleDaoTest {
 
   @Test
   public void select_parameters_by_rule_key() {
-    dbTester.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
-    List<RuleParamDto> ruleDtos = underTest.selectRuleParamsByRuleKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
+    db.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
+    List<RuleParamDto> ruleDtos = underTest.selectRuleParamsByRuleKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
 
     assertThat(ruleDtos.size()).isEqualTo(1);
     RuleParamDto ruleDto = ruleDtos.get(0);
@@ -563,19 +566,19 @@ public class RuleDaoTest {
 
   @Test
   public void select_parameters_by_rule_keys() {
-    dbTester.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
+    db.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
 
-    assertThat(underTest.selectRuleParamsByRuleKeys(dbTester.getSession(),
+    assertThat(underTest.selectRuleParamsByRuleKeys(db.getSession(),
       Arrays.asList(RuleKey.of("checkstyle", "AvoidNull"), RuleKey.of("unused", "Unused")))).hasSize(2);
 
-    assertThat(underTest.selectRuleParamsByRuleKeys(dbTester.getSession(),
+    assertThat(underTest.selectRuleParamsByRuleKeys(db.getSession(),
       singletonList(RuleKey.of("unknown", "Unknown")))).isEmpty();
   }
 
   @Test
   public void insert_parameter() {
-    dbTester.prepareDbUnit(getClass(), "insert_parameter.xml");
-    RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("plugin", "NewRuleKey"));
+    db.prepareDbUnit(getClass(), "insert_parameter.xml");
+    RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("plugin", "NewRuleKey"));
 
     RuleParamDto param = RuleParamDto.createFor(rule1)
       .setName("max")
@@ -583,19 +586,19 @@ public class RuleDaoTest {
       .setDefaultValue("30")
       .setDescription("My Parameter");
 
-    underTest.insertRuleParam(dbTester.getSession(), rule1, param);
-    dbTester.getSession().commit();
+    underTest.insertRuleParam(db.getSession(), rule1, param);
+    db.getSession().commit();
 
-    dbTester.assertDbUnit(getClass(), "insert_parameter-result.xml", "rules_parameters");
+    db.assertDbUnit(getClass(), "insert_parameter-result.xml", "rules_parameters");
   }
 
   @Test
   public void update_parameter() {
-    dbTester.prepareDbUnit(getClass(), "update_parameter.xml");
+    db.prepareDbUnit(getClass(), "update_parameter.xml");
 
-    RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
+    RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
 
-    List<RuleParamDto> params = underTest.selectRuleParamsByRuleKey(dbTester.getSession(), rule1.getKey());
+    List<RuleParamDto> params = underTest.selectRuleParamsByRuleKey(db.getSession(), rule1.getKey());
     assertThat(params).hasSize(1);
 
     RuleParamDto param = Iterables.getFirst(params, null);
@@ -605,20 +608,107 @@ public class RuleDaoTest {
       .setDefaultValue("^[a-z]+(\\.[a-z][a-z0-9]*)*$")
       .setDescription("Regular expression used to check the package names against.");
 
-    underTest.updateRuleParam(dbTester.getSession(), rule1, param);
-    dbTester.getSession().commit();
+    underTest.updateRuleParam(db.getSession(), rule1, param);
+    db.getSession().commit();
 
-    dbTester.assertDbUnit(getClass(), "update_parameter-result.xml", "rules_parameters");
+    db.assertDbUnit(getClass(), "update_parameter-result.xml", "rules_parameters");
   }
 
   @Test
   public void delete_parameter() {
-    dbTester.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
-    assertThat(underTest.selectRuleParamsByRuleKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).hasSize(1);
+    db.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
+    assertThat(underTest.selectRuleParamsByRuleKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).hasSize(1);
 
-    underTest.deleteRuleParam(dbTester.getSession(), 1);
-    dbTester.getSession().commit();
+    underTest.deleteRuleParam(db.getSession(), 1);
+    db.getSession().commit();
 
-    assertThat(underTest.selectRuleParamsByRuleKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).isEmpty();
+    assertThat(underTest.selectRuleParamsByRuleKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).isEmpty();
+  }
+
+  @Test
+  public void scrollIndexingRules_on_empty_table() {
+    Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+
+    underTest.scrollIndexingRules(db.getSession(), accumulator);
+
+    assertThat(accumulator.list).isEmpty();
+  }
+
+  @Test
+  public void scrollIndexingRules() {
+    Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+    RuleDefinitionDto r1 = db.rules().insert();
+    RuleDefinitionDto r2 = db.rules().insert();
+
+    underTest.scrollIndexingRules(db.getSession(), accumulator);
+
+    assertThat(accumulator.list)
+      .extracting(RuleForIndexingDto::getId, RuleForIndexingDto::getRuleKey)
+      .containsExactlyInAnyOrder(tuple(r1.getId(), r1.getKey()), tuple(r2.getId(), r2.getKey()));
+  }
+
+  @Test
+  public void scrollIndexingRulesByKeys() {
+    Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+    RuleDefinitionDto r1 = db.rules().insert();
+    RuleDefinitionDto r2 = db.rules().insert();
+
+    underTest.scrollIndexingRulesByKeys(db.getSession(), singletonList(r1.getKey()), accumulator);
+
+    assertThat(accumulator.list)
+      .extracting(RuleForIndexingDto::getId, RuleForIndexingDto::getRuleKey)
+      .containsExactlyInAnyOrder(tuple(r1.getId(), r1.getKey()));
+  }
+
+  @Test
+  public void scrollIndexingRulesByKeys_scrolls_nothing_if_key_does_not_exist() {
+    Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+    RuleDefinitionDto r1 = db.rules().insert();
+
+    underTest.scrollIndexingRulesByKeys(db.getSession(), singletonList(RuleKey.of("does", "not exist")), accumulator);
+
+    assertThat(accumulator.list).isEmpty();
+  }
+
+  @Test
+  public void scrollIndexingRuleExtensions() {
+    Accumulator<RuleExtensionForIndexingDto> accumulator = new Accumulator<>();
+    RuleDefinitionDto r1 = db.rules().insert();
+    RuleMetadataDto r1Extension = db.rules().insertOrUpdateMetadata(r1, organization, r -> r.setTagsField("t1,t2"));
+    RuleDefinitionDto r2 = db.rules().insert();
+    RuleMetadataDto r2Extension = db.rules().insertOrUpdateMetadata(r2, organization, r -> r.setTagsField("t1,t3"));
+
+    underTest.scrollIndexingRuleExtensions(db.getSession(), accumulator);
+
+    assertThat(accumulator.list)
+      .extracting(RuleExtensionForIndexingDto::getRuleKey, RuleExtensionForIndexingDto::getOrganizationUuid, RuleExtensionForIndexingDto::getTags)
+      .containsExactlyInAnyOrder(
+        tuple(r1.getKey(), organization.getUuid(), r1Extension.getTagsAsString()),
+        tuple(r2.getKey(), organization.getUuid(), r2Extension.getTagsAsString()));
+  }
+
+  @Test
+  public void scrollIndexingRuleExtensionsByIds() {
+    Accumulator<RuleExtensionForIndexingDto> accumulator = new Accumulator<>();
+    RuleDefinitionDto r1 = db.rules().insert();
+    RuleMetadataDto r1Extension = db.rules().insertOrUpdateMetadata(r1, organization, r -> r.setTagsField("t1,t2"));
+    RuleExtensionId r1ExtensionId = new RuleExtensionId(organization.getUuid(), r1.getRepositoryKey(), r1.getRuleKey());
+    RuleDefinitionDto r2 = db.rules().insert();
+    RuleMetadataDto r2Extension = db.rules().insertOrUpdateMetadata(r2, organization, r -> r.setTagsField("t1,t3"));
+
+    underTest.scrollIndexingRuleExtensionsByIds(db.getSession(), singletonList(r1ExtensionId), accumulator);
+
+    assertThat(accumulator.list)
+      .extracting(RuleExtensionForIndexingDto::getRuleKey, RuleExtensionForIndexingDto::getOrganizationUuid, RuleExtensionForIndexingDto::getTags)
+      .containsExactlyInAnyOrder(
+        tuple(r1.getKey(), organization.getUuid(), r1Extension.getTagsAsString()));
+  }
+
+  private static class Accumulator<T>  implements Consumer<T> {
+    private final List<T> list = new ArrayList<>();
+    @Override
+    public void accept(T dto) {
+      list.add(dto);
+    }
   }
 }

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java

@@ -53,6 +53,16 @@ public class CreateEsQueueTable extends DdlChange {
         .setIsNullable(false)
         .setLimit(4000)
         .build())
+      .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
+        .setColumnName("doc_id_type")
+        .setIsNullable(true)
+        .setLimit(20)
+        .build())
+      .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
+        .setColumnName("doc_routing")
+        .setIsNullable(true)
+        .setLimit(4000)
+        .build())
       .addColumn(BigIntegerColumnDef.newBigIntegerColumnDefBuilder()
         .setColumnName("created_at")
         .setIsNullable(false)

server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java

@@ -49,6 +49,8 @@ public class CreateEsQueueTableTest {
     db.assertColumnDefinition(TABLE, "uuid", Types.VARCHAR, 40, false);
     db.assertColumnDefinition(TABLE, "doc_type", Types.VARCHAR, 40, false);
     db.assertColumnDefinition(TABLE, "doc_id", Types.VARCHAR, 4000, false);
+    db.assertColumnDefinition(TABLE, "doc_id_type", Types.VARCHAR, 20, true);
+    db.assertColumnDefinition(TABLE, "doc_routing", Types.VARCHAR, 4000, true);
     db.assertColumnDefinition(TABLE, "created_at", Types.BIGINT, null, false);
   }
 }

server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest/initial.sql

@@ -1,7 +1,9 @@
 CREATE TABLE "ES_QUEUE" (
   "UUID" VARCHAR(40) NOT NULL PRIMARY KEY,
   "DOC_TYPE" VARCHAR(40) NOT NULL,
-  "DOC_UUID" VARCHAR(255) NOT NULL,
+  "DOC_ID" VARCHAR(4000) NOT NULL,
+  "DOC_ID_TYPE" VARCHAR(20),
+  "DOC_ROUTING" VARCHAR(4000),
   "CREATED_AT" BIGINT NOT NULL
 );
 CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID");

server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java

@@ -20,14 +20,11 @@
 package org.sonar.server.es;
 
 import com.google.common.annotations.VisibleForTesting;
-import java.util.Arrays;
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.Nullable;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
@@ -51,12 +48,8 @@ import org.elasticsearch.search.sort.SortOrder;
 import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.core.util.ProgressLogger;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.es.EsQueueDto;
 
 import static java.lang.String.format;
-import static java.util.stream.Collectors.toList;
 
 /**
  * Helper to bulk requests in an efficient way :
@@ -76,22 +69,20 @@ public class BulkIndexer {
   private final EsClient client;
   private final String indexName;
   private final BulkProcessor bulkProcessor;
-  private final AtomicLong counter = new AtomicLong(0L);
-  private final ResilientIndexerResult successCounter = new ResilientIndexerResult();
+  private final IndexingResult result = new IndexingResult();
+  private final IndexingListener indexingListener;
   private final SizeHandler sizeHandler;
-  private final BulkProcessorListener bulkProcessorListener;
-  @Nullable
-  private DbClient dbClient;
-  @Nullable
-  private DbSession dbSession;
-  private Collection<EsQueueDto> esQueueDtos;
 
   public BulkIndexer(EsClient client, String indexName, Size size) {
-    this.dbClient = null;
+    this(client, indexName, size, IndexingListener.noop());
+  }
+
+  public BulkIndexer(EsClient client, String indexName, Size size, IndexingListener indexingListener) {
     this.client = client;
     this.indexName = indexName;
     this.sizeHandler = size.createHandler(Runtime2.INSTANCE);
-    this.bulkProcessorListener = new BulkProcessorListener();
+    this.indexingListener = indexingListener;
+    BulkProcessorListener bulkProcessorListener = new BulkProcessorListener();
     this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener)
       .setBackoffPolicy(BackoffPolicy.exponentialBackoff())
       .setBulkSize(FLUSH_BYTE_SIZE)
@@ -101,42 +92,32 @@ public class BulkIndexer {
   }
 
   public void start() {
+    result.clear();
     sizeHandler.beforeStart(this);
-    counter.set(0L);
-    successCounter.clear();
-  }
-
-  public void start(DbSession dbSession, DbClient dbClient, Collection<EsQueueDto> esQueueDtos) {
-    this.dbClient = dbClient;
-    this.dbSession = dbSession;
-    this.esQueueDtos = esQueueDtos;
-    sizeHandler.beforeStart(this);
-    counter.set(0L);
-    successCounter.clear();
   }
 
   /**
    * @return the number of documents successfully indexed
    */
-  public ResilientIndexerResult stop() {
+  public IndexingResult stop() {
     try {
       bulkProcessor.awaitClose(1, TimeUnit.MINUTES);
     } catch (InterruptedException e) {
       Thread.currentThread().interrupt();
       throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e);
-    } finally {
-      dbSession = null;
     }
     client.prepareRefresh(indexName).get();
     sizeHandler.afterStop(this);
-    return successCounter;
+    return result;
   }
 
   public void add(ActionRequest<?> request) {
+    result.incrementRequests();
     bulkProcessor.add(request);
   }
 
   public void addDeletion(SearchRequestBuilder searchRequest) {
+    // TODO to be replaced by delete_by_query that is back in ES5
     searchRequest
       .addSort("_doc", SortOrder.ASC)
       .setScroll(TimeValue.timeValueMinutes(5))
@@ -171,10 +152,10 @@ public class BulkIndexer {
   }
 
   public void addDeletion(IndexType indexType, String id) {
-    add(client.prepareDelete(indexType, id).setRouting(id).request());
+    add(client.prepareDelete(indexType, id).request());
   }
 
-  public void addDeletion(IndexType indexType, String id, String routing) {
+  public void addDeletion(IndexType indexType, String id, @Nullable String routing) {
     add(client.prepareDelete(indexType, id).setRouting(routing).request());
   }
 
@@ -184,11 +165,11 @@ public class BulkIndexer {
    *
    * Note that the parameter indexName could be removed if progress logs are not needed.
    */
-  public static void delete(EsClient client, String indexName, SearchRequestBuilder searchRequest) {
+  public static IndexingResult delete(EsClient client, String indexName, SearchRequestBuilder searchRequest) {
     BulkIndexer bulk = new BulkIndexer(client, indexName, Size.REGULAR);
     bulk.start();
     bulk.addDeletion(searchRequest);
-    bulk.stop();
+    return bulk.stop();
   }
 
   private final class BulkProcessorListener implements Listener {
@@ -199,37 +180,23 @@ public class BulkIndexer {
 
     @Override
     public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
-      counter.addAndGet(response.getItems().length);
-
+      List<String> successDocIds = new ArrayList<>();
       for (BulkItemResponse item : response.getItems()) {
         if (item.isFailed()) {
           LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage());
-          successCounter.increaseFailure();
         } else {
-          successCounter.increaseSuccess();
+          result.incrementSuccess();
+          successDocIds.add(item.getId());
         }
       }
 
-      deleteSuccessfulItems(response);
+      indexingListener.onSuccess(successDocIds);
     }
 
     @Override
     public void afterBulk(long executionId, BulkRequest req, Throwable e) {
       LOGGER.error("Fail to execute bulk index request: " + req, e);
     }
-
-    private void deleteSuccessfulItems(BulkResponse bulkResponse) {
-      if (esQueueDtos != null) {
-        List<EsQueueDto> itemsToDelete = Arrays.stream(bulkResponse.getItems())
-          .filter(b -> !b.isFailed())
-          .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocId())).findFirst().orElse(null))
-          .filter(Objects::nonNull)
-          .collect(toList());
-
-        dbClient.esQueueDao().delete(dbSession, itemsToDelete);
-        dbSession.commit();
-      }
-    }
   }
 
   public enum Size {
@@ -303,7 +270,7 @@ public class BulkIndexer {
 
     @Override
     void beforeStart(BulkIndexer bulkIndexer) {
-      this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexName), bulkIndexer.counter, LOGGER)
+      this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexName), bulkIndexer.result.total, LOGGER)
         .setPluralLabel("requests");
       this.progress.start();
       Map<String, Object> temporarySettings = new HashMap<>();

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIterator.java → server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java

@@ -17,13 +17,15 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.server.rule.index;
+package org.sonar.server.es;
 
-import java.util.Iterator;
+import java.util.Collection;
 
-public interface RuleIterator extends Iterator<RuleDocWithSystemScope>, AutoCloseable {
+public interface IndexingListener {
 
-  @Override
-  void close();
+  void onSuccess(Collection<String> docIds);
 
+  static IndexingListener noop() {
+    return docIds -> {};
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexerResult.java → server/sonar-server/src/main/java/org/sonar/server/es/IndexingResult.java

@@ -20,40 +20,44 @@
 
 package org.sonar.server.es;
 
-/**
- * The type Resilient indexer result.
- */
-public class ResilientIndexerResult {
-  private long total = 0L;
-  private long failures = 0L;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class IndexingResult {
+
+  // FIXME should be private
+  AtomicLong total = new AtomicLong(0L);
+  private long successes = 0L;
 
-  public ResilientIndexerResult clear() {
-    total = 0L;
-    failures = 0L;
+  IndexingResult clear() {
+    total.set(0L);
+    successes = 0L;
     return this;
   }
 
-  public ResilientIndexerResult increaseFailure() {
-    failures += 1;
-    total += 1;
-    return this;
+  void incrementRequests() {
+    total.incrementAndGet();
   }
 
-  public ResilientIndexerResult increaseSuccess() {
-    total += 1;
+  IndexingResult incrementSuccess() {
+    successes += 1;
     return this;
   }
 
+  public void add(IndexingResult other) {
+    total.addAndGet(other.total.get());
+    successes += other.successes;
+  }
+
   public long getFailures() {
-    return failures;
+    return total.get() - successes;
   }
 
   public long getTotal() {
-    return total;
+    return total.get();
   }
 
   public long getSuccess() {
-    return total - failures;
+    return successes;
   }
 
   /**
@@ -62,16 +66,10 @@ public class ResilientIndexerResult {
    * @see {@link RecoveryIndexer#recover()}
    */
   public double getFailureRatio() {
-    return total == 0 ? 1 : 1.0d * failures / total;
-  }
-
-  public double getSuccessRatio() {
-    return total == 0 ? 0 : 1 -  1.0d * failures / total;
+    return total.get() == 0 ? 1 : (1.0d * getFailures()) / total.get();
   }
 
-  public ResilientIndexerResult add(ResilientIndexerResult other) {
-    this.total += other.getTotal();
-    this.failures += other.getFailures();
-    return this;
+  public boolean isSuccess() {
+    return total.get() == successes;
   }
 }

server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java

@@ -38,6 +38,7 @@ import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.es.EsQueueDto;
+import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.user.index.UserIndexer;
 
@@ -66,15 +67,18 @@ public class RecoveryIndexer implements Startable {
   private final DbClient dbClient;
   private final UserIndexer userIndexer;
   private final RuleIndexer ruleIndexer;
+  private final ActiveRuleIndexer activeRuleIndexer;
   private final long minAgeInMs;
   private final long loopLimit;
 
-  public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer, RuleIndexer ruleIndexer) {
+  public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient,
+    UserIndexer userIndexer, RuleIndexer ruleIndexer, ActiveRuleIndexer activeRuleIndexer) {
     this.system2 = system2;
     this.settings = settings;
     this.dbClient = dbClient;
     this.userIndexer = userIndexer;
     this.ruleIndexer = ruleIndexer;
+    this.activeRuleIndexer = activeRuleIndexer;
     this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS);
     this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT);
   }
@@ -110,11 +114,11 @@ public class RecoveryIndexer implements Startable {
     try (DbSession dbSession = dbClient.openSession(false)) {
       Profiler profiler = Profiler.create(LOGGER).start();
       long beforeDate = system2.now() - minAgeInMs;
-      ResilientIndexerResult result = new ResilientIndexerResult();
+      IndexingResult result = new IndexingResult();
 
       Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
       while (!items.isEmpty()) {
-        ResilientIndexerResult loopResult = new ResilientIndexerResult();
+        IndexingResult loopResult = new IndexingResult();
 
         ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
         for (Map.Entry<EsQueueDto.Type, Collection<EsQueueDto>> entry : itemsByType.asMap().entrySet()) {
@@ -136,7 +140,7 @@ public class RecoveryIndexer implements Startable {
     }
   }
 
-  private ResilientIndexerResult doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) {
+  private IndexingResult doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) {
     LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type);
     switch (type) {
       case USER:
@@ -144,9 +148,11 @@ public class RecoveryIndexer implements Startable {
       case RULE_EXTENSION:
       case RULE:
         return ruleIndexer.index(dbSession, typeItems);
+      case ACTIVE_RULE:
+        return activeRuleIndexer.index(dbSession, typeItems);
       default:
         LOGGER.error(LOG_PREFIX + "ignore {} documents with unsupported type {}", typeItems.size(), type);
-        return new ResilientIndexerResult();
+        return new IndexingResult();
     }
   }
 

server/sonar-server/src/main/java/org/sonar/server/es/ResiliencyIndexingListener.java

@@ -0,0 +1,64 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import org.sonar.core.util.stream.MoreCollectors;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
+
+import static java.util.stream.Collectors.toMap;
+
+/**
+ * Clean-up the db table es_queue when documents
+ * are successfully indexed so that the recovery
+ * daemon does not re-index them.
+ */
+public class ResiliencyIndexingListener implements IndexingListener {
+
+  private final DbClient dbClient;
+  private final DbSession dbSession;
+  private final Collection<EsQueueDto> items;
+
+  public ResiliencyIndexingListener(DbClient dbClient, DbSession dbSession, Collection<EsQueueDto> items) {
+    this.dbClient = dbClient;
+    this.dbSession = dbSession;
+    this.items = items;
+  }
+
+  @Override
+  public void onSuccess(Collection<String> docIds) {
+    if (!docIds.isEmpty()) {
+      Map<String, EsQueueDto> itemsById = items.stream().collect(toMap(EsQueueDto::getDocId, Function.identity()));
+
+      Collection<EsQueueDto> itemsToDelete = docIds
+        .stream()
+        .map(itemsById::get)
+        .filter(Objects::nonNull)
+        .collect(MoreCollectors.toArrayList(docIds.size()));
+      dbClient.esQueueDao().delete(dbSession, itemsToDelete);
+      dbSession.commit();
+    }
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexer.java

@@ -37,5 +37,5 @@ public interface ResilientIndexer {
    * @param items     the items to be indexed
    * @return the number of successful indexation
    */
-  ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items);
+  IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items);
 }

server/sonar-server/src/main/java/org/sonar/server/es/queue/package-info.java

@@ -1,23 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-@ParametersAreNonnullByDefault
-package org.sonar.server.es.queue;
-
-import javax.annotation.ParametersAreNonnullByDefault;

server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java

@@ -153,7 +153,6 @@ import org.sonar.server.qualityprofile.QProfileResetImpl;
 import org.sonar.server.qualityprofile.RuleActivator;
 import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.qualityprofile.ws.OldRestoreAction;
 import org.sonar.server.qualityprofile.ws.ProfilesWs;
 import org.sonar.server.qualityprofile.ws.QProfilesWsModule;
@@ -260,7 +259,6 @@ public class PlatformLevel4 extends PlatformLevel {
 
       // quality profile
       BuiltInQProfileRepositoryImpl.class,
-      ActiveRuleIteratorFactory.class,
       ActiveRuleIndexer.class,
       XMLProfileParser.class,
       XMLProfileSerializer.class,

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/BuiltInQProfileInsertImpl.java

@@ -81,19 +81,19 @@ public class BuiltInQProfileInsertImpl implements BuiltInQProfileInsert {
     Date now = new Date(system2.now());
     RulesProfileDto ruleProfile = insertRulesProfile(dbSession, builtInQProfile, now);
 
-    List<ActiveRuleChange> localChanges = builtInQProfile.getActiveRules()
+    List<ActiveRuleChange> changes = builtInQProfile.getActiveRules()
       .stream()
       .map(activeRule -> insertActiveRule(dbSession, ruleProfile, activeRule, now.getTime()))
       .collect(MoreCollectors.toList());
 
-    localChanges.forEach(change -> dbClient.qProfileChangeDao().insert(batchDbSession, change.toDto(null)));
+    changes.forEach(change -> dbClient.qProfileChangeDao().insert(batchDbSession, change.toDto(null)));
 
     associateToOrganizations(dbSession, batchDbSession, builtInQProfile, ruleProfile);
 
-    dbSession.commit();
+    // TODO batch statements should be executed through dbSession
     batchDbSession.commit();
 
-    activeRuleIndexer.indexRuleProfile(dbSession, ruleProfile);
+    activeRuleIndexer.commitAndIndex(dbSession, changes);
   }
 
   private void associateToOrganizations(DbSession dbSession, DbSession batchDbSession, BuiltInQProfile builtIn, RulesProfileDto rulesProfileDto) {

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/BuiltInQProfileUpdateImpl.java

@@ -62,8 +62,7 @@ public class BuiltInQProfileUpdateImpl implements BuiltInQProfileUpdate {
     toBeDeactivated.forEach(ruleKey ->
       changes.addAll(ruleActivator.deactivateOnBuiltInRulesProfile(dbSession, ruleProfile, ruleKey, false)));
 
-    dbSession.commit();
-    activeRuleIndexer.indexChanges(dbSession, changes);
+    activeRuleIndexer.commitAndIndex(dbSession, changes);
     return changes;
   }
 

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/QProfileFactoryImpl.java

@@ -128,14 +128,16 @@ public class QProfileFactoryImpl implements QProfileFactory {
     db.qualityProfileDao().deleteOrgQProfilesByUuids(dbSession, uuids);
 
     // tables related to rules_profiles and active_rules are deleted
-    // only for custom profiles
+    // only for custom profiles. Built-in profiles are never
+    // deleted from table rules_profiles.
     if (!rulesProfileUuidsOfCustomProfiles.isEmpty()) {
       db.activeRuleDao().deleteParametersByRuleProfileUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
       db.activeRuleDao().deleteByRuleProfileUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
       db.qProfileChangeDao().deleteByRulesProfileUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
       db.qualityProfileDao().deleteRulesProfilesByUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
+      activeRuleIndexer.commitDeletionOfProfiles(dbSession, customProfiles);
+    } else {
+      dbSession.commit();
     }
-    dbSession.commit();
-    activeRuleIndexer.deleteByProfiles(customProfiles);
   }
 }

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/QProfileResetImpl.java

@@ -84,8 +84,7 @@ public class QProfileResetImpl implements QProfileReset {
         // ignore, probably a rule inherited from parent that can't be deactivated
       }
     }
-    dbSession.commit();
-    activeRuleIndexer.indexChanges(dbSession, changes);
+    activeRuleIndexer.commitAndIndex(dbSession, changes);
     return result;
   }
 

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/RuleActivator.java

@@ -83,6 +83,12 @@ public class RuleActivator {
     return doActivate(dbSession, activation, context);
   }
 
+  public List<ActiveRuleChange> activateAndCommit(DbSession dbSession, RuleActivation activation, QProfileDto profile) {
+    List<ActiveRuleChange> changes = activate(dbSession, activation, profile);
+    activeRuleIndexer.commitAndIndex(dbSession, changes);
+    return changes;
+  }
+
   public List<ActiveRuleChange> activate(DbSession dbSession, RuleActivation activation, QProfileDto profile) {
     RuleActivatorContext context = contextFactory.create(dbSession, activation.getRuleKey(), profile, false);
     return doActivate(dbSession, activation, context);
@@ -333,10 +339,9 @@ public class RuleActivator {
    * Deactivate a rule on a Quality profile. Does nothing if the rule is not activated, but
    * fails (fast) if the rule or the profile does not exist.
    */
-  public void deactivateAndUpdateIndex(DbSession dbSession, QProfileDto profile, RuleKey ruleKey) {
+  public void deactivateAndCommit(DbSession dbSession, QProfileDto profile, RuleKey ruleKey) {
     List<ActiveRuleChange> changes = deactivate(dbSession, profile, ruleKey);
-    dbSession.commit();
-    activeRuleIndexer.indexChanges(dbSession, changes);
+    activeRuleIndexer.commitAndIndex(dbSession, changes);
   }
 
   /**
@@ -418,7 +423,7 @@ public class RuleActivator {
     return value;
   }
 
-  public BulkChangeResult bulkActivate(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile, @Nullable String severity) {
+  public BulkChangeResult bulkActivateAndCommit(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile, @Nullable String severity) {
     BulkChangeResult result = new BulkChangeResult();
     Iterator<RuleKey> rules = ruleIndex.searchAll(ruleQuery);
     while (rules.hasNext()) {
@@ -437,12 +442,11 @@ public class RuleActivator {
         result.getErrors().addAll(e.errors());
       }
     }
-    dbSession.commit();
-    activeRuleIndexer.indexChanges(dbSession, result.getChanges());
+    activeRuleIndexer.commitAndIndex(dbSession, result.getChanges());
     return result;
   }
 
-  public BulkChangeResult bulkDeactivate(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile) {
+  public BulkChangeResult bulkDeactivateAndCommit(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile) {
     BulkChangeResult result = new BulkChangeResult();
     Iterator<RuleKey> rules = ruleIndex.searchAll(ruleQuery);
     while (rules.hasNext()) {
@@ -459,12 +463,11 @@ public class RuleActivator {
         result.getErrors().addAll(e.errors());
       }
     }
-    dbSession.commit();
-    activeRuleIndexer.indexChanges(dbSession, result.getChanges());
+    activeRuleIndexer.commitAndIndex(dbSession, result.getChanges());
     return result;
   }
 
-  public List<ActiveRuleChange> setParent(DbSession dbSession, QProfileDto profile, @Nullable QProfileDto parent) {
+  public List<ActiveRuleChange> setParentAndCommit(DbSession dbSession, QProfileDto profile, @Nullable QProfileDto parent) {
     checkRequest(
       parent == null || profile.getLanguage().equals(parent.getLanguage()),
       "Cannot set the profile '%s' as the parent of profile '%s' since their languages differ ('%s' != '%s')",
@@ -492,8 +495,7 @@ public class RuleActivator {
         }
       }
     }
-    dbSession.commit();
-    activeRuleIndexer.indexChanges(dbSession, changes);
+    activeRuleIndexer.commitAndIndex(dbSession, changes);
     return changes;
   }
 

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java

@@ -20,48 +20,64 @@
 package org.sonar.server.qualityprofile.index;
 
 import com.google.common.collect.ImmutableSet;
-import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import javax.annotation.Nullable;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.sonar.api.rule.RuleKey;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
+import org.sonar.db.qualityprofile.IndexedActiveRuleDto;
 import org.sonar.db.qualityprofile.QProfileDto;
 import org.sonar.db.qualityprofile.RulesProfileDto;
+import org.sonar.db.rule.SeverityUtil;
 import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.BulkIndexer.Size;
 import org.sonar.server.es.EsClient;
 import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.IndexingResult;
+import org.sonar.server.es.ResiliencyIndexingListener;
+import org.sonar.server.es.ResilientIndexer;
 import org.sonar.server.es.StartupIndexer;
+import org.sonar.server.qualityprofile.ActiveRule;
 import org.sonar.server.qualityprofile.ActiveRuleChange;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
 import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID;
-import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_KEY;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
 
-public class ActiveRuleIndexer implements StartupIndexer {
+public class ActiveRuleIndexer implements StartupIndexer, ResilientIndexer {
+
+  private static final Logger LOGGER = Loggers.get(ActiveRuleIndexer.class);
+  private static final String ID_TYPE_ACTIVE_RULE_ID = "activeRuleId";
+  private static final String ID_TYPE_RULE_PROFILE_UUID = "ruleProfileUuid";
 
   private final DbClient dbClient;
   private final EsClient esClient;
-  private final ActiveRuleIteratorFactory activeRuleIteratorFactory;
 
-  public ActiveRuleIndexer(DbClient dbClient, EsClient esClient, ActiveRuleIteratorFactory activeRuleIteratorFactory) {
+  public ActiveRuleIndexer(DbClient dbClient, EsClient esClient) {
     this.dbClient = dbClient;
     this.esClient = esClient;
-    this.activeRuleIteratorFactory = activeRuleIteratorFactory;
   }
 
   @Override
   public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
     try (DbSession dbSession = dbClient.openSession(false)) {
-      ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForAll(dbSession);
-      scrollDbAndIndex(dbCursor, Size.LARGE);
+      BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.noop());
+      bulkIndexer.start();
+      dbClient.activeRuleDao().scrollAllForIndexing(dbSession, ar -> bulkIndexer.add(newIndexRequest(ar)));
+      bulkIndexer.stop();
     }
   }
 
@@ -70,79 +86,144 @@ public class ActiveRuleIndexer implements StartupIndexer {
     return ImmutableSet.of(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE);
   }
 
+  public void commitAndIndex(DbSession dbSession, Collection<ActiveRuleChange> changes) {
+    List<EsQueueDto> items = changes.stream()
+      .map(ActiveRuleChange::getActiveRule)
+      .map(ar -> newQueueDto(String.valueOf(ar.getId()), ID_TYPE_ACTIVE_RULE_ID, ar.getRuleKey().toString()))
+      .collect(toArrayList());
+
+    dbClient.esQueueDao().insert(dbSession, items);
+    dbSession.commit();
+    postCommit(dbSession, items);
+  }
+
+  public void commitDeletionOfProfiles(DbSession dbSession, Collection<QProfileDto> profiles) {
+    List<EsQueueDto> items = profiles.stream()
+      .map(QProfileDto::getRulesProfileUuid)
+      .distinct()
+      .map(ruleProfileUuid -> newQueueDto(ruleProfileUuid, ID_TYPE_RULE_PROFILE_UUID, null))
+      .collect(toArrayList());
+
+    dbClient.esQueueDao().insert(dbSession, items);
+    dbSession.commit();
+    postCommit(dbSession, items);
+  }
+
+  /**
+   * Entry point for Byteman tests. See directory tests/resilience.
+   */
+  private void postCommit(DbSession dbSession, Collection<EsQueueDto> items) {
+    index(dbSession, items);
+  }
+
   /**
-   * Important - the existing documents are not deleted, so this method
-   * does not guarantee consistency of index.
+   * @return the number of items that have been successfully indexed
    */
-  public void indexRuleProfile(DbSession dbSession, RulesProfileDto ruleProfile) {
-    try (ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForRuleProfile(dbSession, ruleProfile)) {
-      scrollDbAndIndex(dbCursor, Size.REGULAR);
+  @Override
+  public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+    IndexingResult result = new IndexingResult();
+
+    if (items.isEmpty()) {
+      return result;
     }
+
+    Map<Long, EsQueueDto> activeRuleItems = new HashMap<>();
+    Map<String, EsQueueDto> ruleProfileItems = new HashMap<>();
+    items.forEach(i -> {
+      if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) {
+        ruleProfileItems.put(i.getDocId(), i);
+      } else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) {
+        activeRuleItems.put(Long.parseLong(i.getDocId()), i);
+      } else {
+        LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i);
+        deleteQueueDto(dbSession, i);
+      }
+    });
+
+    if (!activeRuleItems.isEmpty()) {
+      result.add(doIndexActiveRules(dbSession, activeRuleItems));
+    }
+    if (!ruleProfileItems.isEmpty()) {
+      result.add(doIndexRuleProfiles(dbSession, ruleProfileItems));
+    }
+    return result;
   }
 
-  public void indexChanges(DbSession dbSession, List<ActiveRuleChange> changes) {
-    BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
-    bulk.start();
-    List<Integer> idsOfTouchedActiveRules = new ArrayList<>();
-    changes.stream()
-      .filter(c -> c.getActiveRule() != null)
-      .forEach(c -> {
-        if (c.getType().equals(ActiveRuleChange.Type.DEACTIVATED)) {
-          bulk.addDeletion(INDEX_TYPE_ACTIVE_RULE, String.valueOf(c.getActiveRule().getId()), c.getKey().getRuleKey().toString());
-        } else {
-          idsOfTouchedActiveRules.add(c.getActiveRule().getId());
-        }
+  private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) {
+    BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, activeRuleItems.values()));
+    bulkIndexer.start();
+    Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems);
+    dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(),
+      // only index requests, no deletion requests.
+      // Deactivated users are not deleted but updated.
+      i -> {
+        remaining.remove(i.getId());
+        bulkIndexer.add(newIndexRequest(i));
       });
-    try (ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForActiveRules(dbSession, idsOfTouchedActiveRules)) {
-      while (dbCursor.hasNext()) {
-        ActiveRuleDoc activeRule = dbCursor.next();
-        bulk.add(newIndexRequest(activeRule));
+
+    // the remaining ids reference rows that don't exist in db. They must
+    // be deleted from index.
+    remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE,
+      item.getDocId(), item.getDocRouting()));
+    return bulkIndexer.stop();
+  }
+
+  private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) {
+    IndexingResult result = new IndexingResult();
+
+    for (Map.Entry<String, EsQueueDto> entry : ruleProfileItems.entrySet()) {
+      String ruleProfileUUid = entry.getKey();
+      EsQueueDto item = entry.getValue();
+      IndexingResult profileResult;
+
+      RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid);
+      if (profile == null) {
+        // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule
+        SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE)
+          .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid)));
+        profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), search);
+
+      } else {
+        BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.noop());
+        bulkIndexer.start();
+        dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i)));
+        profileResult = bulkIndexer.stop();
+      }
+
+      if (profileResult.isSuccess()) {
+        deleteQueueDto(dbSession, item);
       }
+      result.add(profileResult);
     }
-    bulk.stop();
-  }
 
-  public void deleteByProfiles(Collection<QProfileDto> profiles) {
-    BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
-    bulk.start();
-    profiles.forEach(profile -> {
-      SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE)
-        .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, profile.getRulesProfileUuid())));
-      bulk.addDeletion(search);
-    });
-    bulk.stop();
+    return result;
   }
 
-  public void deleteByRuleKeys(Collection<RuleKey> ruleKeys) {
-    BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
-    bulk.start();
-    ruleKeys.forEach(ruleKey -> {
-      SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE)
-        .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_RULE_KEY, ruleKey.toString())));
-      bulk.addDeletion(search);
-    });
-    bulk.stop();
+  private void deleteQueueDto(DbSession dbSession, EsQueueDto item) {
+    dbClient.esQueueDao().delete(dbSession, item);
+    dbSession.commit();
   }
 
-  private BulkIndexer createBulkIndexer(Size size) {
-    return new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), size);
+  private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) {
+    return new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), size, listener);
   }
 
-  private static IndexRequest newIndexRequest(ActiveRuleDoc doc) {
+  private static IndexRequest newIndexRequest(IndexedActiveRuleDto dto) {
+    ActiveRuleDoc doc = new ActiveRuleDoc(String.valueOf(dto.getId()));
+    doc.setRuleProfileUuid(dto.getRuleProfileUuid());
+    doc.setSeverity(SeverityUtil.getSeverityFromOrdinal(dto.getSeverity()));
+    doc.setRuleKey(RuleKey.of(dto.getRepository(), dto.getKey()));
+    // all the fields must be present, even if value is null
+    String inheritance = dto.getInheritance();
+    doc.setInheritance(inheritance == null ? ActiveRule.Inheritance.NONE.name() : inheritance);
     return new IndexRequest(INDEX_TYPE_ACTIVE_RULE.getIndex(), INDEX_TYPE_ACTIVE_RULE.getType())
       .id(doc.getId())
+      .parent(doc.getParent())
       .routing(doc.getRouting())
-      .parent(doc.getRuleKey().toString())
       .source(doc.getFields());
   }
 
-  private void scrollDbAndIndex(ActiveRuleIterator dbCursor, Size size) {
-    BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), size);
-    bulk.start();
-    while (dbCursor.hasNext()) {
-      ActiveRuleDoc activeRule = dbCursor.next();
-      bulk.add(newIndexRequest(activeRule));
-    }
-    bulk.stop();
+  private static EsQueueDto newQueueDto(String docId, String docIdType, @Nullable String routing) {
+    return EsQueueDto.create(EsQueueDto.Type.ACTIVE_RULE, docId, docIdType, routing);
   }
 }

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIterator.java

@@ -1,27 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.util.Iterator;
-
-public interface ActiveRuleIterator extends Iterator<ActiveRuleDoc>, AutoCloseable {
-  @Override
-  void close();
-}

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIteratorFactory.java

@@ -1,48 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.util.Collection;
-import org.sonar.api.server.ServerSide;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.qualityprofile.RulesProfileDto;
-
-@ServerSide
-public class ActiveRuleIteratorFactory {
-
-  private final DbClient dbClient;
-
-  public ActiveRuleIteratorFactory(DbClient dbClient) {
-    this.dbClient = dbClient;
-  }
-
-  public ActiveRuleIterator createForAll(DbSession dbSession) {
-    return new ActiveRuleIteratorForSingleChunk(dbClient, dbSession);
-  }
-
-  public ActiveRuleIterator createForRuleProfile(DbSession dbSession, RulesProfileDto ruleProfile) {
-    return new ActiveRuleIteratorForSingleChunk(dbClient, dbSession, ruleProfile);
-  }
-
-  public ActiveRuleIterator createForActiveRules(DbSession dbSession, Collection<Integer> activeRuleIds) {
-    return new ActiveRuleIteratorForMultipleChunks(dbClient, dbSession, activeRuleIds);
-  }
-}

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIteratorForMultipleChunks.java

@@ -1,69 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-
-import static java.util.Optional.ofNullable;
-
-public class ActiveRuleIteratorForMultipleChunks implements ActiveRuleIterator {
-
-  private final DbClient dbClient;
-  private final DbSession dbSession;
-  private final Iterator<List<Integer>> iteratorOverChunks;
-  private ActiveRuleIteratorForSingleChunk currentChunk;
-
-  public ActiveRuleIteratorForMultipleChunks(DbClient dbClient, DbSession dbSession, Collection<Integer> activeRuleIds) {
-    this.dbClient = dbClient;
-    this.dbSession = dbSession;
-    this.iteratorOverChunks = DatabaseUtils.toUniqueAndSortedPartitions(activeRuleIds).iterator();
-  }
-
-  @Override
-  public boolean hasNext() {
-    if (currentChunk != null && currentChunk.hasNext()) {
-      return true;
-    }
-    return iteratorOverChunks.hasNext();
-  }
-
-  @Override
-  public ActiveRuleDoc next() {
-    if (currentChunk == null || !currentChunk.hasNext()) {
-      currentChunk = nextChunk();
-    }
-    return currentChunk.next();
-  }
-
-  private ActiveRuleIteratorForSingleChunk nextChunk() {
-    List<Integer> nextInput = iteratorOverChunks.next();
-    return new ActiveRuleIteratorForSingleChunk(dbClient, dbSession, nextInput);
-  }
-
-  @Override
-  public void close() {
-    ofNullable(currentChunk).ifPresent(ActiveRuleIterator::close);
-  }
-}

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIteratorForSingleChunk.java

@@ -1,132 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.List;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-import org.sonar.db.qualityprofile.RulesProfileDto;
-import org.sonar.db.rule.SeverityUtil;
-import org.sonar.server.qualityprofile.ActiveRule;
-
-import static org.apache.commons.lang.StringUtils.repeat;
-
-/**
- * Scrolls over table ISSUES and reads documents to populate
- * the issues index
- */
-class ActiveRuleIteratorForSingleChunk implements ActiveRuleIterator {
-
-  private static final String[] COLUMNS = {
-    "ar.id",
-    "ar.failure_level",
-    "ar.inheritance",
-    "r.plugin_name",
-    "r.plugin_rule_key",
-    "rp.kee"
-  };
-
-  private static final String SQL_ALL = "select " + StringUtils.join(COLUMNS, ",") + " from active_rules ar " +
-    " inner join rules_profiles rp on rp.id = ar.profile_id " +
-    " inner join rules r on r.id = ar.rule_id ";
-
-  private final PreparedStatement stmt;
-  private final ResultSetIterator<ActiveRuleDoc> iterator;
-
-  ActiveRuleIteratorForSingleChunk(DbClient dbClient, DbSession dbSession) {
-    try {
-      stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, SQL_ALL);
-      iterator = new ActiveRuleIteratorInternal(stmt);
-    } catch (Exception e) {
-      throw new IllegalStateException("Fail to prepare SQL request to select all active_rules", e);
-    }
-  }
-
-  ActiveRuleIteratorForSingleChunk(DbClient dbClient, DbSession dbSession, RulesProfileDto ruleProfile) {
-    try {
-      stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, SQL_ALL + " where rp.kee = ?");
-      stmt.setString(1, ruleProfile.getKee());
-      iterator = new ActiveRuleIteratorInternal(stmt);
-    } catch (Exception e) {
-      throw new IllegalStateException("Fail to prepare SQL request to select active_rules of profile " + ruleProfile.getKee(), e);
-    }
-  }
-
-  ActiveRuleIteratorForSingleChunk(DbClient dbClient, DbSession dbSession, List<Integer> activeRuleIds) {
-    try {
-      stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, SQL_ALL + " where ar.id in (" + repeat("?", ",", activeRuleIds.size()) + ")");
-      for (int i = 0; i < activeRuleIds.size(); i++) {
-        stmt.setInt(i + 1, activeRuleIds.get(i));
-      }
-      iterator = new ActiveRuleIteratorInternal(stmt);
-    } catch (Exception e) {
-      throw new IllegalStateException("Fail to prepare SQL request to select active_rules", e);
-    }
-  }
-
-  @Override
-  public boolean hasNext() {
-    return iterator.hasNext();
-  }
-
-  @Override
-  public ActiveRuleDoc next() {
-    return iterator.next();
-  }
-
-  @Override
-  public void close() {
-    try {
-      iterator.close();
-    } finally {
-      DatabaseUtils.closeQuietly(stmt);
-    }
-  }
-
-  private static final class ActiveRuleIteratorInternal extends ResultSetIterator<ActiveRuleDoc> {
-
-    ActiveRuleIteratorInternal(PreparedStatement stmt) throws SQLException {
-      super(stmt);
-    }
-
-    @Override
-    protected ActiveRuleDoc read(ResultSet rs) throws SQLException {
-      long activeRuleId = rs.getLong(1);
-      int severity = rs.getInt(2);
-      String inheritance = rs.getString(3);
-      RuleKey ruleKey = RuleKey.of(rs.getString(4), rs.getString(5));
-      String ruleProfileUuid = rs.getString(6);
-
-      return new ActiveRuleDoc(String.valueOf(activeRuleId))
-        .setRuleProfileUuid(ruleProfileUuid)
-        .setRuleKey(ruleKey)
-        // all the fields must be present, even if value is null
-        .setSeverity(SeverityUtil.getSeverityFromOrdinal(severity))
-        .setInheritance(inheritance == null ? ActiveRule.Inheritance.NONE.name() : inheritance);
-    }
-  }
-}

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/ws/ActivateRuleAction.java

@@ -19,7 +19,6 @@
  */
 package org.sonar.server.qualityprofile.ws;
 
-import java.util.List;
 import java.util.Map;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rule.Severity;
@@ -30,10 +29,8 @@ import org.sonar.api.utils.KeyValueFormat;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.qualityprofile.QProfileDto;
-import org.sonar.server.qualityprofile.ActiveRuleChange;
 import org.sonar.server.qualityprofile.RuleActivation;
 import org.sonar.server.qualityprofile.RuleActivator;
-import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.user.UserSession;
 
 import static java.lang.String.format;
@@ -51,14 +48,12 @@ public class ActivateRuleAction implements QProfileWsAction {
   private final RuleActivator ruleActivator;
   private final UserSession userSession;
   private final QProfileWsSupport wsSupport;
-  private final ActiveRuleIndexer activeRuleIndexer;
 
-  public ActivateRuleAction(DbClient dbClient, RuleActivator ruleActivator, UserSession userSession, QProfileWsSupport wsSupport, ActiveRuleIndexer activeRuleIndexer) {
+  public ActivateRuleAction(DbClient dbClient, RuleActivator ruleActivator, UserSession userSession, QProfileWsSupport wsSupport) {
     this.dbClient = dbClient;
     this.ruleActivator = ruleActivator;
     this.userSession = userSession;
     this.wsSupport = wsSupport;
-    this.activeRuleIndexer = activeRuleIndexer;
   }
 
   public void define(WebService.NewController controller) {
@@ -104,9 +99,7 @@ public class ActivateRuleAction implements QProfileWsAction {
       wsSupport.checkPermission(dbSession, profile);
       wsSupport.checkNotBuiltInt(profile);
       RuleActivation activation = readActivation(request);
-      List<ActiveRuleChange> changes = ruleActivator.activate(dbSession, activation, profile);
-      dbSession.commit();
-      activeRuleIndexer.indexChanges(dbSession, changes);
+      ruleActivator.activateAndCommit(dbSession, activation, profile);
     }
 
     response.noContent();

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/ws/ActivateRulesAction.java

@@ -87,7 +87,7 @@ public class ActivateRulesAction implements QProfileWsAction {
       QProfileDto profile = wsSupport.getProfile(dbSession, fromKey(qualityProfileKey));
       wsSupport.checkPermission(dbSession, profile);
       wsSupport.checkNotBuiltInt(profile);
-      result = ruleActivator.bulkActivate(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile, request.param(PARAM_TARGET_SEVERITY));
+      result = ruleActivator.bulkActivateAndCommit(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile, request.param(PARAM_TARGET_SEVERITY));
     }
 
     writeResponse(result, response);

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/ws/ChangeParentAction.java

@@ -98,13 +98,13 @@ public class ChangeParentAction implements QProfileWsAction {
       String parentKey = request.param(PARAM_PARENT_PROFILE);
       String parentName = request.param(PARAM_PARENT_NAME);
       if (isEmpty(parentKey) && isEmpty(parentName)) {
-        ruleActivator.setParent(dbSession, profile, null);
+        ruleActivator.setParentAndCommit(dbSession, profile, null);
       } else {
         String parentOrganizationKey = parentKey == null ? organization.getKey() : null;
         String parentLanguage = parentKey == null ? request.param(PARAM_LANGUAGE) : null;
         QProfileReference parentRef = QProfileReference.from(parentKey, parentOrganizationKey, parentLanguage, parentName);
         QProfileDto parent = wsSupport.getProfile(dbSession, parentRef);
-        ruleActivator.setParent(dbSession, profile, parent);
+        ruleActivator.setParentAndCommit(dbSession, profile, parent);
       }
 
       response.noContent();

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/ws/CreateAction.java

@@ -132,8 +132,7 @@ public class CreateAction implements QProfileWsAction {
         result.add(exporters.importXml(profile, importerKey, contentToImport, dbSession));
       }
     }
-    dbSession.commit();
-    activeRuleIndexer.indexChanges(dbSession, result.getChanges());
+    activeRuleIndexer.commitAndIndex(dbSession, result.getChanges());
     return buildResponse(result, organization);
   }
 

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/ws/DeactivateRuleAction.java

@@ -79,7 +79,7 @@ public class DeactivateRuleAction implements QProfileWsAction {
       QProfileDto profile = wsSupport.getProfile(dbSession, QProfileReference.fromKey(qualityProfileKey));
       wsSupport.checkPermission(dbSession, profile);
       wsSupport.checkNotBuiltInt(profile);
-      ruleActivator.deactivateAndUpdateIndex(dbSession, profile, ruleKey);
+      ruleActivator.deactivateAndCommit(dbSession, profile, ruleKey);
     }
     response.noContent();
   }

server/sonar-server/src/main/java/org/sonar/server/qualityprofile/ws/DeactivateRulesAction.java

@@ -80,7 +80,7 @@ public class DeactivateRulesAction implements QProfileWsAction {
       QProfileDto profile = wsSupport.getProfile(dbSession, QProfileReference.fromKey(qualityProfileKey));
       wsSupport.checkPermission(dbSession, profile);
       wsSupport.checkNotBuiltInt(profile);
-      result = ruleActivator.bulkDeactivate(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile);
+      result = ruleActivator.bulkDeactivateAndCommit(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile);
     }
     writeResponse(result, response);
   }

server/sonar-server/src/main/java/org/sonar/server/rule/RegisterRules.java

@@ -131,7 +131,7 @@ public class RegisterRules implements Startable {
 
       persistRepositories(dbSession, context.repositories());
       ruleIndexer.commitAndIndex(dbSession, keysToIndex);
-      activeRuleIndexer.indexChanges(dbSession, changes);
+      activeRuleIndexer.commitAndIndex(dbSession, changes);
       profiler.stopDebug();
 
       webServerRuleFinder.startCaching();

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java

@@ -204,32 +204,32 @@ public class RuleDoc extends BaseDoc {
     return ReflectionToStringBuilder.toString(this);
   }
 
-  public static RuleDoc of(RuleForIndexingDto ruleForIndexingDto) {
+  public static RuleDoc of(RuleForIndexingDto dto) {
     RuleDoc ruleDoc = new RuleDoc()
-      .setKey(ruleForIndexingDto.getRuleKey().toString())
-      .setRepository(ruleForIndexingDto.getRepository())
-      .setInternalKey(ruleForIndexingDto.getInternalKey())
-      .setIsTemplate(ruleForIndexingDto.isTemplate())
-      .setLanguage(ruleForIndexingDto.getLanguage())
-      .setName(ruleForIndexingDto.getName())
-      .setRuleKey(ruleForIndexingDto.getPluginRuleKey())
-      .setSeverity(ruleForIndexingDto.getSeverityAsString())
-      .setStatus(ruleForIndexingDto.getStatus().toString())
-      .setType(ruleForIndexingDto.getTypeAsRuleType())
-      .setCreatedAt(ruleForIndexingDto.getCreatedAt())
-      .setUpdatedAt(ruleForIndexingDto.getUpdatedAt());
-
-    if (ruleForIndexingDto.getPluginRuleKey() != null && ruleForIndexingDto.getRepository() != null) {
-      ruleDoc.setTemplateKey(RuleKey.of(ruleForIndexingDto.getPluginRuleKey(), ruleForIndexingDto.getRepository()).toString());
+      .setKey(dto.getRuleKey().toString())
+      .setRepository(dto.getRepository())
+      .setInternalKey(dto.getInternalKey())
+      .setIsTemplate(dto.isTemplate())
+      .setLanguage(dto.getLanguage())
+      .setName(dto.getName())
+      .setRuleKey(dto.getPluginRuleKey())
+      .setSeverity(dto.getSeverityAsString())
+      .setStatus(dto.getStatus().toString())
+      .setType(dto.getTypeAsRuleType())
+      .setCreatedAt(dto.getCreatedAt())
+      .setUpdatedAt(dto.getUpdatedAt());
+
+    if (dto.getTemplateRuleKey() != null && dto.getTemplateRepository() != null) {
+      ruleDoc.setTemplateKey(RuleKey.of(dto.getTemplateRepository(), dto.getTemplateRuleKey()).toString());
     } else {
       ruleDoc.setTemplateKey(null);
     }
 
-    if (ruleForIndexingDto.getDescription() != null && ruleForIndexingDto.getDescriptionFormat() != null) {
-      if (RuleDto.Format.HTML == ruleForIndexingDto.getDescriptionFormat()) {
-        ruleDoc.setHtmlDescription(ruleForIndexingDto.getDescription());
+    if (dto.getDescription() != null && dto.getDescriptionFormat() != null) {
+      if (RuleDto.Format.HTML == dto.getDescriptionFormat()) {
+        ruleDoc.setHtmlDescription(dto.getDescription());
       } else {
-        ruleDoc.setHtmlDescription(Markdown.convertToHtml(ruleForIndexingDto.getDescription()));;
+        ruleDoc.setHtmlDescription(Markdown.convertToHtml(dto.getDescription()));
       }
     }
     return ruleDoc;

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java

@@ -22,16 +22,15 @@ package org.sonar.server.rule.index;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ListMultimap;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import org.elasticsearch.action.index.IndexRequest;
-import org.sonar.db.es.RuleExtensionId;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.es.EsQueueDto;
+import org.sonar.db.es.RuleExtensionId;
 import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.rule.RuleExtensionForIndexingDto;
 import org.sonar.db.rule.RuleForIndexingDto;
@@ -39,15 +38,16 @@ import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.BulkIndexer.Size;
 import org.sonar.server.es.EsClient;
 import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.IndexingResult;
+import org.sonar.server.es.ResiliencyIndexingListener;
 import org.sonar.server.es.ResilientIndexer;
-import org.sonar.server.es.ResilientIndexerResult;
 import org.sonar.server.es.StartupIndexer;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.Collections.singletonList;
 import static java.util.Objects.requireNonNull;
 import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
 
@@ -68,24 +68,27 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
 
   @Override
   public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
-    BulkIndexer bulk = new BulkIndexer(esClient, INDEX, Size.LARGE);
-    bulk.start();
-
-    // index all definitions and system extensions
-    if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE)) {
-      try (RuleIterator rules = new RuleIteratorForSingleChunk(dbClient, null)) {
-        doIndexRuleDefinitions(rules, bulk);
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      BulkIndexer bulk = createBulkIndexer(Size.LARGE, IndexingListener.noop());
+      bulk.start();
+
+      // index all definitions and system extensions
+      if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE)) {
+        dbClient.ruleDao().scrollIndexingRules(dbSession, dto -> {
+          bulk.add(newRuleDocIndexRequest(dto));
+          bulk.add(newRuleExtensionDocIndexRequest(dto));
+        });
       }
-    }
 
-    // index all organization extensions
-    if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE_EXTENSION)) {
-      try (RuleMetadataIterator metadatas = new RuleMetadataIterator(dbClient)) {
-        doIndexRuleExtensions(metadatas, bulk);
+      // index all organization extensions
+      if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE_EXTENSION)) {
+        dbClient.ruleDao().scrollIndexingRuleExtensions(dbSession, dto -> {
+          bulk.add(newRuleExtensionDocIndexRequest(dto));
+        });
       }
-    }
 
-    bulk.stop();
+      bulk.stop();
+    }
   }
 
   public void commitAndIndex(DbSession dbSession, RuleKey ruleKey) {
@@ -123,12 +126,12 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
   }
 
   @Override
-  public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+  public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
     if (items.isEmpty()) {
-      return new ResilientIndexerResult();
+      return new IndexingResult();
     }
 
-    ResilientIndexerResult result = new ResilientIndexerResult();
+    IndexingResult result = new IndexingResult();
 
     ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
 
@@ -138,9 +141,9 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
     return result;
   }
 
-  private ResilientIndexerResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
-    BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR);
-    bulkIndexer.start(dbSession, dbClient, items);
+  private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
+    BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
+    bulkIndexer.start();
 
     Set<RuleKey> rules = items
       .stream()
@@ -151,7 +154,7 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
       .map(i -> RuleKey.parse(i.getDocId()))
       .collect(toHashSet(items.size()));
 
-    dbClient.ruleDao().scrollRuleByRuleKeys(dbSession, rules,
+    dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, rules,
       // only index requests, no deletion requests.
       // Deactivated users are not deleted but updated.
       r -> {
@@ -168,9 +171,9 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
     return bulkIndexer.stop();
   }
 
-  private ResilientIndexerResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) {
-    BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR);
-    bulkIndexer.start(dbSession, dbClient, items);
+  private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) {
+    BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
+    bulkIndexer.start();
 
     Set<RuleExtensionId> docIds = items
       .stream()
@@ -181,15 +184,14 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
       .map(RuleIndexer::explodeRuleExtensionDocId)
       .collect(toHashSet(items.size()));
 
-    dbClient.ruleDao().scrollRuleExtensionByRuleKeys(dbSession, docIds,
+    dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds,
       // only index requests, no deletion requests.
       // Deactivated users are not deleted but updated.
       r -> {
-        docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()) );
+        docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()));
         bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
       });
 
-
     // the remaining items reference rows that don't exist in db. They must
     // be deleted from index.
     docIds.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.getId()));
@@ -197,61 +199,36 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer {
     return bulkIndexer.stop();
   }
 
-  private static void doIndexRuleDefinitions(Iterator<RuleDocWithSystemScope> rules, BulkIndexer bulk) {
-    while (rules.hasNext()) {
-      RuleDocWithSystemScope ruleWithExtension = rules.next();
-      bulk.add(newIndexRequest(ruleWithExtension.getRuleDoc()));
-      bulk.add(newIndexRequest(ruleWithExtension.getRuleExtensionDoc()));
-    }
-  }
-
-  private static void doIndexRuleExtensions(Iterator<RuleExtensionDoc> metadatas, BulkIndexer bulk) {
-    while (metadatas.hasNext()) {
-      RuleExtensionDoc metadata = metadatas.next();
-      bulk.add(newIndexRequest(metadata));
-    }
-  }
-
-  private static IndexRequest newIndexRequest(RuleDoc rule) {
-    return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType())
-      .id(rule.getId())
-      .routing(rule.getRouting())
-      .source(rule.getFields());
-  }
-
-  private static IndexRequest newIndexRequest(RuleExtensionDoc ruleExtension) {
-    return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
-      .id(ruleExtension.getId())
-      .routing(ruleExtension.getRouting())
-      .source(ruleExtension.getFields())
-      .parent(ruleExtension.getParent());
-  }
-
   private static IndexRequest newRuleDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
-    RuleDoc ruleDoc = RuleDoc.of(ruleForIndexingDto);
+    RuleDoc doc = RuleDoc.of(ruleForIndexingDto);
 
-    return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType(), ruleDoc.key().toString())
-      .source(ruleDoc.getFields());
+    return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType())
+      .id(doc.key().toString())
+      .routing(doc.getRouting())
+      .source(doc.getFields());
   }
 
   private static IndexRequest newRuleExtensionDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
     RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleForIndexingDto);
 
-    return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId())
-      .source(ruleExtensionDoc.getFields())
-      .parent(ruleExtensionDoc.getParent());
+    return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
+      .id(ruleExtensionDoc.getId())
+      .routing(ruleExtensionDoc.getRouting())
+      .parent(ruleExtensionDoc.getParent())
+      .source(ruleExtensionDoc.getFields());
   }
 
   private static IndexRequest newRuleExtensionDocIndexRequest(RuleExtensionForIndexingDto ruleExtensionForIndexingDto) {
-    RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleExtensionForIndexingDto);
-
-    return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId())
-      .source(ruleExtensionDoc.getFields())
-      .parent(ruleExtensionDoc.getParent());
+    RuleExtensionDoc doc = RuleExtensionDoc.of(ruleExtensionForIndexingDto);
+    return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
+      .id(doc.getId())
+      .routing(doc.getRouting())
+      .parent(doc.getParent())
+      .source(doc.getFields());
   }
 
-  private BulkIndexer newBulkIndexerForRules(Size bulkSize) {
-    return new BulkIndexer(esClient, INDEX_TYPE_RULE.getIndex(), bulkSize);
+  private BulkIndexer createBulkIndexer(Size bulkSize, IndexingListener listener) {
+    return new BulkIndexer(esClient, INDEX_TYPE_RULE.getIndex(), bulkSize, listener);
   }
 
   private static ListMultimap<EsQueueDto.Type, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) {

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIteratorForMultipleChunks.java

@@ -1,80 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-
-import static java.util.Optional.ofNullable;
-
-public class RuleIteratorForMultipleChunks implements RuleIterator {
-
-  private final DbClient dbClient;
-  private final Iterator<List<RuleKey>> iteratorOverChunks;
-  private RuleIteratorForSingleChunk currentChunk;
-
-  public RuleIteratorForMultipleChunks(DbClient dbClient, Collection<RuleKey> keys) {
-    this.dbClient = dbClient;
-    iteratorOverChunks = DatabaseUtils.toUniqueAndSortedPartitions(keys).iterator();
-  }
-
-  @Override
-  public boolean hasNext() {
-    for (;;) {
-      if (currentChunk != null && currentChunk.hasNext()) {
-        return true;
-      }
-      if (iteratorOverChunks.hasNext()) {
-        currentChunk = nextChunk();
-      } else {
-        return false;
-      }
-    }
-  }
-
-  @Override
-  public RuleDocWithSystemScope next() {
-    for (;;) {
-      if (currentChunk != null && currentChunk.hasNext()) {
-        return currentChunk.next();
-      }
-      if (iteratorOverChunks.hasNext()) {
-        currentChunk = nextChunk();
-      } else {
-        throw new NoSuchElementException();
-      }
-    }
-  }
-
-  private RuleIteratorForSingleChunk nextChunk() {
-    List<RuleKey> nextInput = iteratorOverChunks.next();
-    return new RuleIteratorForSingleChunk(dbClient, nextInput);
-  }
-
-  @Override
-  public void close() {
-    ofNullable(currentChunk).ifPresent(RuleIterator::close);
-  }
-}

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIteratorForSingleChunk.java

@@ -1,211 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.api.rules.RuleType;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-import org.sonar.db.rule.RuleDto;
-import org.sonar.db.rule.SeverityUtil;
-import org.sonar.markdown.Markdown;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static java.util.stream.Collectors.joining;
-
-/**
- * Scrolls over table RULES and reads documents to populate the rules index
- */
-public class RuleIteratorForSingleChunk implements RuleIterator {
-
-  private static final String[] FIELDS = {
-    // column 1
-    "r.plugin_name",
-    "r.plugin_rule_key",
-    "r.name",
-    "r.description",
-    "r.description_format",
-    "r.priority",
-    "r.status",
-    "r.is_template",
-    "r.system_tags",
-    "t.plugin_rule_key",
-
-    // column 11
-    "t.plugin_name",
-    "r.plugin_config_key",
-    "r.language",
-    "r.rule_type",
-    "r.created_at",
-    "r.updated_at",
-  };
-
-  private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM rules r " +
-    "LEFT OUTER JOIN rules t ON t.id=r.template_id";
-  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
-
-  private final DbSession session;
-
-  private final List<RuleKey> ruleKeys;
-
-  private final PreparedStatement stmt;
-  private final ResultSetIterator<RuleDocWithSystemScope> iterator;
-
-  RuleIteratorForSingleChunk(DbClient dbClient, @Nullable List<RuleKey> ruleKeys) {
-    checkArgument(ruleKeys == null || ruleKeys.size() <= DatabaseUtils.PARTITION_SIZE_FOR_ORACLE,
-      "Cannot search for more than " + DatabaseUtils.PARTITION_SIZE_FOR_ORACLE + " rule keys at once. Please provide the keys in smaller chunks.");
-    this.ruleKeys = ruleKeys;
-    this.session = dbClient.openSession(false);
-
-    try {
-      String sql = createSql();
-      stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
-      iterator = createIterator();
-    } catch (Exception e) {
-      session.close();
-      throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
-    }
-  }
-
-  private RuleIteratorInternal createIterator() {
-    try {
-      setParameters(stmt);
-      return new RuleIteratorInternal(stmt);
-    } catch (SQLException e) {
-      DatabaseUtils.closeQuietly(stmt);
-      throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
-    }
-  }
-
-  @Override
-  public boolean hasNext() {
-    return iterator.hasNext();
-  }
-
-  @Override
-  public RuleDocWithSystemScope next() {
-    return iterator.next();
-  }
-
-  private String createSql() {
-    StringBuilder sql = new StringBuilder(SQL_ALL);
-    if (ruleKeys != null && !ruleKeys.isEmpty()) {
-      sql.append(" WHERE ");
-      sql.append(ruleKeys.stream()
-        .map(x -> "(r.plugin_name=? AND r.plugin_rule_key=?)")
-        .collect(joining(" OR "))
-      );
-    }
-    return sql.toString();
-  }
-
-  private void setParameters(PreparedStatement stmt) throws SQLException {
-    AtomicInteger index = new AtomicInteger(1);
-    if (ruleKeys != null && !ruleKeys.isEmpty()) {
-      for (RuleKey ruleKey : ruleKeys) {
-        stmt.setString(index.getAndIncrement(), ruleKey.repository());
-        stmt.setString(index.getAndIncrement(), ruleKey.rule());
-      }
-    }
-  }
-
-  @Override
-  public void close() {
-    try {
-      iterator.close();
-    } finally {
-      DatabaseUtils.closeQuietly(stmt);
-      session.close();
-    }
-  }
-
-  private static final class RuleIteratorInternal extends ResultSetIterator<RuleDocWithSystemScope> {
-
-    public RuleIteratorInternal(PreparedStatement stmt) throws SQLException {
-      super(stmt);
-    }
-
-    @Override
-    protected RuleDocWithSystemScope read(ResultSet rs) throws SQLException {
-      RuleDoc doc = new RuleDoc();
-      RuleExtensionDoc extensionDoc = new RuleExtensionDoc().setScope(RuleExtensionScope.system());
-
-      String repositoryKey = rs.getString(1);
-      String ruleKey = rs.getString(2);
-      RuleKey key = RuleKey.of(repositoryKey, ruleKey);
-      extensionDoc.setRuleKey(key);
-
-      // all the fields must be present, even if value is null
-      doc.setKey(key.toString());
-      doc.setRuleKey(ruleKey);
-      doc.setRepository(repositoryKey);
-      doc.setName(rs.getString(3));
-
-      String description = rs.getString(4);
-      String descriptionFormat = rs.getString(5);
-      if (descriptionFormat != null && description != null) {
-        String htmlDescription;
-        if (RuleDto.Format.HTML == RuleDto.Format.valueOf(descriptionFormat)) {
-          htmlDescription = description;
-        } else {
-          htmlDescription = Markdown.convertToHtml(description);
-        }
-        doc.setHtmlDescription(htmlDescription);
-      }
-
-      doc.setSeverity(SeverityUtil.getSeverityFromOrdinal(rs.getInt(6)));
-      doc.setStatus(rs.getString(7));
-      doc.setIsTemplate(rs.getBoolean(8));
-      extensionDoc.setTags(stringTagsToSet(rs.getString(9)));
-
-      String templateRuleKey = rs.getString(10);
-      String templateRepoKey = rs.getString(11);
-      if (templateRepoKey != null && templateRuleKey != null) {
-        doc.setTemplateKey(RuleKey.of(templateRepoKey, templateRuleKey).toString());
-      } else {
-        doc.setTemplateKey(null);
-      }
-
-      doc.setInternalKey(rs.getString(12));
-      doc.setLanguage(rs.getString(13));
-      doc.setType(RuleType.valueOf(rs.getInt(14)));
-      doc.setCreatedAt(rs.getLong(15));
-      doc.setUpdatedAt(rs.getLong(16));
-
-      return new RuleDocWithSystemScope(doc, extensionDoc);
-    }
-
-    private static Set<String> stringTagsToSet(@Nullable String tags) {
-      return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
-    }
-  }
-}

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleMetadataIterator.java

@@ -1,126 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-package org.sonar.server.rule.index;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Iterator;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-
-/**
- * Scrolls over table RULES_METADATA and reads documents to populate the rule extension index type
- */
-public class RuleMetadataIterator implements Iterator<RuleExtensionDoc>, AutoCloseable {
-
-  private static final String[] FIELDS = {
-    "r.plugin_name",
-    "r.plugin_rule_key",
-    "rm.organization_uuid",
-    "rm.tags"
-  };
-
-  private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM rules r " +
-    "INNER JOIN rules_metadata rm ON rm.rule_id = r.id " +
-    "WHERE rm.tags is not null AND rm.tags != ''";
-  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
-
-  private final DbSession session;
-
-  private final PreparedStatement stmt;
-  private final ResultSetIterator<RuleExtensionDoc> iterator;
-
-  RuleMetadataIterator(DbClient dbClient) {
-    this.session = dbClient.openSession(false);
-
-    try {
-      String sql = SQL_ALL;
-      stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
-      iterator = createIterator();
-    } catch (Exception e) {
-      session.close();
-      throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
-    }
-  }
-
-  private RuleMetadataIteratorInternal createIterator() {
-    try {
-      return new RuleMetadataIteratorInternal(stmt);
-    } catch (SQLException e) {
-      DatabaseUtils.closeQuietly(stmt);
-      throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
-    }
-  }
-
-  @Override
-  public boolean hasNext() {
-    return iterator.hasNext();
-  }
-
-  @Override
-  public RuleExtensionDoc next() {
-    return iterator.next();
-  }
-
-  @Override
-  public void close() {
-    try {
-      iterator.close();
-    } finally {
-      DatabaseUtils.closeQuietly(stmt);
-      session.close();
-    }
-  }
-
-  private static final class RuleMetadataIteratorInternal extends ResultSetIterator<RuleExtensionDoc> {
-
-    public RuleMetadataIteratorInternal(PreparedStatement stmt) throws SQLException {
-      super(stmt);
-    }
-
-    @Override
-    protected RuleExtensionDoc read(ResultSet rs) throws SQLException {
-      RuleExtensionDoc doc = new RuleExtensionDoc();
-
-      String repositoryKey = rs.getString(1);
-      String ruleKey = rs.getString(2);
-      RuleKey key = RuleKey.of(repositoryKey, ruleKey);
-      doc.setRuleKey(key);
-      doc.setScope(RuleExtensionScope.organization(rs.getString(3)));
-      doc.setTags(stringTagsToSet(rs.getString(4)));
-
-      return doc;
-    }
-
-    private static Set<String> stringTagsToSet(@Nullable String tags) {
-      return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
-    }
-  }
-}

server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java

@@ -37,8 +37,10 @@ import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.BulkIndexer.Size;
 import org.sonar.server.es.EsClient;
 import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.ResiliencyIndexingListener;
 import org.sonar.server.es.ResilientIndexer;
-import org.sonar.server.es.ResilientIndexerResult;
+import org.sonar.server.es.IndexingResult;
 import org.sonar.server.es.StartupIndexer;
 
 import static java.util.Collections.singletonList;
@@ -67,7 +69,7 @@ public class UserIndexer implements StartupIndexer, ResilientIndexer {
       ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
       dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByLogin::put);
 
-      BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE);
+      BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.noop());
       bulkIndexer.start();
       dbClient.userDao().scrollAll(dbSession,
         // only index requests, no deletion requests.
@@ -107,9 +109,9 @@ public class UserIndexer implements StartupIndexer, ResilientIndexer {
    * @return the number of items that have been successfully indexed
    */
   @Override
-  public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+  public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
     if (items.isEmpty()) {
-      return new ResilientIndexerResult();
+      return new IndexingResult();
     }
     Set<String> logins = items
       .stream()
@@ -123,8 +125,8 @@ public class UserIndexer implements StartupIndexer, ResilientIndexer {
     ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
     dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put);
 
-    BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR);
-    bulkIndexer.start(dbSession, dbClient, items);
+    BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
+    bulkIndexer.start();
     dbClient.userDao().scrollByLogins(dbSession, logins,
       // only index requests, no deletion requests.
       // Deactivated users are not deleted but updated.
@@ -139,8 +141,8 @@ public class UserIndexer implements StartupIndexer, ResilientIndexer {
     return bulkIndexer.stop();
   }
 
-  private BulkIndexer newBulkIndexer(Size bulkSize) {
-    return new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex(), bulkSize);
+  private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) {
+    return new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex(), bulkSize, listener);
   }
 
   private static IndexRequest newIndexRequest(UserDto user, ListMultimap<String, String> organizationUuidsByLogins) {
@@ -155,6 +157,7 @@ public class UserIndexer implements StartupIndexer, ResilientIndexer {
 
     return new IndexRequest(UserIndexDefinition.INDEX_TYPE_USER.getIndex(), UserIndexDefinition.INDEX_TYPE_USER.getType())
       .id(doc.getId())
+      .routing(doc.getRouting())
       .source(doc.getFields());
   }
 }

server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java

@@ -1,121 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.user.index;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Maps;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Collection;
-import java.util.List;
-import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-import org.sonar.db.es.EsQueueDto;
-import org.sonar.db.user.UserDto;
-
-import static org.apache.commons.lang.StringUtils.repeat;
-import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
-
-/**
- * Scrolls over table USERS and reads documents to populate the user index
- */
-class UserResultSetIterator extends ResultSetIterator<UserDoc> {
-
-  private static final String[] FIELDS = {
-    // column 1
-    "u.login",
-    "u.name",
-    "u.email",
-    "u.active",
-    "u.scm_accounts",
-    "u.created_at",
-    "u.updated_at",
-  };
-
-  private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from users u ";
-
-  private final ListMultimap<String, String> organizationUuidsByLogins;
-
-  private UserResultSetIterator(PreparedStatement stmt, ListMultimap<String, String> organizationUuidsByLogins) throws SQLException {
-    super(stmt);
-    this.organizationUuidsByLogins = organizationUuidsByLogins;
-  }
-
-  static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable Collection<EsQueueDto> esQueueDtos) {
-    try {
-      String sql = SQL_ALL;
-      List<String> logins = null;
-      if (esQueueDtos != null) {
-        logins = esQueueDtos.stream()
-          .filter(i -> i.getDocType() == EsQueueDto.Type.USER)
-          .map(EsQueueDto::getDocId).collect(toArrayList());
-        sql += "where (" + repeat("u.login=?", " or ", logins.size()) + ")";
-      }
-
-      PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
-      setParameters(stmt, logins);
-
-      ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
-      if (esQueueDtos == null) {
-        dbClient.organizationMemberDao().selectAllForUserIndexing(session, organizationUuidsByLogin::put);
-      } else {
-
-        dbClient.organizationMemberDao().selectForUserIndexing(session, logins, organizationUuidsByLogin::put);
-      }
-
-      return new UserResultSetIterator(stmt, organizationUuidsByLogin);
-    } catch (SQLException e) {
-      throw new IllegalStateException("Fail to prepare SQL request to select all users", e);
-    }
-  }
-
-  private static void setParameters(PreparedStatement stmt, @Nullable Collection<String> logins) throws SQLException {
-    if (logins == null) {
-      return;
-    }
-
-    int paramIndex = 1;
-    for (String login : logins) {
-      stmt.setString(paramIndex, login);
-      paramIndex++;
-    }
-  }
-
-  @Override
-  protected UserDoc read(ResultSet rs) throws SQLException {
-    UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(6));
-
-    String login = rs.getString(1);
-
-    // all the keys must be present, even if value is null
-    doc.setLogin(login);
-    doc.setName(rs.getString(2));
-    doc.setEmail(rs.getString(3));
-    doc.setActive(rs.getBoolean(4));
-    doc.setScmAccounts(UserDto.decodeScmAccounts(rs.getString(5)));
-    doc.setOrganizationUuids(organizationUuidsByLogins.get(login));
-    return doc;
-  }
-}

server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java

@@ -82,10 +82,13 @@ public class BulkIndexerTest {
     for (int i = 0; i < 10; i++) {
       indexer.add(newIndexRequest(i));
     }
-    indexer.stop();
+    IndexingResult result = indexer.stop();
 
+    assertThat(result.isSuccess()).isTrue();
+    assertThat(result.getSuccess()).isEqualTo(10);
+    assertThat(result.getFailures()).isEqualTo(0);
+    assertThat(result.getTotal()).isEqualTo(10);
     assertThat(count()).isEqualTo(10);
-
     // replicas are re-enabled
     assertThat(replicas()).isEqualTo(1);
   }

server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java

@@ -65,7 +65,7 @@ public class EsTester extends ExternalResource {
 
   @Override
   protected void before() throws Throwable {
-    truncateIndices();
+    deleteIndices();
 
     if (!indexDefinitions.isEmpty()) {
       container = new ComponentContainer();
@@ -89,10 +89,14 @@ public class EsTester extends ExternalResource {
     }
   }
 
-  private void truncateIndices() {
+  private void deleteIndices() {
     client.nativeClient().admin().indices().prepareDelete("_all").get();
   }
 
+  public void deleteIndex(String indexName) {
+    client.nativeClient().admin().indices().prepareDelete(indexName).get();
+  }
+
   public void putDocuments(String index, String type, BaseDoc... docs) {
     putDocuments(new IndexType(index, type), docs);
   }

server/sonar-server/src/test/java/org/sonar/server/es/IndexingResultTest.java

@@ -0,0 +1,78 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.assertj.core.data.Offset;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class IndexingResultTest {
+
+  private final IndexingResult underTest = new IndexingResult();
+
+  @Test
+  public void test_success() {
+    underTest.incrementRequests();
+    underTest.incrementRequests();
+    underTest.incrementSuccess();
+    underTest.incrementSuccess();
+
+    assertThat(underTest.getFailures()).isEqualTo(0);
+    assertThat(underTest.getSuccess()).isEqualTo(2);
+    assertThat(underTest.getTotal()).isEqualTo(2);
+    assertThat(underTest.getFailureRatio()).isEqualTo(0.0, Offset.offset(0.000001d));
+    assertThat(underTest.isSuccess()).isTrue();
+  }
+
+  @Test
+  public void test_failure() {
+    underTest.incrementRequests();
+    underTest.incrementRequests();
+
+    assertThat(underTest.getFailures()).isEqualTo(2);
+    assertThat(underTest.getSuccess()).isEqualTo(0);
+    assertThat(underTest.getTotal()).isEqualTo(2);
+    assertThat(underTest.getFailureRatio()).isEqualTo(1.0, Offset.offset(0.000001d));
+    assertThat(underTest.isSuccess()).isFalse();
+  }
+
+  @Test
+  public void test_partial_failure() {
+    underTest.incrementRequests();
+    underTest.incrementRequests();
+    underTest.incrementSuccess();
+
+    assertThat(underTest.getFailures()).isEqualTo(1);
+    assertThat(underTest.getSuccess()).isEqualTo(1);
+    assertThat(underTest.getTotal()).isEqualTo(2);
+    assertThat(underTest.getFailureRatio()).isEqualTo(0.5, Offset.offset(0.000001d));
+    assertThat(underTest.isSuccess()).isFalse();
+  }
+
+  @Test
+  public void correctness_even_with_no_data() {
+    assertThat(underTest.getFailures()).isEqualTo(0);
+    assertThat(underTest.getSuccess()).isEqualTo(0);
+    assertThat(underTest.getTotal()).isEqualTo(0);
+    assertThat(underTest.getFailureRatio()).isEqualTo(1);
+    assertThat(underTest.isSuccess()).isTrue();
+  }
+}

server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java

@@ -43,6 +43,7 @@ import org.sonar.db.DbTester;
 import org.sonar.db.es.EsQueueDto;
 import org.sonar.db.rule.RuleDto;
 import org.sonar.db.user.UserDto;
+import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.user.index.UserIndexDefinition;
@@ -73,9 +74,9 @@ public class RecoveryIndexerTest {
   @Rule
   public TestRule safeguard = new DisableOnDebug(new Timeout(60, TimeUnit.SECONDS));
 
-
-  private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class);
   private UserIndexer mockedUserIndexer = mock(UserIndexer.class);
+  private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class);
+  private ActiveRuleIndexer mockedActiveRuleIndexer = mock(ActiveRuleIndexer.class);
   private RecoveryIndexer underTest;
 
   @After
@@ -103,7 +104,7 @@ public class RecoveryIndexerTest {
     Settings settings = new MapSettings()
       .setProperty("sonar.search.recovery.initialDelayInMs", "0")
       .setProperty("sonar.search.recovery.delayInMs", "1");
-    underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer));
+    underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer, mockedActiveRuleIndexer));
     AtomicInteger calls = new AtomicInteger(0);
     doAnswer(invocation -> {
       calls.incrementAndGet();
@@ -234,7 +235,7 @@ public class RecoveryIndexerTest {
   }
 
   @Test
-  public void stop_run_if_too_many_failures() throws Exception {
+  public void stop_run_if_too_many_failures() {
     IntStream.range(0, 10).forEach(i -> createUnindexedUser());
     advanceInTime();
 
@@ -254,7 +255,7 @@ public class RecoveryIndexerTest {
   }
 
   @Test
-  public void do_not_stop_run_if_success_rate_is_greater_than_ratio() throws Exception {
+  public void do_not_stop_run_if_success_rate_is_greater_than_ratio() {
     IntStream.range(0, 10).forEach(i -> createUnindexedUser());
     advanceInTime();
 
@@ -294,7 +295,7 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       called.addAll(items);
       return super.index(dbSession, items);
     }
@@ -308,7 +309,7 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       called.addAll(items);
       return super.index(dbSession, items);
     }
@@ -322,7 +323,7 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       called.addAll(items);
       throw new RuntimeException("boom");
     }
@@ -337,7 +338,7 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+    public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       try {
         if (counter.getCount() == 2) {
           throw new RuntimeException("boom");
@@ -358,15 +359,15 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
-      ResilientIndexerResult result = new ResilientIndexerResult();
+    public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       List<EsQueueDto> filteredItems = items.stream().filter(
         i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList());
+      IndexingResult result = super.index(dbSession, filteredItems);
       if (items.contains(failing)) {
-        result.increaseFailure();
+        result.incrementRequests();
       }
 
-      return result.add(super.index(dbSession, filteredItems));
+      return result;
     }
   }
 
@@ -381,19 +382,17 @@ public class RecoveryIndexerTest {
     }
 
     @Override
-    public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
-      System.out.println("called with " + items.size());
+    public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
       called.addAll(items);
       int success = successfulReturns.next();
-      ResilientIndexerResult result = new ResilientIndexerResult();
+      IndexingResult result = new IndexingResult();
       items.stream().limit(success).forEach(i -> {
-        System.out.println(" + success");
         db.getDbClient().esQueueDao().delete(dbSession, i);
-        result.increaseSuccess();
+        result.incrementSuccess();
         indexed.add(i);
       });
 
-      rangeClosed(1, items.size() - success).forEach(i -> result.increaseFailure());
+      rangeClosed(1, items.size()).forEach(i -> result.incrementRequests());
       dbSession.commit();
       return result;
     }
@@ -434,7 +433,7 @@ public class RecoveryIndexerTest {
   }
 
   private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, Settings settings) {
-    return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer);
+    return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer, mockedActiveRuleIndexer);
   }
 
   private EsQueueDto createUnindexedUser() {

server/sonar-server/src/test/java/org/sonar/server/es/ResilientIndexerResultTest.java

@@ -1,62 +0,0 @@
-package org.sonar.server.es;/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-import java.util.stream.IntStream;
-import org.apache.commons.lang.math.RandomUtils;
-import org.assertj.core.data.Offset;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class ResilientIndexerResultTest {
-
-  private final ResilientIndexerResult underTest = new ResilientIndexerResult();
-
-  @Before
-  public void clear() {
-    underTest.clear();
-  }
-
-  @Test
-  public void ensure_correctness() {
-    int success = 1 + RandomUtils.nextInt(100);
-    int failures = RandomUtils.nextInt(100);
-    IntStream.rangeClosed(1, success).forEach(i -> underTest.increaseSuccess());
-    IntStream.rangeClosed(1, failures).forEach(i -> underTest.increaseFailure());
-
-    assertThat(underTest.getFailures()).isEqualTo(failures);
-    assertThat(underTest.getSuccess()).isEqualTo(success);
-    assertThat(underTest.getTotal()).isEqualTo(success + failures);
-    assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1);
-    assertThat(underTest.getFailureRatio()).isEqualTo(1.0d * failures / (success + failures), Offset.offset(0.000001d));
-    assertThat(underTest.getSuccessRatio()).isEqualTo(1.0d * success / (success + failures), Offset.offset(0.000001d));
-  }
-
-  @Test
-  public void correctness_even_with_no_data() {
-    assertThat(underTest.getFailures()).isEqualTo(0);
-    assertThat(underTest.getSuccess()).isEqualTo(0);
-    assertThat(underTest.getTotal()).isEqualTo(0);
-    assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1);
-    assertThat(underTest.getFailureRatio()).isEqualTo(1);
-    assertThat(underTest.getSuccessRatio()).isEqualTo(0);
-  }
-}

server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java

@@ -86,7 +86,7 @@ public class IssueIndexTest {
   public EsTester esTester = new EsTester(
     new IssueIndexDefinition(settings.asConfig()),
     new ViewIndexDefinition(settings.asConfig()),
-    new RuleIndexDefinition(settings.asConfig());
+    new RuleIndexDefinition(settings.asConfig()));
   @Rule
   public DbTester dbTester = DbTester.create(system2);
   @Rule

server/sonar-server/src/test/java/org/sonar/server/organization/ws/EnableSupportActionTest.java

@@ -27,7 +27,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.mockito.ArgumentCaptor;
-import org.sonar.api.config.MapSettings;
+import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.rule.RuleKey;
 import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.server.ws.WebService;
@@ -71,8 +71,7 @@ public class EnableSupportActionTest {
   @Rule
   public DbTester dbTester = DbTester.create();
   @Rule
-  public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings()));
-
+  public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig()));
 
   private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
   private OrganizationFlags organizationFlags = new OrganizationFlagsImpl(dbTester.getDbClient());
@@ -183,8 +182,7 @@ public class EnableSupportActionTest {
       .containsExactlyInAnyOrder(
         tuple(normal.getKey(), RuleStatus.READY),
         tuple(template.getKey(), RuleStatus.READY),
-        tuple(custom.getKey(), RuleStatus.READY)
-      );
+        tuple(custom.getKey(), RuleStatus.READY));
 
     call();
 
@@ -193,11 +191,10 @@ public class EnableSupportActionTest {
       .containsExactlyInAnyOrder(
         tuple(normal.getKey(), RuleStatus.READY),
         tuple(template.getKey(), RuleStatus.REMOVED),
-        tuple(custom.getKey(), RuleStatus.REMOVED)
-      );
+        tuple(custom.getKey(), RuleStatus.REMOVED));
 
     @SuppressWarnings("unchecked")
-    Class<ArrayList<RuleKey>> listClass = (Class<ArrayList<RuleKey>>)(Class)ArrayList.class;
+    Class<ArrayList<RuleKey>> listClass = (Class<ArrayList<RuleKey>>) (Class) ArrayList.class;
     ArgumentCaptor<ArrayList<RuleKey>> indexedRuleKeys = ArgumentCaptor.forClass(listClass);
     verify(ruleIndexer).commitAndIndex(any(), indexedRuleKeys.capture());
     assertThat(indexedRuleKeys.getValue()).containsExactlyInAnyOrder(template.getKey(), custom.getKey());

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/QProfileFactoryImplTest.java

@@ -49,7 +49,10 @@ import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 
 import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyCollection;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyZeroInteractions;
 
@@ -285,17 +288,17 @@ public class QProfileFactoryImplTest {
   }
 
   private void verifyNoCallsActiveRuleIndexerDelete() {
-    verifyCallActiveRuleIndexerDelete();
+    verify(activeRuleIndexer, never()).commitDeletionOfProfiles(any(DbSession.class), anyCollection());
   }
 
-  private void verifyCallActiveRuleIndexerDelete(String... expectedProfileUuids) {
+  private void verifyCallActiveRuleIndexerDelete(String... expectedRuleProfileUuids) {
     Class<Set<QProfileDto>> setClass = (Class<Set<QProfileDto>>) (Class) Set.class;
     ArgumentCaptor<Set<QProfileDto>> setCaptor = ArgumentCaptor.forClass(setClass);
-    verify(activeRuleIndexer).deleteByProfiles(setCaptor.capture());
+    verify(activeRuleIndexer).commitDeletionOfProfiles(any(DbSession.class), setCaptor.capture());
 
     assertThat(setCaptor.getValue())
       .extracting(QProfileDto::getKee)
-      .containsExactlyInAnyOrder(expectedProfileUuids);
+      .containsExactlyInAnyOrder(expectedRuleProfileUuids);
   }
 
   private void assertThatRulesProfileExists(RulesProfileDto rulesProfile) {
@@ -327,14 +330,6 @@ public class QProfileFactoryImplTest {
     // TODO active_rule_parameters
   }
 
-  private void assertThatCustomProfileIsDefault(OrganizationDto org, QProfileDto profile) {
-    assertThat(db.getDbClient().qualityProfileDao().selectDefaultProfile(dbSession, org, profile.getLanguage())).isEqualTo(profile.getKee());
-  }
-
-  private void assertThatCustomProfileIsAssociatedToProject(OrganizationDto org, QProfileDto profile) {
-    assertThat(db.getDbClient().qualityProfileDao().selectProjectAssociations(dbSession, org, profile, null)).isNotEmpty();
-  }
-
   private static void assertEqual(QProfileDto p1, QProfileDto p2) {
     assertThat(p2.getOrganizationUuid()).isEqualTo(p1.getOrganizationUuid());
     assertThat(p2.getName()).isEqualTo(p1.getName());

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/RuleActivatorTest.java

@@ -48,7 +48,6 @@ import org.sonar.server.es.EsTester;
 import org.sonar.server.es.SearchOptions;
 import org.sonar.server.exceptions.BadRequestException;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -86,8 +85,7 @@ public class RuleActivatorTest {
   public UserSessionRule userSession = UserSessionRule.standalone();
   private RuleIndex ruleIndex = new RuleIndex(es.client());
   private RuleActivatorContextFactory contextFactory = new RuleActivatorContextFactory(db.getDbClient());
-  private ActiveRuleIteratorFactory activeRuleIteratorFactory = new ActiveRuleIteratorFactory(db.getDbClient());
-  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), activeRuleIteratorFactory);
+  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
   private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
   private TypeValidations typeValidations = new TypeValidations(asList(new StringTypeValidation(), new IntegerTypeValidation()));
 
@@ -730,7 +728,7 @@ public class RuleActivatorTest {
   public void unset_parent_when_no_parent_does_not_fail() {
     RuleDefinitionDto rule = createRule();
     QProfileDto profile = createProfile(rule);
-    underTest.setParent(db.getSession(), profile, null);
+    underTest.setParentAndCommit(db.getSession(), profile, null);
   }
 
   @Test
@@ -740,7 +738,7 @@ public class RuleActivatorTest {
 
     expectedException.expect(BadRequestException.class);
     expectedException.expectMessage(" can not be selected as parent of ");
-    underTest.setParent(db.getSession(), profile, profile);
+    underTest.setParentAndCommit(db.getSession(), profile, profile);
   }
 
   @Test
@@ -751,7 +749,7 @@ public class RuleActivatorTest {
 
     expectedException.expect(BadRequestException.class);
     expectedException.expectMessage(" can not be selected as parent of ");
-    underTest.setParent(db.getSession(), parentProfile, childProfile);
+    underTest.setParentAndCommit(db.getSession(), parentProfile, childProfile);
   }
 
   @Test
@@ -763,7 +761,7 @@ public class RuleActivatorTest {
 
     expectedException.expect(BadRequestException.class);
     expectedException.expectMessage(" can not be selected as parent of ");
-    underTest.setParent(db.getSession(), parentProfile, grandchildProfile);
+    underTest.setParentAndCommit(db.getSession(), parentProfile, grandchildProfile);
   }
 
   @Test
@@ -782,7 +780,7 @@ public class RuleActivatorTest {
     expectedException.expect(BadRequestException.class);
     expectedException.expectMessage("Cannot set the profile");
 
-    underTest.setParent(db.getSession(), childProfile, parentProfile);
+    underTest.setParentAndCommit(db.getSession(), childProfile, parentProfile);
   }
 
   @Test
@@ -798,12 +796,12 @@ public class RuleActivatorTest {
     changes = activate(profile2, RuleActivation.create(rule2.getKey()));
     assertThat(changes).hasSize(1);
 
-    changes = underTest.setParent(db.getSession(), profile2, profile1);
+    changes = underTest.setParentAndCommit(db.getSession(), profile2, profile1);
     assertThat(changes).hasSize(1);
     assertThatRuleIsActivated(profile2, rule1, changes, rule1.getSeverityString(), INHERITED, emptyMap());
     assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
 
-    changes = underTest.setParent(db.getSession(), profile2, null);
+    changes = underTest.setParentAndCommit(db.getSession(), profile2, null);
     assertThat(changes).hasSize(1);
     assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
     assertThatRuleIsNotPresent(profile2, rule1);
@@ -821,7 +819,7 @@ public class RuleActivatorTest {
     changes = activate(profile2, RuleActivation.create(rule2.getKey()));
     assertThat(changes).hasSize(1);
 
-    changes = underTest.setParent(db.getSession(), profile2, profile1);
+    changes = underTest.setParentAndCommit(db.getSession(), profile2, profile1);
     assertThat(changes).hasSize(1);
     assertThatRuleIsActivated(profile2, rule1, changes, rule1.getSeverityString(), INHERITED, emptyMap());
     assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
@@ -832,7 +830,7 @@ public class RuleActivatorTest {
     assertThatRuleIsUpdated(profile2, rule1, BLOCKER, ActiveRule.Inheritance.OVERRIDES, emptyMap());
     assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
 
-    changes = underTest.setParent(db.getSession(), profile2, null);
+    changes = underTest.setParentAndCommit(db.getSession(), profile2, null);
     assertThat(changes).hasSize(1);
     // Not testing changes here since severity is not set in changelog
     assertThatRuleIsActivated(profile2, rule1, null, BLOCKER, null, emptyMap());
@@ -856,7 +854,7 @@ public class RuleActivatorTest {
     RuleQuery ruleQuery = new RuleQuery()
       .setRepositories(singletonList(repositoryKey));
 
-    BulkChangeResult bulkChangeResult = underTest.bulkActivate(db.getSession(), ruleQuery, profile, MINOR);
+    BulkChangeResult bulkChangeResult = underTest.bulkActivateAndCommit(db.getSession(), ruleQuery, profile, MINOR);
 
     assertThat(bulkChangeResult.countFailed()).isEqualTo(0);
     assertThat(bulkChangeResult.countSucceeded()).isEqualTo(bulkSize);
@@ -883,7 +881,7 @@ public class RuleActivatorTest {
     RuleQuery ruleQuery = new RuleQuery()
       .setRepositories(singletonList(repositoryKey));
 
-    BulkChangeResult bulkChangeResult = underTest.bulkActivate(db.getSession(), ruleQuery, profile, MINOR);
+    BulkChangeResult bulkChangeResult = underTest.bulkActivateAndCommit(db.getSession(), ruleQuery, profile, MINOR);
 
     assertThat(bulkChangeResult.countFailed()).isEqualTo(0);
     assertThat(bulkChangeResult.countSucceeded()).isEqualTo(bulkSize);
@@ -891,7 +889,7 @@ public class RuleActivatorTest {
     assertThat(db.getDbClient().activeRuleDao().selectByProfile(db.getSession(), profile)).hasSize(bulkSize);
 
     // Now deactivate all rules
-    bulkChangeResult = underTest.bulkDeactivate(db.getSession(), ruleQuery, profile);
+    bulkChangeResult = underTest.bulkDeactivateAndCommit(db.getSession(), ruleQuery, profile);
 
     assertThat(bulkChangeResult.countFailed()).isEqualTo(0);
     assertThat(bulkChangeResult.countSucceeded()).isEqualTo(bulkSize);
@@ -915,7 +913,7 @@ public class RuleActivatorTest {
 
     RuleQuery ruleQuery = new RuleQuery()
       .setQProfile(childProfile);
-    BulkChangeResult bulkChangeResult = underTest.bulkDeactivate(db.getSession(), ruleQuery, childProfile);
+    BulkChangeResult bulkChangeResult = underTest.bulkDeactivateAndCommit(db.getSession(), ruleQuery, childProfile);
 
     assertThat(bulkChangeResult.countFailed()).isEqualTo(1);
     assertThat(bulkChangeResult.countSucceeded()).isEqualTo(0);
@@ -940,7 +938,7 @@ public class RuleActivatorTest {
     RuleQuery query = new RuleQuery()
       .setRuleKey(rule1.getRuleKey())
       .setQProfile(parentProfile);
-    BulkChangeResult result = underTest.bulkActivate(db.getSession(), query, parentProfile, "BLOCKER");
+    BulkChangeResult result = underTest.bulkActivateAndCommit(db.getSession(), query, parentProfile, "BLOCKER");
 
     assertThat(result.getChanges()).hasSize(3);
     assertThat(result.countSucceeded()).isEqualTo(1);
@@ -1057,7 +1055,7 @@ public class RuleActivatorTest {
     db.rules().update(rule1);
 
     QProfileDto childProfile = createProfile(rule1);
-    List<ActiveRuleChange> changes = underTest.setParent(db.getSession(), childProfile, parentProfile);
+    List<ActiveRuleChange> changes = underTest.setParentAndCommit(db.getSession(), childProfile, parentProfile);
 
     assertThatRuleIsNotPresent(childProfile, rule1);
     assertThatRuleIsActivated(childProfile, rule2, changes, rule2.getSeverityString(), INHERITED, emptyMap());

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexerTest.java

@@ -19,28 +19,31 @@
  */
 package org.sonar.server.qualityprofile.index;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
+import org.assertj.core.groups.Tuple;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbTester;
+import org.sonar.db.es.EsQueueDto;
 import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.qualityprofile.ActiveRuleDto;
 import org.sonar.db.qualityprofile.QProfileDto;
-import org.sonar.db.qualityprofile.RulesProfileDto;
 import org.sonar.db.rule.RuleDefinitionDto;
 import org.sonar.server.es.EsTester;
 import org.sonar.server.qualityprofile.ActiveRuleChange;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 
 import static java.util.Arrays.asList;
+import static java.util.Arrays.stream;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.qualityprofile.ActiveRuleChange.Type.ACTIVATED;
-import static org.sonar.server.qualityprofile.ActiveRuleChange.Type.DEACTIVATED;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
 
 public class ActiveRuleIndexerTest {
@@ -53,7 +56,7 @@ public class ActiveRuleIndexerTest {
   @Rule
   public EsTester es = new EsTester(RuleIndexDefinition.createForTest(new MapSettings().asConfig()));
 
-  private ActiveRuleIndexer underTest = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+  private ActiveRuleIndexer underTest = new ActiveRuleIndexer(db.getDbClient(), es.client());
   private RuleDefinitionDto rule1;
   private RuleDefinitionDto rule2;
   private OrganizationDto org;
@@ -71,7 +74,7 @@ public class ActiveRuleIndexerTest {
 
   @Test
   public void getIndexTypes() {
-    assertThat(underTest.getIndexTypes()).containsExactly(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE);
+    assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_ACTIVE_RULE);
   }
 
   @Test
@@ -89,78 +92,111 @@ public class ActiveRuleIndexerTest {
     List<ActiveRuleDoc> docs = es.getDocuments(INDEX_TYPE_ACTIVE_RULE, ActiveRuleDoc.class);
     assertThat(docs).hasSize(1);
     verify(docs.get(0), rule1, profile1, activeRule);
+    assertThatEsQueueTableIsEmpty();
   }
 
   @Test
-  public void deleteByProfiles() throws Exception {
-    ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
-    ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
-    ActiveRuleDto activeRule3 = db.qualityProfiles().activateRule(profile2, rule2);
-    index();
+  public void test_commitAndIndex() {
+    ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+    ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+    ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
 
-    underTest.deleteByProfiles(singletonList(profile2));
+    commitAndIndex(ar1, ar2);
 
-    verifyOnlyIndexed(activeRule1);
+    verifyOnlyIndexed(ar1, ar2);
+    assertThatEsQueueTableIsEmpty();
   }
 
   @Test
-  public void deleteByProfiles_does_nothing_if_profiles_are_not_indexed() throws Exception {
-    ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
-    ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
-    ActiveRuleDto activeRule3 = db.qualityProfiles().activateRule(profile2, rule2);
-    assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+  public void commitAndIndex_empty_list() {
+    ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
 
-    underTest.deleteByProfiles(singletonList(profile2));
+    underTest.commitAndIndex(db.getSession(), Collections.emptyList());
 
     assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+    assertThatEsQueueTableIsEmpty();
   }
 
   @Test
-  public void indexRuleProfile() throws Exception {
-    ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
-    ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
-    ActiveRuleDto activeRule3 = db.qualityProfiles().activateRule(profile2, rule2);
+  public void commitAndIndex_keeps_elements_to_recover_in_ES_QUEUE_on_errors() {
+    ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
+    // force error by deleting the index
+    deleteRulesIndex();
 
-    indexProfile(profile2);
+    commitAndIndex(ar);
 
-    verifyOnlyIndexed(activeRule2, activeRule3);
+    EsQueueDto expectedItem = EsQueueDto.create(EsQueueDto.Type.ACTIVE_RULE, "" + ar.getId(), "activeRuleId", ar.getRuleKey().toString());
+    assertThatEsQueueContainsExactly(expectedItem);
   }
 
   @Test
-  public void indexChanges_puts_documents() throws Exception {
-    ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
-    ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
-    ActiveRuleDto nonIndexed = db.qualityProfiles().activateRule(profile2, rule2);
+  public void commitAndIndex_deletes_the_documents_that_dont_exist_in_database() {
+    ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
+    indexAll();
+    assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
 
-    underTest.indexChanges(db.getSession(), asList(
-      newChange(ACTIVATED, activeRule1), newChange(ACTIVATED, activeRule2)));
+    db.getDbClient().activeRuleDao().delete(db.getSession(), ar.getKey());
+    commitAndIndex(ar);
 
-    verifyOnlyIndexed(activeRule1, activeRule2);
+    assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+    assertThatEsQueueTableIsEmpty();
   }
 
   @Test
-  public void indexChanges_deletes_documents_when_type_is_DEACTIVATED() throws Exception {
-    ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
-    ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
-    underTest.indexChanges(db.getSession(), asList(
-      newChange(ACTIVATED, activeRule1), newChange(ACTIVATED, activeRule2)));
-    assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE)).isEqualTo(2);
+  public void index_fails_and_deletes_doc_if_docIdType_is_unsupported() {
+    EsQueueDto item = EsQueueDto.create(EsQueueDto.Type.ACTIVE_RULE, "the_id", "unsupported", "the_routing");
+    db.getDbClient().esQueueDao().insert(db.getSession(), item);
 
-    underTest.indexChanges(db.getSession(), singletonList(newChange(DEACTIVATED, activeRule1)));
+    underTest.index(db.getSession(), asList(item));
 
-    verifyOnlyIndexed(activeRule2);
+    assertThatEsQueueTableIsEmpty();
+    assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
   }
 
   @Test
-  public void deleteByRuleKeys() {
-    ActiveRuleDto active1 = db.qualityProfiles().activateRule(profile1, rule1);
-    ActiveRuleDto active2 = db.qualityProfiles().activateRule(profile2, rule1);
-    ActiveRuleDto onRule2 = db.qualityProfiles().activateRule(profile2, rule2);
-    index();
+  public void commitDeletionOfProfiles() {
+    ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+    ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+    ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+    indexAll();
+    db.getDbClient().qualityProfileDao().deleteRulesProfilesByUuids(db.getSession(), singletonList(profile2.getRulesProfileUuid()));
+
+    underTest.commitDeletionOfProfiles(db.getSession(), singletonList(profile2));
+
+    verifyOnlyIndexed(ar1);
+  }
+
+  @Test
+  public void commitDeletionOfProfiles_does_nothing_if_profiles_are_not_indexed() {
+    db.qualityProfiles().activateRule(profile1, rule1);
+    indexAll();
+    assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
+
+    underTest.commitDeletionOfProfiles(db.getSession(), singletonList(profile2));
+
+    assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
+  }
 
-    underTest.deleteByRuleKeys(singletonList(rule2.getKey()));
 
-    verifyOnlyIndexed(active1, active2);
+  private void deleteRulesIndex() {
+    es.deleteIndex(RuleIndexDefinition.INDEX_TYPE_RULE.getIndex());
+  }
+
+  private void assertThatEsQueueTableIsEmpty() {
+    assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isEqualTo(0);
+  }
+
+  private void assertThatEsQueueContainsExactly(EsQueueDto expected) {
+    Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), system2.now() + 1_000, 10);
+    assertThat(items)
+      .extracting(EsQueueDto::getDocId, EsQueueDto::getDocIdType, EsQueueDto::getDocRouting)
+      .containsExactlyInAnyOrder(Tuple.tuple(expected.getDocId(), expected.getDocIdType(), expected.getDocRouting()));
+  }
+
+  private void commitAndIndex(ActiveRuleDto... ar) {
+    underTest.commitAndIndex(db.getSession(), stream(ar)
+      .map(a -> new ActiveRuleChange(ActiveRuleChange.Type.ACTIVATED, a))
+      .collect(Collectors.toList()));
   }
 
   private void verifyOnlyIndexed(ActiveRuleDto... expected) {
@@ -171,14 +207,6 @@ public class ActiveRuleIndexerTest {
     }
   }
 
-  private ActiveRuleChange newChange(ActiveRuleChange.Type type, ActiveRuleDto activeRule) {
-    return new ActiveRuleChange(type, activeRule);
-  }
-
-  private void indexProfile(QProfileDto profile) {
-    underTest.indexRuleProfile(db.getSession(), RulesProfileDto.from(profile));
-  }
-
   private void verify(ActiveRuleDoc doc1, RuleDefinitionDto rule, QProfileDto profile, ActiveRuleDto activeRule) {
     assertThat(doc1)
       .matches(doc -> doc.getRuleKey().equals(rule.getKey()))
@@ -188,7 +216,7 @@ public class ActiveRuleIndexerTest {
       .matches(doc -> doc.getSeverity().equals(activeRule.getSeverityString()));
   }
 
-  private void index() {
+  private void indexAll() {
     underTest.indexOnStartup(emptySet());
   }
 

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ActivateRuleActionTest.java

@@ -41,7 +41,6 @@ import org.sonar.server.exceptions.UnauthorizedException;
 import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.RuleActivation;
 import org.sonar.server.qualityprofile.RuleActivator;
-import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.tester.UserSessionRule;
 import org.sonar.server.ws.TestRequest;
 import org.sonar.server.ws.TestResponse;
@@ -68,9 +67,8 @@ public class ActivateRuleActionTest {
   private DbClient dbClient = dbTester.getDbClient();
   private RuleActivator ruleActivator = mock(RuleActivator.class);
   private QProfileWsSupport wsSupport = new QProfileWsSupport(dbClient, userSession, TestDefaultOrganizationProvider.from(dbTester));
-  private ActiveRuleIndexer activeRuleIndexer = mock(ActiveRuleIndexer.class);
 
-  private WsActionTester ws = new WsActionTester(new ActivateRuleAction(dbClient, ruleActivator, userSession, wsSupport, activeRuleIndexer));
+  private WsActionTester ws = new WsActionTester(new ActivateRuleAction(dbClient, ruleActivator, userSession, wsSupport));
 
   private OrganizationDto defaultOrganization;
   private OrganizationDto organization;
@@ -152,7 +150,7 @@ public class ActivateRuleActionTest {
 
     assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
     ArgumentCaptor<RuleActivation> captor = ArgumentCaptor.forClass(RuleActivation.class);
-    verify(ruleActivator).activate(any(DbSession.class), captor.capture(), any(QProfileDto.class));
+    verify(ruleActivator).activateAndCommit(any(DbSession.class), captor.capture(), any(QProfileDto.class));
     RuleActivation value = captor.getValue();
     assertThat(value.getRuleKey()).isEqualTo(ruleKey);
     assertThat(value.getSeverity()).isEqualTo(Severity.BLOCKER);
@@ -176,7 +174,7 @@ public class ActivateRuleActionTest {
 
     assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
     ArgumentCaptor<RuleActivation> captor = ArgumentCaptor.forClass(RuleActivation.class);
-    verify(ruleActivator).activate(any(DbSession.class), captor.capture(), any(QProfileDto.class));
+    verify(ruleActivator).activateAndCommit(any(DbSession.class), captor.capture(), any(QProfileDto.class));
     assertThat(captor.getValue().getRuleKey()).isEqualTo(ruleKey);
     assertThat(captor.getValue().getSeverity()).isEqualTo(Severity.BLOCKER);
     assertThat(captor.getValue().isReset()).isFalse();

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ChangeParentActionTest.java

@@ -53,7 +53,6 @@ import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.RuleActivator;
 import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -104,7 +103,7 @@ public class ChangeParentActionTest {
     EsClient esClient = esTester.client();
     ruleIndex = new RuleIndex(esClient);
     ruleIndexer = new RuleIndexer(esClient, dbClient);
-    activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+    activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
     RuleActivatorContextFactory ruleActivatorContextFactory = new RuleActivatorContextFactory(dbClient);
     TypeValidations typeValidations = new TypeValidations(Collections.emptyList());
     ruleActivator = new RuleActivator(System2.INSTANCE, dbClient, ruleIndex, ruleActivatorContextFactory, typeValidations, activeRuleIndexer, userSessionRule);
@@ -191,7 +190,7 @@ public class ChangeParentActionTest {
     activeRuleIndexer.indexOnStartup(emptySet());
 
     // Set parent 1
-    ruleActivator.setParent(dbSession, child, parent1);
+    ruleActivator.setParentAndCommit(dbSession, child, parent1);
 
     // Set parent 2 through WS
     ws.newRequest()
@@ -219,7 +218,7 @@ public class ChangeParentActionTest {
     activeRuleIndexer.indexOnStartup(emptySet());
 
     // Set parent
-    ruleActivator.setParent(dbSession, child, parent);
+    ruleActivator.setParentAndCommit(dbSession, child, parent);
 
     // Remove parent through WS
     ws.newRequest()
@@ -308,7 +307,7 @@ public class ChangeParentActionTest {
     assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty();
 
     // Set parent
-    ruleActivator.setParent(dbSession, child, parent);
+    ruleActivator.setParentAndCommit(dbSession, child, parent);
 
     // Remove parent
     ws.newRequest()

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/CreateActionTest.java

@@ -53,7 +53,6 @@ import org.sonar.server.qualityprofile.QProfileFactoryImpl;
 import org.sonar.server.qualityprofile.RuleActivator;
 import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -93,7 +92,7 @@ public class CreateActionTest {
   private DbSession dbSession = db.getSession();
   private RuleIndex ruleIndex = new RuleIndex(es.client());
   private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), dbClient);
-  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, es.client(), new ActiveRuleIteratorFactory(dbClient));
+  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, es.client());
   private ProfileImporter[] profileImporters = createImporters();
   private QProfileExporters qProfileExporters = new QProfileExporters(dbClient, null,
     new RuleActivator(mock(System2.class), dbClient, ruleIndex, new RuleActivatorContextFactory(dbClient), null, activeRuleIndexer, userSession),

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/DeactivateRuleActionTest.java

@@ -143,7 +143,7 @@ public class DeactivateRuleActionTest {
     assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
     ArgumentCaptor<RuleKey> ruleKeyCaptor = ArgumentCaptor.forClass(RuleKey.class);
     ArgumentCaptor<QProfileDto> qProfileDtoCaptor = ArgumentCaptor.forClass(QProfileDto.class);
-    verify(ruleActivator).deactivateAndUpdateIndex(any(DbSession.class), qProfileDtoCaptor.capture(), ruleKeyCaptor.capture());
+    verify(ruleActivator).deactivateAndCommit(any(DbSession.class), qProfileDtoCaptor.capture(), ruleKeyCaptor.capture());
     assertThat(ruleKeyCaptor.getValue()).isEqualTo(ruleKey);
     assertThat(qProfileDtoCaptor.getValue().getKee()).isEqualTo(qualityProfile.getKee());
   }
@@ -164,7 +164,7 @@ public class DeactivateRuleActionTest {
     assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
     ArgumentCaptor<RuleKey> captor = ArgumentCaptor.forClass(RuleKey.class);
     ArgumentCaptor<QProfileDto> qProfileDtoCaptor = ArgumentCaptor.forClass(QProfileDto.class);
-    verify(ruleActivator).deactivateAndUpdateIndex(any(DbSession.class), qProfileDtoCaptor.capture(), captor.capture());
+    verify(ruleActivator).deactivateAndCommit(any(DbSession.class), qProfileDtoCaptor.capture(), captor.capture());
     assertThat(captor.getValue()).isEqualTo(ruleKey);
     assertThat(qProfileDtoCaptor.getValue().getKee()).isEqualTo(qualityProfile.getKee());
   }

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/InheritanceActionTest.java

@@ -50,7 +50,6 @@ import org.sonar.server.qualityprofile.RuleActivation;
 import org.sonar.server.qualityprofile.RuleActivator;
 import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -92,7 +91,7 @@ public class InheritanceActionTest {
     dbSession = dbTester.getSession();
     esClient = es.client();
     ruleIndexer = new RuleIndexer(esClient, dbClient);
-    activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+    activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
     TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
     underTest = new InheritanceAction(
       dbClient,
@@ -251,7 +250,7 @@ public class InheritanceActionTest {
   }
 
   private void setParent(QProfileDto profile, QProfileDto parent) {
-    ruleActivator.setParent(dbSession, parent, profile);
+    ruleActivator.setParentAndCommit(dbSession, parent, profile);
   }
 
   private RuleDefinitionDto createRule(String lang, String id) {

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ShowActionTest.java

@@ -35,7 +35,6 @@ import org.sonar.server.es.EsTester;
 import org.sonar.server.exceptions.NotFoundException;
 import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -73,7 +72,7 @@ public class ShowActionTest {
   public ExpectedException expectedException = ExpectedException.none();
 
   private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
-  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
   private RuleIndex ruleIndex = new RuleIndex(es.client());
 
   private WsActionTester ws = new WsActionTester(

server/sonar-server/src/test/java/org/sonar/server/rule/RegisterRulesTest.java

@@ -49,7 +49,6 @@ import org.sonar.server.organization.OrganizationFlags;
 import org.sonar.server.organization.TestOrganizationFlags;
 import org.sonar.server.qualityprofile.RuleActivator;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -99,7 +98,7 @@ public class RegisterRulesTest {
     when(system.now()).thenReturn(DATE1.getTime());
     ruleIndexer = new RuleIndexer(esTester.client(), dbClient);
     ruleIndex = new RuleIndex(esTester.client());
-    activeRuleIndexer = new ActiveRuleIndexer(dbClient, esTester.client(), new ActiveRuleIteratorFactory(dbClient));
+    activeRuleIndexer = new ActiveRuleIndexer(dbClient, esTester.client());
     defaultOrganization = dbTester.getDefaultOrganization();
   }
 

server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java

@@ -44,7 +44,6 @@ import org.sonar.server.es.Facets;
 import org.sonar.server.es.SearchIdResult;
 import org.sonar.server.es.SearchOptions;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 
 import static com.google.common.collect.ImmutableSet.of;
 import static java.util.Arrays.asList;
@@ -104,7 +103,7 @@ public class RuleIndexTest {
   @Before
   public void setUp() {
     ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
-    activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+    activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
     underTest = new RuleIndex(es.client());
   }
 

server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIteratorForSingleChunkTest.java

@@ -1,201 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import com.google.common.collect.Lists;
-import java.util.List;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.api.rule.RuleStatus;
-import org.sonar.api.rule.Severity;
-import org.sonar.api.rules.RuleType;
-import org.sonar.api.utils.System2;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.DbTester;
-import org.sonar.db.rule.RuleDefinitionDto;
-import org.sonar.db.rule.RuleDto;
-import org.sonar.server.exceptions.NotFoundException;
-
-import static com.google.common.collect.Sets.newHashSet;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class RuleIteratorForSingleChunkTest {
-
-  @Rule
-  public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
-  private DbClient dbClient = dbTester.getDbClient();
-  private DbSession dbSession = dbTester.getSession();
-  private RuleDefinitionDto templateRule;
-  private RuleDefinitionDto customRule;
-
-  @Before
-  public void setUp() throws Exception {
-    templateRule = new RuleDefinitionDto()
-        .setRuleKey("S001")
-        .setRepositoryKey("xoo")
-        .setConfigKey("S1")
-        .setName("Null Pointer")
-        .setDescription("S001 desc")
-        .setDescriptionFormat(RuleDto.Format.HTML)
-        .setLanguage("xoo")
-        .setSeverity(Severity.BLOCKER)
-        .setStatus(RuleStatus.READY)
-        .setIsTemplate(true)
-        .setSystemTags(newHashSet("cwe"))
-        .setType(RuleType.BUG)
-        .setCreatedAt(1500000000000L)
-        .setUpdatedAt(1600000000000L);
-
-    customRule = new RuleDefinitionDto()
-        .setRuleKey("S002")
-        .setRepositoryKey("xoo")
-        .setConfigKey("S2")
-        .setName("Slow")
-        .setDescription("*S002 desc*")
-        .setDescriptionFormat(RuleDto.Format.MARKDOWN)
-        .setLanguage("xoo")
-        .setSeverity(Severity.MAJOR)
-        .setStatus(RuleStatus.BETA)
-        .setIsTemplate(false)
-        .setType(RuleType.CODE_SMELL)
-        .setCreatedAt(2000000000000L)
-        .setUpdatedAt(2100000000000L);
-  }
-
-  @Test
-  public void iterator_over_one_rule() {
-    dbTester.rules().insert(templateRule);
-
-    List<RuleDocWithSystemScope> results = getResults();
-
-    assertThat(results).hasSize(1);
-
-    RuleDocWithSystemScope ruleDocWithSystemScope = getRuleDoc(results, templateRule.getRuleKey());
-    RuleDoc templateDoc = ruleDocWithSystemScope.getRuleDoc();
-    RuleExtensionDoc templateExtensionDoc = ruleDocWithSystemScope.getRuleExtensionDoc();
-    assertThat(templateDoc).isNotNull();
-    assertThat(templateDoc.key()).isEqualTo(RuleKey.of("xoo", "S001"));
-    assertThat(templateDoc.ruleKey()).isEqualTo("S001");
-    assertThat(templateDoc.repository()).isEqualTo("xoo");
-    assertThat(templateDoc.internalKey()).isEqualTo("S1");
-    assertThat(templateDoc.name()).isEqualTo("Null Pointer");
-    assertThat(templateDoc.htmlDescription()).isEqualTo("S001 desc");
-    assertThat(templateDoc.language()).isEqualTo("xoo");
-    assertThat(templateDoc.severity()).isEqualTo(Severity.BLOCKER);
-    assertThat(templateDoc.status()).isEqualTo(RuleStatus.READY);
-    assertThat(templateDoc.isTemplate()).isTrue();
-    assertThat(templateExtensionDoc.getTags()).containsOnly("cwe");
-    assertThat(templateDoc.createdAt()).isEqualTo(1500000000000L);
-    assertThat(templateDoc.updatedAt()).isEqualTo(1600000000000L);
-  }
-
-  @Test
-  public void iterator_over_rules() {
-    dbTester.rules().insert(templateRule);
-    dbClient.ruleDao().insert(dbSession, customRule);
-    dbSession.commit();
-
-    List<RuleDocWithSystemScope> results = getResults();
-
-    assertThat(results).hasSize(2);
-
-    RuleDocWithSystemScope templateDocWithSystemScope = getRuleDoc(results, templateRule.getRuleKey());
-    RuleDoc templateDoc = templateDocWithSystemScope.getRuleDoc();
-    RuleExtensionDoc templateExtensionDoc = templateDocWithSystemScope.getRuleExtensionDoc();
-    assertThat(templateDoc.key()).isEqualTo(RuleKey.of("xoo", "S001"));
-    assertThat(templateDoc.ruleKey()).isEqualTo("S001");
-    assertThat(templateDoc.repository()).isEqualTo("xoo");
-    assertThat(templateDoc.internalKey()).isEqualTo("S1");
-    assertThat(templateDoc.name()).isEqualTo("Null Pointer");
-    assertThat(templateDoc.htmlDescription()).isEqualTo("S001 desc");
-    assertThat(templateDoc.language()).isEqualTo("xoo");
-    assertThat(templateDoc.severity()).isEqualTo(Severity.BLOCKER);
-    assertThat(templateDoc.status()).isEqualTo(RuleStatus.READY);
-    assertThat(templateDoc.isTemplate()).isTrue();
-    assertThat(templateExtensionDoc.getTags()).containsOnly("cwe");
-    assertThat(templateDoc.createdAt()).isEqualTo(1500000000000L);
-    assertThat(templateDoc.updatedAt()).isEqualTo(1600000000000L);
-
-    RuleDocWithSystemScope customDocWithSystemScope = getRuleDoc(results, customRule.getRuleKey());
-    RuleDoc customDoc = customDocWithSystemScope.getRuleDoc();
-    RuleExtensionDoc customExtensionDoc = customDocWithSystemScope.getRuleExtensionDoc();
-    assertThat(customDoc.key()).isEqualTo(RuleKey.of("xoo", "S002"));
-    assertThat(customDoc.ruleKey()).isEqualTo("S002");
-    assertThat(customDoc.repository()).isEqualTo("xoo");
-    assertThat(customDoc.internalKey()).isEqualTo("S2");
-    assertThat(customDoc.name()).isEqualTo("Slow");
-    assertThat(customDoc.htmlDescription()).isEqualTo("<strong>S002 desc</strong>");
-    assertThat(customDoc.language()).isEqualTo("xoo");
-    assertThat(customDoc.severity()).isEqualTo(Severity.MAJOR);
-    assertThat(customDoc.status()).isEqualTo(RuleStatus.BETA);
-    assertThat(customDoc.isTemplate()).isFalse();
-    assertThat(customExtensionDoc.getTags()).isEmpty();
-    assertThat(customDoc.createdAt()).isEqualTo(2000000000000L);
-    assertThat(customDoc.updatedAt()).isEqualTo(2100000000000L);
-  }
-
-  @Test
-  public void custom_rule() {
-    dbTester.rules().insert(templateRule);
-    dbClient.ruleDao().insert(dbSession, customRule.setTemplateId(templateRule.getId()));
-    dbSession.commit();
-
-    List<RuleDocWithSystemScope> results = getResults();
-
-    assertThat(results).hasSize(2);
-
-    RuleDocWithSystemScope templateDocWithSystemScope = getRuleDoc(results, templateRule.getRuleKey());
-    RuleDoc templateDoc = templateDocWithSystemScope.getRuleDoc();
-    assertThat(templateDoc.isTemplate()).isTrue();
-    assertThat(templateDoc.templateKey()).isNull();
-
-    RuleDocWithSystemScope customDocWithSystemScope = getRuleDoc(results, customRule.getRuleKey());
-    RuleDoc customDoc = customDocWithSystemScope.getRuleDoc();
-    assertThat(customDoc.isTemplate()).isFalse();
-    assertThat(customDoc.templateKey()).isEqualTo(RuleKey.of("xoo", "S001"));
-  }
-
-  @Test
-  public void removed_rule_is_returned() {
-    dbTester.rules().insert(templateRule.setStatus(RuleStatus.REMOVED));
-    dbSession.commit();
-
-    List<RuleDocWithSystemScope> results = getResults();
-
-    assertThat(results).hasSize(1);
-  }
-
-  private List<RuleDocWithSystemScope> getResults() {
-    return Lists.newArrayList(new RuleIteratorForSingleChunk(dbTester.getDbClient(), null));
-  }
-
-  private RuleDocWithSystemScope getRuleDoc(List<RuleDocWithSystemScope> results, String ruleKey) {
-    RuleDocWithSystemScope rule;
-    rule = results.stream()
-      .filter(r -> ruleKey.equals(r.getRuleDoc().key().rule()))
-      .findAny()
-      .orElseThrow(() -> new NotFoundException("Rule not found in results"));
-    return rule;
-  }
-}

server/sonar-server/src/test/java/org/sonar/server/rule/ws/DeleteActionTest.java

@@ -22,7 +22,7 @@ package org.sonar.server.rule.ws;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
-import org.sonar.api.config.MapSettings;
+import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbClient;
@@ -56,7 +56,7 @@ public class DeleteActionTest {
   @Rule
   public DbTester dbTester = DbTester.create();
   @Rule
-  public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings()));
+  public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig()));
   @Rule
   public ExpectedException thrown = ExpectedException.none();
 

server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java

@@ -55,7 +55,6 @@ import org.sonar.server.qualityprofile.RuleActivation;
 import org.sonar.server.qualityprofile.RuleActivator;
 import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -106,7 +105,7 @@ public class SearchActionTest {
   private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
   private RuleIndex ruleIndex = new RuleIndex(es.client());
   private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
-  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+  private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
   private Languages languages = LanguageTesting.newLanguages(JAVA, "js");
   private ActiveRuleCompleter activeRuleCompleter = new ActiveRuleCompleter(db.getDbClient(), languages);
   private RuleWsSupport wsSupport = new RuleWsSupport(db.getDbClient(), userSession, defaultOrganizationProvider);

server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java

@@ -44,7 +44,6 @@ import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.QProfileTesting;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.text.MacroInterpreter;
@@ -150,7 +149,7 @@ public class ShowActionTest {
       .build();
     Mockito.doReturn(singletonList(active)).when(activeRuleCompleter).completeShow(any(DbSession.class), orgCaptor.capture(), ruleCaptor.capture());
 
-    ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+    ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
     activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes());
 
     TestResponse response = actionTester.newRequest().setMethod("GET")
@@ -185,7 +184,7 @@ public class ShowActionTest {
     RuleMetadataDto ruleMetadata = dbTester.rules().insertOrUpdateMetadata(rule, organization);
 
     dbTester.qualityProfiles().activateRule(profile, rule, a -> a.setSeverity("BLOCKER"));
-    ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+    ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
     activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes());
 
     TestResponse response = actionTester.newRequest().setMethod("GET")

tests/resilience/active_rule_indexer.btm

@@ -0,0 +1,12 @@
+# sonar.web.javaAdditionalOpts=-javaagent:/path/to/byteman-3.0.10/lib/byteman.jar=script:/path/to/active_rule_indexer.btm,boot:/path/to/byteman-3.0.10/lib/byteman.jar
+# sonar.search.recovery.delayInMs=10000
+# sonar.search.recovery.minAgeInMs=30000
+
+RULE make indexing of active rules silently fail
+CLASS ActiveRuleIndexer
+METHOD postCommit
+COMPILE
+AT ENTRY
+IF TRUE
+DO RETURN
+ENDRULE

tests/src/test/java/org/sonarqube/tests/Category5Suite.java

@@ -21,6 +21,7 @@ package org.sonarqube.tests;
 
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
+import org.sonarqube.tests.qualityProfile.ActiveRuleEsResilienceTest;
 import org.sonarqube.tests.qualityProfile.BuiltInQualityProfilesNotificationTest;
 import org.sonarqube.tests.serverSystem.ClusterTest;
 import org.sonarqube.tests.serverSystem.RestartTest;
@@ -51,6 +52,7 @@ import org.sonarqube.tests.user.UserEsResilienceTest;
   SsoAuthenticationTest.class,
   OnboardingTest.class,
   BuiltInQualityProfilesNotificationTest.class,
+  ActiveRuleEsResilienceTest.class,
   UserEsResilienceTest.class
 })
 public class Category5Suite {

tests/src/test/java/org/sonarqube/tests/qualityProfile/ActiveRuleEsResilienceTest.java

@@ -0,0 +1,107 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.tests.qualityProfile;
+
+import com.sonar.orchestrator.Orchestrator;
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.Organizations;
+import org.sonarqube.ws.QualityProfiles;
+import org.sonarqube.ws.client.rule.SearchWsRequest;
+import util.ItUtils;
+
+import static java.lang.String.format;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ActiveRuleEsResilienceTest {
+  private static final String RULE_ONE_BUG_PER_LINE = "xoo:OneBugIssuePerLine";
+
+  @ClassRule
+  public static final Orchestrator orchestrator = Orchestrator.builderEnv()
+    .setServerProperty("sonar.web.javaAdditionalOpts",
+      format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar()))
+    .setServerProperty("sonar.search.recovery.delayInMs", "500")
+    .setServerProperty("sonar.search.recovery.minAgeInMs", "3000")
+    .addPlugin(ItUtils.xooPlugin())
+    .build();
+
+  @Rule
+  public TestRule timeout = new DisableOnDebug(Timeout.builder()
+    .withLookingForStuckThread(true)
+    .withTimeout(60L, TimeUnit.SECONDS)
+    .build());
+
+  @Rule
+  public Tester tester = new Tester(orchestrator);
+
+  @Test
+  public void activation_and_deactivation_of_rule_is_resilient_to_indexing_errors() throws Exception {
+    Organizations.Organization organization = tester.organizations().generate();
+    QualityProfiles.CreateWsResponse.QualityProfile profile = tester.qProfiles().createXooProfile(organization);
+
+    // step 1. activation
+    tester.qProfiles().activateRule(profile.getKey(), RULE_ONE_BUG_PER_LINE);
+
+    assertThat(searchActiveRules(profile)).isEqualTo(0);
+    while (searchActiveRules(profile) == 0) {
+      // rule is indexed by the recovery daemon, which runs every 3 seconds
+      Thread.sleep(500L);
+    }
+    assertThat(searchActiveRules(profile)).isEqualTo(1);
+
+    // step 2. deactivation
+    tester.qProfiles().deactivateRule(profile, RULE_ONE_BUG_PER_LINE);
+    while (searchActiveRules(profile) == 1) {
+      // rule is indexed by the recovery daemon, which runs every 3 seconds
+      Thread.sleep(500L);
+    }
+    assertThat(searchActiveRules(profile)).isEqualTo(0);
+  }
+
+  private long searchActiveRules(QualityProfiles.CreateWsResponse.QualityProfile profile) {
+    SearchWsRequest request = new SearchWsRequest().setActivation(true).setQProfile(profile.getKey());
+    return tester.wsClient().rules().search(request).getRulesCount();
+  }
+
+  private static String findBytemanJar() {
+    // see pom.xml, Maven copies and renames the artifact.
+    File jar = new File("target/byteman.jar");
+    if (!jar.exists()) {
+      throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once.");
+    }
+    return jar.getAbsolutePath();
+  }
+
+  private static String findBytemanScript() {
+    // see pom.xml, Maven copies and renames the artifact.
+    File script = new File("resilience/active_rule_indexer.btm");
+    if (!script.exists()) {
+      throw new IllegalStateException("Can't find " + script);
+    }
+    return script.getAbsolutePath();
+  }
+}