SONAR-8952 only index rules, when they are updated in db

server/sonar-ce/src/main/java/org/sonar/ce/container/ComputeEngineContainerImpl.java

@@ -128,7 +128,6 @@ import org.sonar.server.rule.DefaultRuleFinder;
 import org.sonar.server.rule.DeprecatedRulesDefinitionLoader;
 import org.sonar.server.rule.RuleDefinitionsLoader;
 import org.sonar.server.rule.index.RuleIndex;
-import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.search.EsSearchModule;
 import org.sonar.server.setting.DatabaseSettingLoader;
 import org.sonar.server.setting.DatabaseSettingsEnabler;
@@ -300,7 +299,6 @@ public class ComputeEngineContainerImpl implements ComputeEngineContainer {
       Rules.QProfiles.class,
 
       // rule
-      RuleIndexer.class,
       AnnotationRuleParser.class,
       XMLRuleParser.class,
       DefaultRuleFinder.class,

server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java

@@ -88,7 +88,7 @@ public class ComputeEngineContainerImplTest {
     assertThat(picoContainer.getComponentAdapters())
       .hasSize(
         CONTAINER_ITSELF
-          + 75 // level 4
+          + 74 // level 4
           + 4 // content of CeConfigurationModule
           + 5 // content of CeQueueModule
           + 3 // content of CeHttpModule

server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDto.java

@@ -34,8 +34,17 @@ public class RuleDto {
     HTML, MARKDOWN
   }
 
-  private final RuleDefinitionDto definition = new RuleDefinitionDto();
-  private final RuleMetadataDto metadata = new RuleMetadataDto();
+  private final RuleDefinitionDto definition;
+  private final RuleMetadataDto metadata;
+
+  public RuleDto() {
+    this(new RuleDefinitionDto(), new RuleMetadataDto());
+  }
+
+  public RuleDto(RuleDefinitionDto definition, RuleMetadataDto metadata) {
+    this.definition = definition;
+    this.metadata = metadata;
+  }
 
   public RuleDefinitionDto getDefinition() {
     return definition;

server/sonar-db-dao/src/test/java/org/sonar/db/rule/RuleDbTester.java

@@ -19,6 +19,7 @@
  */
 package org.sonar.db.rule;
 
+import java.util.Arrays;
 import java.util.function.Consumer;
 import org.apache.commons.lang.RandomStringUtils;
 import org.sonar.api.server.rule.RuleParamType;
@@ -62,9 +63,9 @@ public class RuleDbTester {
     });
   }
 
-  public RuleDto insertRule(OrganizationDto organization, Consumer<RuleDto> populateRuleDto) {
+  public RuleDto insertRule(OrganizationDto organization, Consumer<RuleDto>... populaters) {
     RuleDto ruleDto = newRuleDto(organization);
-    populateRuleDto.accept(ruleDto);
+    Arrays.asList(populaters).forEach(populater -> populater.accept(ruleDto));
     return insertRule(ruleDto);
   }
 

server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java

@@ -167,6 +167,7 @@ import org.sonar.server.rule.RuleService;
 import org.sonar.server.rule.RuleUpdater;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
+import org.sonar.server.rule.index.RuleIteratorFactory;
 import org.sonar.server.rule.ws.ActiveRuleCompleter;
 import org.sonar.server.rule.ws.RepositoriesAction;
 import org.sonar.server.rule.ws.RuleMapper;
@@ -280,6 +281,7 @@ public class PlatformLevel4 extends PlatformLevel {
       // rule
       RuleIndexDefinition.class,
       RuleIndexer.class,
+      RuleIteratorFactory.class,
       AnnotationRuleParser.class,
       XMLRuleParser.class,
       DefaultRuleFinder.class,

server/sonar-server/src/main/java/org/sonar/server/rule/RegisterRules.java

@@ -45,12 +45,14 @@ import org.sonar.api.utils.log.Profiler;
 import org.sonar.core.util.stream.Collectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.qualityprofile.ActiveRuleDto;
 import org.sonar.db.qualityprofile.ActiveRuleParamDto;
 import org.sonar.db.rule.RuleDefinitionDto;
 import org.sonar.db.rule.RuleDto.Format;
 import org.sonar.db.rule.RuleParamDto;
 import org.sonar.db.rule.RuleRepositoryDto;
+import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.ActiveRuleChange;
 import org.sonar.server.qualityprofile.RuleActivator;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
@@ -73,9 +75,10 @@ public class RegisterRules implements Startable {
   private final ActiveRuleIndexer activeRuleIndexer;
   private final Languages languages;
   private final System2 system2;
+  private final DefaultOrganizationProvider defaultOrganizationProvider;
 
   public RegisterRules(RuleDefinitionsLoader defLoader, RuleActivator ruleActivator, DbClient dbClient, RuleIndexer ruleIndexer,
-    ActiveRuleIndexer activeRuleIndexer, Languages languages, System2 system2) {
+    ActiveRuleIndexer activeRuleIndexer, Languages languages, System2 system2, DefaultOrganizationProvider defaultOrganizationProvider) {
     this.defLoader = defLoader;
     this.ruleActivator = ruleActivator;
     this.dbClient = dbClient;
@@ -83,34 +86,44 @@ public class RegisterRules implements Startable {
     this.activeRuleIndexer = activeRuleIndexer;
     this.languages = languages;
     this.system2 = system2;
+    this.defaultOrganizationProvider = defaultOrganizationProvider;
   }
 
   @Override
   public void start() {
     Profiler profiler = Profiler.create(LOG).startInfo("Register rules");
-    DbSession session = dbClient.openSession(false);
-    try {
+    try (DbSession session = dbClient.openSession(false)) {
       Map<RuleKey, RuleDefinitionDto> allRules = loadRules(session);
+      List<RuleKey> keysToIndex = new ArrayList<>();
 
       RulesDefinition.Context context = defLoader.load();
       for (RulesDefinition.ExtendedRepository repoDef : getRepositories(context)) {
         if (languages.get(repoDef.language()) != null) {
           for (RulesDefinition.Rule ruleDef : repoDef.rules()) {
-            registerRule(ruleDef, allRules, session);
+            boolean relevantForIndex = registerRule(ruleDef, allRules, session);
+            if (relevantForIndex) {
+              keysToIndex.add(RuleKey.of(ruleDef.repository().key(), ruleDef.key()));
+            }
           }
           session.commit();
         }
       }
-      List<RuleDefinitionDto> activeRules = processRemainingDbRules(allRules.values(), session);
-      List<ActiveRuleChange> changes = removeActiveRulesOnStillExistingRepositories(session, activeRules, context);
+      List<RuleDefinitionDto> removedRules = processRemainingDbRules(allRules.values(), session);
+      List<ActiveRuleChange> changes = removeActiveRulesOnStillExistingRepositories(session, removedRules, context);
       session.commit();
 
       persistRepositories(session, context.repositories());
-      ruleIndexer.index();
+      ruleIndexer.delete(removedRules.stream().map(RuleDefinitionDto::getKey).collect(Collectors.toList(removedRules.size())));
+      ruleIndexer.index(getDefaultOrganization(), keysToIndex);
       activeRuleIndexer.index(changes);
       profiler.stopDebug();
-    } finally {
-      session.close();
+    }
+  }
+
+  private OrganizationDto getDefaultOrganization() {
+    try (DbSession dbSession = dbClient.openSession(false)) {
+      return dbClient.organizationDao().selectByUuid(dbSession, defaultOrganizationProvider.get().getUuid())
+        .orElseThrow(() -> new IllegalStateException("Cannot load default organization"));
     }
   }
 
@@ -129,11 +142,19 @@ public class RegisterRules implements Startable {
     // nothing
   }
 
-  private void registerRule(RulesDefinition.Rule ruleDef, Map<RuleKey, RuleDefinitionDto> allRules, DbSession session) {
+  private boolean registerRule(RulesDefinition.Rule ruleDef, Map<RuleKey, RuleDefinitionDto> allRules, DbSession session) {
     RuleKey ruleKey = RuleKey.of(ruleDef.repository().key(), ruleDef.key());
 
     RuleDefinitionDto existingRule = allRules.remove(ruleKey);
-    RuleDefinitionDto rule = existingRule == null ? createRuleDto(ruleDef, session) : existingRule;
+    boolean newRule;
+    RuleDefinitionDto rule;
+    if (existingRule == null) {
+      rule = createRuleDto(ruleDef, session);
+      newRule = true;
+    } else {
+      rule = existingRule;
+      newRule = false;
+    }
 
     boolean executeUpdate = false;
     if (mergeRule(ruleDef, rule)) {
@@ -153,6 +174,7 @@ public class RegisterRules implements Startable {
     }
 
     mergeParams(ruleDef, rule, session);
+    return newRule || executeUpdate;
   }
 
   private Map<RuleKey, RuleDefinitionDto> loadRules(DbSession session) {
@@ -403,8 +425,8 @@ public class RegisterRules implements Startable {
     rule.setSystemTags(Collections.emptySet());
     update(session, rule);
     // FIXME resetting the tags for all organizations must be handled a different way
-//    rule.setTags(Collections.emptySet());
-//    update(session, rule.getMetadata());
+    // rule.setTags(Collections.emptySet());
+    // update(session, rule.getMetadata());
     removedRules.add(rule);
     if (removedRules.size() % 100 == 0) {
       session.commit();

server/sonar-server/src/main/java/org/sonar/server/rule/RuleCreator.java

@@ -19,11 +19,11 @@
  */
 package org.sonar.server.rule;
 
-import com.google.common.base.Optional;
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
@@ -35,6 +35,7 @@ import org.sonar.api.server.rule.RuleParamType;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.rule.RuleDefinitionDto;
 import org.sonar.db.rule.RuleDto;
 import org.sonar.db.rule.RuleDto.Format;
@@ -72,7 +73,9 @@ public class RuleCreator {
       throw new IllegalArgumentException("Rule template key should not be null");
     }
     String defaultOrganizationUuid = defaultOrganizationProvider.get().getUuid();
-    RuleDto templateRule = dbClient.ruleDao().selectOrFailByKey(dbSession, defaultOrganizationUuid, templateKey);
+    OrganizationDto defaultOrganization = dbClient.organizationDao().selectByUuid(dbSession, defaultOrganizationUuid)
+      .orElseThrow(() -> new IllegalStateException(String.format("Could not find default organization for uuid '%s'", defaultOrganizationUuid)));
+    RuleDto templateRule = dbClient.ruleDao().selectOrFailByKey(dbSession, defaultOrganization.getUuid(), templateKey);
     if (!templateRule.isTemplate()) {
       throw new IllegalArgumentException("This rule is not a template rule: " + templateKey.toString());
     }
@@ -80,15 +83,12 @@ public class RuleCreator {
 
     RuleKey customRuleKey = RuleKey.of(templateRule.getRepositoryKey(), newRule.ruleKey());
 
-    Optional<RuleDefinitionDto> existingRule = loadRule(customRuleKey, dbSession);
-    if (existingRule.isPresent()) {
-      updateExistingRule(existingRule.get(), newRule, dbSession);
-    } else {
-      createCustomRule(customRuleKey, newRule, templateRule, dbSession);
-    }
+    loadRule(customRuleKey, dbSession)
+      .map(existingRule -> updateExistingRule(existingRule, newRule, dbSession))
+      .orElseGet(() -> createCustomRule(customRuleKey, newRule, templateRule, dbSession));
 
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(defaultOrganization, customRuleKey);
     return customRuleKey;
   }
 
@@ -151,7 +151,7 @@ public class RuleCreator {
   }
 
   private Optional<RuleDefinitionDto> loadRule(RuleKey ruleKey, DbSession dbSession) {
-    return dbClient.ruleDao().selectDefinitionByKey(dbSession, ruleKey);
+    return dbClient.ruleDao().selectDefinitionByKey(dbSession, ruleKey).transform(Optional::of).or(Optional::empty);
   }
 
   private RuleKey createCustomRule(RuleKey ruleKey, NewCustomRule newRule, RuleDto templateRuleDto, DbSession dbSession) {
@@ -174,6 +174,7 @@ public class RuleCreator {
       .setCreatedAt(system2.now())
       .setUpdatedAt(system2.now());
     dbClient.ruleDao().insert(dbSession, ruleDefinition);
+
     Set<String> tags = templateRuleDto.getTags();
     if (!tags.isEmpty()) {
       RuleMetadataDto ruleMetadata = new RuleMetadataDto()
@@ -201,7 +202,7 @@ public class RuleCreator {
     dbClient.ruleDao().insertRuleParam(dbSession, ruleDto, ruleParamDto);
   }
 
-  private void updateExistingRule(RuleDefinitionDto ruleDto, NewCustomRule newRule, DbSession dbSession) {
+  private RuleKey updateExistingRule(RuleDefinitionDto ruleDto, NewCustomRule newRule, DbSession dbSession) {
     if (ruleDto.getStatus().equals(RuleStatus.REMOVED)) {
       if (newRule.isPreventReactivation()) {
         throw new ReactivationException(format("A removed rule with the key '%s' already exists", ruleDto.getKey().rule()), ruleDto.getKey());
@@ -213,6 +214,7 @@ public class RuleCreator {
     } else {
       throw new IllegalArgumentException(format("A rule with the key '%s' already exists", ruleDto.getKey().rule()));
     }
+    return ruleDto.getKey();
   }
 
 }

server/sonar-server/src/main/java/org/sonar/server/rule/RuleDeleter.java

@@ -61,7 +61,7 @@ public class RuleDeleter {
       dbClient.ruleDao().update(dbSession, rule);
 
       dbSession.commit();
-      ruleIndexer.index();
+      ruleIndexer.delete(ruleKey);
     }
   }
 }

server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java

@@ -39,6 +39,7 @@ import org.sonar.api.server.debt.DebtRemediationFunction;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.qualityprofile.ActiveRuleDto;
 import org.sonar.db.qualityprofile.ActiveRuleParamDto;
 import org.sonar.db.rule.RuleDefinitionDto;
@@ -77,11 +78,8 @@ public class RuleUpdater {
       return false;
     }
 
-    DbSession dbSession = dbClient.openSession(false);
-    try {
+    try (DbSession dbSession = dbClient.openSession(false)) {
       return update(dbSession, update, userSession);
-    } finally {
-      dbSession.close();
     }
   }
 
@@ -93,13 +91,17 @@ public class RuleUpdater {
       return false;
     }
 
+    String defaultOrganizationUuid = defaultOrganizationProvider.get().getUuid();
+    OrganizationDto organization = dbClient.organizationDao().selectByUuid(dbSession, defaultOrganizationUuid)
+      .orElseThrow(() -> new IllegalStateException(String.format("Could not find default organization '%s'", defaultOrganizationUuid)));
+
     RuleDto rule = getRuleDto(update);
     // validate only the changes, not all the rule fields
     apply(update, rule, userSession);
     update(dbSession, rule);
     updateParameters(dbSession, update, rule);
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, rule.getKey());
     return true;
   }
 

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java

@@ -40,7 +40,7 @@ public class RuleDoc extends BaseDoc {
   }
 
   public RuleDoc() {
-    super(Maps.newHashMap());
+    super(Maps.newHashMapWithExpectedSize(16));
   }
 
   @Override

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java

@@ -19,64 +19,54 @@
  */
 package org.sonar.server.rule.index;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.util.Iterator;
+import java.util.List;
 import org.elasticsearch.action.index.IndexRequest;
-import org.sonar.api.utils.System2;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.server.es.BaseIndexer;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.BulkIndexer.Size;
 import org.sonar.server.es.EsClient;
-import org.sonar.server.organization.DefaultOrganizationProvider;
 
-import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_UPDATED_AT;
 import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
 
-public class RuleIndexer extends BaseIndexer {
+public class RuleIndexer {
 
-  private final DbClient dbClient;
-  private final DefaultOrganizationProvider defaultOrganizationProvider;
+  private final EsClient esClient;
+  private final RuleIteratorFactory ruleIteratorFactory;
 
-  public RuleIndexer(System2 system2, DbClient dbClient, EsClient esClient,
-    DefaultOrganizationProvider defaultOrganizationProvider) {
-    super(system2, esClient, 300, INDEX_TYPE_RULE, FIELD_RULE_UPDATED_AT);
-    this.dbClient = dbClient;
-    this.defaultOrganizationProvider = defaultOrganizationProvider;
+  public RuleIndexer(EsClient esClient, RuleIteratorFactory ruleIteratorFactory) {
+    this.esClient = esClient;
+    this.ruleIteratorFactory = ruleIteratorFactory;
   }
 
-  @Override
-  protected long doIndex(long lastUpdatedAt) {
-    return doIndex(createBulkIndexer(Size.REGULAR), lastUpdatedAt);
+  public void index(OrganizationDto organization, RuleKey ruleKey) {
+    BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
+    try (RuleIterator rules = ruleIteratorFactory.createForKey(organization, ruleKey)) {
+      doIndex(bulk, rules);
+    }
   }
 
-  public void index(Iterator<RuleDoc> rules) {
-    doIndex(createBulkIndexer(Size.REGULAR), rules);
+  public void index(OrganizationDto organization, List<RuleKey> ruleKeys) {
+    BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
+    try (RuleIterator rules = ruleIteratorFactory.createForKeys(organization, ruleKeys)) {
+      doIndex(bulk, rules);
+    }
   }
 
-  private long doIndex(BulkIndexer bulk, long lastUpdatedAt) {
-    long maxDate;
-    try (DbSession dbSession = dbClient.openSession(false)) {
-      String defaultOrganizationUuid = defaultOrganizationProvider.get().getUuid();
-      RuleResultSetIterator rowIt = RuleResultSetIterator.create(dbClient, dbSession, defaultOrganizationUuid, lastUpdatedAt);
-      maxDate = doIndex(bulk, rowIt);
-      rowIt.close();
-      return maxDate;
-    }
+  @VisibleForTesting
+  public void index(Iterator<RuleDoc> rules) {
+    doIndex(createBulkIndexer(Size.REGULAR), rules);
   }
 
-  private static long doIndex(BulkIndexer bulk, Iterator<RuleDoc> rules) {
+  private static void doIndex(BulkIndexer bulk, Iterator<RuleDoc> rules) {
     bulk.start();
-    long maxDate = 0L;
     while (rules.hasNext()) {
       RuleDoc rule = rules.next();
       bulk.add(newIndexRequest(rule));
-
-      // it's more efficient to sort programmatically than in SQL on some databases (MySQL for instance)
-      maxDate = Math.max(maxDate, rule.updatedAt());
     }
     bulk.stop();
-    return maxDate;
   }
 
   private BulkIndexer createBulkIndexer(Size size) {
@@ -88,4 +78,19 @@ public class RuleIndexer extends BaseIndexer {
   private static IndexRequest newIndexRequest(RuleDoc rule) {
     return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType(), rule.key().toString()).source(rule.getFields());
   }
+
+  public void delete(RuleKey ruleKey) {
+    esClient.prepareDelete(INDEX_TYPE_RULE, ruleKey.toString())
+      .setRefresh(true)
+      .get();
+  }
+
+  public void delete(List<RuleKey> rules) {
+    BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
+    bulk.start();
+    for (RuleKey rule : rules) {
+      bulk.addDeletion(INDEX_TYPE_RULE, rule.toString());
+    }
+    bulk.stop();
+  }
 }

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIterator.java

@@ -0,0 +1,29 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.rule.index;
+
+import java.util.Iterator;
+
+public interface RuleIterator extends Iterator<RuleDoc>, AutoCloseable {
+
+  @Override
+  void close();
+
+}

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIteratorFactory.java

@@ -0,0 +1,44 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.rule.index;
+
+import java.util.List;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.db.DbClient;
+import org.sonar.db.organization.OrganizationDto;
+
+import static java.util.Arrays.asList;
+
+public class RuleIteratorFactory {
+
+  private final DbClient dbClient;
+
+  public RuleIteratorFactory(DbClient dbClient) {
+    this.dbClient = dbClient;
+  }
+
+  public RuleIterator createForKey(OrganizationDto organization, RuleKey ruleKey) {
+    return new RuleIteratorForSingleChunk(dbClient, organization, asList(ruleKey));
+  }
+
+  public RuleIterator createForKeys(OrganizationDto organization, List<RuleKey> ruleKeys) {
+    return new RuleIteratorForMultipleChunks(dbClient, organization, ruleKeys);
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIteratorForMultipleChunks.java

@@ -0,0 +1,83 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.rule.index;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.db.DatabaseUtils;
+import org.sonar.db.DbClient;
+import org.sonar.db.organization.OrganizationDto;
+
+import static java.util.Optional.ofNullable;
+
+public class RuleIteratorForMultipleChunks implements RuleIterator {
+
+  private final DbClient dbClient;
+  private final OrganizationDto organization;
+  private final Iterator<List<RuleKey>> iteratorOverChunks;
+  private RuleIteratorForSingleChunk currentChunk;
+
+  public RuleIteratorForMultipleChunks(DbClient dbClient, OrganizationDto organization, Collection<RuleKey> keys) {
+    this.dbClient = dbClient;
+    this.organization = organization;
+    iteratorOverChunks = DatabaseUtils.toUniqueAndSortedPartitions(keys).iterator();
+  }
+
+  @Override
+  public boolean hasNext() {
+    for (;;) {
+      if (currentChunk != null && currentChunk.hasNext()) {
+        return true;
+      }
+      if (iteratorOverChunks.hasNext()) {
+        currentChunk = nextChunk();
+      } else {
+        return false;
+      }
+    }
+  }
+
+  @Override
+  public RuleDoc next() {
+    for (;;) {
+      if (currentChunk != null && currentChunk.hasNext()) {
+        return currentChunk.next();
+      }
+      if (iteratorOverChunks.hasNext()) {
+        currentChunk = nextChunk();
+      } else {
+        throw new NoSuchElementException();
+      }
+    }
+  }
+
+  private RuleIteratorForSingleChunk nextChunk() {
+    List<RuleKey> nextInput = iteratorOverChunks.next();
+    return new RuleIteratorForSingleChunk(dbClient, organization, nextInput);
+  }
+
+  @Override
+  public void close() {
+    ofNullable(currentChunk).ifPresent(RuleIterator::close);
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIteratorForSingleChunk.java

@@ -0,0 +1,217 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.rule.index;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.api.rules.RuleType;
+import org.sonar.db.DatabaseUtils;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.ResultSetIterator;
+import org.sonar.db.organization.OrganizationDto;
+import org.sonar.db.rule.RuleDto;
+import org.sonar.db.rule.SeverityUtil;
+import org.sonar.markdown.Markdown;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.stream.Collectors.joining;
+
+/**
+ * Scrolls over table RULES and reads documents to populate the rules index
+ */
+public class RuleIteratorForSingleChunk implements RuleIterator {
+
+  private static final String[] FIELDS = {
+    // column 1
+    "r.plugin_rule_key",
+    "r.plugin_name",
+    "r.name",
+    "r.description",
+    "r.description_format",
+    "r.priority",
+    "r.status",
+    "r.is_template",
+    "rm.tags",
+    "r.system_tags",
+
+    // column 11
+    "t.plugin_rule_key",
+    "t.plugin_name",
+    "r.plugin_config_key",
+    "r.language",
+    "r.rule_type",
+    "r.created_at",
+    "r.updated_at",
+  };
+
+  private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM rules r " +
+    "LEFT OUTER JOIN rules t ON t.id=r.template_id " +
+    "LEFT OUTER JOIN rules_metadata rm ON rm.rule_id = r.id and rm.organization_uuid=?";
+
+  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
+
+  private final DbSession session;
+
+  private final OrganizationDto organization;
+  private final List<RuleKey> ruleKeys;
+
+  private final PreparedStatement stmt;
+  private final ResultSetIterator<RuleDoc> iterator;
+
+  RuleIteratorForSingleChunk(DbClient dbClient, OrganizationDto organization, @Nullable List<RuleKey> ruleKeys) {
+    checkArgument(ruleKeys == null || ruleKeys.size() <= DatabaseUtils.PARTITION_SIZE_FOR_ORACLE,
+      "Cannot search for more than " + DatabaseUtils.PARTITION_SIZE_FOR_ORACLE + " rule keys at once. Please provide the keys in smaller chunks.");
+    this.organization = organization;
+    this.ruleKeys = ruleKeys;
+    this.session = dbClient.openSession(false);
+
+    try {
+      String sql = createSql();
+      stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
+      iterator = createIterator();
+    } catch (Exception e) {
+      session.close();
+      throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
+    }
+  }
+
+  private RuleIteratorInternal createIterator() {
+    try {
+      setParameters(stmt);
+      return new RuleIteratorInternal(stmt);
+    } catch (SQLException e) {
+      DatabaseUtils.closeQuietly(stmt);
+      throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
+    }
+  }
+
+  @Override
+  public boolean hasNext() {
+    return iterator.hasNext();
+  }
+
+  @Override
+  public RuleDoc next() {
+    return iterator.next();
+  }
+
+  private String createSql() {
+    StringBuilder sql = new StringBuilder(SQL_ALL);
+    if (ruleKeys != null && !ruleKeys.isEmpty()) {
+      sql.append(" WHERE ");
+      sql.append(ruleKeys.stream()
+        .map(x -> "(r.plugin_name=? AND r.plugin_rule_key=?)")
+        .collect(joining(" OR "))
+      );
+    }
+    return sql.toString();
+  }
+
+  private void setParameters(PreparedStatement stmt) throws SQLException {
+    AtomicInteger index = new AtomicInteger(1);
+    stmt.setString(index.getAndIncrement(), organization.getUuid());
+    if (ruleKeys != null && !ruleKeys.isEmpty()) {
+      for (RuleKey ruleKey : ruleKeys) {
+        stmt.setString(index.getAndIncrement(), ruleKey.repository());
+        stmt.setString(index.getAndIncrement(), ruleKey.rule());
+      }
+    }
+  }
+
+  @Override
+  public void close() {
+    try {
+      iterator.close();
+    } finally {
+      DatabaseUtils.closeQuietly(stmt);
+      session.close();
+    }
+  }
+
+  private static final class RuleIteratorInternal extends ResultSetIterator<RuleDoc> {
+
+    public RuleIteratorInternal(PreparedStatement stmt) throws SQLException {
+      super(stmt);
+    }
+
+    @Override
+    protected RuleDoc read(ResultSet rs) throws SQLException {
+      RuleDoc doc = new RuleDoc();
+
+      String ruleKey = rs.getString(1);
+      String repositoryKey = rs.getString(2);
+      RuleKey key = RuleKey.of(repositoryKey, ruleKey);
+
+      // all the fields must be present, even if value is null
+      doc.setKey(key.toString());
+      doc.setRuleKey(ruleKey);
+      doc.setRepository(repositoryKey);
+      doc.setName(rs.getString(3));
+
+      String description = rs.getString(4);
+      String descriptionFormat = rs.getString(5);
+      if (descriptionFormat != null && description != null) {
+        String htmlDescription;
+        if (RuleDto.Format.HTML == RuleDto.Format.valueOf(descriptionFormat)) {
+          htmlDescription = description;
+        } else {
+          htmlDescription = Markdown.convertToHtml(description);
+        }
+        doc.setHtmlDescription(htmlDescription);
+      }
+
+      doc.setSeverity(SeverityUtil.getSeverityFromOrdinal(rs.getInt(6)));
+      doc.setStatus(rs.getString(7));
+      doc.setIsTemplate(rs.getBoolean(8));
+      doc.setAllTags(Sets.union(stringTagsToSet(rs.getString(9)), stringTagsToSet(rs.getString(10))));
+
+      String templateRuleKey = rs.getString(11);
+      String templateRepoKey = rs.getString(12);
+      if (templateRepoKey != null && templateRuleKey != null) {
+        doc.setTemplateKey(RuleKey.of(templateRepoKey, templateRuleKey).toString());
+      } else {
+        doc.setTemplateKey(null);
+      }
+
+      doc.setInternalKey(rs.getString(13));
+      doc.setLanguage(rs.getString(14));
+      doc.setType(RuleType.valueOf(rs.getInt(15)));
+      doc.setCreatedAt(rs.getLong(16));
+      doc.setUpdatedAt(rs.getLong(17));
+
+      return doc;
+    }
+
+    private static Set<String> stringTagsToSet(@Nullable String tags) {
+      return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
+    }
+  }
+}

server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleResultSetIterator.java

@@ -1,144 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.api.rules.RuleType;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-import org.sonar.db.rule.RuleDto;
-import org.sonar.db.rule.SeverityUtil;
-import org.sonar.markdown.Markdown;
-
-/**
- * Scrolls over table RULES and reads documents to populate the rules index
- */
-public class RuleResultSetIterator extends ResultSetIterator<RuleDoc> {
-
-  private static final String[] FIELDS = {
-    // column 1
-    "r.plugin_rule_key",
-    "r.plugin_name",
-    "r.name",
-    "r.description",
-    "r.description_format",
-    "r.priority",
-    "r.status",
-    "r.is_template",
-    "rm.tags",
-    "r.system_tags",
-
-    // column 11
-    "t.plugin_rule_key",
-    "t.plugin_name",
-    "r.plugin_config_key",
-    "r.language",
-    "r.rule_type",
-    "r.created_at",
-    "r.updated_at",
-  };
-
-  private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM rules r " +
-    "LEFT OUTER JOIN rules t ON t.id=r.template_id " +
-    "LEFT OUTER JOIN rules_metadata rm ON rm.rule_id = r.id and rm.organization_uuid=?";
-
-  private static final String SQL_AFTER_DATE = SQL_ALL + " WHERE r.updated_at>?";
-
-  private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
-
-  private RuleResultSetIterator(PreparedStatement stmt) throws SQLException {
-    super(stmt);
-  }
-
-  static RuleResultSetIterator create(DbClient dbClient, DbSession session, String organizationUuid, long afterDate) {
-    try {
-      String sql = afterDate > 0L ? SQL_AFTER_DATE : SQL_ALL;
-      PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
-      stmt.setString(1, organizationUuid);
-      if (afterDate > 0L) {
-        stmt.setLong(2, afterDate);
-      }
-      return new RuleResultSetIterator(stmt);
-    } catch (SQLException e) {
-      throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
-    }
-  }
-
-  @Override
-  protected RuleDoc read(ResultSet rs) throws SQLException {
-    RuleDoc doc = new RuleDoc(Maps.newHashMapWithExpectedSize(16));
-
-    String ruleKey = rs.getString(1);
-    String repositoryKey = rs.getString(2);
-    RuleKey key = RuleKey.of(repositoryKey, ruleKey);
-
-    // all the fields must be present, even if value is null
-    doc.setKey(key.toString());
-    doc.setRuleKey(ruleKey);
-    doc.setRepository(repositoryKey);
-    doc.setName(rs.getString(3));
-
-    String description = rs.getString(4);
-    String descriptionFormat = rs.getString(5);
-    if (descriptionFormat != null) {
-      if (RuleDto.Format.HTML.equals(RuleDto.Format.valueOf(descriptionFormat))) {
-        doc.setHtmlDescription(description);
-      } else {
-        doc.setHtmlDescription(description == null ? null : Markdown.convertToHtml(description));
-      }
-    }
-
-    doc.setSeverity(SeverityUtil.getSeverityFromOrdinal(rs.getInt(6)));
-    doc.setStatus(rs.getString(7));
-    doc.setIsTemplate(rs.getBoolean(8));
-    doc.setAllTags(Sets.union(stringTagsToSet(rs.getString(9)), stringTagsToSet(rs.getString(10))));
-
-    String templateRuleKey = rs.getString(11);
-    String templateRepoKey = rs.getString(12);
-    if (templateRepoKey != null && templateRuleKey != null) {
-      doc.setTemplateKey(RuleKey.of(templateRepoKey, templateRuleKey).toString());
-    } else {
-      doc.setTemplateKey(null);
-    }
-
-    doc.setInternalKey(rs.getString(13));
-    doc.setLanguage(rs.getString(14));
-    doc.setType(RuleType.valueOf(rs.getInt(15)));
-    doc.setCreatedAt(rs.getLong(16));
-    doc.setUpdatedAt(rs.getLong(17));
-
-    return doc;
-  }
-
-  private static Set<String> stringTagsToSet(@Nullable String tags) {
-    return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
-  }
-}

server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java

@@ -47,6 +47,7 @@ import org.sonar.server.es.SearchResult;
 import org.sonar.server.issue.index.IssueDoc;
 import org.sonar.server.issue.index.IssueIndex;
 import org.sonar.server.issue.index.IssueIndexer;
+import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.permission.GroupPermissionChange;
 import org.sonar.server.permission.PermissionChange;
 import org.sonar.server.permission.PermissionUpdater;
@@ -73,6 +74,7 @@ public class IssueServiceMediumTest {
   private DbSession session;
   private IssueService service;
   private RuleIndexer ruleIndexer;
+  private OrganizationDto defaultOrganization;
 
   @Before
   public void setUp() {
@@ -81,6 +83,9 @@ public class IssueServiceMediumTest {
     session = db.openSession(false);
     service = tester.get(IssueService.class);
     ruleIndexer = tester.get(RuleIndexer.class);
+    String defaultOrganizationUuid = tester.get(DefaultOrganizationProvider.class).get().getUuid();
+    defaultOrganization = db.organizationDao().selectByUuid(session, defaultOrganizationUuid)
+      .orElseThrow(() -> new IllegalStateException(String.format("Could not find defautl organization '%s'", defaultOrganizationUuid)));
   }
 
   @After
@@ -90,7 +95,7 @@ public class IssueServiceMediumTest {
 
   @Test
   public void set_tags() {
-    RuleDto rule = newRule();
+    RuleDto rule = newRule(defaultOrganization);
     ComponentDto project = newProject();
     ComponentDto file = newFile(project);
     userSessionRule.logIn("john").addProjectUuidPermissions(UserRole.USER, project.uuid());
@@ -125,7 +130,7 @@ public class IssueServiceMediumTest {
 
   @Test
   public void list_component_tags() {
-    RuleDto rule = newRule();
+    RuleDto rule = newRule(defaultOrganization);
     ComponentDto project = newProject();
     ComponentDto file = newFile(project);
     saveIssue(IssueTesting.newDto(rule, file, project).setTags(ImmutableSet.of("convention", "java8", "bug")));
@@ -145,7 +150,7 @@ public class IssueServiceMediumTest {
 
   @Test
   public void test_listAuthors() {
-    RuleDto rule = newRule();
+    RuleDto rule = newRule(defaultOrganization);
     ComponentDto project = newProject();
     ComponentDto file = newFile(project);
     saveIssue(IssueTesting.newDto(rule, file, project).setAuthorLogin("luke.skywalker"));
@@ -162,7 +167,7 @@ public class IssueServiceMediumTest {
 
   @Test
   public void listAuthors_escapes_regexp_special_characters() {
-    saveIssue(IssueTesting.newDto(newRule(), newFile(newProject()), newProject()).setAuthorLogin("name++"));
+    saveIssue(IssueTesting.newDto(newRule(defaultOrganization), newFile(newProject()), newProject()).setAuthorLogin("name++"));
 
     assertThat(service.listAuthors("invalidRegexp[", 5)).isEmpty();
     assertThat(service.listAuthors("nam+", 5)).isEmpty();
@@ -170,18 +175,18 @@ public class IssueServiceMediumTest {
     assertThat(service.listAuthors(".*", 5)).isEmpty();
   }
 
-  private RuleDto newRule() {
-    return newRule(RuleTesting.newXooX1());
+  private RuleDto newRule(OrganizationDto organization) {
+    return newRule(organization, RuleTesting.newXooX1());
   }
 
-  private RuleDto newRule(RuleDto rule) {
+  private RuleDto newRule(OrganizationDto organization, RuleDto rule) {
     RuleDao ruleDao = tester.get(RuleDao.class);
     ruleDao.insert(session, rule.getDefinition());
     if (rule.getOrganizationUuid() != null) {
       ruleDao.update(session, rule.getMetadata().setRuleId(rule.getId()));
     }
     session.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, rule.getKey());
     return rule;
   }
 
@@ -198,7 +203,7 @@ public class IssueServiceMediumTest {
     // TODO correctly feed default organization. Not a problem as long as issues search does not support "anyone"
     // for each organization
     GroupPermissionChange permissionChange = new GroupPermissionChange(PermissionChange.Operation.ADD, UserRole.USER, new ProjectId(project),
-        GroupIdOrAnyone.forAnyone(organization.getUuid()));
+      GroupIdOrAnyone.forAnyone(organization.getUuid()));
     tester.get(PermissionUpdater.class).apply(session, asList(permissionChange));
     userSessionRule.logIn();
 

server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java

@@ -49,7 +49,6 @@ import org.sonar.server.es.EsTester;
 import org.sonar.server.es.SearchOptions;
 import org.sonar.server.es.SearchResult;
 import org.sonar.server.issue.IssueQuery;
-import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.permission.index.AuthorizationTypeSupport;
 import org.sonar.server.permission.index.PermissionIndexerDao;
 import org.sonar.server.permission.index.PermissionIndexerTester;
@@ -97,7 +96,7 @@ public class IssueIndexTest {
 
   private IssueIndexer issueIndexer = new IssueIndexer(tester.client(), new IssueIteratorFactory(null));
   private ViewIndexer viewIndexer = new ViewIndexer(null, tester.client());
-  private RuleIndexer ruleIndexer = new RuleIndexer(system2, null, tester.client(), TestDefaultOrganizationProvider.fromUuid("org-1"));
+  private RuleIndexer ruleIndexer = new RuleIndexer(tester.client(), null);
   private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(tester, issueIndexer);
 
   private IssueIndex underTest = new IssueIndex(tester.client(), system2, userSessionRule, new AuthorizationTypeSupport(userSessionRule));

server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java

@@ -20,6 +20,7 @@
 package org.sonar.server.issue.ws;
 
 import com.google.common.collect.ImmutableSet;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.config.MapSettings;
@@ -30,16 +31,17 @@ import org.sonar.db.DbTester;
 import org.sonar.db.component.ComponentDto;
 import org.sonar.db.issue.IssueDto;
 import org.sonar.db.issue.IssueTesting;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.rule.RuleDto;
 import org.sonar.server.es.EsTester;
 import org.sonar.server.issue.index.IssueIndex;
 import org.sonar.server.issue.index.IssueIndexDefinition;
 import org.sonar.server.issue.index.IssueIndexer;
 import org.sonar.server.issue.index.IssueIteratorFactory;
-import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.permission.index.AuthorizationTypeSupport;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
+import org.sonar.server.rule.index.RuleIteratorFactory;
 import org.sonar.server.tester.UserSessionRule;
 import org.sonar.server.ws.WsActionTester;
 
@@ -61,10 +63,16 @@ public class TagsActionTest {
   public EsTester es = new EsTester(new IssueIndexDefinition(new MapSettings()), new RuleIndexDefinition(new MapSettings()));
 
   private IssueIndexer issueIndexer = new IssueIndexer(es.client(), new IssueIteratorFactory(db.getDbClient()));
-  private RuleIndexer ruleIndexer = new RuleIndexer(System2.INSTANCE, db.getDbClient(), es.client(), TestDefaultOrganizationProvider.from(db));
+  private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), new RuleIteratorFactory(db.getDbClient()));
   private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new AuthorizationTypeSupport(userSession));
 
   private WsActionTester tester = new WsActionTester(new TagsAction(issueIndex));
+  private OrganizationDto organization;
+
+  @Before
+  public void before() {
+    organization = db.getDefaultOrganization();
+  }
 
   @Test
   public void return_tags_from_issues() throws Exception {
@@ -78,9 +86,10 @@ public class TagsActionTest {
 
   @Test
   public void return_tags_from_rules() throws Exception {
-    db.rules().insertRule(db.getDefaultOrganization(), rule -> rule.setSystemTags(ImmutableSet.of("tag1")).setTags(ImmutableSet.of("tag2")));
-    db.rules().insertRule(db.getDefaultOrganization(), rule -> rule.setSystemTags(ImmutableSet.of("tag3")).setTags(ImmutableSet.of("tag4", "tag5")));
-    ruleIndexer.index();
+    RuleDto r = db.rules().insertRule(organization, rule -> rule.setSystemTags(ImmutableSet.of("tag1")), rule -> rule.setTags(ImmutableSet.of("tag2")));
+    RuleDto r2 = db.rules().insertRule(organization, rule -> rule.setSystemTags(ImmutableSet.of("tag3")), rule -> rule.setTags(ImmutableSet.of("tag4", "tag5")));
+    db.commit();
+    ruleIndexer.index(organization, asList(r.getKey(), r2.getKey()));
 
     String result = tester.newRequest().execute().getInput();
     assertJson(result).isSimilarTo("{\"tags\":[\"tag1\", \"tag2\", \"tag3\", \"tag4\", \"tag5\"]}");
@@ -91,8 +100,8 @@ public class TagsActionTest {
     insertIssueWithBrowsePermission(insertRuleWithoutTags(), "tag1", "tag2");
     insertIssueWithBrowsePermission(insertRuleWithoutTags(), "tag3", "tag4", "tag5");
     issueIndexer.indexOnStartup(null);
-    db.rules().insertRule(db.getDefaultOrganization(), rule -> rule.setSystemTags(ImmutableSet.of("tag6")).setTags(ImmutableSet.of("tag7")));
-    ruleIndexer.index();
+    RuleDto r = db.rules().insertRule(db.getDefaultOrganization(), rule -> rule.setSystemTags(ImmutableSet.of("tag6")).setTags(ImmutableSet.of("tag7")));
+    ruleIndexer.index(organization, r.getKey());
 
     String result = tester.newRequest().execute().getInput();
     assertJson(result).isSimilarTo("{\"tags\":[\"tag1\", \"tag2\", \"tag3\", \"tag4\", \"tag5\", \"tag6\", \"tag7\"]}");
@@ -128,8 +137,8 @@ public class TagsActionTest {
   public void test_example() throws Exception {
     insertIssueWithBrowsePermission(insertRuleWithoutTags(), "convention");
     issueIndexer.indexOnStartup(null);
-    db.rules().insertRule(db.getDefaultOrganization(), rule -> rule.setSystemTags(ImmutableSet.of("cwe")).setTags(ImmutableSet.of("security")));
-    ruleIndexer.index();
+    RuleDto r = db.rules().insertRule(db.getDefaultOrganization(), rule -> rule.setSystemTags(ImmutableSet.of("cwe")).setTags(ImmutableSet.of("security")));
+    ruleIndexer.index(organization, r.getKey());
 
     String result = tester.newRequest().execute().getInput();
     assertJson(result).isSimilarTo(tester.getDef().responseExampleAsString());
@@ -163,7 +172,7 @@ public class TagsActionTest {
   }
 
   private IssueDto insertIssue(RuleDto rule, String... tags) {
-    ComponentDto project = db.components().insertProject();
+    ComponentDto project = db.components().insertProject(organization);
     ComponentDto file = db.components().insertComponent(newFileDto(project));
     IssueDto issueDto = IssueTesting.newDto(rule, file, project).setTags(asList(tags));
     return db.issues().insertIssue(issueDto);

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/QProfileBackuperMediumTest.java

@@ -51,6 +51,7 @@ import org.sonar.server.rule.index.RuleQuery;
 import org.sonar.server.tester.ServerTester;
 import org.sonar.server.tester.UserSessionRule;
 
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
 import static org.sonar.db.qualityprofile.ActiveRuleDto.INHERITED;
@@ -100,9 +101,9 @@ public class QProfileBackuperMediumTest {
       .setName("max").setDefaultValue("10").setType(RuleParamType.INTEGER.type()));
     dbSession.commit();
     dbSession.clearCache();
-    ruleIndexer.index();
     this.organization = OrganizationTesting.newOrganizationDto();
     db.organizationDao().insert(dbSession, organization);
+    ruleIndexer.index(organization, asList(xooRule1.getKey(), xooRule2.getKey()));
   }
 
   @After
@@ -117,7 +118,7 @@ public class QProfileBackuperMediumTest {
     db.ruleDao().insert(dbSession, blahRule.getDefinition());
     dbSession.commit();
     dbSession.clearCache();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, blahRuleKey);
 
     // create profile P1 with rules x2 and x1 activated
     QualityProfileDto profile = newXooP1(organization);

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/QProfileCopierMediumTest.java

@@ -46,6 +46,7 @@ import org.sonar.server.rule.index.RuleQuery;
 import org.sonar.server.tester.ServerTester;
 import org.sonar.server.tester.UserSessionRule;
 
+import static java.util.Arrays.asList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
 import static org.sonar.server.qualityprofile.QProfileTesting.getDefaultOrganization;
@@ -89,8 +90,7 @@ public class QProfileCopierMediumTest {
     sourceProfile = QProfileTesting.newXooP1(organization);
     db.qualityProfileDao().insert(dbSession, sourceProfile);
     dbSession.commit();
-    dbSession.clearCache();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, asList(xooRule1.getKey(), xooRule2.getKey()));
   }
 
   @After

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/RuleActivatorMediumTest.java

@@ -20,10 +20,11 @@
 package org.sonar.server.qualityprofile;
 
 import com.google.common.collect.ImmutableMap;
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 import org.junit.After;
 import org.junit.Before;
@@ -35,6 +36,7 @@ import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.server.rule.RuleParamType;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.qualityprofile.ActiveRuleDto;
 import org.sonar.db.qualityprofile.ActiveRuleKey;
 import org.sonar.db.qualityprofile.ActiveRuleParamDto;
@@ -44,6 +46,7 @@ import org.sonar.db.rule.RuleDto;
 import org.sonar.db.rule.RuleParamDto;
 import org.sonar.server.es.SearchOptions;
 import org.sonar.server.exceptions.BadRequestException;
+import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexer;
@@ -52,6 +55,7 @@ import org.sonar.server.tester.ServerTester;
 import org.sonar.server.tester.UserSessionRule;
 
 import static com.google.common.collect.Lists.newArrayList;
+import static java.util.Arrays.asList;
 import static java.util.Collections.singleton;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.fail;
@@ -95,6 +99,7 @@ public class RuleActivatorMediumTest {
   ActiveRuleIndexer activeRuleIndexer;
 
   QualityProfileDto profileDto;
+  private OrganizationDto organization;
 
   @Before
   public void before() {
@@ -104,6 +109,8 @@ public class RuleActivatorMediumTest {
     ruleActivator = tester.get(RuleActivator.class);
     activeRuleIndexer = tester.get(ActiveRuleIndexer.class);
     ruleIndexer = tester.get(RuleIndexer.class);
+    String defaultOrganizationUuid = tester.get(DefaultOrganizationProvider.class).get().getUuid();
+    organization = db.organizationDao().selectByUuid(dbSession, defaultOrganizationUuid).orElseThrow(() -> new IllegalStateException(String.format("Cannot find default organization '%s'", defaultOrganizationUuid)));
 
     // create pre-defined rules
     RuleDto javaRule = newDto(RuleKey.of("squid", "j1"))
@@ -112,10 +119,11 @@ public class RuleActivatorMediumTest {
     RuleDto xooRule2 = newXooX2().setSeverity("INFO");
     RuleDto xooTemplateRule1 = newTemplateRule(TEMPLATE_RULE_KEY)
       .setSeverity("MINOR").setLanguage("xoo");
-    db.ruleDao().insert(dbSession, javaRule.getDefinition());
-    db.ruleDao().insert(dbSession, xooRule1.getDefinition());
-    db.ruleDao().insert(dbSession, xooRule2.getDefinition());
-    db.ruleDao().insert(dbSession, xooTemplateRule1.getDefinition());
+
+    // store pre-defined rules in database
+    asList(javaRule, xooRule1, xooRule2, xooTemplateRule1).stream()
+      .map(RuleDto::getDefinition)
+      .forEach(definition -> db.ruleDao().insert(dbSession, definition));
     db.ruleDao().insertRuleParam(dbSession, xooRule1.getDefinition(), RuleParamDto.createFor(xooRule1.getDefinition())
       .setName("max").setDefaultValue("10").setType(RuleParamType.INTEGER.type()));
     db.ruleDao().insertRuleParam(dbSession, xooRule1.getDefinition(), RuleParamDto.createFor(xooRule1.getDefinition())
@@ -123,18 +131,22 @@ public class RuleActivatorMediumTest {
     db.ruleDao().insertRuleParam(dbSession, xooTemplateRule1.getDefinition(), RuleParamDto.createFor(xooTemplateRule1.getDefinition())
       .setName("format").setType(RuleParamType.STRING.type()));
 
+    // create custom rule
     RuleDto xooCustomRule1 = newCustomRule(xooTemplateRule1).setRuleKey(CUSTOM_RULE_KEY.rule())
       .setSeverity("MINOR").setLanguage("xoo");
+
+    // store custom rule in database
     db.ruleDao().insert(dbSession, xooCustomRule1.getDefinition());
     db.ruleDao().insertRuleParam(dbSession, xooCustomRule1.getDefinition(), RuleParamDto.createFor(xooTemplateRule1.getDefinition())
       .setName("format").setDefaultValue("txt").setType(RuleParamType.STRING.type()));
 
     // create pre-defined profile P1
-    profileDto = QProfileTesting.newXooP1("org-123");
+    profileDto = QProfileTesting.newXooP1(organization);
     db.qualityProfileDao().insert(dbSession, profileDto);
+
+    // index all rules
     dbSession.commit();
-    dbSession.clearCache();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, asList(javaRule, xooRule1, xooRule2, xooTemplateRule1, xooCustomRule1).stream().map(RuleDto::getKey).collect(Collectors.toList()));
   }
 
   @After
@@ -895,21 +907,24 @@ public class RuleActivatorMediumTest {
   public void bulk_activation() {
     // Generate more rules than the search's max limit
     int bulkSize = SearchOptions.MAX_LIMIT + 10;
+    List<RuleKey> keys = new ArrayList<>();
     for (int i = 0; i < bulkSize; i++) {
-      db.ruleDao().insert(dbSession, newDto(RuleKey.of("bulk", "r_" + i)).setLanguage("xoo").getDefinition());
+      RuleDefinitionDto ruleDefinitionDto = newDto(RuleKey.of("bulk", "r_" + i)).setLanguage("xoo").getDefinition();
+      db.ruleDao().insert(dbSession, ruleDefinitionDto);
+      keys.add(ruleDefinitionDto.getKey());
     }
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, keys);
 
     // 0. No active rules so far (base case) and plenty rules available
     verifyZeroActiveRules(XOO_P1_KEY);
     assertThat(tester.get(RuleIndex.class)
-      .search(new RuleQuery().setRepositories(Arrays.asList("bulk")), new SearchOptions()).getTotal())
+      .search(new RuleQuery().setRepositories(asList("bulk")), new SearchOptions()).getTotal())
         .isEqualTo(bulkSize);
 
     // 1. bulk activate all the rules
     BulkChangeResult result = ruleActivator.bulkActivate(
-      new RuleQuery().setRepositories(Arrays.asList("bulk")), XOO_P1_KEY, "MINOR");
+      new RuleQuery().setRepositories(asList("bulk")), XOO_P1_KEY, "MINOR");
 
     // 2. assert that all activation has been commit to DB and ES
     dbSession.clearCache();
@@ -928,7 +943,6 @@ public class RuleActivatorMediumTest {
     // -> xoo rules x1, x2 and custom1
     dbSession.clearCache();
     assertThat(db.activeRuleDao().selectByProfileKey(dbSession, XOO_P1_KEY)).hasSize(3);
-    assertThat(db.activeRuleDao().selectByProfileKey(dbSession, XOO_P1_KEY)).hasSize(3);
     assertThat(result.countSucceeded()).isEqualTo(3);
     assertThat(result.countFailed()).isGreaterThan(0);
   }

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ChangeParentActionTest.java

@@ -21,6 +21,7 @@ package org.sonar.server.qualityprofile.ws;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Stream;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -33,6 +34,7 @@ import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.rule.Severity;
 import org.sonar.api.server.ws.WebService;
 import org.sonar.api.utils.System2;
+import org.sonar.core.util.stream.Collectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
@@ -54,6 +56,7 @@ import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
+import org.sonar.server.rule.index.RuleIteratorFactory;
 import org.sonar.server.rule.index.RuleQuery;
 import org.sonar.server.tester.UserSessionRule;
 import org.sonar.server.util.TypeValidations;
@@ -101,10 +104,9 @@ public class ChangeParentActionTest {
     ruleIndex = new RuleIndex(esClient);
     TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
     ruleIndexer = new RuleIndexer(
-      System2.INSTANCE,
-      dbClient,
       esClient,
-      defaultOrganizationProvider);
+      new RuleIteratorFactory(dbClient)
+    );
     activeRuleIndexer = new ActiveRuleIndexer(
       System2.INSTANCE,
       dbClient,
@@ -158,7 +160,7 @@ public class ChangeParentActionTest {
 
     RuleDto rule1 = createRule();
     createActiveRule(rule1, parent1);
-    ruleIndexer.index();
+    ruleIndexer.index(organization, rule1.getKey());
     activeRuleIndexer.index();
 
     assertThat(dbClient.activeRuleDao().selectByProfileKey(dbSession, child.getKey())).isEmpty();
@@ -188,7 +190,7 @@ public class ChangeParentActionTest {
     RuleDto rule2 = createRule();
     createActiveRule(rule1, parent1);
     createActiveRule(rule2, parent2);
-    ruleIndexer.index();
+    ruleIndexer.index(organization, Stream.of(rule1, rule2).map(RuleDto::getKey).collect(Collectors.toList()));
     activeRuleIndexer.index();
 
     // Set parent 1
@@ -216,7 +218,7 @@ public class ChangeParentActionTest {
 
     RuleDto rule1 = createRule();
     createActiveRule(rule1, parent);
-    ruleIndexer.index();
+    ruleIndexer.index(organization, rule1.getKey());
     activeRuleIndexer.index();
 
     // Set parent
@@ -245,7 +247,7 @@ public class ChangeParentActionTest {
     RuleDto rule2 = createRule();
     createActiveRule(rule1, parent1);
     createActiveRule(rule2, parent2);
-    ruleIndexer.index();
+    ruleIndexer.index(organization, rule1.getKey());
     activeRuleIndexer.index();
 
     assertThat(dbClient.activeRuleDao().selectByProfileKey(dbSession, child.getKey())).isEmpty();
@@ -304,7 +306,7 @@ public class ChangeParentActionTest {
 
     RuleDto rule1 = createRule();
     createActiveRule(rule1, parent);
-    ruleIndexer.index();
+    ruleIndexer.index(organization, rule1.getKey());
     activeRuleIndexer.index();
 
     assertThat(dbClient.activeRuleDao().selectByProfileKey(dbSession, child.getKey())).isEmpty();

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/CreateActionTest.java

@@ -55,6 +55,7 @@ import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
+import org.sonar.server.rule.index.RuleIteratorFactory;
 import org.sonar.server.rule.index.RuleQuery;
 import org.sonar.server.tester.UserSessionRule;
 import org.sonar.server.ws.TestRequest;
@@ -93,7 +94,7 @@ public class CreateActionTest {
   private DbSession dbSession = dbTester.getSession();
   private RuleIndex ruleIndex = new RuleIndex(esTester.client());
   private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
-  private RuleIndexer ruleIndexer = new RuleIndexer(system2, dbClient, esTester.client(), defaultOrganizationProvider);
+  private RuleIndexer ruleIndexer = new RuleIndexer(esTester.client(), new RuleIteratorFactory(dbClient));
   private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(system2, dbClient, esTester.client());
   private ProfileImporter[] profileImporters = createImporters();
   private QProfileExporters qProfileExporters = new QProfileExporters(dbClient, null,
@@ -242,7 +243,7 @@ public class CreateActionTest {
   private void insertRule(RuleDefinitionDto ruleDto) {
     dbClient.ruleDao().insert(dbSession, ruleDto);
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, ruleDto.getKey());
   }
 
   private CreateWsResponse executeRequest(String name, String language) {

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/InheritanceActionTest.java

@@ -53,11 +53,14 @@ import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
+import org.sonar.server.rule.index.RuleIteratorFactory;
 import org.sonar.server.tester.UserSessionRule;
 import org.sonar.server.util.TypeValidations;
 import org.sonar.server.ws.WsActionTester;
 import org.sonar.test.JsonAssert;
 
+import static java.util.Arrays.asList;
+
 public class InheritanceActionTest {
 
   @Rule
@@ -82,7 +85,7 @@ public class InheritanceActionTest {
     dbClient = dbTester.getDbClient();
     dbSession = dbTester.getSession();
     esClient = esTester.client();
-    ruleIndexer = new RuleIndexer(System2.INSTANCE, dbClient, esClient, TestDefaultOrganizationProvider.fromUuid("org-1"));
+    ruleIndexer = new RuleIndexer(esClient, new RuleIteratorFactory(dbClient));
     activeRuleIndexer = new ActiveRuleIndexer(System2.INSTANCE, dbClient, esClient);
     TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
     underTest = new InheritanceAction(
@@ -116,7 +119,7 @@ public class InheritanceActionTest {
     createActiveRule(rule2, groupWide);
 
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(organization, asList(rule1.getKey(), rule2.getKey(), rule3.getKey()));
     activeRuleIndexer.index();
 
     QualityProfileDto companyWide = createProfile("xoo", "My Company Profile", "xoo-my-company-profile-12345");

server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/QProfilesWsMediumTest.java

@@ -75,7 +75,7 @@ public class QProfilesWsMediumTest {
   private DbClient db;
   private DbSession session;
   private WsTester wsTester;
-  private RuleIndexer ruIndexer = tester.get(RuleIndexer.class);
+  private RuleIndexer ruleIndexer = tester.get(RuleIndexer.class);
   private ActiveRuleIndexer activeRuIndexer = tester.get(ActiveRuleIndexer.class);
   private OrganizationDto organization;
 
@@ -86,7 +86,7 @@ public class QProfilesWsMediumTest {
     wsTester = tester.get(WsTester.class);
     session = db.openSession(false);
 
-    ruIndexer = tester.get(RuleIndexer.class);
+    ruleIndexer = tester.get(RuleIndexer.class);
     activeRuIndexer = tester.get(ActiveRuleIndexer.class);
     organization = OrganizationTesting.newOrganizationDto().setKey("org-123");
     db.organizationDao().insert(session, organization);
@@ -103,7 +103,7 @@ public class QProfilesWsMediumTest {
     RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto");
     createActiveRule(rule, profile);
     session.commit();
-    ruIndexer.index();
+    ruleIndexer.index(organization, rule.getKey());
     activeRuIndexer.index();
 
     // 0. Assert No Active Rule for profile
@@ -132,7 +132,6 @@ public class QProfilesWsMediumTest {
     createActiveRule(rule3, profile);
     createActiveRule(rule1, profile);
     session.commit();
-    ruIndexer.index();
     activeRuIndexer.index();
 
     // 0. Assert No Active Rule for profile
@@ -159,7 +158,6 @@ public class QProfilesWsMediumTest {
     createActiveRule(rule0, php);
     createActiveRule(rule1, php);
     session.commit();
-    ruIndexer.index();
     activeRuIndexer.index();
 
     // 0. Assert No Active Rule for profile
@@ -184,7 +182,6 @@ public class QProfilesWsMediumTest {
     createActiveRule(rule0, profile);
     createActiveRule(rule1, profile);
     session.commit();
-    ruIndexer.index();
     activeRuIndexer.index();
 
     // 0. Assert No Active Rule for profile
@@ -206,7 +203,7 @@ public class QProfilesWsMediumTest {
     QualityProfileDto profile = createProfile("java");
     RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto");
     session.commit();
-    ruIndexer.index();
+    ruleIndexer.index(organization, rule.getKey());
 
     // 0. Assert No Active Rule for profile
     assertThat(db.activeRuleDao().selectByProfileKey(session, profile.getKey())).isEmpty();
@@ -227,7 +224,7 @@ public class QProfilesWsMediumTest {
     QualityProfileDto profile = createProfile("java");
     RuleDefinitionDto rule = createRule("php", "toto");
     session.commit();
-    ruIndexer.index();
+    ruleIndexer.index(organization, rule.getKey());
 
     // 0. Assert No Active Rule for profile
     assertThat(db.activeRuleDao().selectByProfileKey(session, profile.getKey())).isEmpty();
@@ -250,7 +247,7 @@ public class QProfilesWsMediumTest {
     QualityProfileDto profile = createProfile("java");
     RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto");
     session.commit();
-    ruIndexer.index();
+    ruleIndexer.index(organization, rule.getKey());
 
     // 0. Assert No Active Rule for profile
     assertThat(db.activeRuleDao().selectByProfileKey(session, profile.getKey())).isEmpty();
@@ -278,7 +275,6 @@ public class QProfilesWsMediumTest {
     createRule(profile.getLanguage(), "hello");
     createRule(profile.getLanguage(), "world");
     session.commit();
-    ruIndexer.index();
 
     // 0. Assert No Active Rule for profile
     assertThat(db.activeRuleDao().selectByProfileKey(session, profile.getKey())).isEmpty();
@@ -303,7 +299,6 @@ public class QProfilesWsMediumTest {
     createRule(php.getLanguage(), "hello");
     createRule(php.getLanguage(), "world");
     session.commit();
-    ruIndexer.index();
 
     // 0. Assert No Active Rule for profile
     assertThat(db.activeRuleDao().selectByProfileKey(session, php.getKey())).isEmpty();
@@ -327,7 +322,6 @@ public class QProfilesWsMediumTest {
     createRule(profile.getLanguage(), "hello");
     createRule(profile.getLanguage(), "world");
     session.commit();
-    ruIndexer.index();
 
     // 0. Assert No Active Rule for profile
     assertThat(db.activeRuleDao().selectByProfileKey(session, profile.getKey())).isEmpty();
@@ -359,7 +353,6 @@ public class QProfilesWsMediumTest {
     RuleDefinitionDto rule0 = createRule(profile.getLanguage(), "toto");
     RuleDefinitionDto rule1 = createRule(profile.getLanguage(), "tata");
     session.commit();
-    ruIndexer.index();
 
     // 0. Assert No Active Rule for profile
     assertThat(db.activeRuleDao().selectByProfileKey(session, profile.getKey())).isEmpty();
@@ -394,7 +387,6 @@ public class QProfilesWsMediumTest {
     createRule(phpProfile.getLanguage(), "hello");
     createRule(phpProfile.getLanguage(), "world");
     session.commit();
-    ruIndexer.index();
 
     // 1. Activate Rule
     WsTester.TestRequest request = wsTester.newPostRequest(QProfilesWs.API_ENDPOINT, ActivateRulesAction.ACTIVATE_RULES_ACTION);
@@ -423,7 +415,6 @@ public class QProfilesWsMediumTest {
     db.activeRuleDao().insert(session, active2);
 
     session.commit();
-    ruIndexer.index();
     activeRuIndexer.index();
 
     // 0. assert rule child rule is minor
@@ -453,6 +444,8 @@ public class QProfilesWsMediumTest {
       .setSeverity(Severity.BLOCKER)
       .setStatus(RuleStatus.READY);
     db.ruleDao().insert(session, rule.getDefinition());
+    session.commit();
+    ruleIndexer.index(organization, rule.getKey());
     return rule.getDefinition();
   }
 

server/sonar-server/src/test/java/org/sonar/server/rule/RegisterRulesTest.java

@@ -48,10 +48,12 @@ import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.index.RuleIndex;
 import org.sonar.server.rule.index.RuleIndexDefinition;
 import org.sonar.server.rule.index.RuleIndexer;
+import org.sonar.server.rule.index.RuleIteratorFactory;
 import org.sonar.server.rule.index.RuleQuery;
 
 import static com.google.common.collect.Sets.newHashSet;
 import static java.util.Arrays.asList;
+import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -84,7 +86,7 @@ public class RegisterRulesTest {
   @Before
   public void before() {
     when(system.now()).thenReturn(DATE1.getTime());
-    ruleIndexer = new RuleIndexer(system, dbClient, esTester.client(), TestDefaultOrganizationProvider.from(dbTester));
+    ruleIndexer = new RuleIndexer(esTester.client(), new RuleIteratorFactory(dbClient));
     ruleIndex = new RuleIndex(esTester.client());
     activeRuleIndexer = new ActiveRuleIndexer(system, dbClient, esTester.client());
   }
@@ -122,6 +124,47 @@ public class RegisterRulesTest {
     // verify repositories
     assertThat(dbClient.ruleRepositoryDao().selectAll(dbTester.getSession())).extracting(RuleRepositoryDto::getKey).containsOnly("fake");
   }
+  @Test
+  public void insert_then_remove_rule() {
+    String ruleKey = randomAlphanumeric(5);
+
+    // register one rule
+    execute(context -> {
+      RulesDefinition.NewRepository repo = context.createRepository("fake", "java");
+      repo.createRule(ruleKey)
+        .setName(randomAlphanumeric(5))
+        .setHtmlDescription(randomAlphanumeric(20));
+      repo.done();
+    });
+
+    // verify db
+    assertThat(dbClient.ruleDao().selectAllDefinitions(dbTester.getSession()))
+      .extracting(RuleDefinitionDto::getKey)
+      .extracting(RuleKey::rule)
+      .containsExactly(ruleKey);
+
+    // verify index
+    assertThat(ruleIndex.search(new RuleQuery(), new SearchOptions()).getIds())
+      .extracting(RuleKey::rule)
+      .containsExactly(ruleKey);
+
+    // register no rule
+    execute(context -> context.createRepository("fake", "java").done());
+
+    // verify db
+    assertThat(dbClient.ruleDao().selectAllDefinitions(dbTester.getSession()))
+      .extracting(RuleDefinitionDto::getKey)
+      .extracting(RuleKey::rule)
+      .containsExactly(ruleKey);
+    assertThat(dbClient.ruleDao().selectAllDefinitions(dbTester.getSession()))
+      .extracting(RuleDefinitionDto::getStatus)
+      .containsExactly(RuleStatus.REMOVED);
+
+    // verify index
+    assertThat(ruleIndex.search(new RuleQuery(), new SearchOptions()).getIds())
+      .extracting(RuleKey::rule)
+      .isEmpty();
+  }
 
   @Test
   public void delete_repositories_that_have_been_uninstalled() {
@@ -414,7 +457,7 @@ public class RegisterRulesTest {
     Languages languages = mock(Languages.class);
     when(languages.get("java")).thenReturn(mock(Language.class));
 
-    RegisterRules task = new RegisterRules(loader, ruleActivator, dbClient, ruleIndexer, activeRuleIndexer, languages, system);
+    RegisterRules task = new RegisterRules(loader, ruleActivator, dbClient, ruleIndexer, activeRuleIndexer, languages, system, TestDefaultOrganizationProvider.from(dbTester));
     task.start();
     // Execute a commit to refresh session state as the task is using its own session
     dbTester.getSession().commit();

server/sonar-server/src/test/java/org/sonar/server/rule/RuleCreatorMediumTest.java

@@ -519,7 +519,7 @@ public class RuleCreatorMediumTest {
     RuleParamDto ruleParamDto = RuleParamDto.createFor(templateRule.getDefinition()).setName("regex").setType("STRING").setDescription("Reg ex").setDefaultValue(".*");
     dao.insertRuleParam(dbSession, templateRule.getDefinition(), ruleParamDto);
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(defaultOrganization, templateRule.getKey());
     return templateRule;
   }
 
@@ -539,7 +539,7 @@ public class RuleCreatorMediumTest {
       .setName("myIntegers").setType("INTEGER,multiple=true,values=1;2;3").setDescription("My Integers").setDefaultValue("1");
     dao.insertRuleParam(dbSession, templateRule.getDefinition(), ruleParamDto);
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(defaultOrganization, templateRule.getKey());
     return templateRule;
   }
 

server/sonar-server/src/test/java/org/sonar/server/rule/RuleDeleterMediumTest.java

@@ -28,11 +28,13 @@ import org.sonar.api.rule.RuleStatus;
 import org.sonar.api.rule.Severity;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.qualityprofile.QualityProfileDto;
 import org.sonar.db.rule.RuleDao;
 import org.sonar.db.rule.RuleDefinitionDto;
 import org.sonar.db.rule.RuleDto;
 import org.sonar.db.rule.RuleTesting;
+import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.QProfileTesting;
 import org.sonar.server.qualityprofile.RuleActivation;
 import org.sonar.server.qualityprofile.RuleActivator;
@@ -63,6 +65,9 @@ public class RuleDeleterMediumTest {
   ActiveRuleIndexer activeRuleIndexer = tester.get(ActiveRuleIndexer.class);
   RuleDeleter deleter = tester.get(RuleDeleter.class);
   DbSession dbSession = tester.get(DbClient.class).openSession(false);
+  String defaultOrganizationUuid = tester.get(DefaultOrganizationProvider.class).get().getUuid();
+  OrganizationDto organization = db.organizationDao().selectByUuid(dbSession, defaultOrganizationUuid)
+    .orElseThrow(() -> new IllegalStateException(String.format("Cannot find default organization '%s'", defaultOrganizationUuid)));
 
   @Before
   public void before() {
@@ -82,6 +87,11 @@ public class RuleDeleterMediumTest {
       .setCreatedAt(PAST)
       .setUpdatedAt(PAST);
     dao.insert(dbSession, templateRule.getDefinition());
+    dbSession.commit();
+    ruleIndexer.index(organization, templateRule.getKey());
+
+    // Verify in index
+    assertThat(index.searchAll(new RuleQuery())).containsOnly(templateRule.getKey());
 
     // Create custom rule
     RuleDto customRule = RuleTesting.newCustomRule(templateRule)
@@ -89,13 +99,16 @@ public class RuleDeleterMediumTest {
       .setCreatedAt(PAST)
       .setUpdatedAt(PAST);
     dao.insert(dbSession, customRule.getDefinition());
+    dbSession.commit();
+    ruleIndexer.index(organization, customRule.getKey());
+
+    // Verify in index
+    assertThat(index.searchAll(new RuleQuery())).containsOnly(templateRule.getKey(), customRule.getKey());
 
     // Create a quality profile
     QualityProfileDto profileDto = QProfileTesting.newXooP1("org-123");
     db.qualityProfileDao().insert(dbSession, profileDto);
     dbSession.commit();
-    dbSession.clearCache();
-    ruleIndexer.index();
     activeRuleIndexer.index();
 
     // Activate the custom rule

server/sonar-server/src/test/java/org/sonar/server/rule/RuleServiceMediumTest.java

@@ -118,6 +118,6 @@ public class RuleServiceMediumTest {
     dao.insert(dbSession, ruleDto.getDefinition());
     dao.update(dbSession, ruleDto.getMetadata().setRuleId(ruleDto.getId()));
     dbSession.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(defaultOrganization, ruleDto.getKey());
   }
 }

server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java

@@ -37,7 +37,6 @@ import org.sonar.db.rule.RuleTesting;
 import org.sonar.server.es.EsTester;
 import org.sonar.server.es.SearchIdResult;
 import org.sonar.server.es.SearchOptions;
-import org.sonar.server.organization.TestDefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.index.ActiveRuleDoc;
 import org.sonar.server.qualityprofile.index.ActiveRuleDocTesting;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
@@ -86,7 +85,7 @@ public class RuleIndexTest {
 
   @Before
   public void setUp() {
-    ruleIndexer = new RuleIndexer(system2, null, tester.client(), TestDefaultOrganizationProvider.fromUuid("org-1"));
+    ruleIndexer = new RuleIndexer(tester.client(), null);
     activeRuleIndexer = new ActiveRuleIndexer(system2, null, tester.client());
     index = new RuleIndex(tester.client());
   }

server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java

@@ -19,7 +19,7 @@
  */
 package org.sonar.server.rule.index;
 
-import com.google.common.collect.Iterators;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.config.MapSettings;
@@ -30,10 +30,10 @@ import org.sonar.api.utils.System2;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.DbTester;
+import org.sonar.db.organization.OrganizationDto;
 import org.sonar.db.rule.RuleDefinitionDto;
 import org.sonar.db.rule.RuleDto;
 import org.sonar.server.es.EsTester;
-import org.sonar.server.organization.TestDefaultOrganizationProvider;
 
 import static com.google.common.collect.Sets.newHashSet;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -65,11 +65,17 @@ public class RuleIndexerTest {
     .setType(RuleType.BUG)
     .setCreatedAt(1500000000000L)
     .setUpdatedAt(1600000000000L);
+  private OrganizationDto organization;
+
+  @Before
+  public void before() {
+    organization = dbTester.getDefaultOrganization();
+  }
 
   @Test
   public void index_nothing() {
     RuleIndexer indexer = createIndexer();
-    indexer.index(Iterators.emptyIterator());
+//    indexer.index(Iterators.emptyIterator());
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(0L);
   }
 
@@ -79,7 +85,7 @@ public class RuleIndexerTest {
     dbSession.commit();
 
     RuleIndexer indexer = createIndexer();
-    indexer.index();
+    indexer.index(organization, rule.getKey());
 
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
   }
@@ -91,19 +97,19 @@ public class RuleIndexerTest {
     // Create and Index rule
     dbClient.ruleDao().insert(dbSession, rule.setStatus(RuleStatus.READY));
     dbSession.commit();
-    indexer.index();
+    indexer.index(organization, rule.getKey());
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
 
     // Remove rule
     dbTester.getDbClient().ruleDao().update(dbTester.getSession(), rule.setStatus(RuleStatus.READY).setUpdatedAt(2000000000000L));
     dbTester.getSession().commit();
-    indexer.index();
+    indexer.index(organization, rule.getKey());
 
     assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
   }
 
   private RuleIndexer createIndexer() {
-    return new RuleIndexer(system2, dbTester.getDbClient(), esTester.client(), TestDefaultOrganizationProvider.from(dbTester));
+    return new RuleIndexer(esTester.client(), new RuleIteratorFactory(dbTester.getDbClient()));
   }
 
 }

server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIteratorForSingleChunkTest.java

@@ -0,0 +1,195 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.rule.index;
+
+import com.google.common.collect.Lists;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.rule.RuleKey;
+import org.sonar.api.rule.RuleStatus;
+import org.sonar.api.rule.Severity;
+import org.sonar.api.rules.RuleType;
+import org.sonar.api.utils.System2;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.DbTester;
+import org.sonar.db.rule.RuleDefinitionDto;
+import org.sonar.db.rule.RuleDto;
+import org.sonar.server.exceptions.NotFoundException;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class RuleIteratorForSingleChunkTest {
+
+  @Rule
+  public DbTester dbTester = DbTester.create(System2.INSTANCE);
+
+  private DbClient dbClient = dbTester.getDbClient();
+  private DbSession dbSession = dbTester.getSession();
+  private RuleDto templateRule;
+  private RuleDefinitionDto customRule;
+
+  @Before
+  public void setUp() throws Exception {
+    templateRule = new RuleDto()
+        .setRuleKey("S001")
+        .setRepositoryKey("xoo")
+        .setConfigKey("S1")
+        .setName("Null Pointer")
+        .setDescription("S001 desc")
+        .setDescriptionFormat(RuleDto.Format.HTML)
+        .setLanguage("xoo")
+        .setSeverity(Severity.BLOCKER)
+        .setStatus(RuleStatus.READY)
+        .setIsTemplate(true)
+        .setSystemTags(newHashSet("cwe"))
+        .setType(RuleType.BUG)
+        .setCreatedAt(1500000000000L)
+        .setUpdatedAt(1600000000000L)
+        .setOrganizationUuid(dbTester.getDefaultOrganization().getUuid())
+        .setTags(newHashSet("performance"));
+
+    customRule = new RuleDefinitionDto()
+        .setRuleKey("S002")
+        .setRepositoryKey("xoo")
+        .setConfigKey("S2")
+        .setName("Slow")
+        .setDescription("*S002 desc*")
+        .setDescriptionFormat(RuleDto.Format.MARKDOWN)
+        .setLanguage("xoo")
+        .setSeverity(Severity.MAJOR)
+        .setStatus(RuleStatus.BETA)
+        .setIsTemplate(false)
+        .setType(RuleType.CODE_SMELL)
+        .setCreatedAt(2000000000000L)
+        .setUpdatedAt(2100000000000L);
+  }
+
+  @Test
+  public void iterator_over_one_rule() {
+    dbTester.rules().insertRule(templateRule);
+
+    List<RuleDoc> results = getResults();
+
+    assertThat(results).hasSize(1);
+
+    RuleDoc templateDoc = getRuleDoc(results, templateRule.getRuleKey());
+    assertThat(templateDoc).isNotNull();
+    assertThat(templateDoc.key()).isEqualTo(RuleKey.of("xoo", "S001"));
+    assertThat(templateDoc.ruleKey()).isEqualTo("S001");
+    assertThat(templateDoc.repository()).isEqualTo("xoo");
+    assertThat(templateDoc.internalKey()).isEqualTo("S1");
+    assertThat(templateDoc.name()).isEqualTo("Null Pointer");
+    assertThat(templateDoc.htmlDescription()).isEqualTo("S001 desc");
+    assertThat(templateDoc.language()).isEqualTo("xoo");
+    assertThat(templateDoc.severity()).isEqualTo(Severity.BLOCKER);
+    assertThat(templateDoc.status()).isEqualTo(RuleStatus.READY);
+    assertThat(templateDoc.isTemplate()).isTrue();
+    assertThat(templateDoc.allTags()).containsOnly("performance", "cwe");
+    assertThat(templateDoc.createdAt()).isEqualTo(1500000000000L);
+    assertThat(templateDoc.updatedAt()).isEqualTo(1600000000000L);
+  }
+
+  @Test
+  public void iterator_over_rules() {
+    dbTester.rules().insertRule(templateRule);
+    dbClient.ruleDao().insert(dbSession, customRule);
+    dbSession.commit();
+
+    List<RuleDoc> results = getResults();
+
+    assertThat(results).hasSize(2);
+
+    RuleDoc templateDoc = getRuleDoc(results, templateRule.getRuleKey());
+    assertThat(templateDoc.key()).isEqualTo(RuleKey.of("xoo", "S001"));
+    assertThat(templateDoc.ruleKey()).isEqualTo("S001");
+    assertThat(templateDoc.repository()).isEqualTo("xoo");
+    assertThat(templateDoc.internalKey()).isEqualTo("S1");
+    assertThat(templateDoc.name()).isEqualTo("Null Pointer");
+    assertThat(templateDoc.htmlDescription()).isEqualTo("S001 desc");
+    assertThat(templateDoc.language()).isEqualTo("xoo");
+    assertThat(templateDoc.severity()).isEqualTo(Severity.BLOCKER);
+    assertThat(templateDoc.status()).isEqualTo(RuleStatus.READY);
+    assertThat(templateDoc.isTemplate()).isTrue();
+    assertThat(templateDoc.allTags()).containsOnly("performance", "cwe");
+    assertThat(templateDoc.createdAt()).isEqualTo(1500000000000L);
+    assertThat(templateDoc.updatedAt()).isEqualTo(1600000000000L);
+
+    RuleDoc customDoc = getRuleDoc(results, customRule.getRuleKey());
+    assertThat(customDoc.key()).isEqualTo(RuleKey.of("xoo", "S002"));
+    assertThat(customDoc.ruleKey()).isEqualTo("S002");
+    assertThat(customDoc.repository()).isEqualTo("xoo");
+    assertThat(customDoc.internalKey()).isEqualTo("S2");
+    assertThat(customDoc.name()).isEqualTo("Slow");
+    assertThat(customDoc.htmlDescription()).isEqualTo("<strong>S002 desc</strong>");
+    assertThat(customDoc.language()).isEqualTo("xoo");
+    assertThat(customDoc.severity()).isEqualTo(Severity.MAJOR);
+    assertThat(customDoc.status()).isEqualTo(RuleStatus.BETA);
+    assertThat(customDoc.isTemplate()).isFalse();
+    assertThat(customDoc.allTags()).isEmpty();
+    assertThat(customDoc.createdAt()).isEqualTo(2000000000000L);
+    assertThat(customDoc.updatedAt()).isEqualTo(2100000000000L);
+  }
+
+  @Test
+  public void custom_rule() {
+    dbTester.rules().insertRule(templateRule);
+    dbClient.ruleDao().insert(dbSession, customRule.setTemplateId(templateRule.getId()));
+    dbSession.commit();
+
+    List<RuleDoc> results = getResults();
+
+    assertThat(results).hasSize(2);
+
+    RuleDoc templateDoc = getRuleDoc(results, templateRule.getRuleKey());
+    assertThat(templateDoc.isTemplate()).isTrue();
+    assertThat(templateDoc.templateKey()).isNull();
+
+    RuleDoc customDoc = getRuleDoc(results, customRule.getRuleKey());
+    assertThat(customDoc.isTemplate()).isFalse();
+    assertThat(customDoc.templateKey()).isEqualTo(RuleKey.of("xoo", "S001"));
+  }
+
+  @Test
+  public void removed_rule_is_returned() {
+    dbTester.rules().insertRule(templateRule.setStatus(RuleStatus.REMOVED));
+    dbSession.commit();
+
+    List<RuleDoc> results = getResults();
+
+    assertThat(results).hasSize(1);
+  }
+
+  private List<RuleDoc> getResults() {
+    return Lists.newArrayList(new RuleIteratorForSingleChunk(dbTester.getDbClient(), dbTester.getDefaultOrganization(), null));
+  }
+
+  private RuleDoc getRuleDoc(List<RuleDoc> results, String ruleKey) {
+    RuleDoc rule;
+    rule = results.stream()
+      .filter(r -> ruleKey.equals(r.key().rule()))
+      .findAny()
+      .orElseThrow(() -> new NotFoundException("Rule not found in results"));
+    return rule;
+  }
+}

server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleResultSetIteratorTest.java

@@ -1,197 +0,0 @@
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import com.google.common.collect.Maps;
-import java.util.Map;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.api.rule.RuleStatus;
-import org.sonar.api.rule.Severity;
-import org.sonar.api.rules.RuleType;
-import org.sonar.api.utils.System2;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.DbTester;
-import org.sonar.db.rule.RuleDefinitionDto;
-import org.sonar.db.rule.RuleDto;
-
-import static com.google.common.collect.Sets.newHashSet;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class RuleResultSetIteratorTest {
-
-  @Rule
-  public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
-  private DbClient dbClient = dbTester.getDbClient();
-  private DbSession dbSession = dbTester.getSession();
-  private RuleDto templateRule;
-  private RuleDefinitionDto customRule;
-
-  @Before
-  public void setUp() throws Exception {
-    templateRule = new RuleDto()
-        .setRuleKey("S001")
-        .setRepositoryKey("xoo")
-        .setConfigKey("S1")
-        .setName("Null Pointer")
-        .setDescription("S001 desc")
-        .setDescriptionFormat(RuleDto.Format.HTML)
-        .setLanguage("xoo")
-        .setSeverity(Severity.BLOCKER)
-        .setStatus(RuleStatus.READY)
-        .setIsTemplate(true)
-        .setSystemTags(newHashSet("cwe"))
-        .setType(RuleType.BUG)
-        .setCreatedAt(1500000000000L)
-        .setUpdatedAt(1600000000000L)
-        .setOrganizationUuid(dbTester.getDefaultOrganization().getUuid())
-        .setTags(newHashSet("performance"));
-
-    customRule = new RuleDefinitionDto()
-        .setRuleKey("S002")
-        .setRepositoryKey("xoo")
-        .setConfigKey("S2")
-        .setName("Slow")
-        .setDescription("*S002 desc*")
-        .setDescriptionFormat(RuleDto.Format.MARKDOWN)
-        .setLanguage("xoo")
-        .setSeverity(Severity.MAJOR)
-        .setStatus(RuleStatus.BETA)
-        .setIsTemplate(false)
-        .setType(RuleType.CODE_SMELL)
-        .setCreatedAt(2000000000000L)
-        .setUpdatedAt(2100000000000L);
-  }
-
-  @Test
-  public void iterator_over_one_rule() {
-    dbTester.rules().insertRule(templateRule);
-
-    String organizationUuid = dbTester.getDefaultOrganization().getUuid();
-    RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), organizationUuid, 0L);
-    Map<String, RuleDoc> rulesByKey = rulesByKey(it);
-    it.close();
-
-    assertThat(rulesByKey).hasSize(1);
-
-    RuleDoc rule = rulesByKey.get(templateRule.getRuleKey());
-    assertThat(rule).isNotNull();
-    assertThat(rule.key()).isEqualTo(RuleKey.of("xoo", "S001"));
-    assertThat(rule.ruleKey()).isEqualTo("S001");
-    assertThat(rule.repository()).isEqualTo("xoo");
-    assertThat(rule.internalKey()).isEqualTo("S1");
-    assertThat(rule.name()).isEqualTo("Null Pointer");
-    assertThat(rule.htmlDescription()).isEqualTo("S001 desc");
-    assertThat(rule.language()).isEqualTo("xoo");
-    assertThat(rule.severity()).isEqualTo(Severity.BLOCKER);
-    assertThat(rule.status()).isEqualTo(RuleStatus.READY);
-    assertThat(rule.isTemplate()).isTrue();
-    assertThat(rule.allTags()).containsOnly("performance", "cwe");
-    assertThat(rule.createdAt()).isEqualTo(1500000000000L);
-    assertThat(rule.updatedAt()).isEqualTo(1600000000000L);
-  }
-
-  @Test
-  public void iterator_over_rules() {
-    dbTester.rules().insertRule(templateRule);
-    dbClient.ruleDao().insert(dbSession, customRule);
-    dbSession.commit();
-
-    String organizationUuid = dbTester.getDefaultOrganization().getUuid();
-    RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), organizationUuid, 0L);
-    Map<String, RuleDoc> rulesByKey = rulesByKey(it);
-    it.close();
-
-    assertThat(rulesByKey).hasSize(2);
-
-    RuleDoc rule = rulesByKey.get(templateRule.getRuleKey());
-    assertThat(rule.key()).isEqualTo(RuleKey.of("xoo", "S001"));
-    assertThat(rule.ruleKey()).isEqualTo("S001");
-    assertThat(rule.repository()).isEqualTo("xoo");
-    assertThat(rule.internalKey()).isEqualTo("S1");
-    assertThat(rule.name()).isEqualTo("Null Pointer");
-    assertThat(rule.htmlDescription()).isEqualTo("S001 desc");
-    assertThat(rule.language()).isEqualTo("xoo");
-    assertThat(rule.severity()).isEqualTo(Severity.BLOCKER);
-    assertThat(rule.status()).isEqualTo(RuleStatus.READY);
-    assertThat(rule.isTemplate()).isTrue();
-    assertThat(rule.allTags()).containsOnly("performance", "cwe");
-    assertThat(rule.createdAt()).isEqualTo(1500000000000L);
-    assertThat(rule.updatedAt()).isEqualTo(1600000000000L);
-
-    rule = rulesByKey.get(customRule.getRuleKey());
-    assertThat(rule.key()).isEqualTo(RuleKey.of("xoo", "S002"));
-    assertThat(rule.ruleKey()).isEqualTo("S002");
-    assertThat(rule.repository()).isEqualTo("xoo");
-    assertThat(rule.internalKey()).isEqualTo("S2");
-    assertThat(rule.name()).isEqualTo("Slow");
-    assertThat(rule.htmlDescription()).isEqualTo("<strong>S002 desc</strong>");
-    assertThat(rule.language()).isEqualTo("xoo");
-    assertThat(rule.severity()).isEqualTo(Severity.MAJOR);
-    assertThat(rule.status()).isEqualTo(RuleStatus.BETA);
-    assertThat(rule.isTemplate()).isFalse();
-    assertThat(rule.allTags()).isEmpty();
-    assertThat(rule.createdAt()).isEqualTo(2000000000000L);
-    assertThat(rule.updatedAt()).isEqualTo(2100000000000L);
-  }
-
-  @Test
-  public void custom_rule() {
-    dbTester.rules().insertRule(templateRule);
-    dbClient.ruleDao().insert(dbSession, customRule.setTemplateId(templateRule.getId()));
-    dbSession.commit();
-
-    String organizationUuid = dbTester.getDefaultOrganization().getUuid();
-    RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), organizationUuid, 0L);
-    Map<String, RuleDoc> rulesByKey = rulesByKey(it);
-    it.close();
-
-    assertThat(rulesByKey).hasSize(2);
-
-    RuleDoc rule = rulesByKey.get(templateRule.getRuleKey());
-    assertThat(rule.isTemplate()).isTrue();
-    assertThat(rule.templateKey()).isNull();
-
-    rule = rulesByKey.get(customRule.getRuleKey());
-    assertThat(rule.isTemplate()).isFalse();
-    assertThat(rule.templateKey()).isEqualTo(RuleKey.of("xoo", "S001"));
-  }
-
-  @Test
-  public void removed_rule_is_returned() {
-    dbTester.rules().insertRule(templateRule.setStatus(RuleStatus.REMOVED));
-    dbSession.commit();
-
-    String organizationUuid = dbTester.getDefaultOrganization().getUuid();
-    RuleResultSetIterator it = RuleResultSetIterator.create(dbTester.getDbClient(), dbTester.getSession(), organizationUuid, 0L);
-    Map<String, RuleDoc> rulesByKey = rulesByKey(it);
-    it.close();
-
-    assertThat(rulesByKey).hasSize(1);
-  }
-
-  private static Map<String, RuleDoc> rulesByKey(RuleResultSetIterator it) {
-    return Maps.uniqueIndex(it, rule -> rule.key().rule());
-  }
-}

server/sonar-server/src/test/java/org/sonar/server/rule/ws/RulesWsMediumTest.java

@@ -21,12 +21,14 @@ package org.sonar.server.rule.ws;
 
 import com.google.common.collect.ImmutableSet;
 import java.util.Collections;
+import java.util.stream.Stream;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.server.ws.WebService;
+import org.sonar.core.util.stream.Collectors;
 import org.sonar.db.DbClient;
 import org.sonar.db.DbSession;
 import org.sonar.db.organization.OrganizationDto;
@@ -118,8 +120,7 @@ public class RulesWsMediumTest {
     tester.get(ActiveRuleDao.class).insert(session, activeRuleDto);
 
     session.commit();
-    session.clearCache();
-    ruleIndexer.index();
+    ruleIndexer.index(defaultOrganization, rule.getKey());
     activeRuleIndexer.index();
 
     // 1. With Activation
@@ -138,7 +139,7 @@ public class RulesWsMediumTest {
 
   @Test
   public void get_tags() throws Exception {
-    QualityProfileDto profile = QProfileTesting.newXooP1("org-123");
+    QualityProfileDto profile = QProfileTesting.newXooP1(defaultOrganization);
     tester.get(QualityProfileDao.class).insert(session, profile);
 
     RuleDto rule = RuleTesting.newXooX1(defaultOrganization)
@@ -154,7 +155,7 @@ public class RulesWsMediumTest {
     ruleDao.update(session, rule2.getMetadata().setRuleId(rule2.getId()));
 
     session.commit();
-    ruleIndexer.index();
+    ruleIndexer.index(defaultOrganization, Stream.of(rule, rule2).map(RuleDto::getKey).collect(Collectors.toList()));
 
     tester.wsTester().newGetRequest(API_ENDPOINT, API_TAGS_METHOD).execute().assertJson(this.getClass(), "get_tags.json");
     tester.wsTester().newGetRequest(API_ENDPOINT, API_TAGS_METHOD)

server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionMediumTest.java

@@ -81,7 +81,7 @@ public class SearchActionMediumTest {
   private DbSession dbSession;
   private RuleIndexer ruleIndexer;
   private ActiveRuleIndexer activeRuleIndexer;
-  private OrganizationDto defaultOrganization;
+  private OrganizationDto defaultOrganizationDto;
 
   @Before
   public void setUp() {
@@ -92,7 +92,7 @@ public class SearchActionMediumTest {
     ruleIndexer = tester.get(RuleIndexer.class);
     activeRuleIndexer = tester.get(ActiveRuleIndexer.class);
     DefaultOrganization defaultOrganization = tester.get(DefaultOrganizationProvider.class).get();
-    this.defaultOrganization = db.organizationDao().selectByUuid(dbSession, defaultOrganization.getUuid()).get();
+    this.defaultOrganizationDto = db.organizationDao().selectByUuid(dbSession, defaultOrganization.getUuid()).get();
   }
 
   @After
@@ -111,10 +111,9 @@ public class SearchActionMediumTest {
 
   @Test
   public void filter_by_key_rules() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1().getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX2().getDefinition());
+    insertRule(RuleTesting.newXooX1().getDefinition());
+    insertRule(RuleTesting.newXooX2().getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     request.setParam(PARAM_RULE_KEY, RuleTesting.XOO_X1.toString());
@@ -131,16 +130,15 @@ public class SearchActionMediumTest {
 
   @Test
   public void search_2_rules() throws Exception {
-    RuleDto rule1 = RuleTesting.newXooX1(defaultOrganization)
-        .setType(RuleType.BUG);
-    ruleDao.insert(dbSession, rule1.getDefinition());
+    RuleDto rule1 = RuleTesting.newXooX1(defaultOrganizationDto)
+      .setType(RuleType.BUG);
+    insertRule(rule1.getDefinition());
     ruleDao.update(dbSession, rule1.getMetadata().setRuleId(rule1.getId()));
-    RuleDto rule2 = RuleTesting.newXooX2(defaultOrganization)
-        .setType(RuleType.VULNERABILITY);
-    ruleDao.insert(dbSession, rule2.getDefinition());
+    RuleDto rule2 = RuleTesting.newXooX2(defaultOrganizationDto)
+      .setType(RuleType.VULNERABILITY);
+    insertRule(rule2.getDefinition());
     ruleDao.update(dbSession, rule2.getMetadata().setRuleId(rule2.getId()));
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     WsTester.Result result = request.execute();
@@ -150,16 +148,15 @@ public class SearchActionMediumTest {
 
   @Test
   public void search_2_rules_with_fields_selection() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1()
+    insertRule(RuleTesting.newXooX1()
       .setType(RuleType.CODE_SMELL)
       .getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX2()
+    insertRule(RuleTesting.newXooX2()
       .setType(RuleType.BUG)
       .setDescription("A *Xoo* rule")
       .setDescriptionFormat(RuleDto.Format.MARKDOWN)
       .getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD).setParam(WebService.Param.FIELDS, "name,htmlDesc,mdDesc");
     WsTester.Result result = request.execute();
@@ -169,12 +166,11 @@ public class SearchActionMediumTest {
 
   @Test
   public void return_mandatory_fields_even_when_setting_f_param() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1()
+    insertRule(RuleTesting.newXooX1()
       .setName("Rule x1")
       .setType(RuleType.CODE_SMELL)
       .getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD).setParam(WebService.Param.FIELDS, "name");
     WsTester.Result result = request.execute();
@@ -184,9 +180,8 @@ public class SearchActionMediumTest {
 
   @Test
   public void return_lang_field() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1().getDefinition());
+    insertRule(RuleTesting.newXooX1().getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD).setParam(WebService.Param.FIELDS, "lang");
     WsTester.Result result = request.execute();
@@ -199,9 +194,8 @@ public class SearchActionMediumTest {
 
   @Test
   public void return_lang_name_field() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1().getDefinition());
+    insertRule(RuleTesting.newXooX1().getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD).setParam(WebService.Param.FIELDS, "langName");
     WsTester.Result result = request.execute();
@@ -214,9 +208,8 @@ public class SearchActionMediumTest {
 
   @Test
   public void return_lang_key_field_when_language_name_is_not_available() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newDto(RuleKey.of("other", "rule")).setLanguage("unknown").getDefinition());
+    insertRule(RuleTesting.newDto(RuleKey.of("other", "rule")).setLanguage("unknown").getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD).setParam(WebService.Param.FIELDS, "langName");
     WsTester.Result result = request.execute();
@@ -227,17 +220,16 @@ public class SearchActionMediumTest {
 
   @Test
   public void search_debt_rules_with_default_and_overridden_debt_values() throws Exception {
-    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganization)
+    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganizationDto)
       .setDefRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
       .setDefRemediationGapMultiplier("1h")
       .setDefRemediationBaseEffort("15min")
       .setRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
       .setRemediationGapMultiplier("2h")
       .setRemediationBaseEffort("25min");
-    ruleDao.insert(dbSession, ruleDto.getDefinition());
+    insertRule(ruleDto.getDefinition());
     ruleDao.update(dbSession, ruleDto.getMetadata().setRuleId(ruleDto.getId()));
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     request.setParam(WebService.Param.FIELDS, "debtRemFn,debtOverloaded,defaultDebtRemFn");
@@ -247,17 +239,16 @@ public class SearchActionMediumTest {
 
   @Test
   public void search_debt_rules_with_default_linear_offset_and_overridden_constant_debt() throws Exception {
-    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganization)
+    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganizationDto)
       .setDefRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
       .setDefRemediationGapMultiplier("1h")
       .setDefRemediationBaseEffort("15min")
       .setRemediationFunction(DebtRemediationFunction.Type.CONSTANT_ISSUE.name())
       .setRemediationGapMultiplier(null)
       .setRemediationBaseEffort("5min");
-    ruleDao.insert(dbSession, ruleDto.getDefinition());
+    insertRule(ruleDto.getDefinition());
     ruleDao.update(dbSession, ruleDto.getMetadata().setRuleId(ruleDto.getId()));
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     request.setParam(WebService.Param.FIELDS, "debtRemFn,debtOverloaded,defaultDebtRemFn");
@@ -267,17 +258,16 @@ public class SearchActionMediumTest {
 
   @Test
   public void search_debt_rules_with_default_linear_offset_and_overridden_linear_debt() throws Exception {
-    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganization)
+    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganizationDto)
       .setDefRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
       .setDefRemediationGapMultiplier("1h")
       .setDefRemediationBaseEffort("15min")
       .setRemediationFunction(DebtRemediationFunction.Type.LINEAR.name())
       .setRemediationGapMultiplier("1h")
       .setRemediationBaseEffort(null);
-    ruleDao.insert(dbSession, ruleDto.getDefinition());
+    insertRule(ruleDto.getDefinition());
     ruleDao.update(dbSession, ruleDto.getMetadata().setRuleId(ruleDto.getId()));
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     request.setParam(WebService.Param.FIELDS, "debtRemFn,debtOverloaded,defaultDebtRemFn");
@@ -288,12 +278,11 @@ public class SearchActionMediumTest {
   @Test
   public void search_template_rules() throws Exception {
     RuleDto templateRule = RuleTesting.newXooX1().setIsTemplate(true);
-    ruleDao.insert(dbSession, templateRule.getDefinition());
+    insertRule(templateRule.getDefinition());
     RuleDto rule = RuleTesting.newXooX2();
     rule.setTemplateId(templateRule.getId());
-    ruleDao.insert(dbSession, rule.getDefinition());
+    insertRule(rule.getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     request.setParam(WebService.Param.FIELDS, "isTemplate");
@@ -305,10 +294,9 @@ public class SearchActionMediumTest {
   @Test
   public void search_custom_rules_from_template_key() throws Exception {
     RuleDto templateRule = RuleTesting.newXooX1().setIsTemplate(true);
-    ruleDao.insert(dbSession, templateRule.getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX2().setTemplateId(templateRule.getId()).getDefinition());
+    insertRule(templateRule.getDefinition());
+    insertRule(RuleTesting.newXooX2().setTemplateId(templateRule.getId()).getDefinition());
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     request.setParam(WebService.Param.FIELDS, "templateKey");
@@ -323,13 +311,13 @@ public class SearchActionMediumTest {
     tester.get(QualityProfileDao.class).insert(dbSession, profile);
 
     RuleDefinitionDto rule = RuleTesting.newXooX1().getDefinition();
-    ruleDao.insert(dbSession, rule);
+    insertRule(rule);
 
     ActiveRuleDto activeRule = newActiveRule(profile, rule);
     tester.get(ActiveRuleDao.class).insert(dbSession, activeRule);
 
     dbSession.commit();
-    ruleIndexer.index();
+
     activeRuleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -352,7 +340,7 @@ public class SearchActionMediumTest {
     dbSession.commit();
 
     RuleDefinitionDto rule = RuleTesting.newXooX1().getDefinition();
-    ruleDao.insert(dbSession, rule);
+    insertRule(rule);
 
     RuleParamDto param = RuleParamDto.createFor(rule)
       .setDefaultValue("some value")
@@ -394,7 +382,7 @@ public class SearchActionMediumTest {
     tester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule2, activeRuleParam3);
 
     dbSession.commit();
-    ruleIndexer.index();
+
     activeRuleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -424,7 +412,7 @@ public class SearchActionMediumTest {
     dbSession.commit();
 
     RuleDefinitionDto rule = RuleTesting.newXooX1().getDefinition();
-    ruleDao.insert(dbSession, rule);
+    insertRule(rule);
 
     ActiveRuleDto activeRule = newActiveRule(profile, rule);
     tester.get(ActiveRuleDao.class).insert(dbSession, activeRule);
@@ -432,7 +420,7 @@ public class SearchActionMediumTest {
     tester.get(ActiveRuleDao.class).insert(dbSession, activeRule2);
 
     dbSession.commit();
-    ruleIndexer.index();
+
     activeRuleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -449,7 +437,7 @@ public class SearchActionMediumTest {
     QualityProfileDto profile = QProfileTesting.newXooP1("org-123");
     tester.get(QualityProfileDao.class).insert(dbSession, profile);
     RuleDefinitionDto rule = RuleTesting.newXooX1().getDefinition();
-    ruleDao.insert(dbSession, rule);
+    insertRule(rule);
     dbSession.commit();
 
     RuleParamDto param = RuleParamDto.createFor(rule)
@@ -478,7 +466,7 @@ public class SearchActionMediumTest {
     tester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam2);
 
     dbSession.commit();
-    ruleIndexer.index();
+
     activeRuleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -494,12 +482,12 @@ public class SearchActionMediumTest {
   public void get_note_as_markdown_and_html() throws Exception {
     QualityProfileDto profile = QProfileTesting.newXooP1("org-123");
     tester.get(QualityProfileDao.class).insert(dbSession, profile);
-    RuleDto rule = RuleTesting.newXooX1(defaultOrganization).setNoteData("this is *bold*");
-    ruleDao.insert(dbSession, rule.getDefinition());
+    RuleDto rule = RuleTesting.newXooX1(defaultOrganizationDto).setNoteData("this is *bold*");
+    insertRule(rule.getDefinition());
     ruleDao.update(dbSession, rule.getMetadata().setRuleId(rule.getId()));
 
     dbSession.commit();
-    ruleIndexer.index();
+
     activeRuleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -510,15 +498,15 @@ public class SearchActionMediumTest {
 
   @Test
   public void filter_by_tags() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1()
+    insertRule(RuleTesting.newXooX1()
       .setSystemTags(ImmutableSet.of("tag1"))
       .getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX2()
+    insertRule(RuleTesting.newXooX2()
       .setSystemTags(ImmutableSet.of("tag2"))
       .getDefinition());
 
     dbSession.commit();
-    ruleIndexer.index();
+
     activeRuleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -545,12 +533,12 @@ public class SearchActionMediumTest {
 
   @Test
   public void statuses_facet_should_be_sticky() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1().getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX2().setStatus(RuleStatus.BETA).getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX3().setStatus(RuleStatus.DEPRECATED).getDefinition());
+    RuleDefinitionDto definition = RuleTesting.newXooX1().getDefinition();
+    insertRule(definition);
+    insertRule(RuleTesting.newXooX2().setStatus(RuleStatus.BETA).getDefinition());
+    insertRule(RuleTesting.newXooX3().setStatus(RuleStatus.DEPRECATED).getDefinition());
 
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
     request.setParam(PARAM_STATUSES, "DEPRECATED");
@@ -560,18 +548,17 @@ public class SearchActionMediumTest {
 
   @Test
   public void sort_by_name() throws Exception {
-    ruleDao.insert(dbSession, RuleTesting.newXooX1()
+    insertRule(RuleTesting.newXooX1()
       .setName("Dodgy - Consider returning a zero length array rather than null ")
       .getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX2()
+    insertRule(RuleTesting.newXooX2()
       .setName("Bad practice - Creates an empty zip file entry")
       .getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX3()
+    insertRule(RuleTesting.newXooX3()
       .setName("XPath rule")
       .getDefinition());
 
     dbSession.commit();
-    ruleIndexer.index();
 
     // 1. Sort Name Asc
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -595,18 +582,17 @@ public class SearchActionMediumTest {
   @Test
   public void available_since() throws Exception {
     Date since = new Date();
-    ruleDao.insert(dbSession, RuleTesting.newXooX1()
+    insertRule(RuleTesting.newXooX1()
       .setUpdatedAt(since.getTime())
       .setCreatedAt(since.getTime())
       .getDefinition());
-    ruleDao.insert(dbSession, RuleTesting.newXooX2()
+    insertRule(RuleTesting.newXooX2()
       .setUpdatedAt(since.getTime())
       .setCreatedAt(since.getTime())
       .getDefinition());
 
     dbSession.commit();
     dbSession.clearCache();
-    ruleIndexer.index();
 
     // 1. find today's rules
     WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD);
@@ -626,17 +612,16 @@ public class SearchActionMediumTest {
 
   @Test
   public void search_rules_with_deprecated_fields() throws Exception {
-    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganization)
-        .setDefRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
-        .setDefRemediationGapMultiplier("1h")
-        .setDefRemediationBaseEffort("15min")
-        .setRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
-        .setRemediationGapMultiplier("2h")
-        .setRemediationBaseEffort("25min");
-    ruleDao.insert(dbSession, ruleDto.getDefinition());
+    RuleDto ruleDto = RuleTesting.newXooX1(defaultOrganizationDto)
+      .setDefRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
+      .setDefRemediationGapMultiplier("1h")
+      .setDefRemediationBaseEffort("15min")
+      .setRemediationFunction(DebtRemediationFunction.Type.LINEAR_OFFSET.name())
+      .setRemediationGapMultiplier("2h")
+      .setRemediationBaseEffort("25min");
+    insertRule(ruleDto.getDefinition());
     ruleDao.update(dbSession, ruleDto.getMetadata().setRuleId(ruleDto.getId()));
     dbSession.commit();
-    ruleIndexer.index();
 
     WsTester.TestRequest request = tester.wsTester()
       .newGetRequest(API_ENDPOINT, API_SEARCH_METHOD)
@@ -652,4 +637,9 @@ public class SearchActionMediumTest {
       .setSeverity("BLOCKER");
   }
 
+  private void insertRule(RuleDefinitionDto definition) {
+    ruleDao.insert(dbSession, definition);
+    dbSession.commit();
+    ruleIndexer.index(defaultOrganizationDto, definition.getKey());
+  }
 }

server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionMediumTest.java

@@ -46,7 +46,6 @@ import org.sonar.server.organization.DefaultOrganizationProvider;
 import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
 import org.sonar.server.rule.NewCustomRule;
 import org.sonar.server.rule.RuleCreator;
-import org.sonar.server.rule.RuleService;
 import org.sonar.server.rule.index.RuleIndexer;
 import org.sonar.server.tester.ServerTester;
 import org.sonar.server.tester.UserSessionRule;
@@ -68,7 +67,7 @@ public class ShowActionMediumTest {
     .addPermission(ADMINISTER_QUALITY_PROFILES, defaultOrganizationProvider.get().getUuid());
 
   private WsTester wsTester;
-  private RuleService ruleService;
+  private RuleIndexer ruleIndexer;
   private RuleDao ruleDao;
   private DbSession session;
   private OrganizationDto defaultOrganization;
@@ -77,7 +76,7 @@ public class ShowActionMediumTest {
   public void setUp() {
     tester.clearDbAndIndexes();
     wsTester = tester.get(WsTester.class);
-    ruleService = tester.get(RuleService.class);
+    ruleIndexer = tester.get(RuleIndexer.class);
     ruleDao = tester.get(RuleDao.class);
     session = tester.get(DbClient.class).openSession(false);
     defaultOrganization = tester.get(DbClient.class).organizationDao().selectByUuid(session, defaultOrganizationProvider.get().getUuid()).get();
@@ -272,6 +271,8 @@ public class ShowActionMediumTest {
       .setUpdatedAt(new Date().getTime());
     RuleDefinitionDto definition = ruleDto.getDefinition();
     ruleDao.insert(session, definition);
+    session.commit();
+    ruleIndexer.index(defaultOrganization, definition.getKey());
     RuleParamDto regexParam = RuleParamDto.createFor(definition).setName("regex").setType("STRING").setDescription("Reg *exp*").setDefaultValue(".*");
     ruleDao.insertRuleParam(session, definition, regexParam);
 
@@ -292,9 +293,7 @@ public class ShowActionMediumTest {
       .setKey(regexParam.getName())
       .setValue(".*?"));
     session.commit();
-    session.clearCache();
 
-    tester.get(RuleIndexer.class).index();
     tester.get(ActiveRuleIndexer.class).index();
 
     WsTester.TestRequest request = wsTester.newGetRequest("api/rules", "show")