"UUID" VARCHAR(40) NOT NULL PRIMARY KEY,
"DOC_TYPE" VARCHAR(40) NOT NULL,
"DOC_ID" VARCHAR(4000) NOT NULL,
+ "DOC_ID_TYPE" VARCHAR(20),
+ "DOC_ROUTING" VARCHAR(4000),
"CREATED_AT" BIGINT NOT NULL
);
CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID");
*/
package org.sonar.db.es;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
public final class EsQueueDto {
public enum Type {
- USER, RULE, RULE_EXTENSION
+ USER, RULE, RULE_EXTENSION, ACTIVE_RULE
}
private String uuid;
private Type docType;
private String docId;
+ private String docIdType;
+ private String docRouting;
public String getUuid() {
return uuid;
return this;
}
+ @CheckForNull
+ public String getDocIdType() {
+ return docIdType;
+ }
+
+ private EsQueueDto setDocIdType(@Nullable String s) {
+ this.docIdType = s;
+ return this;
+ }
+
+ @CheckForNull
+ public String getDocRouting() {
+ return docRouting;
+ }
+
+ private EsQueueDto setDocRouting(@Nullable String s) {
+ this.docRouting = s;
+ return this;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder("EsQueueDto{");
sb.append("uuid='").append(uuid).append('\'');
sb.append(", docType=").append(docType);
sb.append(", docId='").append(docId).append('\'');
+ sb.append(", docIdType='").append(docIdType).append('\'');
+ sb.append(", docRouting='").append(docRouting).append('\'');
sb.append('}');
return sb.toString();
}
public static EsQueueDto create(Type docType, String docUuid) {
return new EsQueueDto().setDocType(docType).setDocId(docUuid);
}
+
+ public static EsQueueDto create(Type docType, String docId, @Nullable String docIdType, @Nullable String docRouting) {
+ return new EsQueueDto().setDocType(docType)
+ .setDocId(docId).setDocIdType(docIdType).setDocRouting(docRouting);
+ }
}
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.function.Consumer;
import org.sonar.db.Dao;
import org.sonar.db.DatabaseUtils;
import org.sonar.db.DbSession;
import org.sonar.db.rule.RuleParamDto;
import static org.sonar.db.DatabaseUtils.executeLargeInputs;
+import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput;
import static org.sonar.db.KeyLongValue.toMap;
public class ActiveRuleDao implements Dao {
partition -> mapper(dbSession).countActiveRulesByQuery(query.getOrganization().getUuid(), partition, query.getRuleStatus(), query.getInheritance())));
}
+ public void scrollAllForIndexing(DbSession dbSession, Consumer<IndexedActiveRuleDto> consumer) {
+ mapper(dbSession).scrollAllForIndexing(context -> {
+ IndexedActiveRuleDto dto = (IndexedActiveRuleDto) context.getResultObject();
+ consumer.accept(dto);
+ });
+ }
+
+ public void scrollByIdsForIndexing(DbSession dbSession, Collection<Long> ids, Consumer<IndexedActiveRuleDto> consumer) {
+ ActiveRuleMapper mapper = mapper(dbSession);
+ executeLargeInputsWithoutOutput(ids,
+ pageOfIds -> mapper
+ .scrollByIdsForIndexing(pageOfIds, context -> {
+ IndexedActiveRuleDto dto = (IndexedActiveRuleDto) context.getResultObject();
+ consumer.accept(dto);
+ }));
+ }
+
+ public void scrollByRuleProfileForIndexing(DbSession dbSession, String ruleProfileUuid, Consumer<IndexedActiveRuleDto> consumer) {
+ mapper(dbSession).scrollByRuleProfileUuidForIndexing(ruleProfileUuid, context -> {
+ IndexedActiveRuleDto dto = (IndexedActiveRuleDto) context.getResultObject();
+ consumer.accept(dto);
+ });
+ }
+
private static ActiveRuleMapper mapper(DbSession dbSession) {
return dbSession.getMapper(ActiveRuleMapper.class);
}
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.apache.ibatis.annotations.Param;
+import org.apache.ibatis.session.ResultHandler;
import org.sonar.api.rule.RuleStatus;
import org.sonar.db.KeyLongValue;
List<KeyLongValue> countActiveRulesByQuery(@Param("organizationUuid") String organizationUuid, @Param("profileUuids") List<String> profileUuids,
@Nullable @Param("ruleStatus") RuleStatus ruleStatus, @Param("inheritance") String inheritance);
+ void scrollAllForIndexing(ResultHandler handler);
+
+ void scrollByIdsForIndexing(@Param("ids") Collection<Long> ids, ResultHandler handler);
+
+ void scrollByRuleProfileUuidForIndexing(@Param("ruleProfileUuid") String ruleProfileUuid, ResultHandler handler);
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.db.qualityprofile;
+
+import javax.annotation.CheckForNull;
+
+public class IndexedActiveRuleDto {
+ private long id;
+ private int severity;
+ private String inheritance;
+ private String repository;
+ private String key;
+ private String ruleProfileUuid;
+
+ public long getId() {
+ return id;
+ }
+
+ public int getSeverity() {
+ return severity;
+ }
+
+ @CheckForNull
+ public String getInheritance() {
+ return inheritance;
+ }
+
+ public String getRepository() {
+ return repository;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public String getRuleProfileUuid() {
+ return ruleProfileUuid;
+ }
+}
return mapper(dbSession).selectBuiltInRuleProfiles();
}
+ @CheckForNull
+ public RulesProfileDto selectRuleProfile(DbSession dbSession, String ruleProfileUuid) {
+ return mapper(dbSession).selectRuleProfile(ruleProfileUuid);
+ }
+
public void insert(DbSession dbSession, RulesProfileDto dto) {
QualityProfileMapper mapper = mapper(dbSession);
mapper.insertRuleProfile(dto, new Date(system.now()));
List<RulesProfileDto> selectBuiltInRuleProfiles();
+ @CheckForNull
+ RulesProfileDto selectRuleProfile(@Param("uuid") String ruleProfileUuid);
+
List<QProfileDto> selectOrderedByOrganizationUuid(@Param("organizationUuid") String organizationUuid);
@CheckForNull
import java.util.function.Consumer;
import javax.annotation.Nullable;
import org.apache.ibatis.session.ResultHandler;
-import org.sonar.db.es.RuleExtensionId;
import org.sonar.api.rule.RuleKey;
import org.sonar.api.rules.RuleQuery;
import org.sonar.db.Dao;
import org.sonar.db.DbSession;
import org.sonar.db.RowNotFoundException;
+import org.sonar.db.es.RuleExtensionId;
import org.sonar.db.organization.OrganizationDto;
import static com.google.common.base.Preconditions.checkNotNull;
}
}
- public void scrollRuleExtensionByRuleKeys(DbSession dbSession, Collection<RuleExtensionId> ruleExtensionIds, Consumer<RuleExtensionForIndexingDto> consumer) {
+ public void scrollIndexingRuleExtensionsByIds(DbSession dbSession, Collection<RuleExtensionId> ruleExtensionIds, Consumer<RuleExtensionForIndexingDto> consumer) {
RuleMapper mapper = mapper(dbSession);
executeLargeInputsWithoutOutput(ruleExtensionIds,
pageOfRuleExtensionIds -> mapper
- .selectRuleExtensionForIndexingByKeys(pageOfRuleExtensionIds)
+ .selectIndexingRuleExtensionsByIds(pageOfRuleExtensionIds)
.forEach(consumer));
}
- public void scrollRuleByRuleKeys(DbSession dbSession, Collection<RuleKey> ruleKeys, Consumer<RuleForIndexingDto> consumer) {
+ public void scrollIndexingRuleExtensions(DbSession dbSession, Consumer<RuleExtensionForIndexingDto> consumer) {
+ mapper(dbSession).scrollIndexingRuleExtensions(context -> {
+ RuleExtensionForIndexingDto dto = (RuleExtensionForIndexingDto) context.getResultObject();
+ consumer.accept(dto);
+ });
+ }
+
+ public void scrollIndexingRulesByKeys(DbSession dbSession, Collection<RuleKey> ruleKeys, Consumer<RuleForIndexingDto> consumer) {
RuleMapper mapper = mapper(dbSession);
executeLargeInputsWithoutOutput(ruleKeys,
pageOfRuleKeys -> mapper
- .selectRuleForIndexingByKeys(pageOfRuleKeys)
+ .selectIndexingRulesByKeys(pageOfRuleKeys)
.forEach(consumer));
}
+ public void scrollIndexingRules(DbSession dbSession, Consumer<RuleForIndexingDto> consumer) {
+ mapper(dbSession).scrollIndexingRules(context -> {
+ RuleForIndexingDto dto = (RuleForIndexingDto) context.getResultObject();
+ consumer.accept(dto);
+ });
+ }
+
private static RuleMapper mapper(DbSession session) {
return session.getMapper(RuleMapper.class);
}
private boolean isTemplate;
private String systemTags;
private String templateRuleKey;
- private String templateName;
+ private String templateRepository;
private String internalKey;
private String language;
private int type;
return templateRuleKey;
}
- public String getTemplateName() {
- return templateName;
+ public String getTemplateRepository() {
+ return templateRepository;
}
public String getInternalKey() {
List<RuleDefinitionDto> selectDefinitionByKeys(@Param("ruleKeys") List<RuleKey> keys);
- List<RuleForIndexingDto> selectRuleForIndexingByKeys(@Param("ruleKeys") List<RuleKey> keys);
+ void scrollIndexingRules(ResultHandler handler);
- List<RuleExtensionForIndexingDto> selectRuleExtensionForIndexingByKeys(@Param("ruleExtensionIds") List<RuleExtensionId> ruleExtensionIds);
+ List<RuleForIndexingDto> selectIndexingRulesByKeys(@Param("ruleKeys") List<RuleKey> keys);
+
+ void scrollIndexingRuleExtensions(ResultHandler handler);
+
+ List<RuleExtensionForIndexingDto> selectIndexingRuleExtensionsByIds(@Param("ruleExtensionIds") List<RuleExtensionId> ruleExtensionIds);
List<RuleDto> selectByQuery(@Param("organizationUuid") String organizationUuid, @Param("query") RuleQuery ruleQuery);
return tags == null ? new HashSet<>() : new TreeSet<>(Arrays.asList(StringUtils.split(tags, ',')));
}
+ String getTagsAsString() {
+ return tags;
+ }
+
public RuleMetadataDto setTags(Set<String> tags) {
String raw = tags.isEmpty() ? null : StringUtils.join(tags, ',');
checkArgument(raw == null || raw.length() <= 4000, "Rule tags are too long: %s", raw);
uuid,
doc_type as docType,
doc_id as docId,
+ doc_id_type as docIdType,
+ doc_routing as docRouting,
created_at as createdAt
</sql>
uuid,
doc_type,
doc_id,
+ doc_id_type,
+ doc_routing,
created_at
) values (
#{dto.uuid, jdbcType=VARCHAR},
#{dto.docType, jdbcType=VARCHAR},
#{dto.docId, jdbcType=VARCHAR},
+ #{dto.docIdType, jdbcType=VARCHAR},
+ #{dto.docRouting, jdbcType=VARCHAR},
#{now, jdbcType=BIGINT}
)
</insert>
group by oqp.uuid
</select>
+ <select id="scrollAllForIndexing" resultType="org.sonar.db.qualityprofile.IndexedActiveRuleDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+ <include refid="scrollAllForIndexingSql"/>
+ </select>
+
+ <select id="scrollByIdsForIndexing" parameterType="map" resultType="org.sonar.db.qualityprofile.IndexedActiveRuleDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+ <include refid="scrollAllForIndexingSql"/>
+ where ar.id in
+ <foreach collection="ids" open="(" close=")" item="id" separator=",">#{id, jdbcType=BIGINT}</foreach>
+ </select>
+
+ <select id="scrollByRuleProfileUuidForIndexing" parameterType="String" resultType="org.sonar.db.qualityprofile.IndexedActiveRuleDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+ <include refid="scrollAllForIndexingSql"/>
+ where rp.kee = #{ruleProfileUuid, jdbcType=VARCHAR}
+ </select>
+
+ <sql id="scrollAllForIndexingSql">
+ select
+ ar.id as "id",
+ ar.failure_level as "severity",
+ ar.inheritance as "inheritance",
+ r.plugin_name as "repository",
+ r.plugin_rule_key as "key",
+ rp.kee as "ruleProfileUuid"
+ from active_rules ar
+ inner join rules_profiles rp on rp.id = ar.profile_id
+ inner join rules r on r.id = ar.rule_id
+ </sql>
</mapper>
where rp.is_built_in = ${_true}
</select>
+ <select id="selectRuleProfile" resultType="org.sonar.db.qualityprofile.RulesProfileDto">
+ select <include refid="ruleProfileColumns"/>
+ from rules_profiles rp
+ where rp.kee = #{uuid, jdbcType=VARCHAR}
+ </select>
+
<select id="selectOrderedByOrganizationUuid" parameterType="map" resultType="org.sonar.db.qualityprofile.QProfileDto">
select
<include refid="qProfileColumns"/>
and r.plugin_rule_key=#{rule,jdbcType=VARCHAR}
</select>
- <select id="selectRuleExtensionForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto">
- select
- r.plugin_name as "pluginName",
- r.plugin_rule_key as "pluginRuleKey",
- rm.organization_uuid as "organizationUuid",
- rm.tags as "tags"
- from
- rules r
- inner join
- rules_metadata rm on rm.rule_id = r.id
- where
- rm.tags is not null and
- rm.tags != '' and
+ <select id="selectIndexingRuleExtensionsByIds" parameterType="map" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto">
+ <include refid="sqlSelectIndexingRuleExtensions" />
+ and
<foreach collection="ruleExtensionIds" index="index" item="ruleExtId" open="" separator=" or " close="">
- ( r.plugin_name=#{ruleExtId.repositoryName,jdbcType=VARCHAR} and
- r.plugin_rule_key=#{ruleExtId.ruleKey,jdbcType=VARCHAR} and
- rm.organization_uuid=#{ruleExtId.organizationUuid,jdbcType=VARCHAR} )
+ ( r.plugin_name = #{ruleExtId.repositoryName, jdbcType=VARCHAR} and
+ r.plugin_rule_key = #{ruleExtId.ruleKey, jdbcType=VARCHAR} and
+ rm.organization_uuid = #{ruleExtId.organizationUuid, jdbcType=VARCHAR} )
</foreach>
</select>
+ <select id="scrollIndexingRuleExtensions" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+ <include refid="sqlSelectIndexingRuleExtensions" />
+ </select>
+
+ <sql id="sqlSelectIndexingRuleExtensions">
+ select
+ r.plugin_name as "pluginName",
+ r.plugin_rule_key as "pluginRuleKey",
+ rm.organization_uuid as "organizationUuid",
+ rm.tags as "tags"
+ from rules r
+ inner join rules_metadata rm on rm.rule_id = r.id
+ where
+ rm.tags is not null and
+ rm.tags != ''
+ </sql>
+
+ <sql id="sqlSelectIndexingRuleExtensions" databaseId="oracle">
+ select
+ r.plugin_name as "pluginName",
+ r.plugin_rule_key as "pluginRuleKey",
+ rm.organization_uuid as "organizationUuid",
+ rm.tags as "tags"
+ from rules r
+ inner join rules_metadata rm on rm.rule_id = r.id
+ where
+ rm.tags is not null
+ </sql>
+
<select id="selectMetadataByKey" parameterType="map" resultType="org.sonar.db.rule.RuleMetadataDto">
select
rm.rule_id as "ruleId",
</foreach>
</select>
- <select id="selectRuleForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleForIndexingDto">
+ <select id="selectIndexingRulesByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleForIndexingDto">
+ <include refid="sqlSelectIndexingRules"/>
+ where
+ <foreach collection="ruleKeys" index="index" item="ruleKey" open="" separator=" or " close="">
+ (r.plugin_name=#{ruleKey.repository,jdbcType=VARCHAR} and r.plugin_rule_key=#{ruleKey.rule,jdbcType=VARCHAR})
+ </foreach>
+ </select>
+
+ <select id="scrollIndexingRules" resultType="org.sonar.db.rule.RuleForIndexingDto" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY">
+ <include refid="sqlSelectIndexingRules"/>
+ </select>
+
+ <sql id="sqlSelectIndexingRules">
select
r.id as "id",
r.plugin_name as "repository",
r.is_template as "isTemplate",
r.system_tags as "systemTags",
t.plugin_rule_key as "templateRuleKey",
- t.plugin_name as "templateName",
+ t.plugin_name as "templateRepository",
r.plugin_config_key as "internalKey",
r.language as "language",
r.rule_type as "type",
r.created_at as "createdAt",
r.updated_at as "updatedAt"
- from
- rules r
+ from rules r
left outer join rules t on t.id = r.template_id
- where
- <foreach collection="ruleKeys" index="index" item="ruleKey" open="" separator=" or " close="">
- (r.plugin_name=#{ruleKey.repository,jdbcType=VARCHAR} and r.plugin_rule_key=#{ruleKey.rule,jdbcType=VARCHAR})
- </foreach>
- </select>
+ </sql>
<select id="selectByQuery" parameterType="map" resultType="Rule">
select
*/
package org.sonar.db.qualityprofile;
+import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
+import java.util.function.Consumer;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
.containsOnly(entry(profile1.getKee(), 1L));
}
+ @Test
+ public void scrollAllForIndexing_empty_table() {
+ Accumulator accumulator = new Accumulator();
+ underTest.scrollAllForIndexing(dbSession, accumulator);
+ assertThat(accumulator.list).isEmpty();
+ }
+
+ @Test
+ public void scrollAllForIndexing() {
+ ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+ ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+ ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+
+ Accumulator accumulator = new Accumulator();
+ underTest.scrollAllForIndexing(dbSession, accumulator);
+ assertThat(accumulator.list)
+ .extracting(IndexedActiveRuleDto::getId, IndexedActiveRuleDto::getRepository, IndexedActiveRuleDto::getKey, IndexedActiveRuleDto::getRuleProfileUuid,
+ IndexedActiveRuleDto::getSeverity, IndexedActiveRuleDto::getInheritance)
+ .containsExactlyInAnyOrder(
+ tuple((long)ar1.getId(), ar1.getRuleKey().repository(), ar1.getRuleKey().rule(), profile1.getRulesProfileUuid(), ar1.getSeverity(), ar1.getInheritance()),
+ tuple((long)ar2.getId(), ar2.getRuleKey().repository(), ar2.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar2.getSeverity(), ar2.getInheritance()),
+ tuple((long)ar3.getId(), ar3.getRuleKey().repository(), ar3.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar3.getSeverity(), ar3.getInheritance()));
+ }
+
+ @Test
+ public void scrollByIdsForIndexing() {
+ ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+ ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+ ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+
+ Accumulator accumulator = new Accumulator();
+ underTest.scrollByIdsForIndexing(dbSession, asList((long)ar1.getId(), (long)ar2.getId()), accumulator);
+ assertThat(accumulator.list)
+ .extracting(IndexedActiveRuleDto::getId, IndexedActiveRuleDto::getRepository, IndexedActiveRuleDto::getKey, IndexedActiveRuleDto::getRuleProfileUuid,
+ IndexedActiveRuleDto::getSeverity)
+ .containsExactlyInAnyOrder(
+ tuple((long)ar1.getId(), ar1.getRuleKey().repository(), ar1.getRuleKey().rule(), profile1.getRulesProfileUuid(), ar1.getSeverity()),
+ tuple((long)ar2.getId(), ar2.getRuleKey().repository(), ar2.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar2.getSeverity()));
+ }
+
+ @Test
+ public void scrollByRuleProfileForIndexing() {
+ ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+ ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+ ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+
+ Accumulator accumulator = new Accumulator();
+ underTest.scrollByRuleProfileForIndexing(dbSession, profile2.getRulesProfileUuid(), accumulator);
+ assertThat(accumulator.list)
+ .extracting(IndexedActiveRuleDto::getId, IndexedActiveRuleDto::getRepository, IndexedActiveRuleDto::getKey, IndexedActiveRuleDto::getRuleProfileUuid,
+ IndexedActiveRuleDto::getSeverity)
+ .containsExactlyInAnyOrder(
+ tuple((long)ar2.getId(), ar2.getRuleKey().repository(), ar2.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar2.getSeverity()),
+ tuple((long)ar3.getId(), ar3.getRuleKey().repository(), ar3.getRuleKey().rule(), profile2.getRulesProfileUuid(), ar3.getSeverity()));
+ }
+
+ private static class Accumulator implements Consumer<IndexedActiveRuleDto> {
+ private final List<IndexedActiveRuleDto> list = new ArrayList<>();
+
+ @Override
+ public void accept(IndexedActiveRuleDto dto) {
+ list.add(dto);
+ }
+ }
}
assertThat(reloaded.isBuiltIn()).isEqualTo(update.isBuiltIn());
}
+ @Test
+ public void selectRuleProfile() {
+ RulesProfileDto rp = insertRulesProfile();
+
+ assertThat(underTest.selectRuleProfile(dbSession, rp.getKee()).getId()).isEqualTo(rp.getId());
+ assertThat(underTest.selectRuleProfile(dbSession, "missing")).isNull();
+ }
+
@Test
public void deleteRulesProfilesByUuids() {
RulesProfileDto rp1 = insertRulesProfile();
}
@Test
- public void find_all_is_sorted_by_profile_name() {
+ public void selectOrderedByOrganizationUuid_is_sorted_by_profile_name() {
QProfileDto dto1 = new QProfileDto()
.setKee("js_first")
.setRulesProfileUuid("rp-js_first")
}
@Test
- public void get_by_name_and_language() {
+ public void selectByNameAndLanguage() {
List<QProfileDto> sharedData = createSharedData();
QProfileDto dto = underTest.selectByNameAndLanguage(dbSession, organization, "Sonar Way", "java");
}
@Test
- public void get_by_name_and_languages() {
+ public void selectByNameAndLanguages() {
createSharedData();
List<QProfileDto> dtos = underTest.selectByNameAndLanguages(dbSession, organization, "Sonar Way", singletonList("java"));
}
@Test
- public void should_not_find_by_language_in_wrong_organization() {
+ public void should_not_selectByLanguage_in_wrong_organization() {
QProfileDto profile = QualityProfileTesting.newQualityProfileDto()
.setOrganizationUuid(organization.getUuid());
underTest.insert(dbSession, profile);
}
@Test
- public void should_not_find_by_language_with_wrong_language() {
+ public void should_not_selectByLanguage_with_wrong_language() {
QProfileDto profile = QualityProfileTesting.newQualityProfileDto()
.setOrganizationUuid(organization.getUuid());
underTest.insert(dbSession, profile);
}
@Test
- public void find_children() {
+ public void selectChildren() {
QProfileDto original1 = new QProfileDto()
.setKee("java_child1")
.setRulesProfileUuid("rp-java_child1")
import java.util.Collections;
import java.util.List;
import java.util.Optional;
+import java.util.function.Consumer;
import org.apache.ibatis.session.ResultHandler;
import org.junit.Before;
import org.junit.Rule;
import org.sonar.api.utils.System2;
import org.sonar.db.DbTester;
import org.sonar.db.RowNotFoundException;
+import org.sonar.db.es.RuleExtensionId;
import org.sonar.db.organization.OrganizationDto;
import org.sonar.db.organization.OrganizationTesting;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.tuple;
public class RuleDaoTest {
@Rule
public ExpectedException thrown = ExpectedException.none();
@Rule
- public DbTester dbTester = DbTester.create(System2.INSTANCE);
+ public DbTester db = DbTester.create(System2.INSTANCE);
- private RuleDao underTest = dbTester.getDbClient().ruleDao();
+ private RuleDao underTest = db.getDbClient().ruleDao();
private OrganizationDto organization;
@Before
public void before() {
- organization = dbTester.organizations().insert(o -> o.setUuid(ORGANIZATION_UUID));
+ organization = db.organizations().insert(o -> o.setUuid(ORGANIZATION_UUID));
}
@Test
public void selectByKey() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- assertThat(underTest.selectByKey(dbTester.getSession(), organization, RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
+ assertThat(underTest.selectByKey(db.getSession(), organization, RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
- Optional<RuleDto> rule = underTest.selectByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001"));
+ Optional<RuleDto> rule = underTest.selectByKey(db.getSession(), organization, RuleKey.of("java", "S001"));
assertThat(rule.isPresent()).isTrue();
assertThat(rule.get().getId()).isEqualTo(1);
}
@Test
public void selectByKey_populates_organizationUuid_even_when_organization_has_no_metadata() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- assertThat(underTest.selectByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001")).get().getOrganizationUuid())
+ assertThat(underTest.selectByKey(db.getSession(), organization, RuleKey.of("java", "S001")).get().getOrganizationUuid())
.isEqualTo(ORGANIZATION_UUID);
}
@Test
public void selectDefinitionByKey() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- assertThat(underTest.selectDefinitionByKey(dbTester.getSession(), RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
+ assertThat(underTest.selectDefinitionByKey(db.getSession(), RuleKey.of("NOT", "FOUND")).isPresent()).isFalse();
- Optional<RuleDefinitionDto> rule = underTest.selectDefinitionByKey(dbTester.getSession(), RuleKey.of("java", "S001"));
+ Optional<RuleDefinitionDto> rule = underTest.selectDefinitionByKey(db.getSession(), RuleKey.of("java", "S001"));
assertThat(rule.isPresent()).isTrue();
assertThat(rule.get().getId()).isEqualTo(1);
}
@Test
public void selectById() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectById(55l, organizationUuid, dbTester.getSession())).isEmpty();
- Optional<RuleDto> ruleDtoOptional = underTest.selectById(1l, organizationUuid, dbTester.getSession());
+ assertThat(underTest.selectById(55l, organizationUuid, db.getSession())).isEmpty();
+ Optional<RuleDto> ruleDtoOptional = underTest.selectById(1l, organizationUuid, db.getSession());
assertThat(ruleDtoOptional).isPresent();
assertThat(ruleDtoOptional.get().getId()).isEqualTo(1);
}
@Test
public void selectById_populates_organizationUuid_even_when_organization_has_no_metadata() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectById(1l, organizationUuid, dbTester.getSession()).get().getOrganizationUuid())
+ assertThat(underTest.selectById(1l, organizationUuid, db.getSession()).get().getOrganizationUuid())
.isEqualTo(organizationUuid);
}
@Test
public void selectDefinitionById() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- assertThat(underTest.selectDefinitionById(55l, dbTester.getSession())).isEmpty();
- Optional<RuleDefinitionDto> ruleDtoOptional = underTest.selectDefinitionById(1l, dbTester.getSession());
+ assertThat(underTest.selectDefinitionById(55l, db.getSession())).isEmpty();
+ Optional<RuleDefinitionDto> ruleDtoOptional = underTest.selectDefinitionById(1l, db.getSession());
assertThat(ruleDtoOptional).isPresent();
assertThat(ruleDtoOptional.get().getId()).isEqualTo(1);
}
@Test
public void selectByIds() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1))).hasSize(1);
- assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1, 2))).hasSize(2);
- assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1, 2, 3))).hasSize(2);
+ assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1))).hasSize(1);
+ assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1, 2))).hasSize(2);
+ assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1, 2, 3))).hasSize(2);
- assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(123))).isEmpty();
+ assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(123))).isEmpty();
}
@Test
public void selectByIds_populates_organizationUuid_even_when_organization_has_no_metadata() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectByIds(dbTester.getSession(), organizationUuid, asList(1, 2)))
+ assertThat(underTest.selectByIds(db.getSession(), organizationUuid, asList(1, 2)))
.extracting(RuleDto::getOrganizationUuid)
.containsExactly(organizationUuid, organizationUuid);
}
@Test
public void selectDefinitionByIds() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(1))).hasSize(1);
- assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(1, 2))).hasSize(2);
- assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(1, 2, 3))).hasSize(2);
+ assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(1))).hasSize(1);
+ assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(1, 2))).hasSize(2);
+ assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(1, 2, 3))).hasSize(2);
- assertThat(underTest.selectDefinitionByIds(dbTester.getSession(), asList(123))).isEmpty();
+ assertThat(underTest.selectDefinitionByIds(db.getSession(), asList(123))).isEmpty();
}
@Test
public void selectOrFailByKey() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid("org-1");
- RuleDto rule = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001"));
+ RuleDto rule = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("java", "S001"));
assertThat(rule.getId()).isEqualTo(1);
}
@Test
public void selectOrFailByKey_fails_if_rule_not_found() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
thrown.expect(RowNotFoundException.class);
thrown.expectMessage("Rule with key 'NOT:FOUND' does not exist");
OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid("org-1");
- underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("NOT", "FOUND"));
+ underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("NOT", "FOUND"));
}
@Test
public void selectOrFailByKey_populates_organizationUuid_even_when_organization_has_no_metadata() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid(organizationUuid);
- assertThat(underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("java", "S001")).getOrganizationUuid())
+ assertThat(underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("java", "S001")).getOrganizationUuid())
.isEqualTo(organizationUuid);
}
@Test
public void selectOrFailDefinitionByKey_fails_if_rule_not_found() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
thrown.expect(RowNotFoundException.class);
thrown.expectMessage("Rule with key 'NOT:FOUND' does not exist");
- underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("NOT", "FOUND"));
+ underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("NOT", "FOUND"));
}
@Test
public void selectByKeys() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectByKeys(dbTester.getSession(), organizationUuid, Collections.emptyList())).isEmpty();
- assertThat(underTest.selectByKeys(dbTester.getSession(), organizationUuid, asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
+ assertThat(underTest.selectByKeys(db.getSession(), organizationUuid, Collections.emptyList())).isEmpty();
+ assertThat(underTest.selectByKeys(db.getSession(), organizationUuid, asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
- List<RuleDto> rules = underTest.selectByKeys(dbTester.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
+ List<RuleDto> rules = underTest.selectByKeys(db.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
assertThat(rules).hasSize(1);
assertThat(rules.get(0).getId()).isEqualTo(1);
}
@Test
public void selectByKeys_populates_organizationUuid_even_when_organization_has_no_metadata() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectByKeys(dbTester.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER"))))
+ assertThat(underTest.selectByKeys(db.getSession(), organizationUuid, asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER"))))
.extracting(RuleDto::getOrganizationUuid)
.containsExactly(organizationUuid);
}
@Test
public void selectDefinitionByKeys() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- assertThat(underTest.selectDefinitionByKeys(dbTester.getSession(), Collections.emptyList())).isEmpty();
- assertThat(underTest.selectDefinitionByKeys(dbTester.getSession(), asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
+ assertThat(underTest.selectDefinitionByKeys(db.getSession(), Collections.emptyList())).isEmpty();
+ assertThat(underTest.selectDefinitionByKeys(db.getSession(), asList(RuleKey.of("NOT", "FOUND")))).isEmpty();
- List<RuleDefinitionDto> rules = underTest.selectDefinitionByKeys(dbTester.getSession(), asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
+ List<RuleDefinitionDto> rules = underTest.selectDefinitionByKeys(db.getSession(), asList(RuleKey.of("java", "S001"), RuleKey.of("java", "OTHER")));
assertThat(rules).hasSize(1);
assertThat(rules.get(0).getId()).isEqualTo(1);
}
@Test
public void selectAll() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- assertThat(underTest.selectAll(dbTester.getSession(), "org-1"))
+ assertThat(underTest.selectAll(db.getSession(), "org-1"))
.extracting(RuleDto::getId)
.containsOnly(1, 2, 10);
}
@Test
public void selectAll_populates_organizationUuid_even_when_organization_has_no_metadata() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectAll(dbTester.getSession(), organizationUuid))
+ assertThat(underTest.selectAll(db.getSession(), organizationUuid))
.extracting(RuleDto::getOrganizationUuid)
.containsExactly(organizationUuid, organizationUuid, organizationUuid);
}
@Test
public void selectAllDefinitions() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
- List<RuleDefinitionDto> ruleDtos = underTest.selectAllDefinitions(dbTester.getSession());
+ List<RuleDefinitionDto> ruleDtos = underTest.selectAllDefinitions(db.getSession());
assertThat(ruleDtos).extracting("id").containsOnly(1, 2, 10);
}
@Test
public void selectEnabled_with_ResultHandler() {
- dbTester.prepareDbUnit(getClass(), "selectEnabled.xml");
+ db.prepareDbUnit(getClass(), "selectEnabled.xml");
final List<RuleDefinitionDto> rules = new ArrayList<>();
ResultHandler resultHandler = resultContext -> rules.add((RuleDefinitionDto) resultContext.getResultObject());
- underTest.selectEnabled(dbTester.getSession(), resultHandler);
+ underTest.selectEnabled(db.getSession(), resultHandler);
assertThat(rules.size()).isEqualTo(1);
RuleDefinitionDto ruleDto = rules.get(0);
@Test
public void select_by_query() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create())).hasSize(2);
- assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create().withKey("S001"))).hasSize(1);
- assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create().withConfigKey("S1"))).hasSize(1);
- assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create().withRepositoryKey("java"))).hasSize(2);
- assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid,
+ assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create())).hasSize(2);
+ assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create().withKey("S001"))).hasSize(1);
+ assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create().withConfigKey("S1"))).hasSize(1);
+ assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create().withRepositoryKey("java"))).hasSize(2);
+ assertThat(underTest.selectByQuery(db.getSession(), organizationUuid,
RuleQuery.create().withKey("S001").withConfigKey("S1").withRepositoryKey("java"))).hasSize(1);
}
@Test
public void select_by_query_populates_organizationUuid_even_when_organization_has_no_metadata() {
- dbTester.prepareDbUnit(getClass(), "shared.xml");
+ db.prepareDbUnit(getClass(), "shared.xml");
String organizationUuid = "org-1";
- assertThat(underTest.selectByQuery(dbTester.getSession(), organizationUuid, RuleQuery.create()))
+ assertThat(underTest.selectByQuery(db.getSession(), organizationUuid, RuleQuery.create()))
.extracting(RuleDto::getOrganizationUuid)
.containsExactly(organizationUuid, organizationUuid);
}
.setType(RuleType.BUG)
.setCreatedAt(1_500_000_000_000L)
.setUpdatedAt(2_000_000_000_000L);
- underTest.insert(dbTester.getSession(), newRule);
- dbTester.getSession().commit();
+ underTest.insert(db.getSession(), newRule);
+ db.getSession().commit();
- RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("plugin", "NewRuleKey"));
+ RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("plugin", "NewRuleKey"));
assertThat(ruleDto.getId()).isNotNull();
assertThat(ruleDto.getName()).isEqualTo("new name");
assertThat(ruleDto.getDescription()).isEqualTo("new description");
@Test
public void update_RuleDefinitionDto() {
- dbTester.prepareDbUnit(getClass(), "update.xml");
+ db.prepareDbUnit(getClass(), "update.xml");
RuleDefinitionDto ruleToUpdate = new RuleDefinitionDto()
.setId(1)
.setType(RuleType.BUG)
.setUpdatedAt(2_000_000_000_000L);
- underTest.update(dbTester.getSession(), ruleToUpdate);
- dbTester.getSession().commit();
+ underTest.update(db.getSession(), ruleToUpdate);
+ db.getSession().commit();
- RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("plugin", "NewRuleKey"));
+ RuleDefinitionDto ruleDto = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("plugin", "NewRuleKey"));
assertThat(ruleDto.getName()).isEqualTo("new name");
assertThat(ruleDto.getDescription()).isEqualTo("new description");
assertThat(ruleDto.getDescriptionFormat()).isEqualTo(RuleDto.Format.MARKDOWN);
@Test
public void update_RuleMetadataDto_inserts_row_in_RULE_METADATA_if_not_exists_yet() {
- dbTester.prepareDbUnit(getClass(), "update.xml");
+ db.prepareDbUnit(getClass(), "update.xml");
String organizationUuid = "org-1";
RuleMetadataDto metadataToUpdate = new RuleMetadataDto()
.setCreatedAt(3_500_000_000_000L)
.setUpdatedAt(4_000_000_000_000L);
- underTest.insertOrUpdate(dbTester.getSession(), metadataToUpdate);
- dbTester.getSession().commit();
+ underTest.insertOrUpdate(db.getSession(), metadataToUpdate);
+ db.getSession().commit();
OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid(organizationUuid);
- RuleDto ruleDto = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
+ RuleDto ruleDto = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
assertThat(ruleDto.getName()).isEqualTo("Avoid Null");
assertThat(ruleDto.getDescription()).isEqualTo("Should avoid NULL");
assertThat(ruleDto.getDescriptionFormat()).isNull();
@Test
public void update_RuleMetadataDto_updates_row_in_RULE_METADATA_if_already_exists() {
- dbTester.prepareDbUnit(getClass(), "update.xml");
+ db.prepareDbUnit(getClass(), "update.xml");
String organizationUuid = "org-1";
OrganizationDto organization = OrganizationTesting.newOrganizationDto().setUuid(organizationUuid);
RuleMetadataDto metadataV1 = new RuleMetadataDto()
.setCreatedAt(6_500_000_000_000L)
.setUpdatedAt(7_000_000_000_000L);
- underTest.insertOrUpdate(dbTester.getSession(), metadataV1);
- dbTester.commit();
+ underTest.insertOrUpdate(db.getSession(), metadataV1);
+ db.commit();
- assertThat(dbTester.countRowsOfTable("RULES_METADATA")).isEqualTo(1);
- RuleDto ruleDto = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
+ assertThat(db.countRowsOfTable("RULES_METADATA")).isEqualTo(1);
+ RuleDto ruleDto = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
assertThat(ruleDto.getName()).isEqualTo("Avoid Null");
assertThat(ruleDto.getDescription()).isEqualTo("Should avoid NULL");
assertThat(ruleDto.getDescriptionFormat()).isNull();
assertThat(ruleDto.getCreatedAt()).isEqualTo(3_500_000_000_000L);
assertThat(ruleDto.getUpdatedAt()).isEqualTo(4_000_000_000_000L);
- underTest.insertOrUpdate(dbTester.getSession(), metadataV2);
- dbTester.commit();
+ underTest.insertOrUpdate(db.getSession(), metadataV2);
+ db.commit();
- ruleDto = underTest.selectOrFailByKey(dbTester.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
+ ruleDto = underTest.selectOrFailByKey(db.getSession(), organization, RuleKey.of("checkstyle", "AvoidNull"));
assertThat(ruleDto.getName()).isEqualTo("Avoid Null");
assertThat(ruleDto.getDescription()).isEqualTo("Should avoid NULL");
assertThat(ruleDto.getDescriptionFormat()).isNull();
@Test
public void select_parameters_by_rule_key() {
- dbTester.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
- List<RuleParamDto> ruleDtos = underTest.selectRuleParamsByRuleKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
+ db.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
+ List<RuleParamDto> ruleDtos = underTest.selectRuleParamsByRuleKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
assertThat(ruleDtos.size()).isEqualTo(1);
RuleParamDto ruleDto = ruleDtos.get(0);
@Test
public void select_parameters_by_rule_keys() {
- dbTester.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
+ db.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
- assertThat(underTest.selectRuleParamsByRuleKeys(dbTester.getSession(),
+ assertThat(underTest.selectRuleParamsByRuleKeys(db.getSession(),
Arrays.asList(RuleKey.of("checkstyle", "AvoidNull"), RuleKey.of("unused", "Unused")))).hasSize(2);
- assertThat(underTest.selectRuleParamsByRuleKeys(dbTester.getSession(),
+ assertThat(underTest.selectRuleParamsByRuleKeys(db.getSession(),
singletonList(RuleKey.of("unknown", "Unknown")))).isEmpty();
}
@Test
public void insert_parameter() {
- dbTester.prepareDbUnit(getClass(), "insert_parameter.xml");
- RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("plugin", "NewRuleKey"));
+ db.prepareDbUnit(getClass(), "insert_parameter.xml");
+ RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("plugin", "NewRuleKey"));
RuleParamDto param = RuleParamDto.createFor(rule1)
.setName("max")
.setDefaultValue("30")
.setDescription("My Parameter");
- underTest.insertRuleParam(dbTester.getSession(), rule1, param);
- dbTester.getSession().commit();
+ underTest.insertRuleParam(db.getSession(), rule1, param);
+ db.getSession().commit();
- dbTester.assertDbUnit(getClass(), "insert_parameter-result.xml", "rules_parameters");
+ db.assertDbUnit(getClass(), "insert_parameter-result.xml", "rules_parameters");
}
@Test
public void update_parameter() {
- dbTester.prepareDbUnit(getClass(), "update_parameter.xml");
+ db.prepareDbUnit(getClass(), "update_parameter.xml");
- RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
+ RuleDefinitionDto rule1 = underTest.selectOrFailDefinitionByKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"));
- List<RuleParamDto> params = underTest.selectRuleParamsByRuleKey(dbTester.getSession(), rule1.getKey());
+ List<RuleParamDto> params = underTest.selectRuleParamsByRuleKey(db.getSession(), rule1.getKey());
assertThat(params).hasSize(1);
RuleParamDto param = Iterables.getFirst(params, null);
.setDefaultValue("^[a-z]+(\\.[a-z][a-z0-9]*)*$")
.setDescription("Regular expression used to check the package names against.");
- underTest.updateRuleParam(dbTester.getSession(), rule1, param);
- dbTester.getSession().commit();
+ underTest.updateRuleParam(db.getSession(), rule1, param);
+ db.getSession().commit();
- dbTester.assertDbUnit(getClass(), "update_parameter-result.xml", "rules_parameters");
+ db.assertDbUnit(getClass(), "update_parameter-result.xml", "rules_parameters");
}
@Test
public void delete_parameter() {
- dbTester.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
- assertThat(underTest.selectRuleParamsByRuleKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).hasSize(1);
+ db.prepareDbUnit(getClass(), "select_parameters_by_rule_key.xml");
+ assertThat(underTest.selectRuleParamsByRuleKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).hasSize(1);
- underTest.deleteRuleParam(dbTester.getSession(), 1);
- dbTester.getSession().commit();
+ underTest.deleteRuleParam(db.getSession(), 1);
+ db.getSession().commit();
- assertThat(underTest.selectRuleParamsByRuleKey(dbTester.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).isEmpty();
+ assertThat(underTest.selectRuleParamsByRuleKey(db.getSession(), RuleKey.of("checkstyle", "AvoidNull"))).isEmpty();
+ }
+
+ @Test
+ public void scrollIndexingRules_on_empty_table() {
+ Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+
+ underTest.scrollIndexingRules(db.getSession(), accumulator);
+
+ assertThat(accumulator.list).isEmpty();
+ }
+
+ @Test
+ public void scrollIndexingRules() {
+ Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+ RuleDefinitionDto r1 = db.rules().insert();
+ RuleDefinitionDto r2 = db.rules().insert();
+
+ underTest.scrollIndexingRules(db.getSession(), accumulator);
+
+ assertThat(accumulator.list)
+ .extracting(RuleForIndexingDto::getId, RuleForIndexingDto::getRuleKey)
+ .containsExactlyInAnyOrder(tuple(r1.getId(), r1.getKey()), tuple(r2.getId(), r2.getKey()));
+ }
+
+ @Test
+ public void scrollIndexingRulesByKeys() {
+ Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+ RuleDefinitionDto r1 = db.rules().insert();
+ RuleDefinitionDto r2 = db.rules().insert();
+
+ underTest.scrollIndexingRulesByKeys(db.getSession(), singletonList(r1.getKey()), accumulator);
+
+ assertThat(accumulator.list)
+ .extracting(RuleForIndexingDto::getId, RuleForIndexingDto::getRuleKey)
+ .containsExactlyInAnyOrder(tuple(r1.getId(), r1.getKey()));
+ }
+
+ @Test
+ public void scrollIndexingRulesByKeys_scrolls_nothing_if_key_does_not_exist() {
+ Accumulator<RuleForIndexingDto> accumulator = new Accumulator<>();
+ RuleDefinitionDto r1 = db.rules().insert();
+
+ underTest.scrollIndexingRulesByKeys(db.getSession(), singletonList(RuleKey.of("does", "not exist")), accumulator);
+
+ assertThat(accumulator.list).isEmpty();
+ }
+
+ @Test
+ public void scrollIndexingRuleExtensions() {
+ Accumulator<RuleExtensionForIndexingDto> accumulator = new Accumulator<>();
+ RuleDefinitionDto r1 = db.rules().insert();
+ RuleMetadataDto r1Extension = db.rules().insertOrUpdateMetadata(r1, organization, r -> r.setTagsField("t1,t2"));
+ RuleDefinitionDto r2 = db.rules().insert();
+ RuleMetadataDto r2Extension = db.rules().insertOrUpdateMetadata(r2, organization, r -> r.setTagsField("t1,t3"));
+
+ underTest.scrollIndexingRuleExtensions(db.getSession(), accumulator);
+
+ assertThat(accumulator.list)
+ .extracting(RuleExtensionForIndexingDto::getRuleKey, RuleExtensionForIndexingDto::getOrganizationUuid, RuleExtensionForIndexingDto::getTags)
+ .containsExactlyInAnyOrder(
+ tuple(r1.getKey(), organization.getUuid(), r1Extension.getTagsAsString()),
+ tuple(r2.getKey(), organization.getUuid(), r2Extension.getTagsAsString()));
+ }
+
+ @Test
+ public void scrollIndexingRuleExtensionsByIds() {
+ Accumulator<RuleExtensionForIndexingDto> accumulator = new Accumulator<>();
+ RuleDefinitionDto r1 = db.rules().insert();
+ RuleMetadataDto r1Extension = db.rules().insertOrUpdateMetadata(r1, organization, r -> r.setTagsField("t1,t2"));
+ RuleExtensionId r1ExtensionId = new RuleExtensionId(organization.getUuid(), r1.getRepositoryKey(), r1.getRuleKey());
+ RuleDefinitionDto r2 = db.rules().insert();
+ RuleMetadataDto r2Extension = db.rules().insertOrUpdateMetadata(r2, organization, r -> r.setTagsField("t1,t3"));
+
+ underTest.scrollIndexingRuleExtensionsByIds(db.getSession(), singletonList(r1ExtensionId), accumulator);
+
+ assertThat(accumulator.list)
+ .extracting(RuleExtensionForIndexingDto::getRuleKey, RuleExtensionForIndexingDto::getOrganizationUuid, RuleExtensionForIndexingDto::getTags)
+ .containsExactlyInAnyOrder(
+ tuple(r1.getKey(), organization.getUuid(), r1Extension.getTagsAsString()));
+ }
+
+ private static class Accumulator<T> implements Consumer<T> {
+ private final List<T> list = new ArrayList<>();
+ @Override
+ public void accept(T dto) {
+ list.add(dto);
+ }
}
}
.setIsNullable(false)
.setLimit(4000)
.build())
+ .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
+ .setColumnName("doc_id_type")
+ .setIsNullable(true)
+ .setLimit(20)
+ .build())
+ .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder()
+ .setColumnName("doc_routing")
+ .setIsNullable(true)
+ .setLimit(4000)
+ .build())
.addColumn(BigIntegerColumnDef.newBigIntegerColumnDefBuilder()
.setColumnName("created_at")
.setIsNullable(false)
db.assertColumnDefinition(TABLE, "uuid", Types.VARCHAR, 40, false);
db.assertColumnDefinition(TABLE, "doc_type", Types.VARCHAR, 40, false);
db.assertColumnDefinition(TABLE, "doc_id", Types.VARCHAR, 4000, false);
+ db.assertColumnDefinition(TABLE, "doc_id_type", Types.VARCHAR, 20, true);
+ db.assertColumnDefinition(TABLE, "doc_routing", Types.VARCHAR, 4000, true);
db.assertColumnDefinition(TABLE, "created_at", Types.BIGINT, null, false);
}
}
CREATE TABLE "ES_QUEUE" (
"UUID" VARCHAR(40) NOT NULL PRIMARY KEY,
"DOC_TYPE" VARCHAR(40) NOT NULL,
- "DOC_UUID" VARCHAR(255) NOT NULL,
+ "DOC_ID" VARCHAR(4000) NOT NULL,
+ "DOC_ID_TYPE" VARCHAR(20),
+ "DOC_ROUTING" VARCHAR(4000),
"CREATED_AT" BIGINT NOT NULL
);
CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID");
package org.sonar.server.es;
import com.google.common.annotations.VisibleForTesting;
-import java.util.Arrays;
-import java.util.Collection;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.Nullable;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.core.util.ProgressLogger;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.es.EsQueueDto;
import static java.lang.String.format;
-import static java.util.stream.Collectors.toList;
/**
* Helper to bulk requests in an efficient way :
private final EsClient client;
private final String indexName;
private final BulkProcessor bulkProcessor;
- private final AtomicLong counter = new AtomicLong(0L);
- private final ResilientIndexerResult successCounter = new ResilientIndexerResult();
+ private final IndexingResult result = new IndexingResult();
+ private final IndexingListener indexingListener;
private final SizeHandler sizeHandler;
- private final BulkProcessorListener bulkProcessorListener;
- @Nullable
- private DbClient dbClient;
- @Nullable
- private DbSession dbSession;
- private Collection<EsQueueDto> esQueueDtos;
public BulkIndexer(EsClient client, String indexName, Size size) {
- this.dbClient = null;
+ this(client, indexName, size, IndexingListener.noop());
+ }
+
+ public BulkIndexer(EsClient client, String indexName, Size size, IndexingListener indexingListener) {
this.client = client;
this.indexName = indexName;
this.sizeHandler = size.createHandler(Runtime2.INSTANCE);
- this.bulkProcessorListener = new BulkProcessorListener();
+ this.indexingListener = indexingListener;
+ BulkProcessorListener bulkProcessorListener = new BulkProcessorListener();
this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener)
.setBackoffPolicy(BackoffPolicy.exponentialBackoff())
.setBulkSize(FLUSH_BYTE_SIZE)
}
public void start() {
+ result.clear();
sizeHandler.beforeStart(this);
- counter.set(0L);
- successCounter.clear();
- }
-
- public void start(DbSession dbSession, DbClient dbClient, Collection<EsQueueDto> esQueueDtos) {
- this.dbClient = dbClient;
- this.dbSession = dbSession;
- this.esQueueDtos = esQueueDtos;
- sizeHandler.beforeStart(this);
- counter.set(0L);
- successCounter.clear();
}
/**
* @return the number of documents successfully indexed
*/
- public ResilientIndexerResult stop() {
+ public IndexingResult stop() {
try {
bulkProcessor.awaitClose(1, TimeUnit.MINUTES);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e);
- } finally {
- dbSession = null;
}
client.prepareRefresh(indexName).get();
sizeHandler.afterStop(this);
- return successCounter;
+ return result;
}
public void add(ActionRequest<?> request) {
+ result.incrementRequests();
bulkProcessor.add(request);
}
public void addDeletion(SearchRequestBuilder searchRequest) {
+ // TODO to be replaced by delete_by_query that is back in ES5
searchRequest
.addSort("_doc", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(5))
}
public void addDeletion(IndexType indexType, String id) {
- add(client.prepareDelete(indexType, id).setRouting(id).request());
+ add(client.prepareDelete(indexType, id).request());
}
- public void addDeletion(IndexType indexType, String id, String routing) {
+ public void addDeletion(IndexType indexType, String id, @Nullable String routing) {
add(client.prepareDelete(indexType, id).setRouting(routing).request());
}
*
* Note that the parameter indexName could be removed if progress logs are not needed.
*/
- public static void delete(EsClient client, String indexName, SearchRequestBuilder searchRequest) {
+ public static IndexingResult delete(EsClient client, String indexName, SearchRequestBuilder searchRequest) {
BulkIndexer bulk = new BulkIndexer(client, indexName, Size.REGULAR);
bulk.start();
bulk.addDeletion(searchRequest);
- bulk.stop();
+ return bulk.stop();
}
private final class BulkProcessorListener implements Listener {
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
- counter.addAndGet(response.getItems().length);
-
+ List<String> successDocIds = new ArrayList<>();
for (BulkItemResponse item : response.getItems()) {
if (item.isFailed()) {
LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage());
- successCounter.increaseFailure();
} else {
- successCounter.increaseSuccess();
+ result.incrementSuccess();
+ successDocIds.add(item.getId());
}
}
- deleteSuccessfulItems(response);
+ indexingListener.onSuccess(successDocIds);
}
@Override
public void afterBulk(long executionId, BulkRequest req, Throwable e) {
LOGGER.error("Fail to execute bulk index request: " + req, e);
}
-
- private void deleteSuccessfulItems(BulkResponse bulkResponse) {
- if (esQueueDtos != null) {
- List<EsQueueDto> itemsToDelete = Arrays.stream(bulkResponse.getItems())
- .filter(b -> !b.isFailed())
- .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocId())).findFirst().orElse(null))
- .filter(Objects::nonNull)
- .collect(toList());
-
- dbClient.esQueueDao().delete(dbSession, itemsToDelete);
- dbSession.commit();
- }
- }
}
public enum Size {
@Override
void beforeStart(BulkIndexer bulkIndexer) {
- this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexName), bulkIndexer.counter, LOGGER)
+ this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexName), bulkIndexer.result.total, LOGGER)
.setPluralLabel("requests");
this.progress.start();
Map<String, Object> temporarySettings = new HashMap<>();
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.util.Collection;
+
+public interface IndexingListener {
+
+ void onSuccess(Collection<String> docIds);
+
+ static IndexingListener noop() {
+ return docIds -> {};
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.es;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class IndexingResult {
+
+ // FIXME should be private
+ AtomicLong total = new AtomicLong(0L);
+ private long successes = 0L;
+
+ IndexingResult clear() {
+ total.set(0L);
+ successes = 0L;
+ return this;
+ }
+
+ void incrementRequests() {
+ total.incrementAndGet();
+ }
+
+ IndexingResult incrementSuccess() {
+ successes += 1;
+ return this;
+ }
+
+ public void add(IndexingResult other) {
+ total.addAndGet(other.total.get());
+ successes += other.successes;
+ }
+
+ public long getFailures() {
+ return total.get() - successes;
+ }
+
+ public long getTotal() {
+ return total.get();
+ }
+
+ public long getSuccess() {
+ return successes;
+ }
+
+ /**
+ * Get the failure ratio,
+ * if the total is 0, we always return 1 in order to break loop
+ * @see {@link RecoveryIndexer#recover()}
+ */
+ public double getFailureRatio() {
+ return total.get() == 0 ? 1 : (1.0d * getFailures()) / total.get();
+ }
+
+ public boolean isSuccess() {
+ return total.get() == successes;
+ }
+}
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.es.EsQueueDto;
+import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
import org.sonar.server.rule.index.RuleIndexer;
import org.sonar.server.user.index.UserIndexer;
private final DbClient dbClient;
private final UserIndexer userIndexer;
private final RuleIndexer ruleIndexer;
+ private final ActiveRuleIndexer activeRuleIndexer;
private final long minAgeInMs;
private final long loopLimit;
- public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer, RuleIndexer ruleIndexer) {
+ public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient,
+ UserIndexer userIndexer, RuleIndexer ruleIndexer, ActiveRuleIndexer activeRuleIndexer) {
this.system2 = system2;
this.settings = settings;
this.dbClient = dbClient;
this.userIndexer = userIndexer;
this.ruleIndexer = ruleIndexer;
+ this.activeRuleIndexer = activeRuleIndexer;
this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS);
this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT);
}
try (DbSession dbSession = dbClient.openSession(false)) {
Profiler profiler = Profiler.create(LOGGER).start();
long beforeDate = system2.now() - minAgeInMs;
- ResilientIndexerResult result = new ResilientIndexerResult();
+ IndexingResult result = new IndexingResult();
Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
while (!items.isEmpty()) {
- ResilientIndexerResult loopResult = new ResilientIndexerResult();
+ IndexingResult loopResult = new IndexingResult();
ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
for (Map.Entry<EsQueueDto.Type, Collection<EsQueueDto>> entry : itemsByType.asMap().entrySet()) {
}
}
- private ResilientIndexerResult doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) {
+ private IndexingResult doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) {
LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type);
switch (type) {
case USER:
case RULE_EXTENSION:
case RULE:
return ruleIndexer.index(dbSession, typeItems);
+ case ACTIVE_RULE:
+ return activeRuleIndexer.index(dbSession, typeItems);
default:
LOGGER.error(LOG_PREFIX + "ignore {} documents with unsupported type {}", typeItems.size(), type);
- return new ResilientIndexerResult();
+ return new IndexingResult();
}
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+import org.sonar.core.util.stream.MoreCollectors;
+import org.sonar.db.DbClient;
+import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
+
+import static java.util.stream.Collectors.toMap;
+
+/**
+ * Clean-up the db table es_queue when documents
+ * are successfully indexed so that the recovery
+ * daemon does not re-index them.
+ */
+public class ResiliencyIndexingListener implements IndexingListener {
+
+ private final DbClient dbClient;
+ private final DbSession dbSession;
+ private final Collection<EsQueueDto> items;
+
+ public ResiliencyIndexingListener(DbClient dbClient, DbSession dbSession, Collection<EsQueueDto> items) {
+ this.dbClient = dbClient;
+ this.dbSession = dbSession;
+ this.items = items;
+ }
+
+ @Override
+ public void onSuccess(Collection<String> docIds) {
+ if (!docIds.isEmpty()) {
+ Map<String, EsQueueDto> itemsById = items.stream().collect(toMap(EsQueueDto::getDocId, Function.identity()));
+
+ Collection<EsQueueDto> itemsToDelete = docIds
+ .stream()
+ .map(itemsById::get)
+ .filter(Objects::nonNull)
+ .collect(MoreCollectors.toArrayList(docIds.size()));
+ dbClient.esQueueDao().delete(dbSession, itemsToDelete);
+ dbSession.commit();
+ }
+ }
+}
* @param items the items to be indexed
* @return the number of successful indexation
*/
- ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items);
+ IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items);
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.server.es;
-
-/**
- * The type Resilient indexer result.
- */
-public class ResilientIndexerResult {
- private long total = 0L;
- private long failures = 0L;
-
- public ResilientIndexerResult clear() {
- total = 0L;
- failures = 0L;
- return this;
- }
-
- public ResilientIndexerResult increaseFailure() {
- failures += 1;
- total += 1;
- return this;
- }
-
- public ResilientIndexerResult increaseSuccess() {
- total += 1;
- return this;
- }
-
- public long getFailures() {
- return failures;
- }
-
- public long getTotal() {
- return total;
- }
-
- public long getSuccess() {
- return total - failures;
- }
-
- /**
- * Get the failure ratio,
- * if the total is 0, we always return 1 in order to break loop
- * @see {@link RecoveryIndexer#recover()}
- */
- public double getFailureRatio() {
- return total == 0 ? 1 : 1.0d * failures / total;
- }
-
- public double getSuccessRatio() {
- return total == 0 ? 0 : 1 - 1.0d * failures / total;
- }
-
- public ResilientIndexerResult add(ResilientIndexerResult other) {
- this.total += other.getTotal();
- this.failures += other.getFailures();
- return this;
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-@ParametersAreNonnullByDefault
-package org.sonar.server.es.queue;
-
-import javax.annotation.ParametersAreNonnullByDefault;
import org.sonar.server.qualityprofile.RuleActivator;
import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.qualityprofile.ws.OldRestoreAction;
import org.sonar.server.qualityprofile.ws.ProfilesWs;
import org.sonar.server.qualityprofile.ws.QProfilesWsModule;
// quality profile
BuiltInQProfileRepositoryImpl.class,
- ActiveRuleIteratorFactory.class,
ActiveRuleIndexer.class,
XMLProfileParser.class,
XMLProfileSerializer.class,
Date now = new Date(system2.now());
RulesProfileDto ruleProfile = insertRulesProfile(dbSession, builtInQProfile, now);
- List<ActiveRuleChange> localChanges = builtInQProfile.getActiveRules()
+ List<ActiveRuleChange> changes = builtInQProfile.getActiveRules()
.stream()
.map(activeRule -> insertActiveRule(dbSession, ruleProfile, activeRule, now.getTime()))
.collect(MoreCollectors.toList());
- localChanges.forEach(change -> dbClient.qProfileChangeDao().insert(batchDbSession, change.toDto(null)));
+ changes.forEach(change -> dbClient.qProfileChangeDao().insert(batchDbSession, change.toDto(null)));
associateToOrganizations(dbSession, batchDbSession, builtInQProfile, ruleProfile);
- dbSession.commit();
+ // TODO batch statements should be executed through dbSession
batchDbSession.commit();
- activeRuleIndexer.indexRuleProfile(dbSession, ruleProfile);
+ activeRuleIndexer.commitAndIndex(dbSession, changes);
}
private void associateToOrganizations(DbSession dbSession, DbSession batchDbSession, BuiltInQProfile builtIn, RulesProfileDto rulesProfileDto) {
toBeDeactivated.forEach(ruleKey ->
changes.addAll(ruleActivator.deactivateOnBuiltInRulesProfile(dbSession, ruleProfile, ruleKey, false)));
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, changes);
+ activeRuleIndexer.commitAndIndex(dbSession, changes);
return changes;
}
db.qualityProfileDao().deleteOrgQProfilesByUuids(dbSession, uuids);
// tables related to rules_profiles and active_rules are deleted
- // only for custom profiles
+ // only for custom profiles. Built-in profiles are never
+ // deleted from table rules_profiles.
if (!rulesProfileUuidsOfCustomProfiles.isEmpty()) {
db.activeRuleDao().deleteParametersByRuleProfileUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
db.activeRuleDao().deleteByRuleProfileUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
db.qProfileChangeDao().deleteByRulesProfileUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
db.qualityProfileDao().deleteRulesProfilesByUuids(dbSession, rulesProfileUuidsOfCustomProfiles);
+ activeRuleIndexer.commitDeletionOfProfiles(dbSession, customProfiles);
+ } else {
+ dbSession.commit();
}
- dbSession.commit();
- activeRuleIndexer.deleteByProfiles(customProfiles);
}
}
// ignore, probably a rule inherited from parent that can't be deactivated
}
}
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, changes);
+ activeRuleIndexer.commitAndIndex(dbSession, changes);
return result;
}
return doActivate(dbSession, activation, context);
}
+ public List<ActiveRuleChange> activateAndCommit(DbSession dbSession, RuleActivation activation, QProfileDto profile) {
+ List<ActiveRuleChange> changes = activate(dbSession, activation, profile);
+ activeRuleIndexer.commitAndIndex(dbSession, changes);
+ return changes;
+ }
+
public List<ActiveRuleChange> activate(DbSession dbSession, RuleActivation activation, QProfileDto profile) {
RuleActivatorContext context = contextFactory.create(dbSession, activation.getRuleKey(), profile, false);
return doActivate(dbSession, activation, context);
* Deactivate a rule on a Quality profile. Does nothing if the rule is not activated, but
* fails (fast) if the rule or the profile does not exist.
*/
- public void deactivateAndUpdateIndex(DbSession dbSession, QProfileDto profile, RuleKey ruleKey) {
+ public void deactivateAndCommit(DbSession dbSession, QProfileDto profile, RuleKey ruleKey) {
List<ActiveRuleChange> changes = deactivate(dbSession, profile, ruleKey);
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, changes);
+ activeRuleIndexer.commitAndIndex(dbSession, changes);
}
/**
return value;
}
- public BulkChangeResult bulkActivate(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile, @Nullable String severity) {
+ public BulkChangeResult bulkActivateAndCommit(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile, @Nullable String severity) {
BulkChangeResult result = new BulkChangeResult();
Iterator<RuleKey> rules = ruleIndex.searchAll(ruleQuery);
while (rules.hasNext()) {
result.getErrors().addAll(e.errors());
}
}
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, result.getChanges());
+ activeRuleIndexer.commitAndIndex(dbSession, result.getChanges());
return result;
}
- public BulkChangeResult bulkDeactivate(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile) {
+ public BulkChangeResult bulkDeactivateAndCommit(DbSession dbSession, RuleQuery ruleQuery, QProfileDto profile) {
BulkChangeResult result = new BulkChangeResult();
Iterator<RuleKey> rules = ruleIndex.searchAll(ruleQuery);
while (rules.hasNext()) {
result.getErrors().addAll(e.errors());
}
}
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, result.getChanges());
+ activeRuleIndexer.commitAndIndex(dbSession, result.getChanges());
return result;
}
- public List<ActiveRuleChange> setParent(DbSession dbSession, QProfileDto profile, @Nullable QProfileDto parent) {
+ public List<ActiveRuleChange> setParentAndCommit(DbSession dbSession, QProfileDto profile, @Nullable QProfileDto parent) {
checkRequest(
parent == null || profile.getLanguage().equals(parent.getLanguage()),
"Cannot set the profile '%s' as the parent of profile '%s' since their languages differ ('%s' != '%s')",
}
}
}
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, changes);
+ activeRuleIndexer.commitAndIndex(dbSession, changes);
return changes;
}
package org.sonar.server.qualityprofile.index;
import com.google.common.collect.ImmutableSet;
-import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import javax.annotation.Nullable;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.sonar.api.rule.RuleKey;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
+import org.sonar.db.es.EsQueueDto;
+import org.sonar.db.qualityprofile.IndexedActiveRuleDto;
import org.sonar.db.qualityprofile.QProfileDto;
import org.sonar.db.qualityprofile.RulesProfileDto;
+import org.sonar.db.rule.SeverityUtil;
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.BulkIndexer.Size;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.IndexingResult;
+import org.sonar.server.es.ResiliencyIndexingListener;
+import org.sonar.server.es.ResilientIndexer;
import org.sonar.server.es.StartupIndexer;
+import org.sonar.server.qualityprofile.ActiveRule;
import org.sonar.server.qualityprofile.ActiveRuleChange;
import org.sonar.server.rule.index.RuleIndexDefinition;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID;
-import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_KEY;
import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
-public class ActiveRuleIndexer implements StartupIndexer {
+public class ActiveRuleIndexer implements StartupIndexer, ResilientIndexer {
+
+ private static final Logger LOGGER = Loggers.get(ActiveRuleIndexer.class);
+ private static final String ID_TYPE_ACTIVE_RULE_ID = "activeRuleId";
+ private static final String ID_TYPE_RULE_PROFILE_UUID = "ruleProfileUuid";
private final DbClient dbClient;
private final EsClient esClient;
- private final ActiveRuleIteratorFactory activeRuleIteratorFactory;
- public ActiveRuleIndexer(DbClient dbClient, EsClient esClient, ActiveRuleIteratorFactory activeRuleIteratorFactory) {
+ public ActiveRuleIndexer(DbClient dbClient, EsClient esClient) {
this.dbClient = dbClient;
this.esClient = esClient;
- this.activeRuleIteratorFactory = activeRuleIteratorFactory;
}
@Override
public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
try (DbSession dbSession = dbClient.openSession(false)) {
- ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForAll(dbSession);
- scrollDbAndIndex(dbCursor, Size.LARGE);
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.noop());
+ bulkIndexer.start();
+ dbClient.activeRuleDao().scrollAllForIndexing(dbSession, ar -> bulkIndexer.add(newIndexRequest(ar)));
+ bulkIndexer.stop();
}
}
return ImmutableSet.of(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE);
}
+ public void commitAndIndex(DbSession dbSession, Collection<ActiveRuleChange> changes) {
+ List<EsQueueDto> items = changes.stream()
+ .map(ActiveRuleChange::getActiveRule)
+ .map(ar -> newQueueDto(String.valueOf(ar.getId()), ID_TYPE_ACTIVE_RULE_ID, ar.getRuleKey().toString()))
+ .collect(toArrayList());
+
+ dbClient.esQueueDao().insert(dbSession, items);
+ dbSession.commit();
+ postCommit(dbSession, items);
+ }
+
+ public void commitDeletionOfProfiles(DbSession dbSession, Collection<QProfileDto> profiles) {
+ List<EsQueueDto> items = profiles.stream()
+ .map(QProfileDto::getRulesProfileUuid)
+ .distinct()
+ .map(ruleProfileUuid -> newQueueDto(ruleProfileUuid, ID_TYPE_RULE_PROFILE_UUID, null))
+ .collect(toArrayList());
+
+ dbClient.esQueueDao().insert(dbSession, items);
+ dbSession.commit();
+ postCommit(dbSession, items);
+ }
+
+ /**
+ * Entry point for Byteman tests. See directory tests/resilience.
+ */
+ private void postCommit(DbSession dbSession, Collection<EsQueueDto> items) {
+ index(dbSession, items);
+ }
+
/**
- * Important - the existing documents are not deleted, so this method
- * does not guarantee consistency of index.
+ * @return the number of items that have been successfully indexed
*/
- public void indexRuleProfile(DbSession dbSession, RulesProfileDto ruleProfile) {
- try (ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForRuleProfile(dbSession, ruleProfile)) {
- scrollDbAndIndex(dbCursor, Size.REGULAR);
+ @Override
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ IndexingResult result = new IndexingResult();
+
+ if (items.isEmpty()) {
+ return result;
}
+
+ Map<Long, EsQueueDto> activeRuleItems = new HashMap<>();
+ Map<String, EsQueueDto> ruleProfileItems = new HashMap<>();
+ items.forEach(i -> {
+ if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) {
+ ruleProfileItems.put(i.getDocId(), i);
+ } else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) {
+ activeRuleItems.put(Long.parseLong(i.getDocId()), i);
+ } else {
+ LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i);
+ deleteQueueDto(dbSession, i);
+ }
+ });
+
+ if (!activeRuleItems.isEmpty()) {
+ result.add(doIndexActiveRules(dbSession, activeRuleItems));
+ }
+ if (!ruleProfileItems.isEmpty()) {
+ result.add(doIndexRuleProfiles(dbSession, ruleProfileItems));
+ }
+ return result;
}
- public void indexChanges(DbSession dbSession, List<ActiveRuleChange> changes) {
- BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
- bulk.start();
- List<Integer> idsOfTouchedActiveRules = new ArrayList<>();
- changes.stream()
- .filter(c -> c.getActiveRule() != null)
- .forEach(c -> {
- if (c.getType().equals(ActiveRuleChange.Type.DEACTIVATED)) {
- bulk.addDeletion(INDEX_TYPE_ACTIVE_RULE, String.valueOf(c.getActiveRule().getId()), c.getKey().getRuleKey().toString());
- } else {
- idsOfTouchedActiveRules.add(c.getActiveRule().getId());
- }
+ private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) {
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, activeRuleItems.values()));
+ bulkIndexer.start();
+ Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems);
+ dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(),
+ // only index requests, no deletion requests.
+ // Deactivated users are not deleted but updated.
+ i -> {
+ remaining.remove(i.getId());
+ bulkIndexer.add(newIndexRequest(i));
});
- try (ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForActiveRules(dbSession, idsOfTouchedActiveRules)) {
- while (dbCursor.hasNext()) {
- ActiveRuleDoc activeRule = dbCursor.next();
- bulk.add(newIndexRequest(activeRule));
+
+ // the remaining ids reference rows that don't exist in db. They must
+ // be deleted from index.
+ remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE,
+ item.getDocId(), item.getDocRouting()));
+ return bulkIndexer.stop();
+ }
+
+ private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) {
+ IndexingResult result = new IndexingResult();
+
+ for (Map.Entry<String, EsQueueDto> entry : ruleProfileItems.entrySet()) {
+ String ruleProfileUUid = entry.getKey();
+ EsQueueDto item = entry.getValue();
+ IndexingResult profileResult;
+
+ RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid);
+ if (profile == null) {
+ // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule
+ SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE)
+ .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid)));
+ profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), search);
+
+ } else {
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.noop());
+ bulkIndexer.start();
+ dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i)));
+ profileResult = bulkIndexer.stop();
+ }
+
+ if (profileResult.isSuccess()) {
+ deleteQueueDto(dbSession, item);
}
+ result.add(profileResult);
}
- bulk.stop();
- }
- public void deleteByProfiles(Collection<QProfileDto> profiles) {
- BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
- bulk.start();
- profiles.forEach(profile -> {
- SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE)
- .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, profile.getRulesProfileUuid())));
- bulk.addDeletion(search);
- });
- bulk.stop();
+ return result;
}
- public void deleteByRuleKeys(Collection<RuleKey> ruleKeys) {
- BulkIndexer bulk = createBulkIndexer(Size.REGULAR);
- bulk.start();
- ruleKeys.forEach(ruleKey -> {
- SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE)
- .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_RULE_KEY, ruleKey.toString())));
- bulk.addDeletion(search);
- });
- bulk.stop();
+ private void deleteQueueDto(DbSession dbSession, EsQueueDto item) {
+ dbClient.esQueueDao().delete(dbSession, item);
+ dbSession.commit();
}
- private BulkIndexer createBulkIndexer(Size size) {
- return new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), size);
+ private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) {
+ return new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), size, listener);
}
- private static IndexRequest newIndexRequest(ActiveRuleDoc doc) {
+ private static IndexRequest newIndexRequest(IndexedActiveRuleDto dto) {
+ ActiveRuleDoc doc = new ActiveRuleDoc(String.valueOf(dto.getId()));
+ doc.setRuleProfileUuid(dto.getRuleProfileUuid());
+ doc.setSeverity(SeverityUtil.getSeverityFromOrdinal(dto.getSeverity()));
+ doc.setRuleKey(RuleKey.of(dto.getRepository(), dto.getKey()));
+ // all the fields must be present, even if value is null
+ String inheritance = dto.getInheritance();
+ doc.setInheritance(inheritance == null ? ActiveRule.Inheritance.NONE.name() : inheritance);
return new IndexRequest(INDEX_TYPE_ACTIVE_RULE.getIndex(), INDEX_TYPE_ACTIVE_RULE.getType())
.id(doc.getId())
+ .parent(doc.getParent())
.routing(doc.getRouting())
- .parent(doc.getRuleKey().toString())
.source(doc.getFields());
}
- private void scrollDbAndIndex(ActiveRuleIterator dbCursor, Size size) {
- BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE.getIndex(), size);
- bulk.start();
- while (dbCursor.hasNext()) {
- ActiveRuleDoc activeRule = dbCursor.next();
- bulk.add(newIndexRequest(activeRule));
- }
- bulk.stop();
+ private static EsQueueDto newQueueDto(String docId, String docIdType, @Nullable String routing) {
+ return EsQueueDto.create(EsQueueDto.Type.ACTIVE_RULE, docId, docIdType, routing);
}
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.util.Iterator;
-
-public interface ActiveRuleIterator extends Iterator<ActiveRuleDoc>, AutoCloseable {
- @Override
- void close();
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.util.Collection;
-import org.sonar.api.server.ServerSide;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.qualityprofile.RulesProfileDto;
-
-@ServerSide
-public class ActiveRuleIteratorFactory {
-
- private final DbClient dbClient;
-
- public ActiveRuleIteratorFactory(DbClient dbClient) {
- this.dbClient = dbClient;
- }
-
- public ActiveRuleIterator createForAll(DbSession dbSession) {
- return new ActiveRuleIteratorForSingleChunk(dbClient, dbSession);
- }
-
- public ActiveRuleIterator createForRuleProfile(DbSession dbSession, RulesProfileDto ruleProfile) {
- return new ActiveRuleIteratorForSingleChunk(dbClient, dbSession, ruleProfile);
- }
-
- public ActiveRuleIterator createForActiveRules(DbSession dbSession, Collection<Integer> activeRuleIds) {
- return new ActiveRuleIteratorForMultipleChunks(dbClient, dbSession, activeRuleIds);
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-
-import static java.util.Optional.ofNullable;
-
-public class ActiveRuleIteratorForMultipleChunks implements ActiveRuleIterator {
-
- private final DbClient dbClient;
- private final DbSession dbSession;
- private final Iterator<List<Integer>> iteratorOverChunks;
- private ActiveRuleIteratorForSingleChunk currentChunk;
-
- public ActiveRuleIteratorForMultipleChunks(DbClient dbClient, DbSession dbSession, Collection<Integer> activeRuleIds) {
- this.dbClient = dbClient;
- this.dbSession = dbSession;
- this.iteratorOverChunks = DatabaseUtils.toUniqueAndSortedPartitions(activeRuleIds).iterator();
- }
-
- @Override
- public boolean hasNext() {
- if (currentChunk != null && currentChunk.hasNext()) {
- return true;
- }
- return iteratorOverChunks.hasNext();
- }
-
- @Override
- public ActiveRuleDoc next() {
- if (currentChunk == null || !currentChunk.hasNext()) {
- currentChunk = nextChunk();
- }
- return currentChunk.next();
- }
-
- private ActiveRuleIteratorForSingleChunk nextChunk() {
- List<Integer> nextInput = iteratorOverChunks.next();
- return new ActiveRuleIteratorForSingleChunk(dbClient, dbSession, nextInput);
- }
-
- @Override
- public void close() {
- ofNullable(currentChunk).ifPresent(ActiveRuleIterator::close);
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.qualityprofile.index;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.List;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-import org.sonar.db.qualityprofile.RulesProfileDto;
-import org.sonar.db.rule.SeverityUtil;
-import org.sonar.server.qualityprofile.ActiveRule;
-
-import static org.apache.commons.lang.StringUtils.repeat;
-
-/**
- * Scrolls over table ISSUES and reads documents to populate
- * the issues index
- */
-class ActiveRuleIteratorForSingleChunk implements ActiveRuleIterator {
-
- private static final String[] COLUMNS = {
- "ar.id",
- "ar.failure_level",
- "ar.inheritance",
- "r.plugin_name",
- "r.plugin_rule_key",
- "rp.kee"
- };
-
- private static final String SQL_ALL = "select " + StringUtils.join(COLUMNS, ",") + " from active_rules ar " +
- " inner join rules_profiles rp on rp.id = ar.profile_id " +
- " inner join rules r on r.id = ar.rule_id ";
-
- private final PreparedStatement stmt;
- private final ResultSetIterator<ActiveRuleDoc> iterator;
-
- ActiveRuleIteratorForSingleChunk(DbClient dbClient, DbSession dbSession) {
- try {
- stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, SQL_ALL);
- iterator = new ActiveRuleIteratorInternal(stmt);
- } catch (Exception e) {
- throw new IllegalStateException("Fail to prepare SQL request to select all active_rules", e);
- }
- }
-
- ActiveRuleIteratorForSingleChunk(DbClient dbClient, DbSession dbSession, RulesProfileDto ruleProfile) {
- try {
- stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, SQL_ALL + " where rp.kee = ?");
- stmt.setString(1, ruleProfile.getKee());
- iterator = new ActiveRuleIteratorInternal(stmt);
- } catch (Exception e) {
- throw new IllegalStateException("Fail to prepare SQL request to select active_rules of profile " + ruleProfile.getKee(), e);
- }
- }
-
- ActiveRuleIteratorForSingleChunk(DbClient dbClient, DbSession dbSession, List<Integer> activeRuleIds) {
- try {
- stmt = dbClient.getMyBatis().newScrollingSelectStatement(dbSession, SQL_ALL + " where ar.id in (" + repeat("?", ",", activeRuleIds.size()) + ")");
- for (int i = 0; i < activeRuleIds.size(); i++) {
- stmt.setInt(i + 1, activeRuleIds.get(i));
- }
- iterator = new ActiveRuleIteratorInternal(stmt);
- } catch (Exception e) {
- throw new IllegalStateException("Fail to prepare SQL request to select active_rules", e);
- }
- }
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public ActiveRuleDoc next() {
- return iterator.next();
- }
-
- @Override
- public void close() {
- try {
- iterator.close();
- } finally {
- DatabaseUtils.closeQuietly(stmt);
- }
- }
-
- private static final class ActiveRuleIteratorInternal extends ResultSetIterator<ActiveRuleDoc> {
-
- ActiveRuleIteratorInternal(PreparedStatement stmt) throws SQLException {
- super(stmt);
- }
-
- @Override
- protected ActiveRuleDoc read(ResultSet rs) throws SQLException {
- long activeRuleId = rs.getLong(1);
- int severity = rs.getInt(2);
- String inheritance = rs.getString(3);
- RuleKey ruleKey = RuleKey.of(rs.getString(4), rs.getString(5));
- String ruleProfileUuid = rs.getString(6);
-
- return new ActiveRuleDoc(String.valueOf(activeRuleId))
- .setRuleProfileUuid(ruleProfileUuid)
- .setRuleKey(ruleKey)
- // all the fields must be present, even if value is null
- .setSeverity(SeverityUtil.getSeverityFromOrdinal(severity))
- .setInheritance(inheritance == null ? ActiveRule.Inheritance.NONE.name() : inheritance);
- }
- }
-}
*/
package org.sonar.server.qualityprofile.ws;
-import java.util.List;
import java.util.Map;
import org.sonar.api.rule.RuleKey;
import org.sonar.api.rule.Severity;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.qualityprofile.QProfileDto;
-import org.sonar.server.qualityprofile.ActiveRuleChange;
import org.sonar.server.qualityprofile.RuleActivation;
import org.sonar.server.qualityprofile.RuleActivator;
-import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
import org.sonar.server.user.UserSession;
import static java.lang.String.format;
private final RuleActivator ruleActivator;
private final UserSession userSession;
private final QProfileWsSupport wsSupport;
- private final ActiveRuleIndexer activeRuleIndexer;
- public ActivateRuleAction(DbClient dbClient, RuleActivator ruleActivator, UserSession userSession, QProfileWsSupport wsSupport, ActiveRuleIndexer activeRuleIndexer) {
+ public ActivateRuleAction(DbClient dbClient, RuleActivator ruleActivator, UserSession userSession, QProfileWsSupport wsSupport) {
this.dbClient = dbClient;
this.ruleActivator = ruleActivator;
this.userSession = userSession;
this.wsSupport = wsSupport;
- this.activeRuleIndexer = activeRuleIndexer;
}
public void define(WebService.NewController controller) {
wsSupport.checkPermission(dbSession, profile);
wsSupport.checkNotBuiltInt(profile);
RuleActivation activation = readActivation(request);
- List<ActiveRuleChange> changes = ruleActivator.activate(dbSession, activation, profile);
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, changes);
+ ruleActivator.activateAndCommit(dbSession, activation, profile);
}
response.noContent();
QProfileDto profile = wsSupport.getProfile(dbSession, fromKey(qualityProfileKey));
wsSupport.checkPermission(dbSession, profile);
wsSupport.checkNotBuiltInt(profile);
- result = ruleActivator.bulkActivate(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile, request.param(PARAM_TARGET_SEVERITY));
+ result = ruleActivator.bulkActivateAndCommit(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile, request.param(PARAM_TARGET_SEVERITY));
}
writeResponse(result, response);
String parentKey = request.param(PARAM_PARENT_PROFILE);
String parentName = request.param(PARAM_PARENT_NAME);
if (isEmpty(parentKey) && isEmpty(parentName)) {
- ruleActivator.setParent(dbSession, profile, null);
+ ruleActivator.setParentAndCommit(dbSession, profile, null);
} else {
String parentOrganizationKey = parentKey == null ? organization.getKey() : null;
String parentLanguage = parentKey == null ? request.param(PARAM_LANGUAGE) : null;
QProfileReference parentRef = QProfileReference.from(parentKey, parentOrganizationKey, parentLanguage, parentName);
QProfileDto parent = wsSupport.getProfile(dbSession, parentRef);
- ruleActivator.setParent(dbSession, profile, parent);
+ ruleActivator.setParentAndCommit(dbSession, profile, parent);
}
response.noContent();
result.add(exporters.importXml(profile, importerKey, contentToImport, dbSession));
}
}
- dbSession.commit();
- activeRuleIndexer.indexChanges(dbSession, result.getChanges());
+ activeRuleIndexer.commitAndIndex(dbSession, result.getChanges());
return buildResponse(result, organization);
}
QProfileDto profile = wsSupport.getProfile(dbSession, QProfileReference.fromKey(qualityProfileKey));
wsSupport.checkPermission(dbSession, profile);
wsSupport.checkNotBuiltInt(profile);
- ruleActivator.deactivateAndUpdateIndex(dbSession, profile, ruleKey);
+ ruleActivator.deactivateAndCommit(dbSession, profile, ruleKey);
}
response.noContent();
}
QProfileDto profile = wsSupport.getProfile(dbSession, QProfileReference.fromKey(qualityProfileKey));
wsSupport.checkPermission(dbSession, profile);
wsSupport.checkNotBuiltInt(profile);
- result = ruleActivator.bulkDeactivate(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile);
+ result = ruleActivator.bulkDeactivateAndCommit(dbSession, ruleQueryFactory.createRuleQuery(dbSession, request), profile);
}
writeResponse(result, response);
}
persistRepositories(dbSession, context.repositories());
ruleIndexer.commitAndIndex(dbSession, keysToIndex);
- activeRuleIndexer.indexChanges(dbSession, changes);
+ activeRuleIndexer.commitAndIndex(dbSession, changes);
profiler.stopDebug();
webServerRuleFinder.startCaching();
return ReflectionToStringBuilder.toString(this);
}
- public static RuleDoc of(RuleForIndexingDto ruleForIndexingDto) {
+ public static RuleDoc of(RuleForIndexingDto dto) {
RuleDoc ruleDoc = new RuleDoc()
- .setKey(ruleForIndexingDto.getRuleKey().toString())
- .setRepository(ruleForIndexingDto.getRepository())
- .setInternalKey(ruleForIndexingDto.getInternalKey())
- .setIsTemplate(ruleForIndexingDto.isTemplate())
- .setLanguage(ruleForIndexingDto.getLanguage())
- .setName(ruleForIndexingDto.getName())
- .setRuleKey(ruleForIndexingDto.getPluginRuleKey())
- .setSeverity(ruleForIndexingDto.getSeverityAsString())
- .setStatus(ruleForIndexingDto.getStatus().toString())
- .setType(ruleForIndexingDto.getTypeAsRuleType())
- .setCreatedAt(ruleForIndexingDto.getCreatedAt())
- .setUpdatedAt(ruleForIndexingDto.getUpdatedAt());
-
- if (ruleForIndexingDto.getPluginRuleKey() != null && ruleForIndexingDto.getRepository() != null) {
- ruleDoc.setTemplateKey(RuleKey.of(ruleForIndexingDto.getPluginRuleKey(), ruleForIndexingDto.getRepository()).toString());
+ .setKey(dto.getRuleKey().toString())
+ .setRepository(dto.getRepository())
+ .setInternalKey(dto.getInternalKey())
+ .setIsTemplate(dto.isTemplate())
+ .setLanguage(dto.getLanguage())
+ .setName(dto.getName())
+ .setRuleKey(dto.getPluginRuleKey())
+ .setSeverity(dto.getSeverityAsString())
+ .setStatus(dto.getStatus().toString())
+ .setType(dto.getTypeAsRuleType())
+ .setCreatedAt(dto.getCreatedAt())
+ .setUpdatedAt(dto.getUpdatedAt());
+
+ if (dto.getTemplateRuleKey() != null && dto.getTemplateRepository() != null) {
+ ruleDoc.setTemplateKey(RuleKey.of(dto.getTemplateRepository(), dto.getTemplateRuleKey()).toString());
} else {
ruleDoc.setTemplateKey(null);
}
- if (ruleForIndexingDto.getDescription() != null && ruleForIndexingDto.getDescriptionFormat() != null) {
- if (RuleDto.Format.HTML == ruleForIndexingDto.getDescriptionFormat()) {
- ruleDoc.setHtmlDescription(ruleForIndexingDto.getDescription());
+ if (dto.getDescription() != null && dto.getDescriptionFormat() != null) {
+ if (RuleDto.Format.HTML == dto.getDescriptionFormat()) {
+ ruleDoc.setHtmlDescription(dto.getDescription());
} else {
- ruleDoc.setHtmlDescription(Markdown.convertToHtml(ruleForIndexingDto.getDescription()));;
+ ruleDoc.setHtmlDescription(Markdown.convertToHtml(dto.getDescription()));
}
}
return ruleDoc;
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.server.rule.index;
-
-public class RuleDocWithSystemScope {
-
- private final RuleDoc ruleDoc;
- private final RuleExtensionDoc ruleExtensionDoc;
-
- public RuleDocWithSystemScope(RuleDoc ruleDoc, RuleExtensionDoc ruleExtensionDoc) {
- this.ruleDoc = ruleDoc;
- this.ruleExtensionDoc = ruleExtensionDoc;
- }
-
- public RuleDoc getRuleDoc() {
- return ruleDoc;
- }
-
- public RuleExtensionDoc getRuleExtensionDoc() {
- return ruleExtensionDoc;
- }
-}
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ListMultimap;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.elasticsearch.action.index.IndexRequest;
-import org.sonar.db.es.RuleExtensionId;
import org.sonar.api.rule.RuleKey;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.es.EsQueueDto;
+import org.sonar.db.es.RuleExtensionId;
import org.sonar.db.organization.OrganizationDto;
import org.sonar.db.rule.RuleExtensionForIndexingDto;
import org.sonar.db.rule.RuleForIndexingDto;
import org.sonar.server.es.BulkIndexer.Size;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.IndexingResult;
+import org.sonar.server.es.ResiliencyIndexingListener;
import org.sonar.server.es.ResilientIndexer;
-import org.sonar.server.es.ResilientIndexerResult;
import org.sonar.server.es.StartupIndexer;
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Collections.singletonList;
import static java.util.Objects.requireNonNull;
import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX;
import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
@Override
public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
- BulkIndexer bulk = new BulkIndexer(esClient, INDEX, Size.LARGE);
- bulk.start();
-
- // index all definitions and system extensions
- if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE)) {
- try (RuleIterator rules = new RuleIteratorForSingleChunk(dbClient, null)) {
- doIndexRuleDefinitions(rules, bulk);
+ try (DbSession dbSession = dbClient.openSession(false)) {
+ BulkIndexer bulk = createBulkIndexer(Size.LARGE, IndexingListener.noop());
+ bulk.start();
+
+ // index all definitions and system extensions
+ if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE)) {
+ dbClient.ruleDao().scrollIndexingRules(dbSession, dto -> {
+ bulk.add(newRuleDocIndexRequest(dto));
+ bulk.add(newRuleExtensionDocIndexRequest(dto));
+ });
}
- }
- // index all organization extensions
- if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE_EXTENSION)) {
- try (RuleMetadataIterator metadatas = new RuleMetadataIterator(dbClient)) {
- doIndexRuleExtensions(metadatas, bulk);
+ // index all organization extensions
+ if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE_EXTENSION)) {
+ dbClient.ruleDao().scrollIndexingRuleExtensions(dbSession, dto -> {
+ bulk.add(newRuleExtensionDocIndexRequest(dto));
+ });
}
- }
- bulk.stop();
+ bulk.stop();
+ }
}
public void commitAndIndex(DbSession dbSession, RuleKey ruleKey) {
}
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
if (items.isEmpty()) {
- return new ResilientIndexerResult();
+ return new IndexingResult();
}
- ResilientIndexerResult result = new ResilientIndexerResult();
+ IndexingResult result = new IndexingResult();
ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
return result;
}
- private ResilientIndexerResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
- BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR);
- bulkIndexer.start(dbSession, dbClient, items);
+ private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
+ bulkIndexer.start();
Set<RuleKey> rules = items
.stream()
.map(i -> RuleKey.parse(i.getDocId()))
.collect(toHashSet(items.size()));
- dbClient.ruleDao().scrollRuleByRuleKeys(dbSession, rules,
+ dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, rules,
// only index requests, no deletion requests.
// Deactivated users are not deleted but updated.
r -> {
return bulkIndexer.stop();
}
- private ResilientIndexerResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) {
- BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR);
- bulkIndexer.start(dbSession, dbClient, items);
+ private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) {
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
+ bulkIndexer.start();
Set<RuleExtensionId> docIds = items
.stream()
.map(RuleIndexer::explodeRuleExtensionDocId)
.collect(toHashSet(items.size()));
- dbClient.ruleDao().scrollRuleExtensionByRuleKeys(dbSession, docIds,
+ dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds,
// only index requests, no deletion requests.
// Deactivated users are not deleted but updated.
r -> {
- docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()) );
+ docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()));
bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
});
-
// the remaining items reference rows that don't exist in db. They must
// be deleted from index.
docIds.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.getId()));
return bulkIndexer.stop();
}
- private static void doIndexRuleDefinitions(Iterator<RuleDocWithSystemScope> rules, BulkIndexer bulk) {
- while (rules.hasNext()) {
- RuleDocWithSystemScope ruleWithExtension = rules.next();
- bulk.add(newIndexRequest(ruleWithExtension.getRuleDoc()));
- bulk.add(newIndexRequest(ruleWithExtension.getRuleExtensionDoc()));
- }
- }
-
- private static void doIndexRuleExtensions(Iterator<RuleExtensionDoc> metadatas, BulkIndexer bulk) {
- while (metadatas.hasNext()) {
- RuleExtensionDoc metadata = metadatas.next();
- bulk.add(newIndexRequest(metadata));
- }
- }
-
- private static IndexRequest newIndexRequest(RuleDoc rule) {
- return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType())
- .id(rule.getId())
- .routing(rule.getRouting())
- .source(rule.getFields());
- }
-
- private static IndexRequest newIndexRequest(RuleExtensionDoc ruleExtension) {
- return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
- .id(ruleExtension.getId())
- .routing(ruleExtension.getRouting())
- .source(ruleExtension.getFields())
- .parent(ruleExtension.getParent());
- }
-
private static IndexRequest newRuleDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
- RuleDoc ruleDoc = RuleDoc.of(ruleForIndexingDto);
+ RuleDoc doc = RuleDoc.of(ruleForIndexingDto);
- return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType(), ruleDoc.key().toString())
- .source(ruleDoc.getFields());
+ return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType())
+ .id(doc.key().toString())
+ .routing(doc.getRouting())
+ .source(doc.getFields());
}
private static IndexRequest newRuleExtensionDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleForIndexingDto);
- return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId())
- .source(ruleExtensionDoc.getFields())
- .parent(ruleExtensionDoc.getParent());
+ return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
+ .id(ruleExtensionDoc.getId())
+ .routing(ruleExtensionDoc.getRouting())
+ .parent(ruleExtensionDoc.getParent())
+ .source(ruleExtensionDoc.getFields());
}
private static IndexRequest newRuleExtensionDocIndexRequest(RuleExtensionForIndexingDto ruleExtensionForIndexingDto) {
- RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleExtensionForIndexingDto);
-
- return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId())
- .source(ruleExtensionDoc.getFields())
- .parent(ruleExtensionDoc.getParent());
+ RuleExtensionDoc doc = RuleExtensionDoc.of(ruleExtensionForIndexingDto);
+ return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
+ .id(doc.getId())
+ .routing(doc.getRouting())
+ .parent(doc.getParent())
+ .source(doc.getFields());
}
- private BulkIndexer newBulkIndexerForRules(Size bulkSize) {
- return new BulkIndexer(esClient, INDEX_TYPE_RULE.getIndex(), bulkSize);
+ private BulkIndexer createBulkIndexer(Size bulkSize, IndexingListener listener) {
+ return new BulkIndexer(esClient, INDEX_TYPE_RULE.getIndex(), bulkSize, listener);
}
private static ListMultimap<EsQueueDto.Type, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) {
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import java.util.Iterator;
-
-public interface RuleIterator extends Iterator<RuleDocWithSystemScope>, AutoCloseable {
-
- @Override
- void close();
-
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-
-import static java.util.Optional.ofNullable;
-
-public class RuleIteratorForMultipleChunks implements RuleIterator {
-
- private final DbClient dbClient;
- private final Iterator<List<RuleKey>> iteratorOverChunks;
- private RuleIteratorForSingleChunk currentChunk;
-
- public RuleIteratorForMultipleChunks(DbClient dbClient, Collection<RuleKey> keys) {
- this.dbClient = dbClient;
- iteratorOverChunks = DatabaseUtils.toUniqueAndSortedPartitions(keys).iterator();
- }
-
- @Override
- public boolean hasNext() {
- for (;;) {
- if (currentChunk != null && currentChunk.hasNext()) {
- return true;
- }
- if (iteratorOverChunks.hasNext()) {
- currentChunk = nextChunk();
- } else {
- return false;
- }
- }
- }
-
- @Override
- public RuleDocWithSystemScope next() {
- for (;;) {
- if (currentChunk != null && currentChunk.hasNext()) {
- return currentChunk.next();
- }
- if (iteratorOverChunks.hasNext()) {
- currentChunk = nextChunk();
- } else {
- throw new NoSuchElementException();
- }
- }
- }
-
- private RuleIteratorForSingleChunk nextChunk() {
- List<RuleKey> nextInput = iteratorOverChunks.next();
- return new RuleIteratorForSingleChunk(dbClient, nextInput);
- }
-
- @Override
- public void close() {
- ofNullable(currentChunk).ifPresent(RuleIterator::close);
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.api.rules.RuleType;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-import org.sonar.db.rule.RuleDto;
-import org.sonar.db.rule.SeverityUtil;
-import org.sonar.markdown.Markdown;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static java.util.stream.Collectors.joining;
-
-/**
- * Scrolls over table RULES and reads documents to populate the rules index
- */
-public class RuleIteratorForSingleChunk implements RuleIterator {
-
- private static final String[] FIELDS = {
- // column 1
- "r.plugin_name",
- "r.plugin_rule_key",
- "r.name",
- "r.description",
- "r.description_format",
- "r.priority",
- "r.status",
- "r.is_template",
- "r.system_tags",
- "t.plugin_rule_key",
-
- // column 11
- "t.plugin_name",
- "r.plugin_config_key",
- "r.language",
- "r.rule_type",
- "r.created_at",
- "r.updated_at",
- };
-
- private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM rules r " +
- "LEFT OUTER JOIN rules t ON t.id=r.template_id";
- private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
-
- private final DbSession session;
-
- private final List<RuleKey> ruleKeys;
-
- private final PreparedStatement stmt;
- private final ResultSetIterator<RuleDocWithSystemScope> iterator;
-
- RuleIteratorForSingleChunk(DbClient dbClient, @Nullable List<RuleKey> ruleKeys) {
- checkArgument(ruleKeys == null || ruleKeys.size() <= DatabaseUtils.PARTITION_SIZE_FOR_ORACLE,
- "Cannot search for more than " + DatabaseUtils.PARTITION_SIZE_FOR_ORACLE + " rule keys at once. Please provide the keys in smaller chunks.");
- this.ruleKeys = ruleKeys;
- this.session = dbClient.openSession(false);
-
- try {
- String sql = createSql();
- stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
- iterator = createIterator();
- } catch (Exception e) {
- session.close();
- throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
- }
- }
-
- private RuleIteratorInternal createIterator() {
- try {
- setParameters(stmt);
- return new RuleIteratorInternal(stmt);
- } catch (SQLException e) {
- DatabaseUtils.closeQuietly(stmt);
- throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
- }
- }
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public RuleDocWithSystemScope next() {
- return iterator.next();
- }
-
- private String createSql() {
- StringBuilder sql = new StringBuilder(SQL_ALL);
- if (ruleKeys != null && !ruleKeys.isEmpty()) {
- sql.append(" WHERE ");
- sql.append(ruleKeys.stream()
- .map(x -> "(r.plugin_name=? AND r.plugin_rule_key=?)")
- .collect(joining(" OR "))
- );
- }
- return sql.toString();
- }
-
- private void setParameters(PreparedStatement stmt) throws SQLException {
- AtomicInteger index = new AtomicInteger(1);
- if (ruleKeys != null && !ruleKeys.isEmpty()) {
- for (RuleKey ruleKey : ruleKeys) {
- stmt.setString(index.getAndIncrement(), ruleKey.repository());
- stmt.setString(index.getAndIncrement(), ruleKey.rule());
- }
- }
- }
-
- @Override
- public void close() {
- try {
- iterator.close();
- } finally {
- DatabaseUtils.closeQuietly(stmt);
- session.close();
- }
- }
-
- private static final class RuleIteratorInternal extends ResultSetIterator<RuleDocWithSystemScope> {
-
- public RuleIteratorInternal(PreparedStatement stmt) throws SQLException {
- super(stmt);
- }
-
- @Override
- protected RuleDocWithSystemScope read(ResultSet rs) throws SQLException {
- RuleDoc doc = new RuleDoc();
- RuleExtensionDoc extensionDoc = new RuleExtensionDoc().setScope(RuleExtensionScope.system());
-
- String repositoryKey = rs.getString(1);
- String ruleKey = rs.getString(2);
- RuleKey key = RuleKey.of(repositoryKey, ruleKey);
- extensionDoc.setRuleKey(key);
-
- // all the fields must be present, even if value is null
- doc.setKey(key.toString());
- doc.setRuleKey(ruleKey);
- doc.setRepository(repositoryKey);
- doc.setName(rs.getString(3));
-
- String description = rs.getString(4);
- String descriptionFormat = rs.getString(5);
- if (descriptionFormat != null && description != null) {
- String htmlDescription;
- if (RuleDto.Format.HTML == RuleDto.Format.valueOf(descriptionFormat)) {
- htmlDescription = description;
- } else {
- htmlDescription = Markdown.convertToHtml(description);
- }
- doc.setHtmlDescription(htmlDescription);
- }
-
- doc.setSeverity(SeverityUtil.getSeverityFromOrdinal(rs.getInt(6)));
- doc.setStatus(rs.getString(7));
- doc.setIsTemplate(rs.getBoolean(8));
- extensionDoc.setTags(stringTagsToSet(rs.getString(9)));
-
- String templateRuleKey = rs.getString(10);
- String templateRepoKey = rs.getString(11);
- if (templateRepoKey != null && templateRuleKey != null) {
- doc.setTemplateKey(RuleKey.of(templateRepoKey, templateRuleKey).toString());
- } else {
- doc.setTemplateKey(null);
- }
-
- doc.setInternalKey(rs.getString(12));
- doc.setLanguage(rs.getString(13));
- doc.setType(RuleType.valueOf(rs.getInt(14)));
- doc.setCreatedAt(rs.getLong(15));
- doc.setUpdatedAt(rs.getLong(16));
-
- return new RuleDocWithSystemScope(doc, extensionDoc);
- }
-
- private static Set<String> stringTagsToSet(@Nullable String tags) {
- return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
- }
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-package org.sonar.server.rule.index;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableSet;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Iterator;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.db.DatabaseUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-
-/**
- * Scrolls over table RULES_METADATA and reads documents to populate the rule extension index type
- */
-public class RuleMetadataIterator implements Iterator<RuleExtensionDoc>, AutoCloseable {
-
- private static final String[] FIELDS = {
- "r.plugin_name",
- "r.plugin_rule_key",
- "rm.organization_uuid",
- "rm.tags"
- };
-
- private static final String SQL_ALL = "SELECT " + StringUtils.join(FIELDS, ",") + " FROM rules r " +
- "INNER JOIN rules_metadata rm ON rm.rule_id = r.id " +
- "WHERE rm.tags is not null AND rm.tags != ''";
- private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings();
-
- private final DbSession session;
-
- private final PreparedStatement stmt;
- private final ResultSetIterator<RuleExtensionDoc> iterator;
-
- RuleMetadataIterator(DbClient dbClient) {
- this.session = dbClient.openSession(false);
-
- try {
- String sql = SQL_ALL;
- stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
- iterator = createIterator();
- } catch (Exception e) {
- session.close();
- throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
- }
- }
-
- private RuleMetadataIteratorInternal createIterator() {
- try {
- return new RuleMetadataIteratorInternal(stmt);
- } catch (SQLException e) {
- DatabaseUtils.closeQuietly(stmt);
- throw new IllegalStateException("Fail to prepare SQL request to select all rules", e);
- }
- }
-
- @Override
- public boolean hasNext() {
- return iterator.hasNext();
- }
-
- @Override
- public RuleExtensionDoc next() {
- return iterator.next();
- }
-
- @Override
- public void close() {
- try {
- iterator.close();
- } finally {
- DatabaseUtils.closeQuietly(stmt);
- session.close();
- }
- }
-
- private static final class RuleMetadataIteratorInternal extends ResultSetIterator<RuleExtensionDoc> {
-
- public RuleMetadataIteratorInternal(PreparedStatement stmt) throws SQLException {
- super(stmt);
- }
-
- @Override
- protected RuleExtensionDoc read(ResultSet rs) throws SQLException {
- RuleExtensionDoc doc = new RuleExtensionDoc();
-
- String repositoryKey = rs.getString(1);
- String ruleKey = rs.getString(2);
- RuleKey key = RuleKey.of(repositoryKey, ruleKey);
- doc.setRuleKey(key);
- doc.setScope(RuleExtensionScope.organization(rs.getString(3)));
- doc.setTags(stringTagsToSet(rs.getString(4)));
-
- return doc;
- }
-
- private static Set<String> stringTagsToSet(@Nullable String tags) {
- return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags));
- }
- }
-}
import org.sonar.server.es.BulkIndexer.Size;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.ResiliencyIndexingListener;
import org.sonar.server.es.ResilientIndexer;
-import org.sonar.server.es.ResilientIndexerResult;
+import org.sonar.server.es.IndexingResult;
import org.sonar.server.es.StartupIndexer;
import static java.util.Collections.singletonList;
ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByLogin::put);
- BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE);
+ BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.noop());
bulkIndexer.start();
dbClient.userDao().scrollAll(dbSession,
// only index requests, no deletion requests.
* @return the number of items that have been successfully indexed
*/
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
if (items.isEmpty()) {
- return new ResilientIndexerResult();
+ return new IndexingResult();
}
Set<String> logins = items
.stream()
ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put);
- BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR);
- bulkIndexer.start(dbSession, dbClient, items);
+ BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
+ bulkIndexer.start();
dbClient.userDao().scrollByLogins(dbSession, logins,
// only index requests, no deletion requests.
// Deactivated users are not deleted but updated.
return bulkIndexer.stop();
}
- private BulkIndexer newBulkIndexer(Size bulkSize) {
- return new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex(), bulkSize);
+ private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) {
+ return new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex(), bulkSize, listener);
}
private static IndexRequest newIndexRequest(UserDto user, ListMultimap<String, String> organizationUuidsByLogins) {
return new IndexRequest(UserIndexDefinition.INDEX_TYPE_USER.getIndex(), UserIndexDefinition.INDEX_TYPE_USER.getType())
.id(doc.getId())
+ .routing(doc.getRouting())
.source(doc.getFields());
}
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.user.index;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ListMultimap;
-import com.google.common.collect.Maps;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Collection;
-import java.util.List;
-import javax.annotation.Nullable;
-import org.apache.commons.lang.StringUtils;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.ResultSetIterator;
-import org.sonar.db.es.EsQueueDto;
-import org.sonar.db.user.UserDto;
-
-import static org.apache.commons.lang.StringUtils.repeat;
-import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
-
-/**
- * Scrolls over table USERS and reads documents to populate the user index
- */
-class UserResultSetIterator extends ResultSetIterator<UserDoc> {
-
- private static final String[] FIELDS = {
- // column 1
- "u.login",
- "u.name",
- "u.email",
- "u.active",
- "u.scm_accounts",
- "u.created_at",
- "u.updated_at",
- };
-
- private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from users u ";
-
- private final ListMultimap<String, String> organizationUuidsByLogins;
-
- private UserResultSetIterator(PreparedStatement stmt, ListMultimap<String, String> organizationUuidsByLogins) throws SQLException {
- super(stmt);
- this.organizationUuidsByLogins = organizationUuidsByLogins;
- }
-
- static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable Collection<EsQueueDto> esQueueDtos) {
- try {
- String sql = SQL_ALL;
- List<String> logins = null;
- if (esQueueDtos != null) {
- logins = esQueueDtos.stream()
- .filter(i -> i.getDocType() == EsQueueDto.Type.USER)
- .map(EsQueueDto::getDocId).collect(toArrayList());
- sql += "where (" + repeat("u.login=?", " or ", logins.size()) + ")";
- }
-
- PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql);
- setParameters(stmt, logins);
-
- ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
- if (esQueueDtos == null) {
- dbClient.organizationMemberDao().selectAllForUserIndexing(session, organizationUuidsByLogin::put);
- } else {
-
- dbClient.organizationMemberDao().selectForUserIndexing(session, logins, organizationUuidsByLogin::put);
- }
-
- return new UserResultSetIterator(stmt, organizationUuidsByLogin);
- } catch (SQLException e) {
- throw new IllegalStateException("Fail to prepare SQL request to select all users", e);
- }
- }
-
- private static void setParameters(PreparedStatement stmt, @Nullable Collection<String> logins) throws SQLException {
- if (logins == null) {
- return;
- }
-
- int paramIndex = 1;
- for (String login : logins) {
- stmt.setString(paramIndex, login);
- paramIndex++;
- }
- }
-
- @Override
- protected UserDoc read(ResultSet rs) throws SQLException {
- UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(6));
-
- String login = rs.getString(1);
-
- // all the keys must be present, even if value is null
- doc.setLogin(login);
- doc.setName(rs.getString(2));
- doc.setEmail(rs.getString(3));
- doc.setActive(rs.getBoolean(4));
- doc.setScmAccounts(UserDto.decodeScmAccounts(rs.getString(5)));
- doc.setOrganizationUuids(organizationUuidsByLogins.get(login));
- return doc;
- }
-}
for (int i = 0; i < 10; i++) {
indexer.add(newIndexRequest(i));
}
- indexer.stop();
+ IndexingResult result = indexer.stop();
+ assertThat(result.isSuccess()).isTrue();
+ assertThat(result.getSuccess()).isEqualTo(10);
+ assertThat(result.getFailures()).isEqualTo(0);
+ assertThat(result.getTotal()).isEqualTo(10);
assertThat(count()).isEqualTo(10);
-
// replicas are re-enabled
assertThat(replicas()).isEqualTo(1);
}
@Override
protected void before() throws Throwable {
- truncateIndices();
+ deleteIndices();
if (!indexDefinitions.isEmpty()) {
container = new ComponentContainer();
}
}
- private void truncateIndices() {
+ private void deleteIndices() {
client.nativeClient().admin().indices().prepareDelete("_all").get();
}
+ public void deleteIndex(String indexName) {
+ client.nativeClient().admin().indices().prepareDelete(indexName).get();
+ }
+
public void putDocuments(String index, String type, BaseDoc... docs) {
putDocuments(new IndexType(index, type), docs);
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.assertj.core.data.Offset;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class IndexingResultTest {
+
+ private final IndexingResult underTest = new IndexingResult();
+
+ @Test
+ public void test_success() {
+ underTest.incrementRequests();
+ underTest.incrementRequests();
+ underTest.incrementSuccess();
+ underTest.incrementSuccess();
+
+ assertThat(underTest.getFailures()).isEqualTo(0);
+ assertThat(underTest.getSuccess()).isEqualTo(2);
+ assertThat(underTest.getTotal()).isEqualTo(2);
+ assertThat(underTest.getFailureRatio()).isEqualTo(0.0, Offset.offset(0.000001d));
+ assertThat(underTest.isSuccess()).isTrue();
+ }
+
+ @Test
+ public void test_failure() {
+ underTest.incrementRequests();
+ underTest.incrementRequests();
+
+ assertThat(underTest.getFailures()).isEqualTo(2);
+ assertThat(underTest.getSuccess()).isEqualTo(0);
+ assertThat(underTest.getTotal()).isEqualTo(2);
+ assertThat(underTest.getFailureRatio()).isEqualTo(1.0, Offset.offset(0.000001d));
+ assertThat(underTest.isSuccess()).isFalse();
+ }
+
+ @Test
+ public void test_partial_failure() {
+ underTest.incrementRequests();
+ underTest.incrementRequests();
+ underTest.incrementSuccess();
+
+ assertThat(underTest.getFailures()).isEqualTo(1);
+ assertThat(underTest.getSuccess()).isEqualTo(1);
+ assertThat(underTest.getTotal()).isEqualTo(2);
+ assertThat(underTest.getFailureRatio()).isEqualTo(0.5, Offset.offset(0.000001d));
+ assertThat(underTest.isSuccess()).isFalse();
+ }
+
+ @Test
+ public void correctness_even_with_no_data() {
+ assertThat(underTest.getFailures()).isEqualTo(0);
+ assertThat(underTest.getSuccess()).isEqualTo(0);
+ assertThat(underTest.getTotal()).isEqualTo(0);
+ assertThat(underTest.getFailureRatio()).isEqualTo(1);
+ assertThat(underTest.isSuccess()).isTrue();
+ }
+}
import org.sonar.db.es.EsQueueDto;
import org.sonar.db.rule.RuleDto;
import org.sonar.db.user.UserDto;
+import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
import org.sonar.server.user.index.UserIndexDefinition;
@Rule
public TestRule safeguard = new DisableOnDebug(new Timeout(60, TimeUnit.SECONDS));
-
- private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class);
private UserIndexer mockedUserIndexer = mock(UserIndexer.class);
+ private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class);
+ private ActiveRuleIndexer mockedActiveRuleIndexer = mock(ActiveRuleIndexer.class);
private RecoveryIndexer underTest;
@After
Settings settings = new MapSettings()
.setProperty("sonar.search.recovery.initialDelayInMs", "0")
.setProperty("sonar.search.recovery.delayInMs", "1");
- underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer));
+ underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer, mockedActiveRuleIndexer));
AtomicInteger calls = new AtomicInteger(0);
doAnswer(invocation -> {
calls.incrementAndGet();
}
@Test
- public void stop_run_if_too_many_failures() throws Exception {
+ public void stop_run_if_too_many_failures() {
IntStream.range(0, 10).forEach(i -> createUnindexedUser());
advanceInTime();
}
@Test
- public void do_not_stop_run_if_success_rate_is_greater_than_ratio() throws Exception {
+ public void do_not_stop_run_if_success_rate_is_greater_than_ratio() {
IntStream.range(0, 10).forEach(i -> createUnindexedUser());
advanceInTime();
}
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
called.addAll(items);
return super.index(dbSession, items);
}
}
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
called.addAll(items);
return super.index(dbSession, items);
}
}
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
called.addAll(items);
throw new RuntimeException("boom");
}
}
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
try {
if (counter.getCount() == 2) {
throw new RuntimeException("boom");
}
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
- ResilientIndexerResult result = new ResilientIndexerResult();
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
List<EsQueueDto> filteredItems = items.stream().filter(
i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList());
+ IndexingResult result = super.index(dbSession, filteredItems);
if (items.contains(failing)) {
- result.increaseFailure();
+ result.incrementRequests();
}
- return result.add(super.index(dbSession, filteredItems));
+ return result;
}
}
}
@Override
- public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) {
- System.out.println("called with " + items.size());
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
called.addAll(items);
int success = successfulReturns.next();
- ResilientIndexerResult result = new ResilientIndexerResult();
+ IndexingResult result = new IndexingResult();
items.stream().limit(success).forEach(i -> {
- System.out.println(" + success");
db.getDbClient().esQueueDao().delete(dbSession, i);
- result.increaseSuccess();
+ result.incrementSuccess();
indexed.add(i);
});
- rangeClosed(1, items.size() - success).forEach(i -> result.increaseFailure());
+ rangeClosed(1, items.size()).forEach(i -> result.incrementRequests());
dbSession.commit();
return result;
}
}
private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, Settings settings) {
- return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer);
+ return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer, mockedActiveRuleIndexer);
}
private EsQueueDto createUnindexedUser() {
+++ /dev/null
-package org.sonar.server.es;/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-import java.util.stream.IntStream;
-import org.apache.commons.lang.math.RandomUtils;
-import org.assertj.core.data.Offset;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class ResilientIndexerResultTest {
-
- private final ResilientIndexerResult underTest = new ResilientIndexerResult();
-
- @Before
- public void clear() {
- underTest.clear();
- }
-
- @Test
- public void ensure_correctness() {
- int success = 1 + RandomUtils.nextInt(100);
- int failures = RandomUtils.nextInt(100);
- IntStream.rangeClosed(1, success).forEach(i -> underTest.increaseSuccess());
- IntStream.rangeClosed(1, failures).forEach(i -> underTest.increaseFailure());
-
- assertThat(underTest.getFailures()).isEqualTo(failures);
- assertThat(underTest.getSuccess()).isEqualTo(success);
- assertThat(underTest.getTotal()).isEqualTo(success + failures);
- assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1);
- assertThat(underTest.getFailureRatio()).isEqualTo(1.0d * failures / (success + failures), Offset.offset(0.000001d));
- assertThat(underTest.getSuccessRatio()).isEqualTo(1.0d * success / (success + failures), Offset.offset(0.000001d));
- }
-
- @Test
- public void correctness_even_with_no_data() {
- assertThat(underTest.getFailures()).isEqualTo(0);
- assertThat(underTest.getSuccess()).isEqualTo(0);
- assertThat(underTest.getTotal()).isEqualTo(0);
- assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1);
- assertThat(underTest.getFailureRatio()).isEqualTo(1);
- assertThat(underTest.getSuccessRatio()).isEqualTo(0);
- }
-}
public EsTester esTester = new EsTester(
new IssueIndexDefinition(settings.asConfig()),
new ViewIndexDefinition(settings.asConfig()),
- new RuleIndexDefinition(settings.asConfig());
+ new RuleIndexDefinition(settings.asConfig()));
@Rule
public DbTester dbTester = DbTester.create(system2);
@Rule
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.ArgumentCaptor;
-import org.sonar.api.config.MapSettings;
+import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.rule.RuleKey;
import org.sonar.api.rule.RuleStatus;
import org.sonar.api.server.ws.WebService;
@Rule
public DbTester dbTester = DbTester.create();
@Rule
- public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings()));
-
+ public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig()));
private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
private OrganizationFlags organizationFlags = new OrganizationFlagsImpl(dbTester.getDbClient());
.containsExactlyInAnyOrder(
tuple(normal.getKey(), RuleStatus.READY),
tuple(template.getKey(), RuleStatus.READY),
- tuple(custom.getKey(), RuleStatus.READY)
- );
+ tuple(custom.getKey(), RuleStatus.READY));
call();
.containsExactlyInAnyOrder(
tuple(normal.getKey(), RuleStatus.READY),
tuple(template.getKey(), RuleStatus.REMOVED),
- tuple(custom.getKey(), RuleStatus.REMOVED)
- );
+ tuple(custom.getKey(), RuleStatus.REMOVED));
@SuppressWarnings("unchecked")
- Class<ArrayList<RuleKey>> listClass = (Class<ArrayList<RuleKey>>)(Class)ArrayList.class;
+ Class<ArrayList<RuleKey>> listClass = (Class<ArrayList<RuleKey>>) (Class) ArrayList.class;
ArgumentCaptor<ArrayList<RuleKey>> indexedRuleKeys = ArgumentCaptor.forClass(listClass);
verify(ruleIndexer).commitAndIndex(any(), indexedRuleKeys.capture());
assertThat(indexedRuleKeys.getValue()).containsExactlyInAnyOrder(template.getKey(), custom.getKey());
import static java.util.Arrays.asList;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyCollection;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
}
private void verifyNoCallsActiveRuleIndexerDelete() {
- verifyCallActiveRuleIndexerDelete();
+ verify(activeRuleIndexer, never()).commitDeletionOfProfiles(any(DbSession.class), anyCollection());
}
- private void verifyCallActiveRuleIndexerDelete(String... expectedProfileUuids) {
+ private void verifyCallActiveRuleIndexerDelete(String... expectedRuleProfileUuids) {
Class<Set<QProfileDto>> setClass = (Class<Set<QProfileDto>>) (Class) Set.class;
ArgumentCaptor<Set<QProfileDto>> setCaptor = ArgumentCaptor.forClass(setClass);
- verify(activeRuleIndexer).deleteByProfiles(setCaptor.capture());
+ verify(activeRuleIndexer).commitDeletionOfProfiles(any(DbSession.class), setCaptor.capture());
assertThat(setCaptor.getValue())
.extracting(QProfileDto::getKee)
- .containsExactlyInAnyOrder(expectedProfileUuids);
+ .containsExactlyInAnyOrder(expectedRuleProfileUuids);
}
private void assertThatRulesProfileExists(RulesProfileDto rulesProfile) {
// TODO active_rule_parameters
}
- private void assertThatCustomProfileIsDefault(OrganizationDto org, QProfileDto profile) {
- assertThat(db.getDbClient().qualityProfileDao().selectDefaultProfile(dbSession, org, profile.getLanguage())).isEqualTo(profile.getKee());
- }
-
- private void assertThatCustomProfileIsAssociatedToProject(OrganizationDto org, QProfileDto profile) {
- assertThat(db.getDbClient().qualityProfileDao().selectProjectAssociations(dbSession, org, profile, null)).isNotEmpty();
- }
-
private static void assertEqual(QProfileDto p1, QProfileDto p2) {
assertThat(p2.getOrganizationUuid()).isEqualTo(p1.getOrganizationUuid());
assertThat(p2.getName()).isEqualTo(p1.getName());
import org.sonar.server.es.SearchOptions;
import org.sonar.server.exceptions.BadRequestException;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndex;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
public UserSessionRule userSession = UserSessionRule.standalone();
private RuleIndex ruleIndex = new RuleIndex(es.client());
private RuleActivatorContextFactory contextFactory = new RuleActivatorContextFactory(db.getDbClient());
- private ActiveRuleIteratorFactory activeRuleIteratorFactory = new ActiveRuleIteratorFactory(db.getDbClient());
- private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), activeRuleIteratorFactory);
+ private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
private TypeValidations typeValidations = new TypeValidations(asList(new StringTypeValidation(), new IntegerTypeValidation()));
public void unset_parent_when_no_parent_does_not_fail() {
RuleDefinitionDto rule = createRule();
QProfileDto profile = createProfile(rule);
- underTest.setParent(db.getSession(), profile, null);
+ underTest.setParentAndCommit(db.getSession(), profile, null);
}
@Test
expectedException.expect(BadRequestException.class);
expectedException.expectMessage(" can not be selected as parent of ");
- underTest.setParent(db.getSession(), profile, profile);
+ underTest.setParentAndCommit(db.getSession(), profile, profile);
}
@Test
expectedException.expect(BadRequestException.class);
expectedException.expectMessage(" can not be selected as parent of ");
- underTest.setParent(db.getSession(), parentProfile, childProfile);
+ underTest.setParentAndCommit(db.getSession(), parentProfile, childProfile);
}
@Test
expectedException.expect(BadRequestException.class);
expectedException.expectMessage(" can not be selected as parent of ");
- underTest.setParent(db.getSession(), parentProfile, grandchildProfile);
+ underTest.setParentAndCommit(db.getSession(), parentProfile, grandchildProfile);
}
@Test
expectedException.expect(BadRequestException.class);
expectedException.expectMessage("Cannot set the profile");
- underTest.setParent(db.getSession(), childProfile, parentProfile);
+ underTest.setParentAndCommit(db.getSession(), childProfile, parentProfile);
}
@Test
changes = activate(profile2, RuleActivation.create(rule2.getKey()));
assertThat(changes).hasSize(1);
- changes = underTest.setParent(db.getSession(), profile2, profile1);
+ changes = underTest.setParentAndCommit(db.getSession(), profile2, profile1);
assertThat(changes).hasSize(1);
assertThatRuleIsActivated(profile2, rule1, changes, rule1.getSeverityString(), INHERITED, emptyMap());
assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
- changes = underTest.setParent(db.getSession(), profile2, null);
+ changes = underTest.setParentAndCommit(db.getSession(), profile2, null);
assertThat(changes).hasSize(1);
assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
assertThatRuleIsNotPresent(profile2, rule1);
changes = activate(profile2, RuleActivation.create(rule2.getKey()));
assertThat(changes).hasSize(1);
- changes = underTest.setParent(db.getSession(), profile2, profile1);
+ changes = underTest.setParentAndCommit(db.getSession(), profile2, profile1);
assertThat(changes).hasSize(1);
assertThatRuleIsActivated(profile2, rule1, changes, rule1.getSeverityString(), INHERITED, emptyMap());
assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
assertThatRuleIsUpdated(profile2, rule1, BLOCKER, ActiveRule.Inheritance.OVERRIDES, emptyMap());
assertThatRuleIsActivated(profile2, rule2, null, rule2.getSeverityString(), null, emptyMap());
- changes = underTest.setParent(db.getSession(), profile2, null);
+ changes = underTest.setParentAndCommit(db.getSession(), profile2, null);
assertThat(changes).hasSize(1);
// Not testing changes here since severity is not set in changelog
assertThatRuleIsActivated(profile2, rule1, null, BLOCKER, null, emptyMap());
RuleQuery ruleQuery = new RuleQuery()
.setRepositories(singletonList(repositoryKey));
- BulkChangeResult bulkChangeResult = underTest.bulkActivate(db.getSession(), ruleQuery, profile, MINOR);
+ BulkChangeResult bulkChangeResult = underTest.bulkActivateAndCommit(db.getSession(), ruleQuery, profile, MINOR);
assertThat(bulkChangeResult.countFailed()).isEqualTo(0);
assertThat(bulkChangeResult.countSucceeded()).isEqualTo(bulkSize);
RuleQuery ruleQuery = new RuleQuery()
.setRepositories(singletonList(repositoryKey));
- BulkChangeResult bulkChangeResult = underTest.bulkActivate(db.getSession(), ruleQuery, profile, MINOR);
+ BulkChangeResult bulkChangeResult = underTest.bulkActivateAndCommit(db.getSession(), ruleQuery, profile, MINOR);
assertThat(bulkChangeResult.countFailed()).isEqualTo(0);
assertThat(bulkChangeResult.countSucceeded()).isEqualTo(bulkSize);
assertThat(db.getDbClient().activeRuleDao().selectByProfile(db.getSession(), profile)).hasSize(bulkSize);
// Now deactivate all rules
- bulkChangeResult = underTest.bulkDeactivate(db.getSession(), ruleQuery, profile);
+ bulkChangeResult = underTest.bulkDeactivateAndCommit(db.getSession(), ruleQuery, profile);
assertThat(bulkChangeResult.countFailed()).isEqualTo(0);
assertThat(bulkChangeResult.countSucceeded()).isEqualTo(bulkSize);
RuleQuery ruleQuery = new RuleQuery()
.setQProfile(childProfile);
- BulkChangeResult bulkChangeResult = underTest.bulkDeactivate(db.getSession(), ruleQuery, childProfile);
+ BulkChangeResult bulkChangeResult = underTest.bulkDeactivateAndCommit(db.getSession(), ruleQuery, childProfile);
assertThat(bulkChangeResult.countFailed()).isEqualTo(1);
assertThat(bulkChangeResult.countSucceeded()).isEqualTo(0);
RuleQuery query = new RuleQuery()
.setRuleKey(rule1.getRuleKey())
.setQProfile(parentProfile);
- BulkChangeResult result = underTest.bulkActivate(db.getSession(), query, parentProfile, "BLOCKER");
+ BulkChangeResult result = underTest.bulkActivateAndCommit(db.getSession(), query, parentProfile, "BLOCKER");
assertThat(result.getChanges()).hasSize(3);
assertThat(result.countSucceeded()).isEqualTo(1);
db.rules().update(rule1);
QProfileDto childProfile = createProfile(rule1);
- List<ActiveRuleChange> changes = underTest.setParent(db.getSession(), childProfile, parentProfile);
+ List<ActiveRuleChange> changes = underTest.setParentAndCommit(db.getSession(), childProfile, parentProfile);
assertThatRuleIsNotPresent(childProfile, rule1);
assertThatRuleIsActivated(childProfile, rule2, changes, rule2.getSeverityString(), INHERITED, emptyMap());
*/
package org.sonar.server.qualityprofile.index;
+import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
+import org.assertj.core.groups.Tuple;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.System2;
import org.sonar.db.DbTester;
+import org.sonar.db.es.EsQueueDto;
import org.sonar.db.organization.OrganizationDto;
import org.sonar.db.qualityprofile.ActiveRuleDto;
import org.sonar.db.qualityprofile.QProfileDto;
-import org.sonar.db.qualityprofile.RulesProfileDto;
import org.sonar.db.rule.RuleDefinitionDto;
import org.sonar.server.es.EsTester;
import org.sonar.server.qualityprofile.ActiveRuleChange;
import org.sonar.server.rule.index.RuleIndexDefinition;
import static java.util.Arrays.asList;
+import static java.util.Arrays.stream;
import static java.util.Collections.emptySet;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.qualityprofile.ActiveRuleChange.Type.ACTIVATED;
-import static org.sonar.server.qualityprofile.ActiveRuleChange.Type.DEACTIVATED;
import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
public class ActiveRuleIndexerTest {
@Rule
public EsTester es = new EsTester(RuleIndexDefinition.createForTest(new MapSettings().asConfig()));
- private ActiveRuleIndexer underTest = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+ private ActiveRuleIndexer underTest = new ActiveRuleIndexer(db.getDbClient(), es.client());
private RuleDefinitionDto rule1;
private RuleDefinitionDto rule2;
private OrganizationDto org;
@Test
public void getIndexTypes() {
- assertThat(underTest.getIndexTypes()).containsExactly(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE);
+ assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_ACTIVE_RULE);
}
@Test
List<ActiveRuleDoc> docs = es.getDocuments(INDEX_TYPE_ACTIVE_RULE, ActiveRuleDoc.class);
assertThat(docs).hasSize(1);
verify(docs.get(0), rule1, profile1, activeRule);
+ assertThatEsQueueTableIsEmpty();
}
@Test
- public void deleteByProfiles() throws Exception {
- ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
- ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
- ActiveRuleDto activeRule3 = db.qualityProfiles().activateRule(profile2, rule2);
- index();
+ public void test_commitAndIndex() {
+ ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+ ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+ ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
- underTest.deleteByProfiles(singletonList(profile2));
+ commitAndIndex(ar1, ar2);
- verifyOnlyIndexed(activeRule1);
+ verifyOnlyIndexed(ar1, ar2);
+ assertThatEsQueueTableIsEmpty();
}
@Test
- public void deleteByProfiles_does_nothing_if_profiles_are_not_indexed() throws Exception {
- ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
- ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
- ActiveRuleDto activeRule3 = db.qualityProfiles().activateRule(profile2, rule2);
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+ public void commitAndIndex_empty_list() {
+ ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
- underTest.deleteByProfiles(singletonList(profile2));
+ underTest.commitAndIndex(db.getSession(), Collections.emptyList());
assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+ assertThatEsQueueTableIsEmpty();
}
@Test
- public void indexRuleProfile() throws Exception {
- ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
- ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
- ActiveRuleDto activeRule3 = db.qualityProfiles().activateRule(profile2, rule2);
+ public void commitAndIndex_keeps_elements_to_recover_in_ES_QUEUE_on_errors() {
+ ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
+ // force error by deleting the index
+ deleteRulesIndex();
- indexProfile(profile2);
+ commitAndIndex(ar);
- verifyOnlyIndexed(activeRule2, activeRule3);
+ EsQueueDto expectedItem = EsQueueDto.create(EsQueueDto.Type.ACTIVE_RULE, "" + ar.getId(), "activeRuleId", ar.getRuleKey().toString());
+ assertThatEsQueueContainsExactly(expectedItem);
}
@Test
- public void indexChanges_puts_documents() throws Exception {
- ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
- ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
- ActiveRuleDto nonIndexed = db.qualityProfiles().activateRule(profile2, rule2);
+ public void commitAndIndex_deletes_the_documents_that_dont_exist_in_database() {
+ ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
+ indexAll();
+ assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
- underTest.indexChanges(db.getSession(), asList(
- newChange(ACTIVATED, activeRule1), newChange(ACTIVATED, activeRule2)));
+ db.getDbClient().activeRuleDao().delete(db.getSession(), ar.getKey());
+ commitAndIndex(ar);
- verifyOnlyIndexed(activeRule1, activeRule2);
+ assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+ assertThatEsQueueTableIsEmpty();
}
@Test
- public void indexChanges_deletes_documents_when_type_is_DEACTIVATED() throws Exception {
- ActiveRuleDto activeRule1 = db.qualityProfiles().activateRule(profile1, rule1);
- ActiveRuleDto activeRule2 = db.qualityProfiles().activateRule(profile2, rule1);
- underTest.indexChanges(db.getSession(), asList(
- newChange(ACTIVATED, activeRule1), newChange(ACTIVATED, activeRule2)));
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE)).isEqualTo(2);
+ public void index_fails_and_deletes_doc_if_docIdType_is_unsupported() {
+ EsQueueDto item = EsQueueDto.create(EsQueueDto.Type.ACTIVE_RULE, "the_id", "unsupported", "the_routing");
+ db.getDbClient().esQueueDao().insert(db.getSession(), item);
- underTest.indexChanges(db.getSession(), singletonList(newChange(DEACTIVATED, activeRule1)));
+ underTest.index(db.getSession(), asList(item));
- verifyOnlyIndexed(activeRule2);
+ assertThatEsQueueTableIsEmpty();
+ assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
}
@Test
- public void deleteByRuleKeys() {
- ActiveRuleDto active1 = db.qualityProfiles().activateRule(profile1, rule1);
- ActiveRuleDto active2 = db.qualityProfiles().activateRule(profile2, rule1);
- ActiveRuleDto onRule2 = db.qualityProfiles().activateRule(profile2, rule2);
- index();
+ public void commitDeletionOfProfiles() {
+ ActiveRuleDto ar1 = db.qualityProfiles().activateRule(profile1, rule1);
+ ActiveRuleDto ar2 = db.qualityProfiles().activateRule(profile2, rule1);
+ ActiveRuleDto ar3 = db.qualityProfiles().activateRule(profile2, rule2);
+ indexAll();
+ db.getDbClient().qualityProfileDao().deleteRulesProfilesByUuids(db.getSession(), singletonList(profile2.getRulesProfileUuid()));
+
+ underTest.commitDeletionOfProfiles(db.getSession(), singletonList(profile2));
+
+ verifyOnlyIndexed(ar1);
+ }
+
+ @Test
+ public void commitDeletionOfProfiles_does_nothing_if_profiles_are_not_indexed() {
+ db.qualityProfiles().activateRule(profile1, rule1);
+ indexAll();
+ assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
+
+ underTest.commitDeletionOfProfiles(db.getSession(), singletonList(profile2));
+
+ assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
+ }
- underTest.deleteByRuleKeys(singletonList(rule2.getKey()));
- verifyOnlyIndexed(active1, active2);
+ private void deleteRulesIndex() {
+ es.deleteIndex(RuleIndexDefinition.INDEX_TYPE_RULE.getIndex());
+ }
+
+ private void assertThatEsQueueTableIsEmpty() {
+ assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isEqualTo(0);
+ }
+
+ private void assertThatEsQueueContainsExactly(EsQueueDto expected) {
+ Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), system2.now() + 1_000, 10);
+ assertThat(items)
+ .extracting(EsQueueDto::getDocId, EsQueueDto::getDocIdType, EsQueueDto::getDocRouting)
+ .containsExactlyInAnyOrder(Tuple.tuple(expected.getDocId(), expected.getDocIdType(), expected.getDocRouting()));
+ }
+
+ private void commitAndIndex(ActiveRuleDto... ar) {
+ underTest.commitAndIndex(db.getSession(), stream(ar)
+ .map(a -> new ActiveRuleChange(ActiveRuleChange.Type.ACTIVATED, a))
+ .collect(Collectors.toList()));
}
private void verifyOnlyIndexed(ActiveRuleDto... expected) {
}
}
- private ActiveRuleChange newChange(ActiveRuleChange.Type type, ActiveRuleDto activeRule) {
- return new ActiveRuleChange(type, activeRule);
- }
-
- private void indexProfile(QProfileDto profile) {
- underTest.indexRuleProfile(db.getSession(), RulesProfileDto.from(profile));
- }
-
private void verify(ActiveRuleDoc doc1, RuleDefinitionDto rule, QProfileDto profile, ActiveRuleDto activeRule) {
assertThat(doc1)
.matches(doc -> doc.getRuleKey().equals(rule.getKey()))
.matches(doc -> doc.getSeverity().equals(activeRule.getSeverityString()));
}
- private void index() {
+ private void indexAll() {
underTest.indexOnStartup(emptySet());
}
import org.sonar.server.organization.TestDefaultOrganizationProvider;
import org.sonar.server.qualityprofile.RuleActivation;
import org.sonar.server.qualityprofile.RuleActivator;
-import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
import org.sonar.server.tester.UserSessionRule;
import org.sonar.server.ws.TestRequest;
import org.sonar.server.ws.TestResponse;
private DbClient dbClient = dbTester.getDbClient();
private RuleActivator ruleActivator = mock(RuleActivator.class);
private QProfileWsSupport wsSupport = new QProfileWsSupport(dbClient, userSession, TestDefaultOrganizationProvider.from(dbTester));
- private ActiveRuleIndexer activeRuleIndexer = mock(ActiveRuleIndexer.class);
- private WsActionTester ws = new WsActionTester(new ActivateRuleAction(dbClient, ruleActivator, userSession, wsSupport, activeRuleIndexer));
+ private WsActionTester ws = new WsActionTester(new ActivateRuleAction(dbClient, ruleActivator, userSession, wsSupport));
private OrganizationDto defaultOrganization;
private OrganizationDto organization;
assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
ArgumentCaptor<RuleActivation> captor = ArgumentCaptor.forClass(RuleActivation.class);
- verify(ruleActivator).activate(any(DbSession.class), captor.capture(), any(QProfileDto.class));
+ verify(ruleActivator).activateAndCommit(any(DbSession.class), captor.capture(), any(QProfileDto.class));
RuleActivation value = captor.getValue();
assertThat(value.getRuleKey()).isEqualTo(ruleKey);
assertThat(value.getSeverity()).isEqualTo(Severity.BLOCKER);
assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
ArgumentCaptor<RuleActivation> captor = ArgumentCaptor.forClass(RuleActivation.class);
- verify(ruleActivator).activate(any(DbSession.class), captor.capture(), any(QProfileDto.class));
+ verify(ruleActivator).activateAndCommit(any(DbSession.class), captor.capture(), any(QProfileDto.class));
assertThat(captor.getValue().getRuleKey()).isEqualTo(ruleKey);
assertThat(captor.getValue().getSeverity()).isEqualTo(Severity.BLOCKER);
assertThat(captor.getValue().isReset()).isFalse();
import org.sonar.server.qualityprofile.RuleActivator;
import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndex;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
EsClient esClient = esTester.client();
ruleIndex = new RuleIndex(esClient);
ruleIndexer = new RuleIndexer(esClient, dbClient);
- activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+ activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
RuleActivatorContextFactory ruleActivatorContextFactory = new RuleActivatorContextFactory(dbClient);
TypeValidations typeValidations = new TypeValidations(Collections.emptyList());
ruleActivator = new RuleActivator(System2.INSTANCE, dbClient, ruleIndex, ruleActivatorContextFactory, typeValidations, activeRuleIndexer, userSessionRule);
activeRuleIndexer.indexOnStartup(emptySet());
// Set parent 1
- ruleActivator.setParent(dbSession, child, parent1);
+ ruleActivator.setParentAndCommit(dbSession, child, parent1);
// Set parent 2 through WS
ws.newRequest()
activeRuleIndexer.indexOnStartup(emptySet());
// Set parent
- ruleActivator.setParent(dbSession, child, parent);
+ ruleActivator.setParentAndCommit(dbSession, child, parent);
// Remove parent through WS
ws.newRequest()
assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty();
// Set parent
- ruleActivator.setParent(dbSession, child, parent);
+ ruleActivator.setParentAndCommit(dbSession, child, parent);
// Remove parent
ws.newRequest()
import org.sonar.server.qualityprofile.RuleActivator;
import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndex;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
private DbSession dbSession = db.getSession();
private RuleIndex ruleIndex = new RuleIndex(es.client());
private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), dbClient);
- private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, es.client(), new ActiveRuleIteratorFactory(dbClient));
+ private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, es.client());
private ProfileImporter[] profileImporters = createImporters();
private QProfileExporters qProfileExporters = new QProfileExporters(dbClient, null,
new RuleActivator(mock(System2.class), dbClient, ruleIndex, new RuleActivatorContextFactory(dbClient), null, activeRuleIndexer, userSession),
assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
ArgumentCaptor<RuleKey> ruleKeyCaptor = ArgumentCaptor.forClass(RuleKey.class);
ArgumentCaptor<QProfileDto> qProfileDtoCaptor = ArgumentCaptor.forClass(QProfileDto.class);
- verify(ruleActivator).deactivateAndUpdateIndex(any(DbSession.class), qProfileDtoCaptor.capture(), ruleKeyCaptor.capture());
+ verify(ruleActivator).deactivateAndCommit(any(DbSession.class), qProfileDtoCaptor.capture(), ruleKeyCaptor.capture());
assertThat(ruleKeyCaptor.getValue()).isEqualTo(ruleKey);
assertThat(qProfileDtoCaptor.getValue().getKee()).isEqualTo(qualityProfile.getKee());
}
assertThat(response.getStatus()).isEqualTo(HttpURLConnection.HTTP_NO_CONTENT);
ArgumentCaptor<RuleKey> captor = ArgumentCaptor.forClass(RuleKey.class);
ArgumentCaptor<QProfileDto> qProfileDtoCaptor = ArgumentCaptor.forClass(QProfileDto.class);
- verify(ruleActivator).deactivateAndUpdateIndex(any(DbSession.class), qProfileDtoCaptor.capture(), captor.capture());
+ verify(ruleActivator).deactivateAndCommit(any(DbSession.class), qProfileDtoCaptor.capture(), captor.capture());
assertThat(captor.getValue()).isEqualTo(ruleKey);
assertThat(qProfileDtoCaptor.getValue().getKee()).isEqualTo(qualityProfile.getKee());
}
import org.sonar.server.qualityprofile.RuleActivator;
import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndex;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
dbSession = dbTester.getSession();
esClient = es.client();
ruleIndexer = new RuleIndexer(esClient, dbClient);
- activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+ activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester);
underTest = new InheritanceAction(
dbClient,
}
private void setParent(QProfileDto profile, QProfileDto parent) {
- ruleActivator.setParent(dbSession, parent, profile);
+ ruleActivator.setParentAndCommit(dbSession, parent, profile);
}
private RuleDefinitionDto createRule(String lang, String id) {
import org.sonar.server.exceptions.NotFoundException;
import org.sonar.server.organization.TestDefaultOrganizationProvider;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndex;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
public ExpectedException expectedException = ExpectedException.none();
private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
- private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+ private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
private RuleIndex ruleIndex = new RuleIndex(es.client());
private WsActionTester ws = new WsActionTester(
import org.sonar.server.organization.TestOrganizationFlags;
import org.sonar.server.qualityprofile.RuleActivator;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndex;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
when(system.now()).thenReturn(DATE1.getTime());
ruleIndexer = new RuleIndexer(esTester.client(), dbClient);
ruleIndex = new RuleIndex(esTester.client());
- activeRuleIndexer = new ActiveRuleIndexer(dbClient, esTester.client(), new ActiveRuleIteratorFactory(dbClient));
+ activeRuleIndexer = new ActiveRuleIndexer(dbClient, esTester.client());
defaultOrganization = dbTester.getDefaultOrganization();
}
import org.sonar.server.es.SearchIdResult;
import org.sonar.server.es.SearchOptions;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import static com.google.common.collect.ImmutableSet.of;
import static java.util.Arrays.asList;
@Before
public void setUp() {
ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
- activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+ activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
underTest = new RuleIndex(es.client());
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2017 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.rule.index;
-
-import com.google.common.collect.Lists;
-import java.util.List;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.sonar.api.rule.RuleKey;
-import org.sonar.api.rule.RuleStatus;
-import org.sonar.api.rule.Severity;
-import org.sonar.api.rules.RuleType;
-import org.sonar.api.utils.System2;
-import org.sonar.db.DbClient;
-import org.sonar.db.DbSession;
-import org.sonar.db.DbTester;
-import org.sonar.db.rule.RuleDefinitionDto;
-import org.sonar.db.rule.RuleDto;
-import org.sonar.server.exceptions.NotFoundException;
-
-import static com.google.common.collect.Sets.newHashSet;
-import static org.assertj.core.api.Assertions.assertThat;
-
-public class RuleIteratorForSingleChunkTest {
-
- @Rule
- public DbTester dbTester = DbTester.create(System2.INSTANCE);
-
- private DbClient dbClient = dbTester.getDbClient();
- private DbSession dbSession = dbTester.getSession();
- private RuleDefinitionDto templateRule;
- private RuleDefinitionDto customRule;
-
- @Before
- public void setUp() throws Exception {
- templateRule = new RuleDefinitionDto()
- .setRuleKey("S001")
- .setRepositoryKey("xoo")
- .setConfigKey("S1")
- .setName("Null Pointer")
- .setDescription("S001 desc")
- .setDescriptionFormat(RuleDto.Format.HTML)
- .setLanguage("xoo")
- .setSeverity(Severity.BLOCKER)
- .setStatus(RuleStatus.READY)
- .setIsTemplate(true)
- .setSystemTags(newHashSet("cwe"))
- .setType(RuleType.BUG)
- .setCreatedAt(1500000000000L)
- .setUpdatedAt(1600000000000L);
-
- customRule = new RuleDefinitionDto()
- .setRuleKey("S002")
- .setRepositoryKey("xoo")
- .setConfigKey("S2")
- .setName("Slow")
- .setDescription("*S002 desc*")
- .setDescriptionFormat(RuleDto.Format.MARKDOWN)
- .setLanguage("xoo")
- .setSeverity(Severity.MAJOR)
- .setStatus(RuleStatus.BETA)
- .setIsTemplate(false)
- .setType(RuleType.CODE_SMELL)
- .setCreatedAt(2000000000000L)
- .setUpdatedAt(2100000000000L);
- }
-
- @Test
- public void iterator_over_one_rule() {
- dbTester.rules().insert(templateRule);
-
- List<RuleDocWithSystemScope> results = getResults();
-
- assertThat(results).hasSize(1);
-
- RuleDocWithSystemScope ruleDocWithSystemScope = getRuleDoc(results, templateRule.getRuleKey());
- RuleDoc templateDoc = ruleDocWithSystemScope.getRuleDoc();
- RuleExtensionDoc templateExtensionDoc = ruleDocWithSystemScope.getRuleExtensionDoc();
- assertThat(templateDoc).isNotNull();
- assertThat(templateDoc.key()).isEqualTo(RuleKey.of("xoo", "S001"));
- assertThat(templateDoc.ruleKey()).isEqualTo("S001");
- assertThat(templateDoc.repository()).isEqualTo("xoo");
- assertThat(templateDoc.internalKey()).isEqualTo("S1");
- assertThat(templateDoc.name()).isEqualTo("Null Pointer");
- assertThat(templateDoc.htmlDescription()).isEqualTo("S001 desc");
- assertThat(templateDoc.language()).isEqualTo("xoo");
- assertThat(templateDoc.severity()).isEqualTo(Severity.BLOCKER);
- assertThat(templateDoc.status()).isEqualTo(RuleStatus.READY);
- assertThat(templateDoc.isTemplate()).isTrue();
- assertThat(templateExtensionDoc.getTags()).containsOnly("cwe");
- assertThat(templateDoc.createdAt()).isEqualTo(1500000000000L);
- assertThat(templateDoc.updatedAt()).isEqualTo(1600000000000L);
- }
-
- @Test
- public void iterator_over_rules() {
- dbTester.rules().insert(templateRule);
- dbClient.ruleDao().insert(dbSession, customRule);
- dbSession.commit();
-
- List<RuleDocWithSystemScope> results = getResults();
-
- assertThat(results).hasSize(2);
-
- RuleDocWithSystemScope templateDocWithSystemScope = getRuleDoc(results, templateRule.getRuleKey());
- RuleDoc templateDoc = templateDocWithSystemScope.getRuleDoc();
- RuleExtensionDoc templateExtensionDoc = templateDocWithSystemScope.getRuleExtensionDoc();
- assertThat(templateDoc.key()).isEqualTo(RuleKey.of("xoo", "S001"));
- assertThat(templateDoc.ruleKey()).isEqualTo("S001");
- assertThat(templateDoc.repository()).isEqualTo("xoo");
- assertThat(templateDoc.internalKey()).isEqualTo("S1");
- assertThat(templateDoc.name()).isEqualTo("Null Pointer");
- assertThat(templateDoc.htmlDescription()).isEqualTo("S001 desc");
- assertThat(templateDoc.language()).isEqualTo("xoo");
- assertThat(templateDoc.severity()).isEqualTo(Severity.BLOCKER);
- assertThat(templateDoc.status()).isEqualTo(RuleStatus.READY);
- assertThat(templateDoc.isTemplate()).isTrue();
- assertThat(templateExtensionDoc.getTags()).containsOnly("cwe");
- assertThat(templateDoc.createdAt()).isEqualTo(1500000000000L);
- assertThat(templateDoc.updatedAt()).isEqualTo(1600000000000L);
-
- RuleDocWithSystemScope customDocWithSystemScope = getRuleDoc(results, customRule.getRuleKey());
- RuleDoc customDoc = customDocWithSystemScope.getRuleDoc();
- RuleExtensionDoc customExtensionDoc = customDocWithSystemScope.getRuleExtensionDoc();
- assertThat(customDoc.key()).isEqualTo(RuleKey.of("xoo", "S002"));
- assertThat(customDoc.ruleKey()).isEqualTo("S002");
- assertThat(customDoc.repository()).isEqualTo("xoo");
- assertThat(customDoc.internalKey()).isEqualTo("S2");
- assertThat(customDoc.name()).isEqualTo("Slow");
- assertThat(customDoc.htmlDescription()).isEqualTo("<strong>S002 desc</strong>");
- assertThat(customDoc.language()).isEqualTo("xoo");
- assertThat(customDoc.severity()).isEqualTo(Severity.MAJOR);
- assertThat(customDoc.status()).isEqualTo(RuleStatus.BETA);
- assertThat(customDoc.isTemplate()).isFalse();
- assertThat(customExtensionDoc.getTags()).isEmpty();
- assertThat(customDoc.createdAt()).isEqualTo(2000000000000L);
- assertThat(customDoc.updatedAt()).isEqualTo(2100000000000L);
- }
-
- @Test
- public void custom_rule() {
- dbTester.rules().insert(templateRule);
- dbClient.ruleDao().insert(dbSession, customRule.setTemplateId(templateRule.getId()));
- dbSession.commit();
-
- List<RuleDocWithSystemScope> results = getResults();
-
- assertThat(results).hasSize(2);
-
- RuleDocWithSystemScope templateDocWithSystemScope = getRuleDoc(results, templateRule.getRuleKey());
- RuleDoc templateDoc = templateDocWithSystemScope.getRuleDoc();
- assertThat(templateDoc.isTemplate()).isTrue();
- assertThat(templateDoc.templateKey()).isNull();
-
- RuleDocWithSystemScope customDocWithSystemScope = getRuleDoc(results, customRule.getRuleKey());
- RuleDoc customDoc = customDocWithSystemScope.getRuleDoc();
- assertThat(customDoc.isTemplate()).isFalse();
- assertThat(customDoc.templateKey()).isEqualTo(RuleKey.of("xoo", "S001"));
- }
-
- @Test
- public void removed_rule_is_returned() {
- dbTester.rules().insert(templateRule.setStatus(RuleStatus.REMOVED));
- dbSession.commit();
-
- List<RuleDocWithSystemScope> results = getResults();
-
- assertThat(results).hasSize(1);
- }
-
- private List<RuleDocWithSystemScope> getResults() {
- return Lists.newArrayList(new RuleIteratorForSingleChunk(dbTester.getDbClient(), null));
- }
-
- private RuleDocWithSystemScope getRuleDoc(List<RuleDocWithSystemScope> results, String ruleKey) {
- RuleDocWithSystemScope rule;
- rule = results.stream()
- .filter(r -> ruleKey.equals(r.getRuleDoc().key().rule()))
- .findAny()
- .orElseThrow(() -> new NotFoundException("Rule not found in results"));
- return rule;
- }
-}
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import org.sonar.api.config.MapSettings;
+import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.rule.RuleStatus;
import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
@Rule
public DbTester dbTester = DbTester.create();
@Rule
- public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings()));
+ public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig()));
@Rule
public ExpectedException thrown = ExpectedException.none();
import org.sonar.server.qualityprofile.RuleActivator;
import org.sonar.server.qualityprofile.RuleActivatorContextFactory;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndex;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
private RuleIndex ruleIndex = new RuleIndex(es.client());
private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient());
- private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client(), new ActiveRuleIteratorFactory(db.getDbClient()));
+ private ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(db.getDbClient(), es.client());
private Languages languages = LanguageTesting.newLanguages(JAVA, "js");
private ActiveRuleCompleter activeRuleCompleter = new ActiveRuleCompleter(db.getDbClient(), languages);
private RuleWsSupport wsSupport = new RuleWsSupport(db.getDbClient(), userSession, defaultOrganizationProvider);
import org.sonar.server.organization.TestDefaultOrganizationProvider;
import org.sonar.server.qualityprofile.QProfileTesting;
import org.sonar.server.qualityprofile.index.ActiveRuleIndexer;
-import org.sonar.server.qualityprofile.index.ActiveRuleIteratorFactory;
import org.sonar.server.rule.index.RuleIndexDefinition;
import org.sonar.server.rule.index.RuleIndexer;
import org.sonar.server.text.MacroInterpreter;
.build();
Mockito.doReturn(singletonList(active)).when(activeRuleCompleter).completeShow(any(DbSession.class), orgCaptor.capture(), ruleCaptor.capture());
- ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+ ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes());
TestResponse response = actionTester.newRequest().setMethod("GET")
RuleMetadataDto ruleMetadata = dbTester.rules().insertOrUpdateMetadata(rule, organization);
dbTester.qualityProfiles().activateRule(profile, rule, a -> a.setSeverity("BLOCKER"));
- ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient));
+ ActiveRuleIndexer activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient);
activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes());
TestResponse response = actionTester.newRequest().setMethod("GET")
--- /dev/null
+# sonar.web.javaAdditionalOpts=-javaagent:/path/to/byteman-3.0.10/lib/byteman.jar=script:/path/to/active_rule_indexer.btm,boot:/path/to/byteman-3.0.10/lib/byteman.jar
+# sonar.search.recovery.delayInMs=10000
+# sonar.search.recovery.minAgeInMs=30000
+
+RULE make indexing of active rules silently fail
+CLASS ActiveRuleIndexer
+METHOD postCommit
+COMPILE
+AT ENTRY
+IF TRUE
+DO RETURN
+ENDRULE
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
+import org.sonarqube.tests.qualityProfile.ActiveRuleEsResilienceTest;
import org.sonarqube.tests.qualityProfile.BuiltInQualityProfilesNotificationTest;
import org.sonarqube.tests.serverSystem.ClusterTest;
import org.sonarqube.tests.serverSystem.RestartTest;
SsoAuthenticationTest.class,
OnboardingTest.class,
BuiltInQualityProfilesNotificationTest.class,
+ ActiveRuleEsResilienceTest.class,
UserEsResilienceTest.class
})
public class Category5Suite {
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonarqube.tests.qualityProfile;
+
+import com.sonar.orchestrator.Orchestrator;
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.Organizations;
+import org.sonarqube.ws.QualityProfiles;
+import org.sonarqube.ws.client.rule.SearchWsRequest;
+import util.ItUtils;
+
+import static java.lang.String.format;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ActiveRuleEsResilienceTest {
+ private static final String RULE_ONE_BUG_PER_LINE = "xoo:OneBugIssuePerLine";
+
+ @ClassRule
+ public static final Orchestrator orchestrator = Orchestrator.builderEnv()
+ .setServerProperty("sonar.web.javaAdditionalOpts",
+ format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar()))
+ .setServerProperty("sonar.search.recovery.delayInMs", "500")
+ .setServerProperty("sonar.search.recovery.minAgeInMs", "3000")
+ .addPlugin(ItUtils.xooPlugin())
+ .build();
+
+ @Rule
+ public TestRule timeout = new DisableOnDebug(Timeout.builder()
+ .withLookingForStuckThread(true)
+ .withTimeout(60L, TimeUnit.SECONDS)
+ .build());
+
+ @Rule
+ public Tester tester = new Tester(orchestrator);
+
+ @Test
+ public void activation_and_deactivation_of_rule_is_resilient_to_indexing_errors() throws Exception {
+ Organizations.Organization organization = tester.organizations().generate();
+ QualityProfiles.CreateWsResponse.QualityProfile profile = tester.qProfiles().createXooProfile(organization);
+
+ // step 1. activation
+ tester.qProfiles().activateRule(profile.getKey(), RULE_ONE_BUG_PER_LINE);
+
+ assertThat(searchActiveRules(profile)).isEqualTo(0);
+ while (searchActiveRules(profile) == 0) {
+ // rule is indexed by the recovery daemon, which runs every 3 seconds
+ Thread.sleep(500L);
+ }
+ assertThat(searchActiveRules(profile)).isEqualTo(1);
+
+ // step 2. deactivation
+ tester.qProfiles().deactivateRule(profile, RULE_ONE_BUG_PER_LINE);
+ while (searchActiveRules(profile) == 1) {
+ // rule is indexed by the recovery daemon, which runs every 3 seconds
+ Thread.sleep(500L);
+ }
+ assertThat(searchActiveRules(profile)).isEqualTo(0);
+ }
+
+ private long searchActiveRules(QualityProfiles.CreateWsResponse.QualityProfile profile) {
+ SearchWsRequest request = new SearchWsRequest().setActivation(true).setQProfile(profile.getKey());
+ return tester.wsClient().rules().search(request).getRulesCount();
+ }
+
+ private static String findBytemanJar() {
+ // see pom.xml, Maven copies and renames the artifact.
+ File jar = new File("target/byteman.jar");
+ if (!jar.exists()) {
+ throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once.");
+ }
+ return jar.getAbsolutePath();
+ }
+
+ private static String findBytemanScript() {
+ // see pom.xml, Maven copies and renames the artifact.
+ File script = new File("resilience/active_rule_indexer.btm");
+ if (!script.exists()) {
+ throw new IllegalStateException("Can't find " + script);
+ }
+ return script.getAbsolutePath();
+ }
+}
projects / sonarqube.git / commitdiff