diff options
45 files changed, 1143 insertions, 369 deletions
diff --git a/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl b/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl index 202db3d5e1f..3942cb98f86 100644 --- a/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl +++ b/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl @@ -662,7 +662,7 @@ CREATE INDEX "CE_TASK_UUID" ON "WEBHOOK_DELIVERIES" ("CE_TASK_UUID"); CREATE TABLE "ES_QUEUE" ( "UUID" VARCHAR(40) NOT NULL PRIMARY KEY, "DOC_TYPE" VARCHAR(40) NOT NULL, - "DOC_UUID" VARCHAR(255) NOT NULL, + "DOC_ID" VARCHAR(4000) NOT NULL, "CREATED_AT" BIGINT NOT NULL ); CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID"); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java index feb3148fa57..35c51c7f2e5 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java @@ -22,12 +22,12 @@ package org.sonar.db.es; public final class EsQueueDto { public enum Type { - USER + USER, RULE, RULE_EXTENSION } private String uuid; private Type docType; - private String docUuid; + private String docId; public String getUuid() { return uuid; @@ -47,12 +47,12 @@ public final class EsQueueDto { return this; } - public String getDocUuid() { - return docUuid; + public String getDocId() { + return docId; } - private EsQueueDto setDocUuid(String s) { - this.docUuid = s; + private EsQueueDto setDocId(String s) { + this.docId = s; return this; } @@ -61,12 +61,31 @@ public final class EsQueueDto { StringBuilder sb = new StringBuilder("EsQueueDto{"); sb.append("uuid='").append(uuid).append('\''); sb.append(", docType=").append(docType); - sb.append(", docUuid='").append(docUuid).append('\''); + sb.append(", docId='").append(docId).append('\''); sb.append('}'); return sb.toString(); } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof EsQueueDto)) { + return false; + } + + EsQueueDto that = (EsQueueDto) o; + + return uuid.equals(that.uuid); + } + + @Override + public int hashCode() { + return uuid.hashCode(); + } + public static EsQueueDto create(Type docType, String docUuid) { - return new EsQueueDto().setDocType(docType).setDocUuid(docUuid); + return new EsQueueDto().setDocType(docType).setDocId(docUuid); } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java new file mode 100644 index 00000000000..7f821be5889 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java @@ -0,0 +1,88 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.db.es; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; + +public class RuleExtensionId { + private final String organizationUuid; + private final String repositoryName; + private final String ruleKey; + private final String id; + + private static final Splitter ID_SPLITTER = Splitter.on(CharMatcher.anyOf(":|")); + + public RuleExtensionId(String organizationUuid, String repositoryName, String ruleKey) { + this.organizationUuid = organizationUuid; + this.repositoryName = repositoryName; + this.ruleKey = ruleKey; + this.id = format("%s:%s|%s",repositoryName,ruleKey,organizationUuid); + } + + public RuleExtensionId(String ruleExtensionId) { + List<String> splittedId = ID_SPLITTER.splitToList(ruleExtensionId); + checkArgument(splittedId.size() == 3, "Incorrect Id %s", ruleExtensionId); + this.id = ruleExtensionId; + this.repositoryName = splittedId.get(0); + this.ruleKey = splittedId.get(1); + this.organizationUuid = splittedId.get(2); + } + + public String getOrganizationUuid() { + return organizationUuid; + } + + public String getRepositoryName() { + return repositoryName; + } + + public String getRuleKey() { + return ruleKey; + } + + public String getId() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RuleExtensionId)) { + return false; + } + + RuleExtensionId that = (RuleExtensionId) o; + + return id.equals(that.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java index e4902888070..94e47de330f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java @@ -22,8 +22,10 @@ package org.sonar.db.rule; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.ibatis.session.ResultHandler; +import org.sonar.db.es.RuleExtensionId; import org.sonar.api.rule.RuleKey; import org.sonar.api.rules.RuleQuery; import org.sonar.db.Dao; @@ -34,6 +36,7 @@ import org.sonar.db.organization.OrganizationDto; import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Optional.ofNullable; import static org.sonar.db.DatabaseUtils.executeLargeInputs; +import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput; public class RuleDao implements Dao { @@ -149,6 +152,24 @@ public class RuleDao implements Dao { } } + public void scrollRuleExtensionByRuleKeys(DbSession dbSession, Collection<RuleExtensionId> ruleExtensionIds, Consumer<RuleExtensionForIndexingDto> consumer) { + RuleMapper mapper = mapper(dbSession); + + executeLargeInputsWithoutOutput(ruleExtensionIds, + pageOfRuleExtensionIds -> mapper + .selectRuleExtensionForIndexingByKeys(pageOfRuleExtensionIds) + .forEach(consumer)); + } + + public void scrollRuleByRuleKeys(DbSession dbSession, Collection<RuleKey> ruleKeys, Consumer<RuleForIndexingDto> consumer) { + RuleMapper mapper = mapper(dbSession); + + executeLargeInputsWithoutOutput(ruleKeys, + pageOfRuleKeys -> mapper + .selectRuleForIndexingByKeys(pageOfRuleKeys) + .forEach(consumer)); + } + private static RuleMapper mapper(DbSession session) { return session.getMapper(RuleMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java new file mode 100644 index 00000000000..ef127131794 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java @@ -0,0 +1,81 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.db.rule; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import org.sonar.api.rule.RuleKey; + +public class RuleExtensionForIndexingDto { + + private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + + private String pluginName; + private String pluginRuleKey; + private String organizationUuid; + private String tags; + + public String getPluginName() { + return pluginName; + } + + public RuleExtensionForIndexingDto setPluginName(String pluginName) { + this.pluginName = pluginName; + return this; + } + + public String getPluginRuleKey() { + return pluginRuleKey; + } + + public RuleExtensionForIndexingDto setPluginRuleKey(String pluginRuleKey) { + this.pluginRuleKey = pluginRuleKey; + return this; + } + + public String getOrganizationUuid() { + return organizationUuid; + } + + public RuleExtensionForIndexingDto setOrganizationUuid(String organizationUuid) { + this.organizationUuid = organizationUuid; + return this; + } + + public String getTags() { + return tags; + } + + public RuleExtensionForIndexingDto setTags(String tags) { + this.tags = tags; + return this; + } + + public RuleKey getRuleKey() { + return RuleKey.of(pluginName, pluginRuleKey); + } + + public Set<String> getTagsAsSet() { + return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags)); + } + +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java new file mode 100644 index 00000000000..52459e6338e --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java @@ -0,0 +1,135 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.db.rule; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rules.RuleType; + +public class RuleForIndexingDto { + + private Integer id; + private String repository; + private String pluginRuleKey; + private String name; + private String description; + private RuleDto.Format descriptionFormat; + private Integer severity; + private RuleStatus status; + private boolean isTemplate; + private String systemTags; + private String templateRuleKey; + private String templateName; + private String internalKey; + private String language; + private int type; + private long createdAt; + private long updatedAt; + + private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + + public Integer getId() { + return id; + } + + public String getRepository() { + return repository; + } + + public String getPluginRuleKey() { + return pluginRuleKey; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public RuleDto.Format getDescriptionFormat() { + return descriptionFormat; + } + + public Integer getSeverity() { + return severity; + } + + public RuleStatus getStatus() { + return status; + } + + public boolean isTemplate() { + return isTemplate; + } + + public String getSystemTags() { + return systemTags; + } + + public String getTemplateRuleKey() { + return templateRuleKey; + } + + public String getTemplateName() { + return templateName; + } + + public String getInternalKey() { + return internalKey; + } + + public String getLanguage() { + return language; + } + + public int getType() { + return type; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getUpdatedAt() { + return updatedAt; + } + + public RuleType getTypeAsRuleType() { + return RuleType.valueOf(type); + } + + public String getSeverityAsString() { + return severity != null ? SeverityUtil.getSeverityFromOrdinal(severity) : null; + } + + public RuleKey getRuleKey() { + return RuleKey.of(repository, pluginRuleKey); + } + + public Set<String> getSystemTagsAsSet() { + return ImmutableSet.copyOf(TAGS_SPLITTER.split(systemTags == null ? "" : systemTags)); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java index 77ab19067d1..d5a98e60200 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java @@ -22,6 +22,7 @@ package org.sonar.db.rule; import java.util.List; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.session.ResultHandler; +import org.sonar.db.es.RuleExtensionId; import org.sonar.api.rule.RuleKey; import org.sonar.api.rules.RuleQuery; @@ -51,6 +52,10 @@ public interface RuleMapper { List<RuleDefinitionDto> selectDefinitionByKeys(@Param("ruleKeys") List<RuleKey> keys); + List<RuleForIndexingDto> selectRuleForIndexingByKeys(@Param("ruleKeys") List<RuleKey> keys); + + List<RuleExtensionForIndexingDto> selectRuleExtensionForIndexingByKeys(@Param("ruleExtensionIds") List<RuleExtensionId> ruleExtensionIds); + List<RuleDto> selectByQuery(@Param("organizationUuid") String organizationUuid, @Param("query") RuleQuery ruleQuery); void insertDefinition(RuleDefinitionDto ruleDefinitionDto); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml index 08b3761380c..62ec6e3421b 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml @@ -6,7 +6,7 @@ <sql id="esQueueColumns"> uuid, doc_type as docType, - doc_uuid as docUuid, + doc_id as docId, created_at as createdAt </sql> @@ -14,12 +14,12 @@ insert into es_queue ( uuid, doc_type, - doc_uuid, + doc_id, created_at ) values ( #{dto.uuid, jdbcType=VARCHAR}, #{dto.docType, jdbcType=VARCHAR}, - #{dto.docUuid, jdbcType=VARCHAR}, + #{dto.docId, jdbcType=VARCHAR}, #{now, jdbcType=BIGINT} ) </insert> diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml index 25ef4923224..0e29a3e0d94 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml @@ -139,6 +139,26 @@ and r.plugin_rule_key=#{rule,jdbcType=VARCHAR} </select> + <select id="selectRuleExtensionForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleExtensionForIndexingDto"> + select + r.plugin_name as "pluginName", + r.plugin_rule_key as "pluginRuleKey", + rm.organization_uuid as "organizationUuid", + rm.tags as "tags" + from + rules r + inner join + rules_metadata rm on rm.rule_id = r.id + where + rm.tags is not null and + rm.tags != '' and + <foreach collection="ruleExtensionIds" index="index" item="ruleExtId" open="" separator=" or " close=""> + ( r.plugin_name=#{ruleExtId.repositoryName,jdbcType=VARCHAR} and + r.plugin_rule_key=#{ruleExtId.ruleKey,jdbcType=VARCHAR} and + rm.organization_uuid=#{ruleExtId.organizationUuid,jdbcType=VARCHAR} ) + </foreach> + </select> + <select id="selectMetadataByKey" parameterType="map" resultType="org.sonar.db.rule.RuleMetadataDto"> select rm.rule_id as "ruleId", @@ -185,6 +205,34 @@ </foreach> </select> + <select id="selectRuleForIndexingByKeys" parameterType="map" resultType="org.sonar.db.rule.RuleForIndexingDto"> + select + r.id as "id", + r.plugin_name as "repository", + r.plugin_rule_key as "pluginRuleKey", + r.name as "name", + r.description as "description", + r.description_format as "descriptionFormat", + r.priority as "severity", + r.status as "status", + r.is_template as "isTemplate", + r.system_tags as "systemTags", + t.plugin_rule_key as "templateRuleKey", + t.plugin_name as "templateName", + r.plugin_config_key as "internalKey", + r.language as "language", + r.rule_type as "type", + r.created_at as "createdAt", + r.updated_at as "updatedAt" + from + rules r + left outer join rules t on t.id = r.template_id + where + <foreach collection="ruleKeys" index="index" item="ruleKey" open="" separator=" or " close=""> + (r.plugin_name=#{ruleKey.repository,jdbcType=VARCHAR} and r.plugin_rule_key=#{ruleKey.rule,jdbcType=VARCHAR}) + </foreach> + </select> + <select id="selectByQuery" parameterType="map" resultType="Rule"> select <include refid="selectJoinedTablesColumns"/> diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java index a8e8bc70bc9..78eaf82f6f5 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java @@ -49,9 +49,9 @@ public class CreateEsQueueTable extends DdlChange { .setLimit(40) .build()) .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder() - .setColumnName("doc_uuid") + .setColumnName("doc_id") .setIsNullable(false) - .setLimit(255) + .setLimit(4000) .build()) .addColumn(BigIntegerColumnDef.newBigIntegerColumnDefBuilder() .setColumnName("created_at") diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java index 23a950006f5..aa8603a17e1 100644 --- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java @@ -48,7 +48,7 @@ public class CreateEsQueueTableTest { db.assertColumnDefinition(TABLE, "uuid", Types.VARCHAR, 40, false); db.assertColumnDefinition(TABLE, "doc_type", Types.VARCHAR, 40, false); - db.assertColumnDefinition(TABLE, "doc_uuid", Types.VARCHAR, 255, false); + db.assertColumnDefinition(TABLE, "doc_id", Types.VARCHAR, 4000, false); db.assertColumnDefinition(TABLE, "created_at", Types.BIGINT, null, false); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index 8aaff572899..dbf1e2132b9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -77,7 +77,7 @@ public class BulkIndexer { private final String indexName; private final BulkProcessor bulkProcessor; private final AtomicLong counter = new AtomicLong(0L); - private final AtomicLong successCounter = new AtomicLong(0L); + private final ResilientIndexerResult successCounter = new ResilientIndexerResult(); private final SizeHandler sizeHandler; private final BulkProcessorListener bulkProcessorListener; @Nullable @@ -103,7 +103,7 @@ public class BulkIndexer { public void start() { sizeHandler.beforeStart(this); counter.set(0L); - successCounter.set(0L); + successCounter.clear(); } public void start(DbSession dbSession, DbClient dbClient, Collection<EsQueueDto> esQueueDtos) { @@ -112,13 +112,13 @@ public class BulkIndexer { this.esQueueDtos = esQueueDtos; sizeHandler.beforeStart(this); counter.set(0L); - successCounter.set(0L); + successCounter.clear(); } /** * @return the number of documents successfully indexed */ - public long stop() { + public ResilientIndexerResult stop() { try { bulkProcessor.awaitClose(1, TimeUnit.MINUTES); } catch (InterruptedException e) { @@ -129,7 +129,7 @@ public class BulkIndexer { } client.prepareRefresh(indexName).get(); sizeHandler.afterStop(this); - return successCounter.get(); + return successCounter; } public void add(ActionRequest<?> request) { @@ -204,8 +204,9 @@ public class BulkIndexer { for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage()); + successCounter.increaseFailure(); } else { - successCounter.incrementAndGet(); + successCounter.increaseSuccess(); } } @@ -221,7 +222,7 @@ public class BulkIndexer { if (esQueueDtos != null) { List<EsQueueDto> itemsToDelete = Arrays.stream(bulkResponse.getItems()) .filter(b -> !b.isFailed()) - .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocUuid())).findFirst().orElse(null)) + .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocId())).findFirst().orElse(null)) .filter(Objects::nonNull) .collect(toList()); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java index c7ffdb9b2d6..3c7c5ab6c69 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java @@ -38,6 +38,7 @@ import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.es.EsQueueDto; +import org.sonar.server.rule.index.RuleIndexer; import org.sonar.server.user.index.UserIndexer; import static java.lang.String.format; @@ -64,14 +65,16 @@ public class RecoveryIndexer implements Startable { private final Settings settings; private final DbClient dbClient; private final UserIndexer userIndexer; + private final RuleIndexer ruleIndexer; private final long minAgeInMs; private final long loopLimit; - public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer) { + public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer, RuleIndexer ruleIndexer) { this.system2 = system2; this.settings = settings; this.dbClient = dbClient; this.userIndexer = userIndexer; + this.ruleIndexer = ruleIndexer; this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS); this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT); } @@ -107,42 +110,43 @@ public class RecoveryIndexer implements Startable { try (DbSession dbSession = dbClient.openSession(false)) { Profiler profiler = Profiler.create(LOGGER).start(); long beforeDate = system2.now() - minAgeInMs; - long total = 0L; - long totalSuccess = 0L; + ResilientIndexerResult result = new ResilientIndexerResult(); Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); while (!items.isEmpty()) { - total += items.size(); - long loopSuccess = 0L; + ResilientIndexerResult loopResult = new ResilientIndexerResult(); ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items); for (Map.Entry<EsQueueDto.Type, Collection<EsQueueDto>> entry : itemsByType.asMap().entrySet()) { - loopSuccess += doIndex(dbSession, entry.getKey(), entry.getValue()); + loopResult.add(doIndex(dbSession, entry.getKey(), entry.getValue())); } - totalSuccess += loopSuccess; - if (1.0d * (items.size() - loopSuccess) / items.size() >= CIRCUIT_BREAKER_IN_PERCENT) { - LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", items.size() - loopSuccess, items.size()); + result.add(loopResult); + if (loopResult.getFailureRatio() >= CIRCUIT_BREAKER_IN_PERCENT) { + LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal()); break; } items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); } - if (total > 0L) { - profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", total, total - totalSuccess)); + if (result.getTotal() > 0L) { + profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures())); } } catch (Throwable t) { LOGGER.error(LOG_PREFIX + "fail to recover documents", t); } } - private long doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) { + private ResilientIndexerResult doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) { LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type); switch (type) { case USER: return userIndexer.index(dbSession, typeItems); + case RULE_EXTENSION: + case RULE: + return ruleIndexer.index(dbSession, typeItems); default: LOGGER.error(LOG_PREFIX + "ignore {} documents with unsupported type {}", typeItems.size(), type); - return 0; + return new ResilientIndexerResult(); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexer.java new file mode 100644 index 00000000000..5e818aac748 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexer.java @@ -0,0 +1,41 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.es; + +import java.util.Collection; +import org.sonar.db.DbSession; +import org.sonar.db.es.EsQueueDto; + +/** + * This kind of indexers that are resilient + */ +public interface ResilientIndexer { + + /** + * Index the items and delete them from es_queue table when the indexation + * is done, keep it if there is a failure on the item of the collection + * + * @param dbSession the db session + * @param items the items to be indexed + * @return the number of successful indexation + */ + ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items); +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexerResult.java b/server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexerResult.java new file mode 100644 index 00000000000..c176a819bda --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexerResult.java @@ -0,0 +1,77 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.es; + +/** + * The type Resilient indexer result. + */ +public class ResilientIndexerResult { + private long total = 0L; + private long failures = 0L; + + public ResilientIndexerResult clear() { + total = 0L; + failures = 0L; + return this; + } + + public ResilientIndexerResult increaseFailure() { + failures += 1; + total += 1; + return this; + } + + public ResilientIndexerResult increaseSuccess() { + total += 1; + return this; + } + + public long getFailures() { + return failures; + } + + public long getTotal() { + return total; + } + + public long getSuccess() { + return total - failures; + } + + /** + * Get the failure ratio, + * if the total is 0, we always return 1 in order to break loop + * @see {@link RecoveryIndexer#recover()} + */ + public double getFailureRatio() { + return total == 0 ? 1 : 1.0d * failures / total; + } + + public double getSuccessRatio() { + return total == 0 ? 0 : 1 - 1.0d * failures / total; + } + + public ResilientIndexerResult add(ResilientIndexerResult other) { + this.total += other.getTotal(); + this.failures += other.getFailures(); + return this; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/EnableSupportAction.java b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/EnableSupportAction.java index 4d36a672126..b33d761fea8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/EnableSupportAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/EnableSupportAction.java @@ -89,8 +89,7 @@ public class EnableSupportAction implements OrganizationsWsAction { createDefaultMembersGroup(dbSession); List<RuleKey> disabledTemplateAndCustomRuleKeys = disableTemplateRulesAndCustomRules(dbSession); enableFeature(dbSession); - dbSession.commit(); - ruleIndexer.indexRuleDefinitions(disabledTemplateAndCustomRuleKeys); + ruleIndexer.commitAndIndex(dbSession, disabledTemplateAndCustomRuleKeys); } } response.noContent(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/RegisterRules.java b/server/sonar-server/src/main/java/org/sonar/server/rule/RegisterRules.java index c714e91142d..822ef0b315f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/RegisterRules.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/RegisterRules.java @@ -130,7 +130,7 @@ public class RegisterRules implements Startable { keysToIndex.addAll(removedRules.stream().map(RuleDefinitionDto::getKey).collect(Collectors.toList())); persistRepositories(dbSession, context.repositories()); - ruleIndexer.indexRuleDefinitions(keysToIndex); + ruleIndexer.commitAndIndex(dbSession, keysToIndex); activeRuleIndexer.indexChanges(dbSession, changes); profiler.stopDebug(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/RuleCreator.java b/server/sonar-server/src/main/java/org/sonar/server/rule/RuleCreator.java index a1bba61a376..1752bbd4c97 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/RuleCreator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/RuleCreator.java @@ -86,8 +86,7 @@ public class RuleCreator { .map(existingRule -> updateExistingRule(existingRule, newRule, dbSession)) .orElseGet(() -> createCustomRule(customRuleKey, newRule, templateRule, dbSession)); - dbSession.commit(); - ruleIndexer.indexRuleDefinition(customRuleKey); + ruleIndexer.commitAndIndex(dbSession, customRuleKey); return customRuleKey; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java b/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java index 2d5d3d2da8d..70e11dee084 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java @@ -32,7 +32,6 @@ import java.util.Set; import java.util.function.Consumer; import javax.annotation.Nonnull; import org.apache.commons.lang.builder.EqualsBuilder; -import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.api.rule.Severity; import org.sonar.api.server.ServerSide; @@ -82,11 +81,9 @@ public class RuleUpdater { apply(update, rule, userSession); update(dbSession, rule); updateParameters(dbSession, organization, update, rule); - dbSession.commit(); + ruleIndexer.commitAndIndex(dbSession, rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, organization, rule.getKey()); - RuleKey ruleKey = rule.getKey(); - ruleIndexer.indexRuleDefinition(ruleKey); - ruleIndexer.indexRuleExtension(organization, ruleKey); return true; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java index a13c452c811..601e6118f79 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleDoc.java @@ -27,6 +27,9 @@ import org.apache.commons.lang.builder.ReflectionToStringBuilder; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.api.rules.RuleType; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleForIndexingDto; +import org.sonar.markdown.Markdown; import org.sonar.server.es.BaseDoc; /** @@ -201,4 +204,34 @@ public class RuleDoc extends BaseDoc { return ReflectionToStringBuilder.toString(this); } + public static RuleDoc of(RuleForIndexingDto ruleForIndexingDto) { + RuleDoc ruleDoc = new RuleDoc() + .setKey(ruleForIndexingDto.getRuleKey().toString()) + .setRepository(ruleForIndexingDto.getRepository()) + .setInternalKey(ruleForIndexingDto.getInternalKey()) + .setIsTemplate(ruleForIndexingDto.isTemplate()) + .setLanguage(ruleForIndexingDto.getLanguage()) + .setName(ruleForIndexingDto.getName()) + .setRuleKey(ruleForIndexingDto.getPluginRuleKey()) + .setSeverity(ruleForIndexingDto.getSeverityAsString()) + .setStatus(ruleForIndexingDto.getStatus().toString()) + .setType(ruleForIndexingDto.getTypeAsRuleType()) + .setCreatedAt(ruleForIndexingDto.getCreatedAt()) + .setUpdatedAt(ruleForIndexingDto.getUpdatedAt()); + + if (ruleForIndexingDto.getPluginRuleKey() != null && ruleForIndexingDto.getRepository() != null) { + ruleDoc.setTemplateKey(RuleKey.of(ruleForIndexingDto.getPluginRuleKey(), ruleForIndexingDto.getRepository()).toString()); + } else { + ruleDoc.setTemplateKey(null); + } + + if (ruleForIndexingDto.getDescription() != null && ruleForIndexingDto.getDescriptionFormat() != null) { + if (RuleDto.Format.HTML == ruleForIndexingDto.getDescriptionFormat()) { + ruleDoc.setHtmlDescription(ruleForIndexingDto.getDescription()); + } else { + ruleDoc.setHtmlDescription(Markdown.convertToHtml(ruleForIndexingDto.getDescription()));; + } + } + return ruleDoc; + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java index 21c0316a9ec..226e3cda99f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java @@ -24,6 +24,8 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang.builder.ReflectionToStringBuilder; import org.sonar.api.rule.RuleKey; +import org.sonar.db.rule.RuleExtensionForIndexingDto; +import org.sonar.db.rule.RuleForIndexingDto; import org.sonar.db.rule.RuleMetadataDto; import org.sonar.server.es.BaseDoc; @@ -86,6 +88,20 @@ public class RuleExtensionDoc extends BaseDoc { .setTags(ruleExtension.getTags()); } + public static RuleExtensionDoc of(RuleForIndexingDto rule) { + return new RuleExtensionDoc() + .setRuleKey(rule.getRuleKey()) + .setScope(RuleExtensionScope.system()) + .setTags(rule.getSystemTagsAsSet()); + } + + public static RuleExtensionDoc of(RuleExtensionForIndexingDto rule) { + return new RuleExtensionDoc() + .setRuleKey(rule.getRuleKey()) + .setScope(RuleExtensionScope.organization(rule.getOrganizationUuid())) + .setTags(rule.getTagsAsSet()); + } + @Override public String toString() { return ReflectionToStringBuilder.toString(this); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java index ae950c9d135..6f1c21d2d43 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java @@ -20,26 +20,38 @@ package org.sonar.server.rule.index; import com.google.common.collect.ImmutableSet; -import java.util.Arrays; +import com.google.common.collect.ListMultimap; +import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Set; import org.elasticsearch.action.index.IndexRequest; +import org.sonar.db.es.RuleExtensionId; import org.sonar.api.rule.RuleKey; +import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.es.EsQueueDto; import org.sonar.db.organization.OrganizationDto; +import org.sonar.db.rule.RuleExtensionForIndexingDto; +import org.sonar.db.rule.RuleForIndexingDto; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; +import org.sonar.server.es.ResilientIndexer; +import org.sonar.server.es.ResilientIndexerResult; import org.sonar.server.es.StartupIndexer; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Collections.singletonList; +import static java.util.Objects.requireNonNull; +import static org.sonar.core.util.stream.MoreCollectors.toHashSet; +import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX; import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE; import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION; -public class RuleIndexer implements StartupIndexer { +public class RuleIndexer implements StartupIndexer, ResilientIndexer { private final EsClient esClient; private final DbClient dbClient; @@ -56,7 +68,7 @@ public class RuleIndexer implements StartupIndexer { @Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - BulkIndexer bulk = new BulkIndexer(esClient, RuleIndexDefinition.INDEX, Size.LARGE); + BulkIndexer bulk = new BulkIndexer(esClient, INDEX, Size.LARGE); bulk.start(); // index all definitions and system extensions @@ -76,35 +88,113 @@ public class RuleIndexer implements StartupIndexer { bulk.stop(); } - public void indexRuleDefinition(RuleKey ruleKey) { - indexRuleDefinitions(singletonList(ruleKey)); + public void commitAndIndex(DbSession dbSession, RuleKey ruleKey) { + commitAndIndex(dbSession, singletonList(ruleKey)); } - public void indexRuleDefinitions(List<RuleKey> ruleKeys) { - BulkIndexer bulk = new BulkIndexer(esClient, RuleIndexDefinition.INDEX, Size.REGULAR); - bulk.start(); + public void commitAndIndex(DbSession dbSession, Collection<RuleKey> ruleDtos) { + List<EsQueueDto> items = ruleDtos.stream() + .map(key -> EsQueueDto.create(EsQueueDto.Type.RULE, key.toString())) + .collect(MoreCollectors.toArrayList()); - try (RuleIterator rules = new RuleIteratorForMultipleChunks(dbClient, ruleKeys)) { - doIndexRuleDefinitions(rules, bulk); - } + dbClient.esQueueDao().insert(dbSession, items); + dbSession.commit(); + postCommit(dbSession, ruleDtos, items); + } - bulk.stop(); + public void commitAndIndex(DbSession dbSession, OrganizationDto organizationDto, RuleKey ruleKey) { + List<EsQueueDto> items = singletonList(EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, ruleKey + "|" + organizationDto.getUuid())); + + dbClient.esQueueDao().insert(dbSession, items); + dbSession.commit(); + postCommit(dbSession, ruleKey, organizationDto, items); } - public void indexRuleExtension(OrganizationDto organization, RuleKey ruleKey) { - try (DbSession dbSession = dbClient.openSession(false)) { - dbClient.ruleDao() - .selectMetadataByKey(dbSession, ruleKey, organization) - .map(ruleExtension -> RuleExtensionDoc.of(ruleKey, RuleExtensionScope.organization(organization), ruleExtension)) - .map(Arrays::asList) - .map(List::iterator) - .ifPresent(metadata -> { - BulkIndexer bulk = new BulkIndexer(esClient, RuleIndexDefinition.INDEX, Size.REGULAR); - bulk.start(); - doIndexRuleExtensions(metadata, bulk); - bulk.stop(); - }); + /** + * Entry point for Byteman tests. See directory tests/resilience. + * The parameter "ruleKeys" is used only by the Byteman script. + */ + private void postCommit(DbSession dbSession, Collection<RuleKey> ruleKeys, Collection<EsQueueDto> items) { + index(dbSession, items); + } + + private void postCommit(DbSession dbSession, RuleKey ruleKeys, OrganizationDto organizationDto, Collection<EsQueueDto> items) { + index(dbSession, items); + } + + @Override + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { + if (items.isEmpty()) { + return new ResilientIndexerResult(); } + + ResilientIndexerResult result = new ResilientIndexerResult(); + + ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items); + + result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE))); + result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION))); + + return result; + } + + private ResilientIndexerResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) { + BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR); + bulkIndexer.start(dbSession, dbClient, items); + + Set<RuleKey> rules = items + .stream() + .filter(i -> { + requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing"); + return i.getDocType() == EsQueueDto.Type.RULE; + }) + .map(i -> RuleKey.parse(i.getDocId())) + .collect(toHashSet(items.size())); + + dbClient.ruleDao().scrollRuleByRuleKeys(dbSession, rules, + // only index requests, no deletion requests. + // Deactivated users are not deleted but updated. + r -> { + rules.remove(r.getRuleKey()); + bulkIndexer.add(newRuleDocIndexRequest(r)); + bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); + }); + + // the remaining items reference rows that don't exist in db. They must + // be deleted from index. + rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString())); + rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.toString())); + + return bulkIndexer.stop(); + } + + private ResilientIndexerResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) { + BulkIndexer bulkIndexer = newBulkIndexerForRules(Size.REGULAR); + bulkIndexer.start(dbSession, dbClient, items); + + Set<RuleExtensionId> docIds = items + .stream() + .filter(i -> { + requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing"); + return i.getDocType() == EsQueueDto.Type.RULE_EXTENSION; + }) + .map(RuleIndexer::explodeRuleExtensionDocId) + .collect(toHashSet(items.size())); + + dbClient.ruleDao().scrollRuleExtensionByRuleKeys(dbSession, docIds, + // only index requests, no deletion requests. + // Deactivated users are not deleted but updated. + r -> { + docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()) ); + bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); + }); + + + // the remaining items reference rows that don't exist in db. They must + // be deleted from index. + docIds.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.getId())); + + return bulkIndexer.stop(); } private static void doIndexRuleDefinitions(Iterator<RuleDocWithSystemScope> rules, BulkIndexer bulk) { @@ -136,4 +226,40 @@ public class RuleIndexer implements StartupIndexer { .source(ruleExtension.getFields()) .parent(ruleExtension.getParent()); } + + private static IndexRequest newRuleDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) { + RuleDoc ruleDoc = RuleDoc.of(ruleForIndexingDto); + + return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType(), ruleDoc.key().toString()) + .source(ruleDoc.getFields()); + } + + private static IndexRequest newRuleExtensionDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) { + RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleForIndexingDto); + + return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId()) + .source(ruleExtensionDoc.getFields()) + .parent(ruleExtensionDoc.getParent()); + } + + private static IndexRequest newRuleExtensionDocIndexRequest(RuleExtensionForIndexingDto ruleExtensionForIndexingDto) { + RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleExtensionForIndexingDto); + + return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType(), ruleExtensionDoc.getId()) + .source(ruleExtensionDoc.getFields()) + .parent(ruleExtensionDoc.getParent()); + } + + private BulkIndexer newBulkIndexerForRules(Size bulkSize) { + return new BulkIndexer(esClient, INDEX_TYPE_RULE.getIndex(), bulkSize); + } + + private static ListMultimap<EsQueueDto.Type, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) { + return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType)); + } + + private static RuleExtensionId explodeRuleExtensionDocId(EsQueueDto esQueueDto) { + checkArgument(esQueueDto.getDocType() == EsQueueDto.Type.RULE_EXTENSION); + return new RuleExtensionId(esQueueDto.getDocId()); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/ws/DeleteAction.java b/server/sonar-server/src/main/java/org/sonar/server/rule/ws/DeleteAction.java index 85da5137d03..9f48cb1bfe4 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/ws/DeleteAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/ws/DeleteAction.java @@ -88,9 +88,7 @@ public class DeleteAction implements RulesWsAction { rule.setStatus(RuleStatus.REMOVED); rule.setUpdatedAt(system2.now()); dbClient.ruleDao().update(dbSession, rule); - - dbSession.commit(); - ruleIndexer.indexRuleDefinition(ruleKey); + ruleIndexer.commitAndIndex(dbSession, ruleKey); } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 6f99292e7cb..b6ae239b4af 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -37,6 +37,8 @@ import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; +import org.sonar.server.es.ResilientIndexer; +import org.sonar.server.es.ResilientIndexerResult; import org.sonar.server.es.StartupIndexer; import static java.util.Collections.singletonList; @@ -44,7 +46,7 @@ import static java.util.Objects.requireNonNull; import static org.sonar.core.util.stream.MoreCollectors.toHashSet; import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER; -public class UserIndexer implements StartupIndexer { +public class UserIndexer implements StartupIndexer, ResilientIndexer { private final DbClient dbClient; private final EsClient esClient; @@ -104,17 +106,18 @@ public class UserIndexer implements StartupIndexer { /** * @return the number of items that have been successfully indexed */ - public long index(DbSession dbSession, Collection<EsQueueDto> items) { + @Override + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { if (items.isEmpty()) { - return 0L; + return new ResilientIndexerResult(); } Set<String> logins = items .stream() .filter(i -> { - requireNonNull(i.getDocUuid(), () -> "BUG - " + i + " has not been persisted before indexing"); - return true; + requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing"); + return i.getDocType() == EsQueueDto.Type.USER; }) - .map(EsQueueDto::getDocUuid) + .map(EsQueueDto::getDocId) .collect(toHashSet(items.size())); ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java index 3bdf905f729..22b07b634c7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java @@ -70,7 +70,7 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> { if (esQueueDtos != null) { logins = esQueueDtos.stream() .filter(i -> i.getDocType() == EsQueueDto.Type.USER) - .map(EsQueueDto::getDocUuid).collect(toArrayList()); + .map(EsQueueDto::getDocId).collect(toArrayList()); sql += "where (" + repeat("u.login=?", " or ", logins.size()) + ")"; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java index a1c5519384a..dd8abe6998a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -20,21 +20,15 @@ package org.sonar.server.es; import com.google.common.collect.ImmutableMap; -import java.util.ArrayList; -import java.util.Collection; -import java.util.stream.IntStream; -import org.apache.commons.lang.math.RandomUtils; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.sonar.api.utils.internal.TestSystem2; import org.sonar.db.DbTester; -import org.sonar.db.es.EsQueueDto; import org.sonar.server.es.BulkIndexer.Size; import static org.assertj.core.api.Assertions.assertThat; @@ -114,42 +108,6 @@ public class BulkIndexerTest { assertThat(count()).isEqualTo(removeFrom); } - @Test - @Ignore - public void when_index_is_done_EsQueues_must_be_deleted() { - BulkIndexer indexer = new BulkIndexer(esTester.client(), INDEX, Size.REGULAR); - int nbOfDelete = 10 + RandomUtils.nextInt(10); - int nbOfInsert = 10 + RandomUtils.nextInt(10); - int nbOfDocumentNotToBeDeleted = 10 + RandomUtils.nextInt(10); - Collection<EsQueueDto> esQueueDtos = new ArrayList<>(); - - // Those documents must be kept - FakeDoc[] docs = new FakeDoc[nbOfDocumentNotToBeDeleted]; - for (int i = 1; i <= nbOfDocumentNotToBeDeleted; i++) { - docs[i] = FakeIndexDefinition.newDoc(-i); - } - esTester.putDocuments(INDEX_TYPE_FAKE, docs); - - // Create nbOfDelete documents to be deleted - docs = new FakeDoc[nbOfDelete]; - for (int i = 1; i <= nbOfDelete; i++) { - docs[i] = FakeIndexDefinition.newDoc(i); - } - esTester.putDocuments(INDEX_TYPE_FAKE, docs); - assertThat(count()).isEqualTo(nbOfDelete + nbOfDocumentNotToBeDeleted); - - indexer.start(dbTester.getSession(), dbTester.getDbClient(), esQueueDtos); - // Create nbOfDelete for old Documents - IntStream.rangeClosed(1, nbOfDelete).forEach( - i -> indexer.addDeletion(INDEX_TYPE_FAKE, "" + i)); - // Create nbOfInsert for new Documents - IntStream.rangeClosed(nbOfDelete + 1, nbOfInsert).forEach( - i -> indexer.add(newIndexRequest(i))); - indexer.stop(); - - assertThat(count()).isEqualTo(nbOfInsert + nbOfDocumentNotToBeDeleted); - } - private long count() { return esTester.countDocuments("fakes", "fake"); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java index bb43319c9e4..f83015e5aac 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java @@ -30,6 +30,7 @@ import java.util.stream.IntStream; import org.junit.After; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.DisableOnDebug; import org.junit.rules.TestRule; import org.junit.rules.Timeout; import org.sonar.api.config.Settings; @@ -40,10 +41,14 @@ import org.sonar.api.utils.log.LoggerLevel; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.es.EsQueueDto; +import org.sonar.db.rule.RuleDto; import org.sonar.db.user.UserDto; +import org.sonar.server.rule.index.RuleIndexDefinition; +import org.sonar.server.rule.index.RuleIndexer; import org.sonar.server.user.index.UserIndexDefinition; import org.sonar.server.user.index.UserIndexer; +import static java.util.stream.IntStream.rangeClosed; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -57,16 +62,20 @@ public class RecoveryIndexerTest { private static final long PAST = 1_000L; private TestSystem2 system2 = new TestSystem2().setNow(PAST); + private MapSettings emptySettings = new MapSettings(); @Rule - public final EsTester es = new EsTester(new UserIndexDefinition(new MapSettings().asConfig())); + public final EsTester es = new EsTester(new UserIndexDefinition(emptySettings.asConfig()), new RuleIndexDefinition(emptySettings.asConfig())); @Rule public final DbTester db = DbTester.create(system2); @Rule public final LogTester logTester = new LogTester().setLevel(TRACE); @Rule - public TestRule safeguard = new Timeout(60, TimeUnit.SECONDS); + public TestRule safeguard = new DisableOnDebug(new Timeout(60, TimeUnit.SECONDS)); + + private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class); + private UserIndexer mockedUserIndexer = mock(UserIndexer.class); private RecoveryIndexer underTest; @After @@ -79,7 +88,8 @@ public class RecoveryIndexerTest { @Test public void display_default_configuration_at_startup() { UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client()); - underTest = newRecoveryIndexer(userIndexer, new MapSettings()); + RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient()); + underTest = newRecoveryIndexer(userIndexer, ruleIndexer, emptySettings); underTest.start(); @@ -93,7 +103,7 @@ public class RecoveryIndexerTest { Settings settings = new MapSettings() .setProperty("sonar.search.recovery.initialDelayInMs", "0") .setProperty("sonar.search.recovery.delayInMs", "1"); - underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mock(UserIndexer.class))); + underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer)); AtomicInteger calls = new AtomicInteger(0); doAnswer(invocation -> { calls.incrementAndGet(); @@ -109,13 +119,33 @@ public class RecoveryIndexerTest { } @Test - public void successfully_index_old_records() { + public void successfully_index_RULE_records() { + EsQueueDto item1 = createUnindexedRule(); + EsQueueDto item2 = createUnindexedRule(); + + ProxyRuleIndexer ruleIndexer = new ProxyRuleIndexer(); + advanceInTime(); + + underTest = newRecoveryIndexer(mockedUserIndexer, ruleIndexer); + underTest.recover(); + + assertThatQueueHasSize(0); + assertThat(ruleIndexer.called) + .extracting(EsQueueDto::getUuid) + .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid()); + + assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 2 RULE"); + assertThatLogsContain(INFO, "Elasticsearch recovery - 4 documents processed [0 failures]"); + } + + @Test + public void successfully_index_USER_records() { EsQueueDto item1 = createUnindexedUser(); EsQueueDto item2 = createUnindexedUser(); ProxyUserIndexer userIndexer = new ProxyUserIndexer(); advanceInTime(); - underTest = newRecoveryIndexer(userIndexer); + underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer); underTest.recover(); assertThatQueueHasSize(0); @@ -134,7 +164,7 @@ public class RecoveryIndexerTest { ProxyUserIndexer userIndexer = new ProxyUserIndexer(); // do not advance in time - underTest = newRecoveryIndexer(userIndexer); + underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer); underTest.recover(); assertThatQueueHasSize(2); @@ -161,7 +191,7 @@ public class RecoveryIndexerTest { FailingOnceUserIndexer failingOnceUserIndexer = new FailingOnceUserIndexer(); advanceInTime(); - underTest = newRecoveryIndexer(failingOnceUserIndexer); + underTest = newRecoveryIndexer(failingOnceUserIndexer, mockedRuleIndexer); underTest.recover(); // No rows treated @@ -175,7 +205,7 @@ public class RecoveryIndexerTest { advanceInTime(); FailingUserIndexer userIndexer = new FailingUserIndexer(); - underTest = newRecoveryIndexer(userIndexer); + underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer); underTest.start(); // all runs fail, but they are still scheduled @@ -191,7 +221,7 @@ public class RecoveryIndexerTest { advanceInTime(); FailingOnceUserIndexer userIndexer = new FailingOnceUserIndexer(); - underTest = newRecoveryIndexer(userIndexer); + underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer); underTest.start(); // first run fails, second run succeeds @@ -213,7 +243,7 @@ public class RecoveryIndexerTest { PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(1); Settings settings = new MapSettings() .setProperty("sonar.search.recovery.loopLimit", "3"); - underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings); + underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings); underTest.recover(); assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [2/3 documents], waiting for next run"); @@ -233,7 +263,7 @@ public class RecoveryIndexerTest { PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(4, 4, 2); Settings settings = new MapSettings() .setProperty("sonar.search.recovery.loopLimit", "5"); - underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings); + underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings); underTest.recover(); assertThatLogsDoNotContain(ERROR, "too many failures"); @@ -249,7 +279,7 @@ public class RecoveryIndexerTest { advanceInTime(); FailingAlwaysOnSameElementIndexer indexer = new FailingAlwaysOnSameElementIndexer(buggy); - underTest = newRecoveryIndexer(indexer); + underTest = newRecoveryIndexer(indexer, mockedRuleIndexer); underTest.recover(); assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [1/1 documents], waiting for next run"); @@ -264,7 +294,21 @@ public class RecoveryIndexerTest { } @Override - public long index(DbSession dbSession, Collection<EsQueueDto> items) { + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { + called.addAll(items); + return super.index(dbSession, items); + } + } + + private class ProxyRuleIndexer extends RuleIndexer { + private final List<EsQueueDto> called = new ArrayList<>(); + + ProxyRuleIndexer() { + super(es.client(), db.getDbClient()); + } + + @Override + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { called.addAll(items); return super.index(dbSession, items); } @@ -278,7 +322,7 @@ public class RecoveryIndexerTest { } @Override - public long index(DbSession dbSession, Collection<EsQueueDto> items) { + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { called.addAll(items); throw new RuntimeException("boom"); } @@ -293,7 +337,7 @@ public class RecoveryIndexerTest { } @Override - public long index(DbSession dbSession, Collection<EsQueueDto> items) { + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { try { if (counter.getCount() == 2) { throw new RuntimeException("boom"); @@ -314,10 +358,15 @@ public class RecoveryIndexerTest { } @Override - public long index(DbSession dbSession, Collection<EsQueueDto> items) { + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { + ResilientIndexerResult result = new ResilientIndexerResult(); List<EsQueueDto> filteredItems = items.stream().filter( i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList()); - return super.index(dbSession, filteredItems); + if (items.contains(failing)) { + result.increaseFailure(); + } + + return result.add(super.index(dbSession, filteredItems)); } } @@ -332,17 +381,21 @@ public class RecoveryIndexerTest { } @Override - public long index(DbSession dbSession, Collection<EsQueueDto> items) { + public ResilientIndexerResult index(DbSession dbSession, Collection<EsQueueDto> items) { System.out.println("called with " + items.size()); called.addAll(items); int success = successfulReturns.next(); + ResilientIndexerResult result = new ResilientIndexerResult(); items.stream().limit(success).forEach(i -> { System.out.println(" + success"); db.getDbClient().esQueueDao().delete(dbSession, i); + result.increaseSuccess(); indexed.add(i); }); + + rangeClosed(1, items.size() - success).forEach(i -> result.increaseFailure()); dbSession.commit(); - return success; + return result; } } @@ -368,19 +421,20 @@ public class RecoveryIndexerTest { private RecoveryIndexer newRecoveryIndexer() { UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client()); - return newRecoveryIndexer(userIndexer); + RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient()); + return newRecoveryIndexer(userIndexer, ruleIndexer); } - private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer) { + private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer) { Settings settings = new MapSettings() .setProperty("sonar.search.recovery.initialDelayInMs", "0") .setProperty("sonar.search.recovery.delayInMs", "1") .setProperty("sonar.search.recovery.minAgeInMs", "1"); - return newRecoveryIndexer(userIndexer, settings); + return newRecoveryIndexer(userIndexer, ruleIndexer, settings); } - private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, Settings settings) { - return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer); + private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, Settings settings) { + return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer); } private EsQueueDto createUnindexedUser() { @@ -391,4 +445,13 @@ public class RecoveryIndexerTest { return item; } + + private EsQueueDto createUnindexedRule() { + RuleDto rule = db.rules().insertRule(); + EsQueueDto item = EsQueueDto.create(EsQueueDto.Type.RULE, rule.getKey().toString()); + db.getDbClient().esQueueDao().insert(db.getSession(), item); + db.commit(); + + return item; + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/ResilientIndexerResultTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/ResilientIndexerResultTest.java new file mode 100644 index 00000000000..6a3a03c3ba1 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/es/ResilientIndexerResultTest.java @@ -0,0 +1,62 @@ +package org.sonar.server.es;/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +import java.util.stream.IntStream; +import org.apache.commons.lang.math.RandomUtils; +import org.assertj.core.data.Offset; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ResilientIndexerResultTest { + + private final ResilientIndexerResult underTest = new ResilientIndexerResult(); + + @Before + public void clear() { + underTest.clear(); + } + + @Test + public void ensure_correctness() { + int success = 1 + RandomUtils.nextInt(100); + int failures = RandomUtils.nextInt(100); + IntStream.rangeClosed(1, success).forEach(i -> underTest.increaseSuccess()); + IntStream.rangeClosed(1, failures).forEach(i -> underTest.increaseFailure()); + + assertThat(underTest.getFailures()).isEqualTo(failures); + assertThat(underTest.getSuccess()).isEqualTo(success); + assertThat(underTest.getTotal()).isEqualTo(success + failures); + assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1); + assertThat(underTest.getFailureRatio()).isEqualTo(1.0d * failures / (success + failures), Offset.offset(0.000001d)); + assertThat(underTest.getSuccessRatio()).isEqualTo(1.0d * success / (success + failures), Offset.offset(0.000001d)); + } + + @Test + public void correctness_even_with_no_data() { + assertThat(underTest.getFailures()).isEqualTo(0); + assertThat(underTest.getSuccess()).isEqualTo(0); + assertThat(underTest.getTotal()).isEqualTo(0); + assertThat(underTest.getFailureRatio() + underTest.getSuccessRatio()).isEqualTo(1); + assertThat(underTest.getFailureRatio()).isEqualTo(1); + assertThat(underTest.getSuccessRatio()).isEqualTo(0); + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java index f3c1174e9ee..526768f9998 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/IssueServiceMediumTest.java @@ -141,7 +141,7 @@ public class IssueServiceMediumTest { ruleDao.insertOrUpdate(session, rule.getMetadata().setRuleId(rule.getId())); } session.commit(); - ruleIndexer.indexRuleDefinition(rule.getDefinition().getKey()); + //FIXME ruleIndexer.commitAndIndex(dbSession, rule.getDefinition().getKey()); return rule; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index 1e9dcb16efa..45ed66c7cab 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -81,25 +81,25 @@ import static org.sonar.server.issue.IssueDocTesting.newDoc; public class IssueIndexTest { private System2 system2 = mock(System2.class); - + private MapSettings settings = new MapSettings(); @Rule - public EsTester tester = new EsTester( - new IssueIndexDefinition(new MapSettings().asConfig()), - new ViewIndexDefinition(new MapSettings().asConfig()), - new RuleIndexDefinition(new MapSettings().asConfig())); + public EsTester esTester = new EsTester( + new IssueIndexDefinition(settings.asConfig()), + new ViewIndexDefinition(settings.asConfig()), + new RuleIndexDefinition(settings.asConfig()); @Rule - public DbTester db = DbTester.create(system2); + public DbTester dbTester = DbTester.create(system2); @Rule public UserSessionRule userSessionRule = UserSessionRule.standalone(); @Rule public ExpectedException expectedException = ExpectedException.none(); - private IssueIndexer issueIndexer = new IssueIndexer(tester.client(), new IssueIteratorFactory(null)); - private ViewIndexer viewIndexer = new ViewIndexer(null, tester.client()); - private RuleIndexer ruleIndexer = new RuleIndexer(tester.client(), db.getDbClient()); - private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(tester, issueIndexer); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(null)); + private ViewIndexer viewIndexer = new ViewIndexer(null, esTester.client()); + private RuleIndexer ruleIndexer = new RuleIndexer(esTester.client(), dbTester.getDbClient()); + private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(esTester, issueIndexer); - private IssueIndex underTest = new IssueIndex(tester.client(), system2, userSessionRule, new AuthorizationTypeSupport(userSessionRule)); + private IssueIndex underTest = new IssueIndex(esTester.client(), system2, userSessionRule, new AuthorizationTypeSupport(userSessionRule)); @Before public void setUp() { @@ -1329,13 +1329,11 @@ public class IssueIndexTest { @Test public void list_tags() { - RuleDefinitionDto r1 = db.rules().insert(); - ruleIndexer.indexRuleDefinition(r1.getKey()); - - RuleDefinitionDto r2 = db.rules().insert(); - ruleIndexer.indexRuleDefinition(r2.getKey()); + RuleDefinitionDto r1 = dbTester.rules().insert(); + RuleDefinitionDto r2 = dbTester.rules().insert(); + ruleIndexer.commitAndIndex(dbTester.getSession(), asList(r1.getKey(), r2.getKey())); - OrganizationDto org = db.organizations().insert(); + OrganizationDto org = dbTester.organizations().insert(); ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); ComponentDto file = newFileDto(project, null); indexIssues( diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java index f93118a2b02..b32d3e58cd1 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java @@ -56,25 +56,26 @@ import static org.sonar.test.JsonAssert.assertJson; public class TagsActionTest { + private MapSettings settings = new MapSettings(); @Rule public UserSessionRule userSession = UserSessionRule.standalone(); @Rule - public DbTester db = DbTester.create(); + public DbTester dbTester = DbTester.create(); @Rule - public EsTester es = new EsTester(new IssueIndexDefinition(new MapSettings().asConfig()), new RuleIndexDefinition(new MapSettings().asConfig())); + public EsTester esTester = new EsTester(new IssueIndexDefinition(settings.asConfig()), new RuleIndexDefinition(settings.asConfig())); - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), new IssueIteratorFactory(db.getDbClient())); - private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient()); - private PermissionIndexerTester permissionIndexerTester = new PermissionIndexerTester(es, issueIndexer); - private IssueIndex issueIndex = new IssueIndex(es.client(), System2.INSTANCE, userSession, new AuthorizationTypeSupport(userSession)); - private RuleIndex ruleIndex = new RuleIndex(es.client()); + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(dbTester.getDbClient())); + private RuleIndexer ruleIndexer = new RuleIndexer(esTester.client(), dbTester.getDbClient()); + private PermissionIndexerTester permissionIndexerTester = new PermissionIndexerTester(esTester, issueIndexer); + private IssueIndex issueIndex = new IssueIndex(esTester.client(), System2.INSTANCE, userSession, new AuthorizationTypeSupport(userSession)); + private RuleIndex ruleIndex = new RuleIndex(esTester.client()); - private WsActionTester tester = new WsActionTester(new TagsAction(issueIndex, ruleIndex, db.getDbClient(), TestDefaultOrganizationProvider.from(db))); + private WsActionTester tester = new WsActionTester(new TagsAction(issueIndex, ruleIndex, dbTester.getDbClient(), TestDefaultOrganizationProvider.from(dbTester))); private OrganizationDto organization; @Before public void before() { - organization = db.organizations().insert(); + organization = dbTester.organizations().insert(); } @Test @@ -92,15 +93,15 @@ public class TagsActionTest { @Test public void return_tags_from_rules() throws Exception { userSession.logIn(); - RuleDefinitionDto r = db.rules().insert(setSystemTags("tag1")); - ruleIndexer.indexRuleDefinition(r.getKey()); - db.rules().insertOrUpdateMetadata(r, organization, setTags("tag2")); - ruleIndexer.indexRuleExtension(organization, r.getKey()); + RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag1")); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); + dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag2")); + ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); - RuleDefinitionDto r2 = db.rules().insert(setSystemTags("tag3")); - ruleIndexer.indexRuleDefinition(r2.getKey()); - db.rules().insertOrUpdateMetadata(r2, organization, setTags("tag4", "tag5")); - ruleIndexer.indexRuleExtension(organization, r2.getKey()); + RuleDefinitionDto r2 = dbTester.rules().insert(setSystemTags("tag3")); + ruleIndexer.commitAndIndex(dbTester.getSession(), r2.getKey()); + dbTester.rules().insertOrUpdateMetadata(r2, organization, setTags("tag4", "tag5")); + ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r2.getKey()); String result = tester.newRequest() .setParam("organization", organization.getKey()) @@ -114,10 +115,10 @@ public class TagsActionTest { insertIssueWithBrowsePermission(insertRuleWithoutTags(), "tag1", "tag2"); insertIssueWithBrowsePermission(insertRuleWithoutTags(), "tag3", "tag4", "tag5"); - RuleDefinitionDto r = db.rules().insert(setSystemTags("tag6")); - ruleIndexer.indexRuleDefinition(r.getKey()); - db.rules().insertOrUpdateMetadata(r, organization, setTags("tag7")); - ruleIndexer.indexRuleExtension(organization, r.getKey()); + RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag6")); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); + dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag7")); + ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); String result = tester.newRequest() .setParam("organization", organization.getKey()) @@ -176,10 +177,10 @@ public class TagsActionTest { userSession.logIn(); insertIssueWithBrowsePermission(insertRuleWithoutTags(), "convention"); - RuleDefinitionDto r = db.rules().insert(setSystemTags("cwe")); - ruleIndexer.indexRuleDefinition(r.getKey()); - db.rules().insertOrUpdateMetadata(r, organization, setTags("security")); - ruleIndexer.indexRuleExtension(organization, r.getKey()); + RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("cwe")); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); + dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("security")); + ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); String result = tester.newRequest() .setParam("organization", organization.getKey()) @@ -220,7 +221,7 @@ public class TagsActionTest { } private RuleDefinitionDto insertRuleWithoutTags() { - return db.rules().insert(setSystemTags()); + return dbTester.rules().insert(setSystemTags()); } private void insertIssueWithBrowsePermission(RuleDefinitionDto rule, String... tags) { @@ -229,8 +230,8 @@ public class TagsActionTest { } private IssueDto insertIssueWithoutBrowsePermission(RuleDefinitionDto rule, String... tags) { - IssueDto issue = db.issues().insertIssue(organization, i -> i.setRule(rule).setTags(asList(tags))); - ComponentDto project = db.getDbClient().componentDao().selectByUuid(db.getSession(), issue.getProjectUuid()).get(); + IssueDto issue = dbTester.issues().insertIssue(organization, i -> i.setRule(rule).setTags(asList(tags))); + ComponentDto project = dbTester.getDbClient().componentDao().selectByUuid(dbTester.getSession(), issue.getProjectUuid()).get(); userSession.addProjectPermission(USER, project); issueIndexer.index(Collections.singletonList(issue.getKey())); return issue; diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/EnableSupportActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/EnableSupportActionTest.java index 900a24451ba..8e7dd830227 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/EnableSupportActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/EnableSupportActionTest.java @@ -27,6 +27,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.ArgumentCaptor; +import org.sonar.api.config.MapSettings; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.api.server.ws.WebService; @@ -39,12 +40,14 @@ import org.sonar.db.permission.template.PermissionTemplateGroupDto; import org.sonar.db.rule.RuleDefinitionDto; import org.sonar.db.user.GroupDto; import org.sonar.db.user.UserDto; +import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.exceptions.UnauthorizedException; import org.sonar.server.organization.DefaultOrganizationProvider; import org.sonar.server.organization.OrganizationFlags; import org.sonar.server.organization.OrganizationFlagsImpl; import org.sonar.server.organization.TestDefaultOrganizationProvider; +import org.sonar.server.rule.index.RuleIndexDefinition; import org.sonar.server.rule.index.RuleIndexer; import org.sonar.server.tester.UserSessionRule; import org.sonar.server.usergroups.DefaultGroupCreatorImpl; @@ -54,7 +57,8 @@ import org.sonar.server.ws.WsActionTester; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; -import static org.mockito.Mockito.mock; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.sonar.db.permission.OrganizationPermission.ADMINISTER; @@ -65,20 +69,23 @@ public class EnableSupportActionTest { @Rule public UserSessionRule userSession = UserSessionRule.standalone(); @Rule - public DbTester db = DbTester.create(); + public DbTester dbTester = DbTester.create(); + @Rule + public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings())); + - private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); - private OrganizationFlags organizationFlags = new OrganizationFlagsImpl(db.getDbClient()); - private RuleIndexer ruleIndexer = mock(RuleIndexer.class); - private EnableSupportAction underTest = new EnableSupportAction(userSession, db.getDbClient(), defaultOrganizationProvider, organizationFlags, - new DefaultGroupCreatorImpl(db.getDbClient()), new DefaultGroupFinder(db.getDbClient()), ruleIndexer); + private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester); + private OrganizationFlags organizationFlags = new OrganizationFlagsImpl(dbTester.getDbClient()); + private RuleIndexer ruleIndexer = spy(new RuleIndexer(esTester.client(), dbTester.getDbClient())); + private EnableSupportAction underTest = new EnableSupportAction(userSession, dbTester.getDbClient(), defaultOrganizationProvider, organizationFlags, + new DefaultGroupCreatorImpl(dbTester.getDbClient()), new DefaultGroupFinder(dbTester.getDbClient()), ruleIndexer); private WsActionTester tester = new WsActionTester(underTest); @Test public void enabling_support_saves_internal_property_and_flags_caller_as_root() { - UserDto user = db.users().insertUser(); - UserDto otherUser = db.users().insertUser(); - db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users"); + UserDto user = dbTester.users().insertUser(); + UserDto otherUser = dbTester.users().insertUser(); + dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users"); verifyFeatureEnabled(false); verifyRoot(user, false); verifyRoot(otherUser, false); @@ -93,48 +100,48 @@ public class EnableSupportActionTest { @Test public void enabling_support_creates_default_members_group_and_associate_org_members() throws Exception { - OrganizationDto defaultOrganization = db.getDefaultOrganization(); - OrganizationDto anotherOrganization = db.organizations().insert(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - UserDto userInAnotherOrganization = db.users().insertUser(); - db.organizations().addMember(defaultOrganization, user1); - db.organizations().addMember(defaultOrganization, user2); - db.organizations().addMember(anotherOrganization, userInAnotherOrganization); - db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users"); + OrganizationDto defaultOrganization = dbTester.getDefaultOrganization(); + OrganizationDto anotherOrganization = dbTester.organizations().insert(); + UserDto user1 = dbTester.users().insertUser(); + UserDto user2 = dbTester.users().insertUser(); + UserDto userInAnotherOrganization = dbTester.users().insertUser(); + dbTester.organizations().addMember(defaultOrganization, user1); + dbTester.organizations().addMember(defaultOrganization, user2); + dbTester.organizations().addMember(anotherOrganization, userInAnotherOrganization); + dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users"); logInAsSystemAdministrator(user1.getLogin()); call(); - Optional<Integer> defaultGroupId = db.getDbClient().organizationDao().getDefaultGroupId(db.getSession(), defaultOrganization.getUuid()); + Optional<Integer> defaultGroupId = dbTester.getDbClient().organizationDao().getDefaultGroupId(dbTester.getSession(), defaultOrganization.getUuid()); assertThat(defaultGroupId).isPresent(); - GroupDto membersGroup = db.getDbClient().groupDao().selectById(db.getSession(), defaultGroupId.get()); + GroupDto membersGroup = dbTester.getDbClient().groupDao().selectById(dbTester.getSession(), defaultGroupId.get()); assertThat(membersGroup).isNotNull(); assertThat(membersGroup.getName()).isEqualTo("Members"); - assertThat(db.getDbClient().groupMembershipDao().selectGroupIdsByUserId(db.getSession(), user1.getId())).containsOnly(defaultGroupId.get()); - assertThat(db.getDbClient().groupMembershipDao().selectGroupIdsByUserId(db.getSession(), user2.getId())).containsOnly(defaultGroupId.get()); - assertThat(db.getDbClient().groupMembershipDao().selectGroupIdsByUserId(db.getSession(), userInAnotherOrganization.getId())).isEmpty(); + assertThat(dbTester.getDbClient().groupMembershipDao().selectGroupIdsByUserId(dbTester.getSession(), user1.getId())).containsOnly(defaultGroupId.get()); + assertThat(dbTester.getDbClient().groupMembershipDao().selectGroupIdsByUserId(dbTester.getSession(), user2.getId())).containsOnly(defaultGroupId.get()); + assertThat(dbTester.getDbClient().groupMembershipDao().selectGroupIdsByUserId(dbTester.getSession(), userInAnotherOrganization.getId())).isEmpty(); } @Test public void enabling_support_copy_sonar_users_permissions_to_members_group() throws Exception { - OrganizationDto defaultOrganization = db.getDefaultOrganization(); - UserDto user = db.users().insertUser(); - GroupDto sonarUsersGroup = db.users().insertDefaultGroup(defaultOrganization, "sonar-users"); - ComponentDto project = db.components().insertPrivateProject(defaultOrganization); - db.users().insertPermissionOnGroup(sonarUsersGroup, "user"); - db.users().insertProjectPermissionOnGroup(sonarUsersGroup, "codeviewer", project); + OrganizationDto defaultOrganization = dbTester.getDefaultOrganization(); + UserDto user = dbTester.users().insertUser(); + GroupDto sonarUsersGroup = dbTester.users().insertDefaultGroup(defaultOrganization, "sonar-users"); + ComponentDto project = dbTester.components().insertPrivateProject(defaultOrganization); + dbTester.users().insertPermissionOnGroup(sonarUsersGroup, "user"); + dbTester.users().insertProjectPermissionOnGroup(sonarUsersGroup, "codeviewer", project); // Should be ignored - GroupDto anotherGroup = db.users().insertGroup(); - db.users().insertPermissionOnGroup(anotherGroup, "admin"); + GroupDto anotherGroup = dbTester.users().insertGroup(); + dbTester.users().insertPermissionOnGroup(anotherGroup, "admin"); logInAsSystemAdministrator(user.getLogin()); call(); - int defaultGroupId = db.getDbClient().organizationDao().getDefaultGroupId(db.getSession(), defaultOrganization.getUuid()).get(); + int defaultGroupId = dbTester.getDbClient().organizationDao().getDefaultGroupId(dbTester.getSession(), defaultOrganization.getUuid()).get(); assertThat(defaultGroupId).isNotEqualTo(sonarUsersGroup.getId()); List<GroupPermissionDto> result = new ArrayList<>(); - db.getDbClient().groupPermissionDao().selectAllPermissionsByGroupId(db.getSession(), defaultOrganization.getUuid(), defaultGroupId, + dbTester.getDbClient().groupPermissionDao().selectAllPermissionsByGroupId(dbTester.getSession(), defaultOrganization.getUuid(), defaultGroupId, context -> result.add((GroupPermissionDto) context.getResultObject())); assertThat(result).extracting(GroupPermissionDto::getResourceId, GroupPermissionDto::getRole).containsOnly( tuple(null, "user"), tuple(project.getId(), "codeviewer")); @@ -142,36 +149,36 @@ public class EnableSupportActionTest { @Test public void enabling_support_copy_sonar_users_permission_templates_to_members_group() throws Exception { - OrganizationDto defaultOrganization = db.getDefaultOrganization(); - UserDto user = db.users().insertUser(); - GroupDto sonarUsersGroup = db.users().insertDefaultGroup(defaultOrganization, "sonar-users"); - PermissionTemplateDto permissionTemplate = db.permissionTemplates().insertTemplate(db.getDefaultOrganization()); - db.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "user"); - db.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "admin"); + OrganizationDto defaultOrganization = dbTester.getDefaultOrganization(); + UserDto user = dbTester.users().insertUser(); + GroupDto sonarUsersGroup = dbTester.users().insertDefaultGroup(defaultOrganization, "sonar-users"); + PermissionTemplateDto permissionTemplate = dbTester.permissionTemplates().insertTemplate(dbTester.getDefaultOrganization()); + dbTester.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "user"); + dbTester.permissionTemplates().addGroupToTemplate(permissionTemplate, sonarUsersGroup, "admin"); // Should be ignored - GroupDto otherGroup = db.users().insertGroup(); - db.permissionTemplates().addGroupToTemplate(permissionTemplate, otherGroup, "user"); + GroupDto otherGroup = dbTester.users().insertGroup(); + dbTester.permissionTemplates().addGroupToTemplate(permissionTemplate, otherGroup, "user"); logInAsSystemAdministrator(user.getLogin()); call(); - int defaultGroupId = db.getDbClient().organizationDao().getDefaultGroupId(db.getSession(), defaultOrganization.getUuid()).get(); - assertThat(db.getDbClient().permissionTemplateDao().selectAllGroupPermissionTemplatesByGroupId(db.getSession(), defaultGroupId)) + int defaultGroupId = dbTester.getDbClient().organizationDao().getDefaultGroupId(dbTester.getSession(), defaultOrganization.getUuid()).get(); + assertThat(dbTester.getDbClient().permissionTemplateDao().selectAllGroupPermissionTemplatesByGroupId(dbTester.getSession(), defaultGroupId)) .extracting(PermissionTemplateGroupDto::getGroupId, PermissionTemplateGroupDto::getPermission) .containsOnly(tuple(defaultGroupId, "user"), tuple(defaultGroupId, "admin")); } @Test public void enabling_organizations_should_remove_template_rule_and_custom_rule() { - RuleDefinitionDto normal = db.rules().insert(); - RuleDefinitionDto template = db.rules().insert(r -> r.setIsTemplate(true)); - RuleDefinitionDto custom = db.rules().insert(r -> r.setTemplateId(template.getId())); + RuleDefinitionDto normal = dbTester.rules().insert(); + RuleDefinitionDto template = dbTester.rules().insert(r -> r.setIsTemplate(true)); + RuleDefinitionDto custom = dbTester.rules().insert(r -> r.setTemplateId(template.getId())); - UserDto user = db.users().insertUser(); - db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users"); + UserDto user = dbTester.users().insertUser(); + dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users"); logInAsSystemAdministrator(user.getLogin()); - assertThat(db.getDbClient().ruleDao().selectAllDefinitions(db.getSession())) + assertThat(dbTester.getDbClient().ruleDao().selectAllDefinitions(dbTester.getSession())) .extracting(RuleDefinitionDto::getKey, RuleDefinitionDto::getStatus) .containsExactlyInAnyOrder( tuple(normal.getKey(), RuleStatus.READY), @@ -181,7 +188,7 @@ public class EnableSupportActionTest { call(); - assertThat(db.getDbClient().ruleDao().selectAllDefinitions(db.getSession())) + assertThat(dbTester.getDbClient().ruleDao().selectAllDefinitions(dbTester.getSession())) .extracting(RuleDefinitionDto::getKey, RuleDefinitionDto::getStatus) .containsExactlyInAnyOrder( tuple(normal.getKey(), RuleStatus.READY), @@ -192,15 +199,15 @@ public class EnableSupportActionTest { @SuppressWarnings("unchecked") Class<ArrayList<RuleKey>> listClass = (Class<ArrayList<RuleKey>>)(Class)ArrayList.class; ArgumentCaptor<ArrayList<RuleKey>> indexedRuleKeys = ArgumentCaptor.forClass(listClass); - verify(ruleIndexer).indexRuleDefinitions(indexedRuleKeys.capture()); + verify(ruleIndexer).commitAndIndex(any(), indexedRuleKeys.capture()); assertThat(indexedRuleKeys.getValue()).containsExactlyInAnyOrder(template.getKey(), custom.getKey()); } @Test public void throw_IAE_when_members_group_already_exists() throws Exception { - UserDto user = db.users().insertUser(); - db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users"); - db.users().insertGroup(db.getDefaultOrganization(), "Members"); + UserDto user = dbTester.users().insertUser(); + dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users"); + dbTester.users().insertGroup(dbTester.getDefaultOrganization(), "Members"); logInAsSystemAdministrator(user.getLogin()); expectedException.expect(IllegalArgumentException.class); @@ -231,7 +238,7 @@ public class EnableSupportActionTest { @Test public void throw_ISE_when_default_organization_has_not_default_group() { - UserDto user = db.users().insertUser(); + UserDto user = dbTester.users().insertUser(); logInAsSystemAdministrator(user.getLogin()); expectedException.expect(IllegalStateException.class); @@ -242,7 +249,7 @@ public class EnableSupportActionTest { @Test public void do_nothing_if_support_is_already_enabled() { - db.users().insertDefaultGroup(db.getDefaultOrganization(), "sonar-users"); + dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users"); logInAsSystemAdministrator("foo"); call(); @@ -265,7 +272,7 @@ public class EnableSupportActionTest { } private void logInAsSystemAdministrator(String login) { - userSession.logIn(login).addPermission(ADMINISTER, db.getDefaultOrganization()); + userSession.logIn(login).addPermission(ADMINISTER, dbTester.getDefaultOrganization()); } private void call() { @@ -274,10 +281,10 @@ public class EnableSupportActionTest { } private void verifyFeatureEnabled(boolean enabled) { - assertThat(organizationFlags.isEnabled(db.getSession())).isEqualTo(enabled); + assertThat(organizationFlags.isEnabled(dbTester.getSession())).isEqualTo(enabled); } private void verifyRoot(UserDto user, boolean root) { - db.rootFlag().verify(user.getLogin(), root); + dbTester.rootFlag().verify(user.getLogin(), root); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ChangeParentActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ChangeParentActionTest.java index 4416f3eb8a8..edcd0787fe9 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ChangeParentActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/ChangeParentActionTest.java @@ -21,7 +21,6 @@ package org.sonar.server.qualityprofile.ws; import java.util.Collections; import java.util.List; -import java.util.stream.Stream; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,7 +33,6 @@ import org.sonar.api.rule.RuleStatus; import org.sonar.api.rule.Severity; import org.sonar.api.server.ws.WebService; import org.sonar.api.utils.System2; -import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; @@ -65,6 +63,7 @@ import org.sonar.server.util.TypeValidations; import org.sonar.server.ws.TestRequest; import org.sonar.server.ws.WsActionTester; +import static java.util.Arrays.asList; import static java.util.Collections.emptySet; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; @@ -158,7 +157,7 @@ public class ChangeParentActionTest { RuleDefinitionDto rule1 = createRule(); createActiveRule(rule1, parent1); - ruleIndexer.indexRuleDefinition(rule1.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule1.getKey()); activeRuleIndexer.indexOnStartup(emptySet()); assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty(); @@ -188,7 +187,7 @@ public class ChangeParentActionTest { RuleDefinitionDto rule2 = createRule(); createActiveRule(rule1, parent1); createActiveRule(rule2, parent2); - ruleIndexer.indexRuleDefinitions(Stream.of(rule1, rule2).map(RuleDefinitionDto::getKey).collect(MoreCollectors.toList())); + ruleIndexer.commitAndIndex(dbSession, asList(rule1.getKey(), rule2.getKey())); activeRuleIndexer.indexOnStartup(emptySet()); // Set parent 1 @@ -216,7 +215,7 @@ public class ChangeParentActionTest { RuleDefinitionDto rule1 = createRule(); createActiveRule(rule1, parent); - ruleIndexer.indexRuleDefinition(rule1.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule1.getKey()); activeRuleIndexer.indexOnStartup(emptySet()); // Set parent @@ -244,7 +243,7 @@ public class ChangeParentActionTest { RuleDefinitionDto rule2 = createRule(); createActiveRule(rule1, parent1); createActiveRule(rule2, parent2); - ruleIndexer.indexRuleDefinition(rule1.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule1.getKey()); activeRuleIndexer.indexOnStartup(emptySet()); assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty(); @@ -303,7 +302,7 @@ public class ChangeParentActionTest { RuleDefinitionDto rule1 = createRule(); createActiveRule(rule1, parent); - ruleIndexer.indexRuleDefinition(rule1.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule1.getKey()); activeRuleIndexer.indexOnStartup(emptySet()); assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, child.getKee())).isEmpty(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/CreateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/CreateActionTest.java index 8c560ae1b93..33c34e2daa3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/CreateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/CreateActionTest.java @@ -252,7 +252,7 @@ public class CreateActionTest { private void insertRule(RuleDefinitionDto ruleDto) { dbClient.ruleDao().insert(dbSession, ruleDto); dbSession.commit(); - ruleIndexer.indexRuleDefinition(ruleDto.getKey()); + ruleIndexer.commitAndIndex(dbSession, ruleDto.getKey()); } private CreateWsResponse executeRequest(String name, String language) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/InheritanceActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/InheritanceActionTest.java index 4e21a465e21..65841db6c72 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/InheritanceActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/InheritanceActionTest.java @@ -59,6 +59,7 @@ import org.sonar.server.util.TypeValidations; import org.sonar.server.ws.WsActionTester; import org.sonarqube.ws.QualityProfiles.InheritanceWsResponse; +import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.sonar.server.qualityprofile.QProfileTesting.newQProfileDto; import static org.sonar.test.JsonAssert.assertJson; @@ -68,7 +69,7 @@ import static org.sonarqube.ws.client.qualityprofile.QualityProfileWsParameters. public class InheritanceActionTest { @Rule - public DbTester db = DbTester.create(); + public DbTester dbTester = DbTester.create(); @Rule public EsTester es = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig())); @Rule @@ -87,12 +88,12 @@ public class InheritanceActionTest { @Before public void setUp() { - dbClient = db.getDbClient(); - dbSession = db.getSession(); + dbClient = dbTester.getDbClient(); + dbSession = dbTester.getSession(); esClient = es.client(); ruleIndexer = new RuleIndexer(esClient, dbClient); activeRuleIndexer = new ActiveRuleIndexer(dbClient, esClient, new ActiveRuleIteratorFactory(dbClient)); - TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); + TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(dbTester); underTest = new InheritanceAction( dbClient, new QProfileWsSupport(dbClient, userSession, defaultOrganizationProvider), @@ -106,7 +107,7 @@ public class InheritanceActionTest { new TypeValidations(new ArrayList<>()), activeRuleIndexer, userSession); - organization = db.organizations().insert(); + organization = dbTester.organizations().insert(); } @Test @@ -118,7 +119,7 @@ public class InheritanceActionTest { /* * sonar way (2) <- companyWide (2) <- buWide (2, 1 overriding) <- (forProject1 (2), forProject2 (2)) */ - QProfileDto sonarway = db.qualityProfiles().insert(organization, p -> p.setKee("xoo-sonar-way").setLanguage("xoo").setName("Sonar way").setIsBuiltIn(true)); + QProfileDto sonarway = dbTester.qualityProfiles().insert(organization, p -> p.setKee("xoo-sonar-way").setLanguage("xoo").setName("Sonar way").setIsBuiltIn(true)); ActiveRuleDto activeRule1 = createActiveRule(rule1, sonarway); ActiveRuleDto activeRule2 = createActiveRule(rule2, sonarway); @@ -153,22 +154,18 @@ public class InheritanceActionTest { @Test public void inheritance_parent_child() throws Exception { - RuleDefinitionDto rule1 = db.rules().insert(); - ruleIndexer.indexRuleDefinition(rule1.getKey()); - - RuleDefinitionDto rule2 = db.rules().insert(); - ruleIndexer.indexRuleDefinition(rule1.getKey()); - - RuleDefinitionDto rule3 = db.rules().insert(); - ruleIndexer.indexRuleDefinition(rule1.getKey()); - - QProfileDto parent = db.qualityProfiles().insert(organization); - db.qualityProfiles().activateRule(parent, rule1); - db.qualityProfiles().activateRule(parent, rule2); + RuleDefinitionDto rule1 = dbTester.rules().insert(); + RuleDefinitionDto rule2 = dbTester.rules().insert(); + RuleDefinitionDto rule3 = dbTester.rules().insert(); + ruleIndexer.commitAndIndex(dbTester.getSession(), asList(rule1.getKey(), rule2.getKey(), rule3.getKey())); + + QProfileDto parent = dbTester.qualityProfiles().insert(organization); + dbTester.qualityProfiles().activateRule(parent, rule1); + dbTester.qualityProfiles().activateRule(parent, rule2); long parentRules = 2; - QProfileDto child = db.qualityProfiles().insert(organization, q -> q.setParentKee(parent.getKee())); - db.qualityProfiles().activateRule(child, rule3); + QProfileDto child = dbTester.qualityProfiles().insert(organization, q -> q.setParentKee(parent.getKee())); + dbTester.qualityProfiles().activateRule(child, rule3); long childRules = 1; activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes()); @@ -191,11 +188,11 @@ public class InheritanceActionTest { @Test public void inheritance_ignores_removed_rules() throws Exception { - RuleDefinitionDto rule = db.rules().insert(r -> r.setStatus(RuleStatus.REMOVED)); - ruleIndexer.indexRuleDefinition(rule.getKey()); + RuleDefinitionDto rule = dbTester.rules().insert(r -> r.setStatus(RuleStatus.REMOVED)); + ruleIndexer.commitAndIndex(dbTester.getSession(), rule.getKey()); - QProfileDto profile = db.qualityProfiles().insert(organization); - db.qualityProfiles().activateRule(profile, rule); + QProfileDto profile = dbTester.qualityProfiles().insert(organization); + dbTester.qualityProfiles().activateRule(profile, rule); long activeRules = 0; activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes()); @@ -266,8 +263,7 @@ public class InheritanceActionTest { .setUpdatedAt(now) .setCreatedAt(now); dbClient.ruleDao().insert(dbSession, rule); - dbSession.commit(); - ruleIndexer.indexRuleDefinition(rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule.getKey()); return rule; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/QProfilesWsMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/QProfilesWsMediumTest.java index 624c04de5fb..172e67b4f68 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/QProfilesWsMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/qualityprofile/ws/QProfilesWsMediumTest.java @@ -109,8 +109,7 @@ public class QProfilesWsMediumTest { QProfileDto profile = createProfile("java"); RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto"); createActiveRule(rule, profile); - dbSession.commit(); - ruleIndexer.indexRuleDefinition(rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule.getKey()); activeRuleIndexer.indexOnStartup(activeRuleIndexer.getIndexTypes()); // 0. Assert No Active Rule for profile @@ -209,8 +208,7 @@ public class QProfilesWsMediumTest { public void activate_rule() throws Exception { QProfileDto profile = createProfile("java"); RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto"); - dbSession.commit(); - ruleIndexer.indexRuleDefinition(rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule.getKey()); // 0. Assert No Active Rule for profile assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, profile.getKee())).isEmpty(); @@ -230,8 +228,7 @@ public class QProfilesWsMediumTest { public void activate_rule_diff_languages() throws Exception { QProfileDto profile = createProfile("java"); RuleDefinitionDto rule = createRule("php", "toto"); - dbSession.commit(); - ruleIndexer.indexRuleDefinition(rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule.getKey()); // 0. Assert No Active Rule for profile assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, profile.getKee())).isEmpty(); @@ -253,8 +250,7 @@ public class QProfilesWsMediumTest { public void activate_rule_override_severity() throws Exception { QProfileDto profile = createProfile("java"); RuleDefinitionDto rule = createRule(profile.getLanguage(), "toto"); - dbSession.commit(); - ruleIndexer.indexRuleDefinition(rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule.getKey()); // 0. Assert No Active Rule for profile assertThat(dbClient.activeRuleDao().selectByProfileUuid(dbSession, profile.getKee())).isEmpty(); @@ -456,8 +452,7 @@ public class QProfilesWsMediumTest { .setSeverity(Severity.BLOCKER) .setStatus(RuleStatus.READY); dbClient.ruleDao().insert(dbSession, rule); - dbSession.commit(); - ruleIndexer.indexRuleDefinition(rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule.getKey()); return rule; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/RuleCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/RuleCreatorTest.java index b85b1fea380..9dbda10be8d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/RuleCreatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/RuleCreatorTest.java @@ -63,17 +63,17 @@ public class RuleCreatorTest { public ExpectedException expectedException = ExpectedException.none(); @Rule - public DbTester db = DbTester.create(system2); + public DbTester dbTester = DbTester.create(system2); @Rule public EsTester es = new EsTester(new RuleIndexDefinition(new MapSettings().asConfig())); private RuleIndex ruleIndex = new RuleIndex(es.client()); - private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient()); - private DbSession dbSession = db.getSession(); + private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), dbTester.getDbClient()); + private DbSession dbSession = dbTester.getSession(); - private RuleCreator underTest = new RuleCreator(system2, new RuleIndexer(es.client(), db.getDbClient()), db.getDbClient(), newFullTypeValidations(), - TestDefaultOrganizationProvider.from(db)); + private RuleCreator underTest = new RuleCreator(system2, new RuleIndexer(es.client(), dbTester.getDbClient()), dbTester.getDbClient(), newFullTypeValidations(), + TestDefaultOrganizationProvider.from(dbTester)); @Test public void create_custom_rule() { @@ -88,7 +88,7 @@ public class RuleCreatorTest { .setParameters(ImmutableMap.of("regex", "a.*")); RuleKey customRuleKey = underTest.create(dbSession, newRule); - RuleDto rule = db.getDbClient().ruleDao().selectOrFailByKey(dbSession, db.getDefaultOrganization(), customRuleKey); + RuleDto rule = dbTester.getDbClient().ruleDao().selectOrFailByKey(dbSession, dbTester.getDefaultOrganization(), customRuleKey); assertThat(rule).isNotNull(); assertThat(rule.getKey()).isEqualTo(RuleKey.of("java", "CUSTOM_RULE")); assertThat(rule.getTemplateId()).isEqualTo(templateRule.getId()); @@ -105,7 +105,7 @@ public class RuleCreatorTest { assertThat(rule.getTags()).containsOnly("usertag1", "usertag2"); assertThat(rule.getSystemTags()).containsOnly("tag1", "tag4"); - List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); + List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); assertThat(params).hasSize(1); RuleParamDto param = params.get(0); @@ -132,7 +132,7 @@ public class RuleCreatorTest { RuleKey customRuleKey = underTest.create(dbSession, newRule); - List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); + List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); assertThat(params).hasSize(1); RuleParamDto param = params.get(0); assertThat(param.getName()).isEqualTo("regex"); @@ -153,7 +153,7 @@ public class RuleCreatorTest { RuleKey customRuleKey = underTest.create(dbSession, newRule); - List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); + List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); assertThat(params).hasSize(1); RuleParamDto param = params.get(0); assertThat(param.getName()).isEqualTo("myIntegers"); @@ -175,7 +175,7 @@ public class RuleCreatorTest { RuleKey customRuleKey = underTest.create(dbSession, newRule); - List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); + List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); assertThat(params).hasSize(1); RuleParamDto param = params.get(0); assertThat(param.getName()).isEqualTo("myIntegers"); @@ -237,8 +237,8 @@ public class RuleCreatorTest { .setDescription("Old description") .setDescriptionFormat(Format.MARKDOWN) .setSeverity(Severity.INFO); - db.rules().insert(rule.getDefinition()); - db.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*")); + dbTester.rules().insert(rule.getDefinition()); + dbTester.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*")); dbSession.commit(); // Create custom rule with same key, but with different values @@ -250,7 +250,7 @@ public class RuleCreatorTest { .setParameters(ImmutableMap.of("regex", "c.*")); RuleKey customRuleKey = underTest.create(dbSession, newRule); - RuleDefinitionDto result = db.getDbClient().ruleDao().selectOrFailDefinitionByKey(dbSession, customRuleKey); + RuleDefinitionDto result = dbTester.getDbClient().ruleDao().selectOrFailDefinitionByKey(dbSession, customRuleKey); assertThat(result.getKey()).isEqualTo(RuleKey.of("java", key)); assertThat(result.getStatus()).isEqualTo(RuleStatus.READY); @@ -259,7 +259,7 @@ public class RuleCreatorTest { assertThat(result.getDescription()).isEqualTo("Old description"); assertThat(result.getSeverityString()).isEqualTo(Severity.INFO); - List<RuleParamDto> params = db.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); + List<RuleParamDto> params = dbTester.getDbClient().ruleDao().selectRuleParamsByRuleKey(dbSession, customRuleKey); assertThat(params).hasSize(1); assertThat(params.get(0).getDefaultValue()).isEqualTo("a.*"); } @@ -276,8 +276,8 @@ public class RuleCreatorTest { .setName("Old name") .setDescription("Old description") .setSeverity(Severity.INFO); - db.rules().insert(rule.getDefinition()); - db.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*")); + dbTester.rules().insert(rule.getDefinition()); + dbTester.rules().insertRuleParam(rule.getDefinition(), param -> param.setDefaultValue("a.*")); dbSession.commit(); // Create custom rule with same key, but with different values @@ -427,7 +427,7 @@ public class RuleCreatorTest { public void fail_to_create_custom_rule_when_wrong_rule_template() { // insert rule RuleDefinitionDto rule = newRule(RuleKey.of("java", "S001")).setIsTemplate(false); - db.rules().insert(rule); + dbTester.rules().insert(rule); dbSession.commit(); expectedException.expect(IllegalArgumentException.class); @@ -458,7 +458,7 @@ public class RuleCreatorTest { } private RuleDto createTemplateRule() { - RuleDto templateRule = RuleTesting.newDto(RuleKey.of("java", "S001"), db.getDefaultOrganization()) + RuleDto templateRule = RuleTesting.newDto(RuleKey.of("java", "S001"), dbTester.getDefaultOrganization()) .setIsTemplate(true) .setLanguage("java") .setConfigKey("S001") @@ -470,10 +470,10 @@ public class RuleCreatorTest { .setSystemTags(Sets.newHashSet("tag1", "tag4")) .setCreatedAt(new Date().getTime()) .setUpdatedAt(new Date().getTime()); - db.rules().insert(templateRule.getDefinition()); - db.rules().insertOrUpdateMetadata(templateRule.getMetadata().setRuleId(templateRule.getId())); - db.rules().insertRuleParam(templateRule.getDefinition(), param -> param.setName("regex").setType("STRING").setDescription("Reg ex").setDefaultValue(".*")); - ruleIndexer.indexRuleDefinition(templateRule.getDefinition().getKey()); + dbTester.rules().insert(templateRule.getDefinition()); + dbTester.rules().insertOrUpdateMetadata(templateRule.getMetadata().setRuleId(templateRule.getId())); + dbTester.rules().insertRuleParam(templateRule.getDefinition(), param -> param.setName("regex").setType("STRING").setDescription("Reg ex").setDefaultValue(".*")); + ruleIndexer.commitAndIndex(dbTester.getSession(), templateRule.getDefinition().getKey()); return templateRule; } @@ -488,9 +488,9 @@ public class RuleCreatorTest { .setGapDescription("desc") .setCreatedAt(new Date().getTime()) .setUpdatedAt(new Date().getTime()); - db.rules().insert(templateRule); - db.rules().insertRuleParam(templateRule, param -> param.setName("myIntegers").setType("INTEGER,multiple=true,values=1;2;3").setDescription("My Integers").setDefaultValue("1")); - ruleIndexer.indexRuleDefinition(templateRule.getKey()); + dbTester.rules().insert(templateRule); + dbTester.rules().insertRuleParam(templateRule, param -> param.setName("myIntegers").setType("INTEGER,multiple=true,values=1;2;3").setDescription("My Integers").setDefaultValue("1")); + ruleIndexer.commitAndIndex(dbTester.getSession(), templateRule.getKey()); return templateRule; } @@ -505,9 +505,9 @@ public class RuleCreatorTest { .setGapDescription("desc") .setCreatedAt(new Date().getTime()) .setUpdatedAt(new Date().getTime()); - db.rules().insert(templateRule); - db.rules().insertRuleParam(templateRule, param -> param.setName("first").setType("INTEGER").setDescription("First integer").setDefaultValue("0")); - db.rules().insertRuleParam(templateRule, param -> param.setName("second").setType("INTEGER").setDescription("Second integer").setDefaultValue("0")); + dbTester.rules().insert(templateRule); + dbTester.rules().insertRuleParam(templateRule, param -> param.setName("first").setType("INTEGER").setDescription("First integer").setDefaultValue("0")); + dbTester.rules().insertRuleParam(templateRule, param -> param.setName("second").setType("INTEGER").setDescription("Second integer").setDefaultValue("0")); return templateRule; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java index 93e8ccba225..95a47a88e51 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java @@ -20,6 +20,7 @@ package org.sonar.server.rule.index; import com.google.common.collect.ImmutableSet; +import java.util.Collections; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; @@ -68,16 +69,14 @@ public class RuleIndexerTest { @Test public void index_nothing() { - // underTest.index(Iterators.emptyIterator()); + underTest.index(dbSession, Collections.emptyList()); assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(0L); } @Test public void index() { dbClient.ruleDao().insert(dbSession, rule); - dbSession.commit(); - - underTest.indexRuleDefinition(rule.getKey()); + underTest.commitAndIndex(dbSession, rule.getKey()); assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); } @@ -87,13 +86,12 @@ public class RuleIndexerTest { // Create and Index rule dbClient.ruleDao().insert(dbSession, rule.setStatus(RuleStatus.READY)); dbSession.commit(); - underTest.indexRuleDefinition(rule.getKey()); + underTest.commitAndIndex(dbTester.getSession(), rule.getKey()); assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); // Remove rule dbTester.getDbClient().ruleDao().update(dbTester.getSession(), rule.setStatus(RuleStatus.READY).setUpdatedAt(2000000000000L)); - dbTester.getSession().commit(); - underTest.indexRuleDefinition(rule.getKey()); + underTest.commitAndIndex(dbTester.getSession(), rule.getKey()); assertThat(esTester.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); } @@ -101,10 +99,10 @@ public class RuleIndexerTest { @Test public void index_rule_extension_with_long_id() { RuleDefinitionDto rule = dbTester.rules().insert(r -> r.setRuleKey(RuleTesting.randomRuleKeyOfMaximumLength())); - underTest.indexRuleDefinition(rule.getKey()); + underTest.commitAndIndex(dbTester.getSession(), rule.getKey()); OrganizationDto organization = dbTester.organizations().insert(); dbTester.rules().insertOrUpdateMetadata(rule, organization, m -> m.setTags(ImmutableSet.of("bla"))); - underTest.indexRuleExtension(organization, rule.getKey()); + underTest.commitAndIndex(dbTester.getSession(), organization, rule.getKey()); RuleExtensionDoc doc = new RuleExtensionDoc() .setRuleKey(rule.getKey()) diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/DeleteActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/DeleteActionTest.java index 6bd1543aea8..06971a1b467 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/DeleteActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/DeleteActionTest.java @@ -19,42 +19,50 @@ */ package org.sonar.server.rule.ws; +import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.mockito.Mockito; +import org.sonar.api.config.MapSettings; import org.sonar.api.rule.RuleStatus; import org.sonar.api.utils.System2; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.rule.RuleDefinitionDto; +import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.exceptions.UnauthorizedException; import org.sonar.server.organization.DefaultOrganizationProvider; import org.sonar.server.organization.TestDefaultOrganizationProvider; import org.sonar.server.qualityprofile.RuleActivator; +import org.sonar.server.rule.index.RuleIndexDefinition; import org.sonar.server.rule.index.RuleIndexer; import org.sonar.server.tester.UserSessionRule; import org.sonar.server.ws.WsActionTester; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import static org.sonar.db.permission.OrganizationPermission.ADMINISTER_QUALITY_PROFILES; public class DeleteActionTest { private static final long PAST = 10000L; - @org.junit.Rule + @Rule public UserSessionRule userSession = UserSessionRule.standalone(); - @org.junit.Rule + @Rule public DbTester dbTester = DbTester.create(); - @org.junit.Rule + @Rule + public EsTester esTester = new EsTester(new RuleIndexDefinition(new MapSettings())); + @Rule public ExpectedException thrown = ExpectedException.none(); private DbClient dbClient = dbTester.getDbClient(); private DbSession dbSession = dbTester.getSession(); - private RuleIndexer ruleIndexer = mock(RuleIndexer.class); + private RuleIndexer ruleIndexer = spy(new RuleIndexer(esTester.client(), dbClient)); private RuleActivator ruleActivator = mock(RuleActivator.class); private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.fromUuid("ORG1"); private RuleWsSupport ruleWsSupport = new RuleWsSupport(mock(DbClient.class), userSession, defaultOrganizationProvider); @@ -79,8 +87,7 @@ public class DeleteActionTest { .setParam("key", customRule.getKey().toString()) .execute(); - Mockito.verify(ruleIndexer).indexRuleDefinition(eq(customRule.getKey())); - Mockito.verifyNoMoreInteractions(ruleIndexer); + verify(ruleIndexer).commitAndIndex(any(), eq(customRule.getKey())); // Verify custom rule has status REMOVED RuleDefinitionDto customRuleReloaded = dbClient.ruleDao().selectOrFailDefinitionByKey(dbSession, customRule.getKey()); diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionMediumTest.java index f0de64894ce..1d633e366f8 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionMediumTest.java @@ -26,10 +26,10 @@ public class SearchActionMediumTest { // @Test // public void search_profile_active_rules_with_inheritance() throws Exception { // QProfileDto profile = QProfileTesting.newXooP1(defaultOrganizationDto); - // tester.get(QualityProfileDao.class).insert(dbSession, profile); + // esTester.get(QualityProfileDao.class).insert(dbSession, profile); // // QProfileDto profile2 = QProfileTesting.newXooP2(defaultOrganizationDto).setParentKee(profile.getKee()); - // tester.get(QualityProfileDao.class).insert(dbSession, profile2); + // esTester.get(QualityProfileDao.class).insert(dbSession, profile2); // // dbSession.commit(); // @@ -37,15 +37,15 @@ public class SearchActionMediumTest { // insertRule(rule); // // ActiveRuleDto activeRule = newActiveRule(profile, rule); - // tester.get(ActiveRuleDao.class).insert(dbSession, activeRule); + // esTester.get(ActiveRuleDao.class).insert(dbSession, activeRule); // ActiveRuleDto activeRule2 = newActiveRule(profile2, rule).setInheritance(ActiveRuleDto.OVERRIDES).setSeverity(Severity.CRITICAL); - // tester.get(ActiveRuleDao.class).insert(dbSession, activeRule2); + // esTester.get(ActiveRuleDao.class).insert(dbSession, activeRule2); // // dbSession.commit(); // // activeRuleIndexer.index(); // - // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.TEXT_QUERY, "x1"); // request.setParam(PARAM_ACTIVATION, "true"); // request.setParam(PARAM_QPROFILE, profile2.getKee()); @@ -57,7 +57,7 @@ public class SearchActionMediumTest { // @Test // public void search_all_active_rules_params() throws Exception { // QProfileDto profile = QProfileTesting.newXooP1(defaultOrganizationDto); - // tester.get(QualityProfileDao.class).insert(dbSession, profile); + // esTester.get(QualityProfileDao.class).insert(dbSession, profile); // RuleDefinitionDto rule = RuleTesting.newXooX1().getDefinition(); // insertRule(rule); // dbSession.commit(); @@ -77,21 +77,21 @@ public class SearchActionMediumTest { // ruleDao.insertRuleParam(dbSession, rule, param2); // // ActiveRuleDto activeRule = newActiveRule(profile, rule); - // tester.get(ActiveRuleDao.class).insert(dbSession, activeRule); + // esTester.get(ActiveRuleDao.class).insert(dbSession, activeRule); // // ActiveRuleParamDto activeRuleParam = ActiveRuleParamDto.createFor(param) // .setValue("The VALUE"); - // tester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam); + // esTester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam); // // ActiveRuleParamDto activeRuleParam2 = ActiveRuleParamDto.createFor(param2) // .setValue("The Other Value"); - // tester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam2); + // esTester.get(ActiveRuleDao.class).insertParam(dbSession, activeRule, activeRuleParam2); // // dbSession.commit(); // // activeRuleIndexer.index(); // - // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.TEXT_QUERY, "x1"); // request.setParam(PARAM_ACTIVATION, "true"); // request.setParam(WebService.Param.FIELDS, "params"); @@ -103,7 +103,7 @@ public class SearchActionMediumTest { // @Test // public void get_note_as_markdown_and_html() throws Exception { // QProfileDto profile = QProfileTesting.newXooP1("org-123"); - // tester.get(QualityProfileDao.class).insert(dbSession, profile); + // esTester.get(QualityProfileDao.class).insert(dbSession, profile); // RuleDto rule = RuleTesting.newXooX1(defaultOrganizationDto).setNoteData("this is *bold*"); // insertRule(rule.getDefinition()); // ruleDao.insertOrUpdate(dbSession, rule.getMetadata().setRuleId(rule.getId())); @@ -112,7 +112,7 @@ public class SearchActionMediumTest { // // activeRuleIndexer.index(); // - // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.FIELDS, "htmlNote, mdNote"); // WsTester.Result result = request.execute(); // result.assertJson(this.getClass(), "get_note_as_markdown_and_html.json"); @@ -128,7 +128,7 @@ public class SearchActionMediumTest { // // activeRuleIndexer.index(); // - // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(PARAM_TAGS, "tag1"); // request.setParam(WebService.Param.FIELDS, "sysTags, tags"); // request.setParam(WebService.Param.FACETS, "tags"); @@ -138,7 +138,7 @@ public class SearchActionMediumTest { // // @Test // public void severities_facet_should_have_all_severities() throws Exception { - // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.FACETS, "severities"); // request.execute().assertJson(this.getClass(), "severities_facet.json"); // } @@ -160,7 +160,7 @@ public class SearchActionMediumTest { // dbSession.commit(); // // // 1. Sort Name Asc - // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.FIELDS, ""); // request.setParam(WebService.Param.SORT, "name"); // request.setParam(WebService.Param.ASCENDING, "true"); @@ -169,7 +169,7 @@ public class SearchActionMediumTest { // result.assertJson("{\"total\":3,\"p\":1,\"ps\":100,\"rules\":[{\"key\":\"xoo:x2\"},{\"key\":\"xoo:x1\"},{\"key\":\"xoo:x3\"}]}"); // // // 2. Sort Name DESC - // request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.FIELDS, ""); // request.setParam(WebService.Param.SORT, RuleIndexDefinition.FIELD_RULE_NAME); // request.setParam(WebService.Param.ASCENDING, "false"); @@ -194,7 +194,7 @@ public class SearchActionMediumTest { // dbSession.clearCache(); // // // 1. find today's rules - // WsTester.TestRequest request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // WsTester.TestRequest request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.FIELDS, ""); // request.setParam(PARAM_AVAILABLE_SINCE, DateUtils.formatDate(since)); // request.setParam(WebService.Param.SORT, RuleIndexDefinition.FIELD_RULE_KEY); @@ -202,7 +202,7 @@ public class SearchActionMediumTest { // result.assertJson("{\"total\":2,\"p\":1,\"ps\":100,\"rules\":[{\"key\":\"xoo:x1\"},{\"key\":\"xoo:x2\"}]}"); // // // 2. no rules since tomorrow - // request = tester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); + // request = esTester.wsTester().newGetRequest(API_ENDPOINT, API_SEARCH_METHOD); // request.setParam(WebService.Param.FIELDS, ""); // request.setParam(PARAM_AVAILABLE_SINCE, DateUtils.formatDate(DateUtils.addDays(since, 1))); // result = request.execute(); @@ -222,7 +222,7 @@ public class SearchActionMediumTest { // ruleDao.insertOrUpdate(dbSession, ruleDto.getMetadata().setRuleId(ruleDto.getId())); // dbSession.commit(); // - // WsTester.TestRequest request = tester.wsTester() + // WsTester.TestRequest request = esTester.wsTester() // .newGetRequest(API_ENDPOINT, API_SEARCH_METHOD) // .setParam(WebService.Param.FIELDS, "name,defaultDebtRemFn,debtRemFn,effortToFixDescription,debtOverloaded"); // WsTester.Result result = request.execute(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java index 3ab344e19f4..9ce85332e56 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java @@ -739,7 +739,7 @@ public class SearchActionTest { @SafeVarargs private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer<RuleMetadataDto>... populaters) { RuleMetadataDto metadata = db.rules().insertOrUpdateMetadata(rule, organization, populaters); - ruleIndexer.indexRuleExtension(organization, rule.getKey()); + ruleIndexer.commitAndIndex(db.getSession(), organization, rule.getKey()); return metadata; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionMediumTest.java index 38ed5a92b2c..889552c2a06 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionMediumTest.java @@ -271,7 +271,7 @@ public class ShowActionMediumTest { .setUpdatedAt(new Date().getTime()); ruleDao.insert(session, ruleDto); session.commit(); - ruleIndexer.indexRuleDefinition(ruleDto.getKey()); + ruleIndexer.commitAndIndex(session, ruleDto.getKey()); RuleParamDto regexParam = RuleParamDto.createFor(ruleDto).setName("regex").setType("STRING").setDescription("Reg *exp*").setDefaultValue(".*"); ruleDao.insertRuleParam(session, ruleDto, regexParam); diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java index ce6cdb53d0e..22ff5c7c42a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java @@ -230,14 +230,14 @@ public class ShowActionTest { private RuleDefinitionDto insertRule() { RuleDefinitionDto rule = dbTester.rules().insert(); - ruleIndexer.indexRuleDefinition(rule.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), rule.getKey()); return rule; } @SafeVarargs private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer<RuleMetadataDto>... populaters) { RuleMetadataDto metadata = dbTester.rules().insertOrUpdateMetadata(rule, organization, populaters); - ruleIndexer.indexRuleExtension(organization, rule.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), organization, rule.getKey()); return metadata; } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java index c32e97894ad..a75116f809b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java @@ -99,7 +99,7 @@ public class TagsActionTest { @Test public void return_system_tag() throws Exception { RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag")); - ruleIndexer.indexRuleDefinition(r.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); String result = tester.newRequest().execute().getInput(); assertJson(result).isSimilarTo("{\"tags\":[\"tag\"]}"); @@ -108,9 +108,9 @@ public class TagsActionTest { @Test public void return_tag() throws Exception { RuleDefinitionDto r = dbTester.rules().insert(setSystemTags()); - ruleIndexer.indexRuleDefinition(r.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag")); - ruleIndexer.indexRuleExtension(organization, r.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); String result = tester.newRequest().setParam("organization", organization.getKey()).execute().getInput(); assertJson(result).isSimilarTo("{\"tags\":[\"tag\"]}"); diff --git a/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java b/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java index 10c3a441a53..6232cf1c2d5 100644 --- a/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java +++ b/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java @@ -92,8 +92,7 @@ public class UserEsResilienceTest { public void creation_and_update_of_user_are_resilient_to_indexing_crash() throws Exception { String login = "crash"; - // creation of user succeeds but index is not up-to-date (indexing - // crashes are not propagated to web services) + // creation of user succeeds in db but indexing crashes --> ws fails expectHttpError(500, () -> tester.users().generate(u -> u.setLogin(login))); // user exists in db, it can't be created again. |