From 4b08d04bb8db93c40fa7cffe05ed51dbdac4b81d Mon Sep 17 00:00:00 2001 From: Eric Hartmann Date: Thu, 29 Jun 2017 21:41:53 +0200 Subject: [PATCH] SONAR-9480 make WS operations on rules resilient to Elasticsearch errors --- .../org/sonar/db/version/schema-h2.ddl | 2 +- .../main/java/org/sonar/db/es/EsQueueDto.java | 35 +++- .../java/org/sonar/db/es/RuleExtensionId.java | 88 +++++++++ .../main/java/org/sonar/db/rule/RuleDao.java | 21 +++ .../db/rule/RuleExtensionForIndexingDto.java | 81 ++++++++ .../org/sonar/db/rule/RuleForIndexingDto.java | 135 ++++++++++++++ .../java/org/sonar/db/rule/RuleMapper.java | 5 + .../org/sonar/db/es/EsQueueMapper.xml | 6 +- .../org/sonar/db/rule/RuleMapper.xml | 48 +++++ .../version/v65/CreateEsQueueTable.java | 4 +- .../version/v65/CreateEsQueueTableTest.java | 2 +- .../java/org/sonar/server/es/BulkIndexer.java | 15 +- .../org/sonar/server/es/RecoveryIndexer.java | 30 +-- .../org/sonar/server/es/ResilientIndexer.java | 41 ++++ .../server/es/ResilientIndexerResult.java | 77 ++++++++ .../organization/ws/EnableSupportAction.java | 3 +- .../org/sonar/server/rule/RegisterRules.java | 2 +- .../org/sonar/server/rule/RuleCreator.java | 3 +- .../org/sonar/server/rule/RuleUpdater.java | 7 +- .../org/sonar/server/rule/index/RuleDoc.java | 33 ++++ .../server/rule/index/RuleExtensionDoc.java | 16 ++ .../sonar/server/rule/index/RuleIndexer.java | 176 +++++++++++++++--- .../sonar/server/rule/ws/DeleteAction.java | 4 +- .../sonar/server/user/index/UserIndexer.java | 15 +- .../user/index/UserResultSetIterator.java | 2 +- .../org/sonar/server/es/BulkIndexerTest.java | 42 ----- .../sonar/server/es/RecoveryIndexerTest.java | 113 ++++++++--- .../server/es/ResilientIndexerResultTest.java | 62 ++++++ .../server/issue/IssueServiceMediumTest.java | 2 +- .../server/issue/index/IssueIndexTest.java | 32 ++-- .../sonar/server/issue/ws/TagsActionTest.java | 57 +++--- .../ws/EnableSupportActionTest.java | 127 +++++++------ .../ws/ChangeParentActionTest.java | 13 +- .../qualityprofile/ws/CreateActionTest.java | 2 +- .../ws/InheritanceActionTest.java | 48 +++-- .../ws/QProfilesWsMediumTest.java | 15 +- .../sonar/server/rule/RuleCreatorTest.java | 56 +++--- .../server/rule/index/RuleIndexerTest.java | 16 +- .../server/rule/ws/DeleteActionTest.java | 21 ++- .../rule/ws/SearchActionMediumTest.java | 38 ++-- .../server/rule/ws/SearchActionTest.java | 2 +- .../server/rule/ws/ShowActionMediumTest.java | 2 +- .../sonar/server/rule/ws/ShowActionTest.java | 4 +- .../sonar/server/rule/ws/TagsActionTest.java | 6 +- .../tests/user/UserEsResilienceTest.java | 3 +- 45 files changed, 1143 insertions(+), 369 deletions(-) create mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java create mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java create mode 100644 server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexer.java create mode 100644 server/sonar-server/src/main/java/org/sonar/server/es/ResilientIndexerResult.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/es/ResilientIndexerResultTest.java diff --git a/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl b/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl index 202db3d5e1f..3942cb98f86 100644 --- a/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl +++ b/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl @@ -662,7 +662,7 @@ CREATE INDEX "CE_TASK_UUID" ON "WEBHOOK_DELIVERIES" ("CE_TASK_UUID"); CREATE TABLE "ES_QUEUE" ( "UUID" VARCHAR(40) NOT NULL PRIMARY KEY, "DOC_TYPE" VARCHAR(40) NOT NULL, - "DOC_UUID" VARCHAR(255) NOT NULL, + "DOC_ID" VARCHAR(4000) NOT NULL, "CREATED_AT" BIGINT NOT NULL ); CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID"); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java index feb3148fa57..35c51c7f2e5 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java @@ -22,12 +22,12 @@ package org.sonar.db.es; public final class EsQueueDto { public enum Type { - USER + USER, RULE, RULE_EXTENSION } private String uuid; private Type docType; - private String docUuid; + private String docId; public String getUuid() { return uuid; @@ -47,12 +47,12 @@ public final class EsQueueDto { return this; } - public String getDocUuid() { - return docUuid; + public String getDocId() { + return docId; } - private EsQueueDto setDocUuid(String s) { - this.docUuid = s; + private EsQueueDto setDocId(String s) { + this.docId = s; return this; } @@ -61,12 +61,31 @@ public final class EsQueueDto { StringBuilder sb = new StringBuilder("EsQueueDto{"); sb.append("uuid='").append(uuid).append('\''); sb.append(", docType=").append(docType); - sb.append(", docUuid='").append(docUuid).append('\''); + sb.append(", docId='").append(docId).append('\''); sb.append('}'); return sb.toString(); } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof EsQueueDto)) { + return false; + } + + EsQueueDto that = (EsQueueDto) o; + + return uuid.equals(that.uuid); + } + + @Override + public int hashCode() { + return uuid.hashCode(); + } + public static EsQueueDto create(Type docType, String docUuid) { - return new EsQueueDto().setDocType(docType).setDocUuid(docUuid); + return new EsQueueDto().setDocType(docType).setDocId(docUuid); } } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java new file mode 100644 index 00000000000..7f821be5889 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/RuleExtensionId.java @@ -0,0 +1,88 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.db.es; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Splitter; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; + +public class RuleExtensionId { + private final String organizationUuid; + private final String repositoryName; + private final String ruleKey; + private final String id; + + private static final Splitter ID_SPLITTER = Splitter.on(CharMatcher.anyOf(":|")); + + public RuleExtensionId(String organizationUuid, String repositoryName, String ruleKey) { + this.organizationUuid = organizationUuid; + this.repositoryName = repositoryName; + this.ruleKey = ruleKey; + this.id = format("%s:%s|%s",repositoryName,ruleKey,organizationUuid); + } + + public RuleExtensionId(String ruleExtensionId) { + List splittedId = ID_SPLITTER.splitToList(ruleExtensionId); + checkArgument(splittedId.size() == 3, "Incorrect Id %s", ruleExtensionId); + this.id = ruleExtensionId; + this.repositoryName = splittedId.get(0); + this.ruleKey = splittedId.get(1); + this.organizationUuid = splittedId.get(2); + } + + public String getOrganizationUuid() { + return organizationUuid; + } + + public String getRepositoryName() { + return repositoryName; + } + + public String getRuleKey() { + return ruleKey; + } + + public String getId() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RuleExtensionId)) { + return false; + } + + RuleExtensionId that = (RuleExtensionId) o; + + return id.equals(that.id); + } + + @Override + public int hashCode() { + return id.hashCode(); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java index e4902888070..94e47de330f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDao.java @@ -22,8 +22,10 @@ package org.sonar.db.rule; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.ibatis.session.ResultHandler; +import org.sonar.db.es.RuleExtensionId; import org.sonar.api.rule.RuleKey; import org.sonar.api.rules.RuleQuery; import org.sonar.db.Dao; @@ -34,6 +36,7 @@ import org.sonar.db.organization.OrganizationDto; import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Optional.ofNullable; import static org.sonar.db.DatabaseUtils.executeLargeInputs; +import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput; public class RuleDao implements Dao { @@ -149,6 +152,24 @@ public class RuleDao implements Dao { } } + public void scrollRuleExtensionByRuleKeys(DbSession dbSession, Collection ruleExtensionIds, Consumer consumer) { + RuleMapper mapper = mapper(dbSession); + + executeLargeInputsWithoutOutput(ruleExtensionIds, + pageOfRuleExtensionIds -> mapper + .selectRuleExtensionForIndexingByKeys(pageOfRuleExtensionIds) + .forEach(consumer)); + } + + public void scrollRuleByRuleKeys(DbSession dbSession, Collection ruleKeys, Consumer consumer) { + RuleMapper mapper = mapper(dbSession); + + executeLargeInputsWithoutOutput(ruleKeys, + pageOfRuleKeys -> mapper + .selectRuleForIndexingByKeys(pageOfRuleKeys) + .forEach(consumer)); + } + private static RuleMapper mapper(DbSession session) { return session.getMapper(RuleMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java new file mode 100644 index 00000000000..ef127131794 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleExtensionForIndexingDto.java @@ -0,0 +1,81 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.db.rule; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import org.sonar.api.rule.RuleKey; + +public class RuleExtensionForIndexingDto { + + private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + + private String pluginName; + private String pluginRuleKey; + private String organizationUuid; + private String tags; + + public String getPluginName() { + return pluginName; + } + + public RuleExtensionForIndexingDto setPluginName(String pluginName) { + this.pluginName = pluginName; + return this; + } + + public String getPluginRuleKey() { + return pluginRuleKey; + } + + public RuleExtensionForIndexingDto setPluginRuleKey(String pluginRuleKey) { + this.pluginRuleKey = pluginRuleKey; + return this; + } + + public String getOrganizationUuid() { + return organizationUuid; + } + + public RuleExtensionForIndexingDto setOrganizationUuid(String organizationUuid) { + this.organizationUuid = organizationUuid; + return this; + } + + public String getTags() { + return tags; + } + + public RuleExtensionForIndexingDto setTags(String tags) { + this.tags = tags; + return this; + } + + public RuleKey getRuleKey() { + return RuleKey.of(pluginName, pluginRuleKey); + } + + public Set getTagsAsSet() { + return ImmutableSet.copyOf(TAGS_SPLITTER.split(tags == null ? "" : tags)); + } + +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java new file mode 100644 index 00000000000..52459e6338e --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleForIndexingDto.java @@ -0,0 +1,135 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.db.rule; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rules.RuleType; + +public class RuleForIndexingDto { + + private Integer id; + private String repository; + private String pluginRuleKey; + private String name; + private String description; + private RuleDto.Format descriptionFormat; + private Integer severity; + private RuleStatus status; + private boolean isTemplate; + private String systemTags; + private String templateRuleKey; + private String templateName; + private String internalKey; + private String language; + private int type; + private long createdAt; + private long updatedAt; + + private static final Splitter TAGS_SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + + public Integer getId() { + return id; + } + + public String getRepository() { + return repository; + } + + public String getPluginRuleKey() { + return pluginRuleKey; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public RuleDto.Format getDescriptionFormat() { + return descriptionFormat; + } + + public Integer getSeverity() { + return severity; + } + + public RuleStatus getStatus() { + return status; + } + + public boolean isTemplate() { + return isTemplate; + } + + public String getSystemTags() { + return systemTags; + } + + public String getTemplateRuleKey() { + return templateRuleKey; + } + + public String getTemplateName() { + return templateName; + } + + public String getInternalKey() { + return internalKey; + } + + public String getLanguage() { + return language; + } + + public int getType() { + return type; + } + + public long getCreatedAt() { + return createdAt; + } + + public long getUpdatedAt() { + return updatedAt; + } + + public RuleType getTypeAsRuleType() { + return RuleType.valueOf(type); + } + + public String getSeverityAsString() { + return severity != null ? SeverityUtil.getSeverityFromOrdinal(severity) : null; + } + + public RuleKey getRuleKey() { + return RuleKey.of(repository, pluginRuleKey); + } + + public Set getSystemTagsAsSet() { + return ImmutableSet.copyOf(TAGS_SPLITTER.split(systemTags == null ? "" : systemTags)); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java index 77ab19067d1..d5a98e60200 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleMapper.java @@ -22,6 +22,7 @@ package org.sonar.db.rule; import java.util.List; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.session.ResultHandler; +import org.sonar.db.es.RuleExtensionId; import org.sonar.api.rule.RuleKey; import org.sonar.api.rules.RuleQuery; @@ -51,6 +52,10 @@ public interface RuleMapper { List selectDefinitionByKeys(@Param("ruleKeys") List keys); + List selectRuleForIndexingByKeys(@Param("ruleKeys") List keys); + + List selectRuleExtensionForIndexingByKeys(@Param("ruleExtensionIds") List ruleExtensionIds); + List selectByQuery(@Param("organizationUuid") String organizationUuid, @Param("query") RuleQuery ruleQuery); void insertDefinition(RuleDefinitionDto ruleDefinitionDto); diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml index 08b3761380c..62ec6e3421b 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml @@ -6,7 +6,7 @@ uuid, doc_type as docType, - doc_uuid as docUuid, + doc_id as docId, created_at as createdAt @@ -14,12 +14,12 @@ insert into es_queue ( uuid, doc_type, - doc_uuid, + doc_id, created_at ) values ( #{dto.uuid, jdbcType=VARCHAR}, #{dto.docType, jdbcType=VARCHAR}, - #{dto.docUuid, jdbcType=VARCHAR}, + #{dto.docId, jdbcType=VARCHAR}, #{now, jdbcType=BIGINT} ) diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml index 25ef4923224..0e29a3e0d94 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/rule/RuleMapper.xml @@ -139,6 +139,26 @@ and r.plugin_rule_key=#{rule,jdbcType=VARCHAR} + + + +