diff options
11 files changed, 1073 insertions, 917 deletions
diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDto.java index b01a4220169..ca6e20244c1 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/rule/RuleDto.java @@ -37,6 +37,11 @@ import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.api.rules.CleanCodeAttribute; import org.sonar.api.rules.RuleType; +import org.sonar.api.server.debt.DebtRemediationFunction; +import org.sonar.api.server.rule.RulesDefinition; +import org.sonar.api.utils.System2; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.DbSession; import org.sonar.db.issue.ImpactDto; import static com.google.common.base.Preconditions.checkArgument; @@ -44,6 +49,7 @@ import static java.lang.String.format; import static java.util.Arrays.asList; import static java.util.Collections.emptySet; import static java.util.Optional.ofNullable; +import static org.apache.commons.lang.StringUtils.isNotEmpty; import static org.sonar.db.rule.RuleDescriptionSectionDto.DEFAULT_KEY; public class RuleDto { @@ -652,6 +658,48 @@ public class RuleDto { return strings == null || strings.isEmpty() ? null : String.join(",", strings); } + public static RuleDto from(RulesDefinition.Rule ruleDef, Set<RuleDescriptionSectionDto> ruleDescriptionSectionDtos, UuidFactory uuidFactory, + long now) { + RuleDto ruleDto = new RuleDto() + .setUuid(uuidFactory.create()) + .setRuleKey(RuleKey.of(ruleDef.repository().key(), ruleDef.key())) + .setPluginKey(ruleDef.pluginKey()) + .setIsTemplate(ruleDef.template()) + .setConfigKey(ruleDef.internalKey()) + .setLanguage(ruleDef.repository().language()) + .setName(ruleDef.name()) + .setSeverity(ruleDef.severity()) + .setStatus(ruleDef.status()) + .setGapDescription(ruleDef.gapDescription()) + .setSystemTags(ruleDef.tags()) + .setSecurityStandards(ruleDef.securityStandards()) + .setType(RuleType.valueOf(ruleDef.type().name())) + .setScope(Scope.valueOf(ruleDef.scope().name())) + .setIsExternal(ruleDef.repository().isExternal()) + .setIsAdHoc(false) + .setCreatedAt(now) + .setUpdatedAt(now) + .setEducationPrinciples(ruleDef.educationPrincipleKeys()); + + if (isNotEmpty(ruleDef.htmlDescription())) { + ruleDto.setDescriptionFormat(Format.HTML); + } else if (isNotEmpty(ruleDef.markdownDescription())) { + ruleDto.setDescriptionFormat(Format.MARKDOWN); + } + + ruleDescriptionSectionDtos.forEach(ruleDto::addRuleDescriptionSectionDto); + + DebtRemediationFunction debtRemediationFunction = ruleDef.debtRemediationFunction(); + if (debtRemediationFunction != null) { + ruleDto.setDefRemediationFunction(debtRemediationFunction.type().name()); + ruleDto.setDefRemediationGapMultiplier(debtRemediationFunction.gapMultiplier()); + ruleDto.setDefRemediationBaseEffort(debtRemediationFunction.baseEffort()); + ruleDto.setGapDescription(ruleDef.gapDescription()); + } + + return ruleDto; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof RuleDto)) { diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/RegisterRules.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/RegisterRules.java deleted file mode 100644 index 81bc58417b5..00000000000 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/RegisterRules.java +++ /dev/null @@ -1,903 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2023 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.rule; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.apache.commons.lang.StringUtils; -import org.sonar.api.Startable; -import org.sonar.api.resources.Languages; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.rule.RuleScope; -import org.sonar.api.rule.RuleStatus; -import org.sonar.api.rules.RuleType; -import org.sonar.api.server.debt.DebtRemediationFunction; -import org.sonar.api.server.rule.RulesDefinition; -import org.sonar.api.utils.System2; -import org.sonar.api.utils.log.Logger; -import org.sonar.api.utils.log.Loggers; -import org.sonar.api.utils.log.Profiler; -import org.sonar.core.util.UuidFactory; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.qualityprofile.ActiveRuleDto; -import org.sonar.db.qualityprofile.ActiveRuleParamDto; -import org.sonar.db.rule.DeprecatedRuleKeyDto; -import org.sonar.db.rule.RuleDescriptionSectionDto; -import org.sonar.db.rule.RuleDto; -import org.sonar.db.rule.RuleDto.Format; -import org.sonar.db.rule.RuleDto.Scope; -import org.sonar.db.rule.RuleParamDto; -import org.sonar.db.rule.RuleRepositoryDto; -import org.sonar.server.es.metadata.MetadataIndex; -import org.sonar.server.qualityprofile.ActiveRuleChange; -import org.sonar.server.qualityprofile.QProfileRules; -import org.sonar.server.qualityprofile.index.ActiveRuleIndexer; -import org.sonar.server.rule.index.RuleIndexer; - -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Sets.difference; -import static com.google.common.collect.Sets.intersection; -import static java.lang.String.format; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableMap; -import static org.apache.commons.lang.StringUtils.isNotEmpty; - -/** - * Register rules at server startup - */ -public class RegisterRules implements Startable { - - private static final Logger LOG = Loggers.get(RegisterRules.class); - - private final RuleDefinitionsLoader defLoader; - private final QProfileRules qProfileRules; - private final DbClient dbClient; - private final RuleIndexer ruleIndexer; - private final ActiveRuleIndexer activeRuleIndexer; - private final Languages languages; - private final System2 system2; - private final WebServerRuleFinder webServerRuleFinder; - private final UuidFactory uuidFactory; - private final MetadataIndex metadataIndex; - private final RuleDescriptionSectionsGeneratorResolver ruleDescriptionSectionsGeneratorResolver; - - - public RegisterRules(RuleDefinitionsLoader defLoader, QProfileRules qProfileRules, DbClient dbClient, RuleIndexer ruleIndexer, - ActiveRuleIndexer activeRuleIndexer, Languages languages, System2 system2, - WebServerRuleFinder webServerRuleFinder, UuidFactory uuidFactory, MetadataIndex metadataIndex, - RuleDescriptionSectionsGeneratorResolver ruleDescriptionSectionsGeneratorResolver) { - this.defLoader = defLoader; - this.qProfileRules = qProfileRules; - this.dbClient = dbClient; - this.ruleIndexer = ruleIndexer; - this.activeRuleIndexer = activeRuleIndexer; - this.languages = languages; - this.system2 = system2; - this.webServerRuleFinder = webServerRuleFinder; - this.uuidFactory = uuidFactory; - this.metadataIndex = metadataIndex; - this.ruleDescriptionSectionsGeneratorResolver = ruleDescriptionSectionsGeneratorResolver; - } - - @Override - public void start() { - Profiler profiler = Profiler.create(LOG).startInfo("Register rules"); - try (DbSession dbSession = dbClient.openSession(true)) { - RulesDefinition.Context ruleDefinitionContext = defLoader.load(); - List<RulesDefinition.Repository> repositories = ruleDefinitionContext.repositories(); - RegisterRulesContext registerRulesContext = createRegisterRulesContext(dbSession); - - verifyRuleKeyConsistency(repositories, registerRulesContext); - - for (RulesDefinition.ExtendedRepository repoDef : repositories) { - if (languages.get(repoDef.language()) != null) { - registerRules(registerRulesContext, repoDef.rules(), dbSession); - dbSession.commit(); - } - } - processRemainingDbRules(registerRulesContext, dbSession); - List<ActiveRuleChange> changes = removeActiveRulesOnStillExistingRepositories(dbSession, registerRulesContext, repositories); - dbSession.commit(); - - persistRepositories(dbSession, ruleDefinitionContext.repositories()); - // FIXME lack of resiliency, active rules index is corrupted if rule index fails - // to be updated. Only a single DB commit should be executed. - ruleIndexer.commitAndIndex(dbSession, registerRulesContext.getAllModified().map(RuleDto::getUuid).collect(Collectors.toSet())); - changes.forEach(arChange -> dbClient.qProfileChangeDao().insert(dbSession, arChange.toDto(null))); - activeRuleIndexer.commitAndIndex(dbSession, changes); - registerRulesContext.getRenamed().forEach(e -> LOG.info("Rule {} re-keyed to {}", e.getValue(), e.getKey().getKey())); - profiler.stopDebug(); - - if (!registerRulesContext.hasDbRules()) { - Stream.concat(ruleIndexer.getIndexTypes().stream(), activeRuleIndexer.getIndexTypes().stream()) - .forEach(t -> metadataIndex.setInitialized(t, true)); - } - - webServerRuleFinder.startCaching(); - } - } - - private RegisterRulesContext createRegisterRulesContext(DbSession dbSession) { - Map<RuleKey, RuleDto> allRules = dbClient.ruleDao().selectAll(dbSession).stream() - .collect(Collectors.toMap(RuleDto::getKey, Function.identity())); - Map<String, Set<SingleDeprecatedRuleKey>> existingDeprecatedKeysById = loadDeprecatedRuleKeys(dbSession); - Map<String, List<RuleParamDto>> ruleParamsByRuleUuid = loadAllRuleParameters(dbSession); - return new RegisterRulesContext(allRules, existingDeprecatedKeysById, ruleParamsByRuleUuid); - } - - private Map<String, List<RuleParamDto>> loadAllRuleParameters(DbSession dbSession) { - return dbClient.ruleDao().selectAllRuleParams(dbSession).stream() - .collect(Collectors.groupingBy(RuleParamDto::getRuleUuid)); - } - - private Map<String, Set<SingleDeprecatedRuleKey>> loadDeprecatedRuleKeys(DbSession dbSession) { - return dbClient.ruleDao().selectAllDeprecatedRuleKeys(dbSession).stream() - .map(SingleDeprecatedRuleKey::from) - .collect(Collectors.groupingBy(SingleDeprecatedRuleKey::getRuleUuid, Collectors.toSet())); - } - - private static class RegisterRulesContext { - // initial immutable data - private final Map<RuleKey, RuleDto> dbRules; - private final Set<RuleDto> known; - private final Map<String, Set<SingleDeprecatedRuleKey>> dbDeprecatedKeysByUuid; - private final Map<String, List<RuleParamDto>> ruleParamsByRuleUuid; - private final Map<RuleKey, RuleDto> dbRulesByDbDeprecatedKey; - // mutable data - private final Set<RuleDto> created = new HashSet<>(); - private final Map<RuleDto, RuleKey> renamed = new HashMap<>(); - private final Set<RuleDto> updated = new HashSet<>(); - private final Set<RuleDto> unchanged = new HashSet<>(); - private final Set<RuleDto> removed = new HashSet<>(); - - private RegisterRulesContext(Map<RuleKey, RuleDto> dbRules, Map<String, Set<SingleDeprecatedRuleKey>> dbDeprecatedKeysByUuid, - Map<String, List<RuleParamDto>> ruleParamsByRuleUuid) { - this.dbRules = ImmutableMap.copyOf(dbRules); - this.known = ImmutableSet.copyOf(dbRules.values()); - this.dbDeprecatedKeysByUuid = dbDeprecatedKeysByUuid; - this.ruleParamsByRuleUuid = ruleParamsByRuleUuid; - this.dbRulesByDbDeprecatedKey = buildDbRulesByDbDeprecatedKey(dbDeprecatedKeysByUuid, dbRules); - } - - private static Map<RuleKey, RuleDto> buildDbRulesByDbDeprecatedKey(Map<String, Set<SingleDeprecatedRuleKey>> dbDeprecatedKeysByUuid, - Map<RuleKey, RuleDto> dbRules) { - Map<String, RuleDto> dbRulesByRuleUuid = dbRules.values().stream() - .collect(Collectors.toMap(RuleDto::getUuid, Function.identity())); - - Map<RuleKey, RuleDto> rulesByKey = new LinkedHashMap<>(); - for (Map.Entry<String, Set<SingleDeprecatedRuleKey>> entry : dbDeprecatedKeysByUuid.entrySet()) { - String ruleUuid = entry.getKey(); - RuleDto rule = dbRulesByRuleUuid.get(ruleUuid); - if (rule == null) { - LOG.warn("Could not retrieve rule with uuid %s referenced by a deprecated rule key. " + - "The following deprecated rule keys seem to be referencing a non-existing rule", - ruleUuid, entry.getValue()); - } else { - entry.getValue().forEach(d -> rulesByKey.put(d.getOldRuleKeyAsRuleKey(), rule)); - } - } - return unmodifiableMap(rulesByKey); - } - - private boolean hasDbRules() { - return !dbRules.isEmpty(); - } - - private Optional<RuleDto> getDbRuleFor(RulesDefinition.Rule ruleDef) { - RuleKey ruleKey = RuleKey.of(ruleDef.repository().key(), ruleDef.key()); - Optional<RuleDto> res = Stream.concat(Stream.of(ruleKey), ruleDef.deprecatedRuleKeys().stream()) - .map(dbRules::get) - .filter(Objects::nonNull) - .findFirst(); - // may occur in case of plugin downgrade - if (res.isEmpty()) { - return Optional.ofNullable(dbRulesByDbDeprecatedKey.get(ruleKey)); - } - return res; - } - - private Map<RuleKey, SingleDeprecatedRuleKey> getDbDeprecatedKeysByOldRuleKey() { - return dbDeprecatedKeysByUuid.values().stream() - .flatMap(Collection::stream) - .collect(Collectors.toMap(SingleDeprecatedRuleKey::getOldRuleKeyAsRuleKey, Function.identity())); - } - - private Set<SingleDeprecatedRuleKey> getDBDeprecatedKeysFor(RuleDto rule) { - return dbDeprecatedKeysByUuid.getOrDefault(rule.getUuid(), emptySet()); - } - - private List<RuleParamDto> getRuleParametersFor(String ruleUuid) { - return ruleParamsByRuleUuid.getOrDefault(ruleUuid, emptyList()); - } - - private Stream<RuleDto> getRemaining() { - Set<RuleDto> res = new HashSet<>(dbRules.values()); - res.removeAll(unchanged); - res.removeAll(renamed.keySet()); - res.removeAll(updated); - res.removeAll(removed); - return res.stream(); - } - - private Stream<RuleDto> getRemoved() { - return removed.stream(); - } - - public Stream<Map.Entry<RuleDto, RuleKey>> getRenamed() { - return renamed.entrySet().stream(); - } - - private Stream<RuleDto> getAllModified() { - return Stream.of( - created.stream(), - updated.stream(), - removed.stream(), - renamed.keySet().stream()) - .flatMap(s -> s); - } - - private boolean isCreated(RuleDto ruleDto) { - return created.contains(ruleDto); - } - - private boolean isRenamed(RuleDto ruleDto) { - return renamed.containsKey(ruleDto); - } - - private boolean isUpdated(RuleDto ruleDto) { - return updated.contains(ruleDto); - } - - private void created(RuleDto ruleDto) { - checkState(!known.contains(ruleDto), "known RuleDto can't be created"); - created.add(ruleDto); - } - - private void renamed(RuleDto ruleDto) { - ensureKnown(ruleDto); - renamed.put(ruleDto, ruleDto.getKey()); - } - - private void updated(RuleDto ruleDto) { - ensureKnown(ruleDto); - updated.add(ruleDto); - } - - private void removed(RuleDto ruleDto) { - ensureKnown(ruleDto); - removed.add(ruleDto); - } - - private void unchanged(RuleDto ruleDto) { - ensureKnown(ruleDto); - unchanged.add(ruleDto); - } - - private void ensureKnown(RuleDto ruleDto) { - checkState(known.contains(ruleDto), "unknown RuleDto"); - } - } - - private void persistRepositories(DbSession dbSession, List<RulesDefinition.Repository> repositories) { - List<String> keys = repositories.stream().map(RulesDefinition.Repository::key).toList(); - Set<String> existingKeys = dbClient.ruleRepositoryDao().selectAllKeys(dbSession); - - Map<Boolean, List<RuleRepositoryDto>> dtos = repositories.stream() - .map(r -> new RuleRepositoryDto(r.key(), r.language(), r.name())) - .collect(Collectors.groupingBy(i -> existingKeys.contains(i.getKey()))); - - dbClient.ruleRepositoryDao().update(dbSession, dtos.getOrDefault(true, emptyList())); - dbClient.ruleRepositoryDao().insert(dbSession, dtos.getOrDefault(false, emptyList())); - dbClient.ruleRepositoryDao().deleteIfKeyNotIn(dbSession, keys); - dbSession.commit(); - } - - @Override - public void stop() { - // nothing - } - - private void registerRules(RegisterRulesContext context, List<RulesDefinition.Rule> ruleDefs, DbSession session) { - Map<RulesDefinition.Rule, RuleDto> dtos = new LinkedHashMap<>(ruleDefs.size()); - - for (RulesDefinition.Rule ruleDef : ruleDefs) { - RuleKey ruleKey = RuleKey.of(ruleDef.repository().key(), ruleDef.key()); - RuleDto ruleDto = findOrCreateRuleDto(context, session, ruleDef); - dtos.put(ruleDef, ruleDto); - - // we must detect renaming __before__ we modify the DTO - if (!ruleDto.getKey().equals(ruleKey)) { - context.renamed(ruleDto); - ruleDto.setRuleKey(ruleKey); - } - - if (anyMerge(ruleDef, ruleDto)) { - context.updated(ruleDto); - } - - if (context.isUpdated(ruleDto) || context.isRenamed(ruleDto)) { - update(session, ruleDto); - } else if (!context.isCreated(ruleDto)) { - context.unchanged(ruleDto); - } - } - - for (Map.Entry<RulesDefinition.Rule, RuleDto> e : dtos.entrySet()) { - mergeParams(context, e.getKey(), e.getValue(), session); - updateDeprecatedKeys(context, e.getKey(), e.getValue(), session); - } - } - - @Nonnull - private RuleDto findOrCreateRuleDto(RegisterRulesContext context, DbSession session, RulesDefinition.Rule ruleDef) { - return context.getDbRuleFor(ruleDef) - .orElseGet(() -> { - RuleDto newRule = createRuleDto(ruleDef, session); - context.created(newRule); - return newRule; - }); - } - - private RuleDto createRuleDto(RulesDefinition.Rule ruleDef, DbSession session) { - RuleDto ruleDto = new RuleDto() - .setUuid(uuidFactory.create()) - .setRuleKey(RuleKey.of(ruleDef.repository().key(), ruleDef.key())) - .setPluginKey(ruleDef.pluginKey()) - .setIsTemplate(ruleDef.template()) - .setConfigKey(ruleDef.internalKey()) - .setLanguage(ruleDef.repository().language()) - .setName(ruleDef.name()) - .setSeverity(ruleDef.severity()) - .setStatus(ruleDef.status()) - .setGapDescription(ruleDef.gapDescription()) - .setSystemTags(ruleDef.tags()) - .setSecurityStandards(ruleDef.securityStandards()) - .setType(RuleType.valueOf(ruleDef.type().name())) - .setScope(toDtoScope(ruleDef.scope())) - .setIsExternal(ruleDef.repository().isExternal()) - .setIsAdHoc(false) - .setCreatedAt(system2.now()) - .setUpdatedAt(system2.now()) - .setEducationPrinciples(ruleDef.educationPrincipleKeys()); - - if (isNotEmpty(ruleDef.htmlDescription())) { - ruleDto.setDescriptionFormat(Format.HTML); - } else if (isNotEmpty(ruleDef.markdownDescription())) { - ruleDto.setDescriptionFormat(Format.MARKDOWN); - } - - generateRuleDescriptionSections(ruleDef) - .forEach(ruleDto::addRuleDescriptionSectionDto); - - DebtRemediationFunction debtRemediationFunction = ruleDef.debtRemediationFunction(); - if (debtRemediationFunction != null) { - ruleDto.setDefRemediationFunction(debtRemediationFunction.type().name()); - ruleDto.setDefRemediationGapMultiplier(debtRemediationFunction.gapMultiplier()); - ruleDto.setDefRemediationBaseEffort(debtRemediationFunction.baseEffort()); - ruleDto.setGapDescription(ruleDef.gapDescription()); - } - - dbClient.ruleDao().insert(session, ruleDto); - return ruleDto; - } - - private Set<RuleDescriptionSectionDto> generateRuleDescriptionSections(RulesDefinition.Rule ruleDef) { - RuleDescriptionSectionsGenerator descriptionSectionGenerator = ruleDescriptionSectionsGeneratorResolver.getRuleDescriptionSectionsGenerator(ruleDef); - return descriptionSectionGenerator.generateSections(ruleDef); - } - - private static Scope toDtoScope(RuleScope scope) { - switch (scope) { - case ALL: - return Scope.ALL; - case MAIN: - return Scope.MAIN; - case TEST: - return Scope.TEST; - default: - throw new IllegalArgumentException("Unknown rule scope: " + scope); - } - } - - private boolean anyMerge(RulesDefinition.Rule ruleDef, RuleDto ruleDto) { - boolean ruleMerged = mergeRule(ruleDef, ruleDto); - boolean debtDefinitionsMerged = mergeDebtDefinitions(ruleDef, ruleDto); - boolean tagsMerged = mergeTags(ruleDef, ruleDto); - boolean securityStandardsMerged = mergeSecurityStandards(ruleDef, ruleDto); - boolean educationPrinciplesMerged = mergeEducationPrinciples(ruleDef, ruleDto); - return ruleMerged || debtDefinitionsMerged || tagsMerged || securityStandardsMerged || educationPrinciplesMerged; - } - - private boolean mergeRule(RulesDefinition.Rule def, RuleDto dto) { - boolean changed = false; - if (!Objects.equals(dto.getName(), def.name())) { - dto.setName(def.name()); - changed = true; - } - if (mergeDescription(def, dto)) { - changed = true; - } - if (!Objects.equals(dto.getPluginKey(), def.pluginKey())) { - dto.setPluginKey(def.pluginKey()); - changed = true; - } - if (!Objects.equals(dto.getConfigKey(), def.internalKey())) { - dto.setConfigKey(def.internalKey()); - changed = true; - } - String severity = def.severity(); - if (!Objects.equals(dto.getSeverityString(), severity)) { - dto.setSeverity(severity); - changed = true; - } - boolean isTemplate = def.template(); - if (isTemplate != dto.isTemplate()) { - dto.setIsTemplate(isTemplate); - changed = true; - } - if (def.status() != dto.getStatus()) { - dto.setStatus(def.status()); - changed = true; - } - if (!Objects.equals(dto.getScope().name(), def.scope().name())) { - dto.setScope(toDtoScope(def.scope())); - changed = true; - } - if (!Objects.equals(dto.getLanguage(), def.repository().language())) { - dto.setLanguage(def.repository().language()); - changed = true; - } - RuleType type = RuleType.valueOf(def.type().name()); - if (!Objects.equals(dto.getType(), type.getDbConstant())) { - dto.setType(type); - changed = true; - } - if (dto.isAdHoc()) { - dto.setIsAdHoc(false); - changed = true; - } - return changed; - } - - private boolean mergeDescription(RulesDefinition.Rule rule, RuleDto ruleDto) { - Set<RuleDescriptionSectionDto> newRuleDescriptionSectionDtos = generateRuleDescriptionSections(rule); - if (ruleDescriptionSectionsUnchanged(ruleDto, newRuleDescriptionSectionDtos)) { - return false; - } - ruleDto.replaceRuleDescriptionSectionDtos(newRuleDescriptionSectionDtos); - if (containsHtmlDescription(rule)) { - ruleDto.setDescriptionFormat(Format.HTML); - return true; - } else if (isNotEmpty(rule.markdownDescription())) { - ruleDto.setDescriptionFormat(Format.MARKDOWN); - return true; - } - return false; - } - - private static boolean containsHtmlDescription(RulesDefinition.Rule rule) { - return isNotEmpty(rule.htmlDescription()) || !rule.ruleDescriptionSections().isEmpty(); - } - - private static boolean ruleDescriptionSectionsUnchanged(RuleDto ruleDto, Set<RuleDescriptionSectionDto> newRuleDescriptionSectionDtos) { - if (ruleDto.getRuleDescriptionSectionDtos().size() != newRuleDescriptionSectionDtos.size()) { - return false; - } - return ruleDto.getRuleDescriptionSectionDtos().stream() - .allMatch(sectionDto -> contains(newRuleDescriptionSectionDtos, sectionDto)); - } - - private static boolean contains(Set<RuleDescriptionSectionDto> sectionDtos, RuleDescriptionSectionDto sectionDto) { - return sectionDtos.stream() - .filter(s -> s.getKey().equals(sectionDto.getKey()) && s.getContent().equals(sectionDto.getContent())) - .anyMatch(s -> Objects.equals(s.getContext(), sectionDto.getContext())); - } - - private static boolean mergeDebtDefinitions(RulesDefinition.Rule def, RuleDto dto) { - // Debt definitions are set to null if the sub-characteristic and the remediation function are null - DebtRemediationFunction debtRemediationFunction = def.debtRemediationFunction(); - boolean hasDebt = debtRemediationFunction != null; - if (hasDebt) { - return mergeDebtDefinitions(dto, - debtRemediationFunction.type().name(), - debtRemediationFunction.gapMultiplier(), - debtRemediationFunction.baseEffort(), - def.gapDescription()); - } - return mergeDebtDefinitions(dto, null, null, null, null); - } - - private static boolean mergeDebtDefinitions(RuleDto dto, @Nullable String remediationFunction, - @Nullable String remediationCoefficient, @Nullable String remediationOffset, @Nullable String gapDescription) { - boolean changed = false; - - if (!Objects.equals(dto.getDefRemediationFunction(), remediationFunction)) { - dto.setDefRemediationFunction(remediationFunction); - changed = true; - } - if (!Objects.equals(dto.getDefRemediationGapMultiplier(), remediationCoefficient)) { - dto.setDefRemediationGapMultiplier(remediationCoefficient); - changed = true; - } - if (!Objects.equals(dto.getDefRemediationBaseEffort(), remediationOffset)) { - dto.setDefRemediationBaseEffort(remediationOffset); - changed = true; - } - if (!Objects.equals(dto.getGapDescription(), gapDescription)) { - dto.setGapDescription(gapDescription); - changed = true; - } - return changed; - } - - private void mergeParams(RegisterRulesContext context, RulesDefinition.Rule ruleDef, RuleDto rule, DbSession session) { - List<RuleParamDto> paramDtos = context.getRuleParametersFor(rule.getUuid()); - Map<String, RuleParamDto> existingParamsByName = new HashMap<>(); - - Profiler profiler = Profiler.create(Loggers.get(getClass())); - for (RuleParamDto paramDto : paramDtos) { - RulesDefinition.Param paramDef = ruleDef.param(paramDto.getName()); - if (paramDef == null) { - profiler.start(); - dbClient.activeRuleDao().deleteParamsByRuleParam(session, paramDto); - profiler.stopDebug(format("Propagate deleted param with name %s to active rules of rule %s", paramDto.getName(), rule.getKey())); - dbClient.ruleDao().deleteRuleParam(session, paramDto.getUuid()); - } else { - if (mergeParam(paramDto, paramDef)) { - dbClient.ruleDao().updateRuleParam(session, rule, paramDto); - } - existingParamsByName.put(paramDto.getName(), paramDto); - } - } - - // Create newly parameters - for (RulesDefinition.Param param : ruleDef.params()) { - RuleParamDto paramDto = existingParamsByName.get(param.key()); - if (paramDto != null) { - continue; - } - paramDto = RuleParamDto.createFor(rule) - .setName(param.key()) - .setDescription(param.description()) - .setDefaultValue(param.defaultValue()) - .setType(param.type().toString()); - dbClient.ruleDao().insertRuleParam(session, rule, paramDto); - if (StringUtils.isEmpty(param.defaultValue())) { - continue; - } - // Propagate the default value to existing active rule parameters - profiler.start(); - for (ActiveRuleDto activeRule : dbClient.activeRuleDao().selectByRuleUuid(session, rule.getUuid())) { - ActiveRuleParamDto activeParam = ActiveRuleParamDto.createFor(paramDto).setValue(param.defaultValue()); - dbClient.activeRuleDao().insertParam(session, activeRule, activeParam); - } - profiler.stopDebug(format("Propagate new param with name %s to active rules of rule %s", paramDto.getName(), rule.getKey())); - } - } - - private static boolean mergeParam(RuleParamDto paramDto, RulesDefinition.Param paramDef) { - boolean changed = false; - if (!Objects.equals(paramDto.getType(), paramDef.type().toString())) { - paramDto.setType(paramDef.type().toString()); - changed = true; - } - if (!Objects.equals(paramDto.getDefaultValue(), paramDef.defaultValue())) { - paramDto.setDefaultValue(paramDef.defaultValue()); - changed = true; - } - if (!Objects.equals(paramDto.getDescription(), paramDef.description())) { - paramDto.setDescription(paramDef.description()); - changed = true; - } - return changed; - } - - private void updateDeprecatedKeys(RegisterRulesContext context, RulesDefinition.Rule ruleDef, RuleDto rule, DbSession dbSession) { - - Set<SingleDeprecatedRuleKey> deprecatedRuleKeysFromDefinition = SingleDeprecatedRuleKey.from(ruleDef); - Set<SingleDeprecatedRuleKey> deprecatedRuleKeysFromDB = context.getDBDeprecatedKeysFor(rule); - - // DeprecatedKeys that must be deleted - List<String> uuidsToBeDeleted = difference(deprecatedRuleKeysFromDB, deprecatedRuleKeysFromDefinition).stream() - .map(SingleDeprecatedRuleKey::getUuid) - .toList(); - - dbClient.ruleDao().deleteDeprecatedRuleKeys(dbSession, uuidsToBeDeleted); - - // DeprecatedKeys that must be created - Sets.SetView<SingleDeprecatedRuleKey> deprecatedRuleKeysToBeCreated = difference(deprecatedRuleKeysFromDefinition, deprecatedRuleKeysFromDB); - - deprecatedRuleKeysToBeCreated - .forEach(r -> dbClient.ruleDao().insert(dbSession, new DeprecatedRuleKeyDto() - .setUuid(uuidFactory.create()) - .setRuleUuid(rule.getUuid()) - .setOldRepositoryKey(r.getOldRepositoryKey()) - .setOldRuleKey(r.getOldRuleKey()) - .setCreatedAt(system2.now()))); - } - - private static boolean mergeTags(RulesDefinition.Rule ruleDef, RuleDto dto) { - boolean changed = false; - - if (RuleStatus.REMOVED == ruleDef.status()) { - dto.setSystemTags(emptySet()); - changed = true; - } else if (dto.getSystemTags().size() != ruleDef.tags().size() || - !dto.getSystemTags().containsAll(ruleDef.tags())) { - dto.setSystemTags(ruleDef.tags()); - changed = true; - } - return changed; - } - - private static boolean mergeSecurityStandards(RulesDefinition.Rule ruleDef, RuleDto dto) { - boolean changed = false; - - if (RuleStatus.REMOVED == ruleDef.status()) { - dto.setSecurityStandards(emptySet()); - changed = true; - } else if (dto.getSecurityStandards().size() != ruleDef.securityStandards().size() || - !dto.getSecurityStandards().containsAll(ruleDef.securityStandards())) { - dto.setSecurityStandards(ruleDef.securityStandards()); - changed = true; - } - return changed; - } - - private static boolean mergeEducationPrinciples(RulesDefinition.Rule ruleDef, RuleDto dto) { - boolean changed = false; - if (dto.getEducationPrinciples().size() != ruleDef.educationPrincipleKeys().size() || - !dto.getEducationPrinciples().containsAll(ruleDef.educationPrincipleKeys())) { - dto.setEducationPrinciples(ruleDef.educationPrincipleKeys()); - changed = true; - } - return changed; - } - - private void processRemainingDbRules(RegisterRulesContext recorder, DbSession dbSession) { - // custom rules check status of template, so they must be processed at the end - List<RuleDto> customRules = new ArrayList<>(); - - recorder.getRemaining().forEach(rule -> { - if (rule.isCustomRule()) { - customRules.add(rule); - } else if (!rule.isAdHoc() && rule.getStatus() != RuleStatus.REMOVED) { - removeRule(dbSession, recorder, rule); - } - }); - - for (RuleDto customRule : customRules) { - String templateUuid = customRule.getTemplateUuid(); - checkNotNull(templateUuid, "Template uuid of the custom rule '%s' is null", customRule); - Optional<RuleDto> template = dbClient.ruleDao().selectByUuid(templateUuid, dbSession); - if (template.isPresent() && template.get().getStatus() != RuleStatus.REMOVED) { - if (updateCustomRuleFromTemplateRule(customRule, template.get())) { - recorder.updated(customRule); - update(dbSession, customRule); - } - } else { - removeRule(dbSession, recorder, customRule); - } - } - - dbSession.commit(); - } - - private void removeRule(DbSession session, RegisterRulesContext recorder, RuleDto rule) { - LOG.info(format("Disable rule %s", rule.getKey())); - rule.setStatus(RuleStatus.REMOVED); - rule.setSystemTags(emptySet()); - update(session, rule); - // FIXME resetting the tags for all organizations must be handled a different way - // rule.setTags(Collections.emptySet()); - // update(session, rule.getMetadata()); - recorder.removed(rule); - if (recorder.getRemoved().count() % 100 == 0) { - session.commit(); - } - } - - private static boolean updateCustomRuleFromTemplateRule(RuleDto customRule, RuleDto templateRule) { - boolean changed = false; - if (!Objects.equals(customRule.getLanguage(), templateRule.getLanguage())) { - customRule.setLanguage(templateRule.getLanguage()); - changed = true; - } - if (!Objects.equals(customRule.getConfigKey(), templateRule.getConfigKey())) { - customRule.setConfigKey(templateRule.getConfigKey()); - changed = true; - } - if (!Objects.equals(customRule.getPluginKey(), templateRule.getPluginKey())) { - customRule.setPluginKey(templateRule.getPluginKey()); - changed = true; - } - if (!Objects.equals(customRule.getDefRemediationFunction(), templateRule.getDefRemediationFunction())) { - customRule.setDefRemediationFunction(templateRule.getDefRemediationFunction()); - changed = true; - } - if (!Objects.equals(customRule.getDefRemediationGapMultiplier(), templateRule.getDefRemediationGapMultiplier())) { - customRule.setDefRemediationGapMultiplier(templateRule.getDefRemediationGapMultiplier()); - changed = true; - } - if (!Objects.equals(customRule.getDefRemediationBaseEffort(), templateRule.getDefRemediationBaseEffort())) { - customRule.setDefRemediationBaseEffort(templateRule.getDefRemediationBaseEffort()); - changed = true; - } - if (!Objects.equals(customRule.getGapDescription(), templateRule.getGapDescription())) { - customRule.setGapDescription(templateRule.getGapDescription()); - changed = true; - } - if (customRule.getStatus() != templateRule.getStatus()) { - customRule.setStatus(templateRule.getStatus()); - changed = true; - } - if (!Objects.equals(customRule.getSeverityString(), templateRule.getSeverityString())) { - customRule.setSeverity(templateRule.getSeverityString()); - changed = true; - } - if (!Objects.equals(customRule.getRepositoryKey(), templateRule.getRepositoryKey())) { - customRule.setRepositoryKey(templateRule.getRepositoryKey()); - changed = true; - } - return changed; - } - - /** - * SONAR-4642 - * <p/> - * Remove active rules on repositories that still exists. - * <p/> - * For instance, if the javascript repository do not provide anymore some rules, active rules related to this rules will be removed. - * But if the javascript repository do not exists anymore, then related active rules will not be removed. - * <p/> - * The side effect of this approach is that extended repositories will not be managed the same way. - * If an extended repository do not exists anymore, then related active rules will be removed. - */ - private List<ActiveRuleChange> removeActiveRulesOnStillExistingRepositories(DbSession dbSession, RegisterRulesContext recorder, List<RulesDefinition.Repository> context) { - Set<String> existingAndRenamedRepositories = getExistingAndRenamedRepositories(recorder, context); - List<ActiveRuleChange> changes = new ArrayList<>(); - Profiler profiler = Profiler.create(Loggers.get(getClass())); - - recorder.getRemoved() - .filter(rule -> existingAndRenamedRepositories.contains(rule.getRepositoryKey())) - .forEach(rule -> { - // SONAR-4642 Remove active rules only when repository still exists - profiler.start(); - changes.addAll(qProfileRules.deleteRule(dbSession, rule)); - profiler.stopDebug(format("Remove active rule for rule %s", rule.getKey())); - }); - - return changes; - } - - private static Set<String> getExistingAndRenamedRepositories(RegisterRulesContext recorder, Collection<RulesDefinition.Repository> context) { - return Stream.concat( - context.stream().map(RulesDefinition.ExtendedRepository::key), - recorder.getRenamed().map(Map.Entry::getValue).map(RuleKey::repository)) - .collect(Collectors.toSet()); - } - - private void update(DbSession session, RuleDto rule) { - rule.setUpdatedAt(system2.now()); - dbClient.ruleDao().update(session, rule); - } - - private static void verifyRuleKeyConsistency(List<RulesDefinition.Repository> repositories, RegisterRulesContext registerRulesContext) { - List<RulesDefinition.Rule> definedRules = repositories.stream() - .flatMap(r -> r.rules().stream()) - .toList(); - - Set<RuleKey> definedRuleKeys = definedRules.stream() - .map(r -> RuleKey.of(r.repository().key(), r.key())) - .collect(Collectors.toSet()); - - List<RuleKey> definedDeprecatedRuleKeys = definedRules.stream() - .flatMap(r -> r.deprecatedRuleKeys().stream()) - .toList(); - - // Find duplicates in declared deprecated rule keys - Set<RuleKey> duplicates = findDuplicates(definedDeprecatedRuleKeys); - checkState(duplicates.isEmpty(), "The following deprecated rule keys are declared at least twice [%s]", - lazyToString(() -> duplicates.stream().map(RuleKey::toString).collect(Collectors.joining(",")))); - - // Find rule keys that are both deprecated and used - Set<RuleKey> intersection = intersection(new HashSet<>(definedRuleKeys), new HashSet<>(definedDeprecatedRuleKeys)).immutableCopy(); - checkState(intersection.isEmpty(), "The following rule keys are declared both as deprecated and used key [%s]", - lazyToString(() -> intersection.stream().map(RuleKey::toString).collect(Collectors.joining(",")))); - - // Find incorrect usage of deprecated keys - Map<RuleKey, SingleDeprecatedRuleKey> dbDeprecatedRuleKeysByOldRuleKey = registerRulesContext.getDbDeprecatedKeysByOldRuleKey(); - - Set<String> incorrectRuleKeyMessage = definedRules.stream() - .flatMap(r -> filterInvalidDeprecatedRuleKeys(dbDeprecatedRuleKeysByOldRuleKey, r)) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); - - checkState(incorrectRuleKeyMessage.isEmpty(), "An incorrect state of deprecated rule keys has been detected.\n %s", - lazyToString(() -> String.join("\n", incorrectRuleKeyMessage))); - } - - private static Stream<String> filterInvalidDeprecatedRuleKeys(Map<RuleKey, SingleDeprecatedRuleKey> dbDeprecatedRuleKeysByOldRuleKey, RulesDefinition.Rule rule) { - return rule.deprecatedRuleKeys().stream() - .map(rk -> { - SingleDeprecatedRuleKey singleDeprecatedRuleKey = dbDeprecatedRuleKeysByOldRuleKey.get(rk); - if (singleDeprecatedRuleKey == null) { - // new deprecated rule key : OK - return null; - } - RuleKey parentRuleKey = RuleKey.of(rule.repository().key(), rule.key()); - if (parentRuleKey.equals(singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey())) { - // same parent : OK - return null; - } - if (rule.deprecatedRuleKeys().contains(singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey())) { - // the new rule is deprecating the old parentRuleKey : OK - return null; - } - return format("The deprecated rule key [%s] was previously deprecated by [%s]. [%s] should be a deprecated key of [%s],", - rk.toString(), - singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey().toString(), - singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey().toString(), - RuleKey.of(rule.repository().key(), rule.key()).toString()); - }); - } - - private static Object lazyToString(Supplier<String> toString) { - return new Object() { - @Override - public String toString() { - return toString.get(); - } - }; - } - - private static <T> Set<T> findDuplicates(Collection<T> list) { - Set<T> duplicates = new HashSet<>(); - Set<T> uniques = new HashSet<>(); - - list.forEach(t -> { - if (!uniques.add(t)) { - duplicates.add(t); - } - }); - - return duplicates; - } -} diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/RuleDescriptionSectionsGeneratorResolver.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/RuleDescriptionSectionsGeneratorResolver.java index d3fd65836b5..417dd32b2fc 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/RuleDescriptionSectionsGeneratorResolver.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/RuleDescriptionSectionsGeneratorResolver.java @@ -21,6 +21,7 @@ package org.sonar.server.rule; import java.util.Set; import org.sonar.api.server.rule.RulesDefinition; +import org.sonar.db.rule.RuleDescriptionSectionDto; import static java.util.stream.Collectors.toSet; import static org.sonar.api.utils.Preconditions.checkState; @@ -32,7 +33,7 @@ public class RuleDescriptionSectionsGeneratorResolver { this.ruleDescriptionSectionsGenerators = ruleDescriptionSectionsGenerators; } - RuleDescriptionSectionsGenerator getRuleDescriptionSectionsGenerator(RulesDefinition.Rule ruleDef) { + public RuleDescriptionSectionsGenerator getRuleDescriptionSectionsGenerator(RulesDefinition.Rule ruleDef) { Set<RuleDescriptionSectionsGenerator> generatorsFound = ruleDescriptionSectionsGenerators.stream() .filter(generator -> generator.isGeneratorForRule(ruleDef)) .collect(toSet()); @@ -41,4 +42,8 @@ public class RuleDescriptionSectionsGeneratorResolver { return generatorsFound.iterator().next(); } + public Set<RuleDescriptionSectionDto> generateFor(RulesDefinition.Rule ruleDef) { + return getRuleDescriptionSectionsGenerator(ruleDef).generateSections(ruleDef); + } + } diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesKeyVerifier.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesKeyVerifier.java new file mode 100644 index 00000000000..888bdb0b125 --- /dev/null +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesKeyVerifier.java @@ -0,0 +1,121 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.registration; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.server.rule.RulesDefinition; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Sets.intersection; +import static java.lang.String.format; + +public class RulesKeyVerifier { + + void verifyRuleKeyConsistency(List<RulesDefinition.Repository> repositories, RulesRegistrationContext rulesRegistrationContext) { + List<RulesDefinition.Rule> definedRules = repositories.stream() + .flatMap(r -> r.rules().stream()) + .toList(); + + Set<RuleKey> definedRuleKeys = definedRules.stream() + .map(r -> RuleKey.of(r.repository().key(), r.key())) + .collect(Collectors.toSet()); + + List<RuleKey> definedDeprecatedRuleKeys = definedRules.stream() + .flatMap(r -> r.deprecatedRuleKeys().stream()) + .toList(); + + // Find duplicates in declared deprecated rule keys + Set<RuleKey> duplicates = findDuplicates(definedDeprecatedRuleKeys); + checkState(duplicates.isEmpty(), "The following deprecated rule keys are declared at least twice [%s]", + lazyToString(() -> duplicates.stream().map(RuleKey::toString).collect(Collectors.joining(",")))); + + // Find rule keys that are both deprecated and used + Set<RuleKey> intersection = intersection(new HashSet<>(definedRuleKeys), new HashSet<>(definedDeprecatedRuleKeys)).immutableCopy(); + checkState(intersection.isEmpty(), "The following rule keys are declared both as deprecated and used key [%s]", + lazyToString(() -> intersection.stream().map(RuleKey::toString).collect(Collectors.joining(",")))); + + // Find incorrect usage of deprecated keys + Map<RuleKey, SingleDeprecatedRuleKey> dbDeprecatedRuleKeysByOldRuleKey = rulesRegistrationContext.getDbDeprecatedKeysByOldRuleKey(); + + Set<String> incorrectRuleKeyMessage = definedRules.stream() + .flatMap(r -> filterInvalidDeprecatedRuleKeys(dbDeprecatedRuleKeysByOldRuleKey, r)) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + checkState(incorrectRuleKeyMessage.isEmpty(), "An incorrect state of deprecated rule keys has been detected.\n %s", + lazyToString(() -> String.join("\n", incorrectRuleKeyMessage))); + } + + private static Stream<String> filterInvalidDeprecatedRuleKeys(Map<RuleKey, SingleDeprecatedRuleKey> dbDeprecatedRuleKeysByOldRuleKey, RulesDefinition.Rule rule) { + return rule.deprecatedRuleKeys().stream() + .map(rk -> { + SingleDeprecatedRuleKey singleDeprecatedRuleKey = dbDeprecatedRuleKeysByOldRuleKey.get(rk); + if (singleDeprecatedRuleKey == null) { + // new deprecated rule key : OK + return null; + } + RuleKey parentRuleKey = RuleKey.of(rule.repository().key(), rule.key()); + if (parentRuleKey.equals(singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey())) { + // same parent : OK + return null; + } + if (rule.deprecatedRuleKeys().contains(singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey())) { + // the new rule is deprecating the old parentRuleKey : OK + return null; + } + return format("The deprecated rule key [%s] was previously deprecated by [%s]. [%s] should be a deprecated key of [%s],", + rk.toString(), + singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey().toString(), + singleDeprecatedRuleKey.getNewRuleKeyAsRuleKey().toString(), + RuleKey.of(rule.repository().key(), rule.key()).toString()); + }); + } + + private static Object lazyToString(Supplier<String> toString) { + return new Object() { + @Override + public String toString() { + return toString.get(); + } + }; + } + + private static <T> Set<T> findDuplicates(Collection<T> list) { + Set<T> duplicates = new HashSet<>(); + Set<T> uniques = new HashSet<>(); + + list.forEach(t -> { + if (!uniques.add(t)) { + duplicates.add(t); + } + }); + + return duplicates; + } +} diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesRegistrant.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesRegistrant.java new file mode 100644 index 00000000000..4f03212ecaf --- /dev/null +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesRegistrant.java @@ -0,0 +1,327 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.registration; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import org.sonar.api.Startable; +import org.sonar.api.resources.Languages; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.server.rule.RulesDefinition; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.api.utils.log.Profiler; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleRepositoryDto; +import org.sonar.server.es.metadata.MetadataIndex; +import org.sonar.server.qualityprofile.ActiveRuleChange; +import org.sonar.server.qualityprofile.QProfileRules; +import org.sonar.server.qualityprofile.index.ActiveRuleIndexer; +import org.sonar.server.rule.RuleDefinitionsLoader; +import org.sonar.server.rule.RuleDescriptionSectionsGeneratorResolver; +import org.sonar.server.rule.WebServerRuleFinder; +import org.sonar.server.rule.index.RuleIndexer; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; + +/** + * Registers rules at server startup + */ +public class RulesRegistrant implements Startable { + + private static final Logger LOG = Loggers.get(RulesRegistrant.class); + + private final RuleDefinitionsLoader defLoader; + private final QProfileRules qProfileRules; + private final DbClient dbClient; + private final RuleIndexer ruleIndexer; + private final ActiveRuleIndexer activeRuleIndexer; + private final Languages languages; + private final System2 system2; + private final WebServerRuleFinder webServerRuleFinder; + private final UuidFactory uuidFactory; + private final MetadataIndex metadataIndex; + private final RuleDescriptionSectionsGeneratorResolver ruleDescriptionSectionsGeneratorResolver; + private final RulesKeyVerifier rulesKeyVerifier; + private final StartupRuleUpdater startupRuleUpdater; + + public RulesRegistrant(RuleDefinitionsLoader defLoader, QProfileRules qProfileRules, DbClient dbClient, RuleIndexer ruleIndexer, + ActiveRuleIndexer activeRuleIndexer, Languages languages, System2 system2, + WebServerRuleFinder webServerRuleFinder, UuidFactory uuidFactory, MetadataIndex metadataIndex, + RuleDescriptionSectionsGeneratorResolver ruleDescriptionSectionsGeneratorResolver, + RulesKeyVerifier rulesKeyVerifier, StartupRuleUpdater startupRuleUpdater) { + this.defLoader = defLoader; + this.qProfileRules = qProfileRules; + this.dbClient = dbClient; + this.ruleIndexer = ruleIndexer; + this.activeRuleIndexer = activeRuleIndexer; + this.languages = languages; + this.system2 = system2; + this.webServerRuleFinder = webServerRuleFinder; + this.uuidFactory = uuidFactory; + this.metadataIndex = metadataIndex; + this.ruleDescriptionSectionsGeneratorResolver = ruleDescriptionSectionsGeneratorResolver; + this.rulesKeyVerifier = rulesKeyVerifier; + this.startupRuleUpdater = startupRuleUpdater; + } + + @Override + public void start() { + Profiler profiler = Profiler.create(LOG).startInfo("Register rules"); + try (DbSession dbSession = dbClient.openSession(true)) { + List<RulesDefinition.Repository> repositories = defLoader.load().repositories(); + RulesRegistrationContext rulesRegistrationContext = RulesRegistrationContext.create(dbClient, dbSession); + rulesKeyVerifier.verifyRuleKeyConsistency(repositories, rulesRegistrationContext); + + for (RulesDefinition.ExtendedRepository repoDef : repositories) { + if (languages.get(repoDef.language()) != null) { + registerRules(rulesRegistrationContext, repoDef.rules(), dbSession); + dbSession.commit(); + } + } + processRemainingDbRules(rulesRegistrationContext, dbSession); + List<ActiveRuleChange> changes = removeActiveRulesOnStillExistingRepositories(dbSession, rulesRegistrationContext, repositories); + dbSession.commit(); + + persistRepositories(dbSession, repositories); + // FIXME lack of resiliency, active rules index is corrupted if rule index fails + // to be updated. Only a single DB commit should be executed. + ruleIndexer.commitAndIndex(dbSession, rulesRegistrationContext.getAllModified().map(RuleDto::getUuid).collect(Collectors.toSet())); + changes.forEach(arChange -> dbClient.qProfileChangeDao().insert(dbSession, arChange.toDto(null))); + activeRuleIndexer.commitAndIndex(dbSession, changes); + rulesRegistrationContext.getRenamed().forEach(e -> LOG.info("Rule {} re-keyed to {}", e.getValue(), e.getKey().getKey())); + profiler.stopDebug(); + + if (!rulesRegistrationContext.hasDbRules()) { + Stream.concat(ruleIndexer.getIndexTypes().stream(), activeRuleIndexer.getIndexTypes().stream()) + .forEach(t -> metadataIndex.setInitialized(t, true)); + } + + webServerRuleFinder.startCaching(); + } + } + + private void persistRepositories(DbSession dbSession, List<RulesDefinition.Repository> repositories) { + List<String> keys = repositories.stream().map(RulesDefinition.Repository::key).toList(); + Set<String> existingKeys = dbClient.ruleRepositoryDao().selectAllKeys(dbSession); + + Map<Boolean, List<RuleRepositoryDto>> dtos = repositories.stream() + .map(r -> new RuleRepositoryDto(r.key(), r.language(), r.name())) + .collect(Collectors.groupingBy(i -> existingKeys.contains(i.getKey()))); + + dbClient.ruleRepositoryDao().update(dbSession, dtos.getOrDefault(true, emptyList())); + dbClient.ruleRepositoryDao().insert(dbSession, dtos.getOrDefault(false, emptyList())); + dbClient.ruleRepositoryDao().deleteIfKeyNotIn(dbSession, keys); + dbSession.commit(); + } + + @Override + public void stop() { + // nothing + } + + private void registerRules(RulesRegistrationContext context, List<RulesDefinition.Rule> ruleDefs, DbSession session) { + Map<RulesDefinition.Rule, RuleDto> dtos = new LinkedHashMap<>(ruleDefs.size()); + + for (RulesDefinition.Rule ruleDef : ruleDefs) { + RuleKey ruleKey = RuleKey.of(ruleDef.repository().key(), ruleDef.key()); + RuleDto ruleDto = findOrCreateRuleDto(context, session, ruleDef); + dtos.put(ruleDef, ruleDto); + + // we must detect renaming __before__ we modify the DTO + if (!ruleDto.getKey().equals(ruleKey)) { + context.renamed(ruleDto); + ruleDto.setRuleKey(ruleKey); + } + + if (startupRuleUpdater.findChangesAndUpdateRule(ruleDef, ruleDto)) { + context.updated(ruleDto); + } + + if (context.isUpdated(ruleDto) || context.isRenamed(ruleDto)) { + update(session, ruleDto); + } else if (!context.isCreated(ruleDto)) { + context.unchanged(ruleDto); + } + } + + for (Map.Entry<RulesDefinition.Rule, RuleDto> e : dtos.entrySet()) { + startupRuleUpdater.mergeParams(context, e.getKey(), e.getValue(), session); + startupRuleUpdater.updateDeprecatedKeys(context, e.getKey(), e.getValue(), session); + } + } + + @Nonnull + private RuleDto findOrCreateRuleDto(RulesRegistrationContext context, DbSession session, RulesDefinition.Rule ruleDef) { + return context.getDbRuleFor(ruleDef) + .orElseGet(() -> { + RuleDto newRule = RuleDto.from(ruleDef, ruleDescriptionSectionsGeneratorResolver.generateFor(ruleDef), uuidFactory, system2.now()); + dbClient.ruleDao().insert(session, newRule); + context.created(newRule); + return newRule; + }); + } + + private void processRemainingDbRules(RulesRegistrationContext recorder, DbSession dbSession) { + // custom rules check status of template, so they must be processed at the end + List<RuleDto> customRules = new ArrayList<>(); + + recorder.getRemaining().forEach(rule -> { + if (rule.isCustomRule()) { + customRules.add(rule); + } else if (!rule.isAdHoc() && rule.getStatus() != RuleStatus.REMOVED) { + removeRule(dbSession, recorder, rule); + } + }); + + for (RuleDto customRule : customRules) { + String templateUuid = customRule.getTemplateUuid(); + checkNotNull(templateUuid, "Template uuid of the custom rule '%s' is null", customRule); + Optional<RuleDto> template = dbClient.ruleDao().selectByUuid(templateUuid, dbSession); + if (template.isPresent() && template.get().getStatus() != RuleStatus.REMOVED) { + if (updateCustomRuleFromTemplateRule(customRule, template.get())) { + recorder.updated(customRule); + update(dbSession, customRule); + } + } else { + removeRule(dbSession, recorder, customRule); + } + } + + dbSession.commit(); + } + + private void removeRule(DbSession session, RulesRegistrationContext recorder, RuleDto rule) { + LOG.info(format("Disable rule %s", rule.getKey())); + rule.setStatus(RuleStatus.REMOVED); + rule.setSystemTags(emptySet()); + update(session, rule); + // FIXME resetting the tags for all organizations must be handled a different way + // rule.setTags(Collections.emptySet()); + // update(session, rule.getMetadata()); + recorder.removed(rule); + if (recorder.getRemoved().count() % 100 == 0) { + session.commit(); + } + } + + private static boolean updateCustomRuleFromTemplateRule(RuleDto customRule, RuleDto templateRule) { + boolean changed = false; + if (!Objects.equals(customRule.getLanguage(), templateRule.getLanguage())) { + customRule.setLanguage(templateRule.getLanguage()); + changed = true; + } + if (!Objects.equals(customRule.getConfigKey(), templateRule.getConfigKey())) { + customRule.setConfigKey(templateRule.getConfigKey()); + changed = true; + } + if (!Objects.equals(customRule.getPluginKey(), templateRule.getPluginKey())) { + customRule.setPluginKey(templateRule.getPluginKey()); + changed = true; + } + if (!Objects.equals(customRule.getDefRemediationFunction(), templateRule.getDefRemediationFunction())) { + customRule.setDefRemediationFunction(templateRule.getDefRemediationFunction()); + changed = true; + } + if (!Objects.equals(customRule.getDefRemediationGapMultiplier(), templateRule.getDefRemediationGapMultiplier())) { + customRule.setDefRemediationGapMultiplier(templateRule.getDefRemediationGapMultiplier()); + changed = true; + } + if (!Objects.equals(customRule.getDefRemediationBaseEffort(), templateRule.getDefRemediationBaseEffort())) { + customRule.setDefRemediationBaseEffort(templateRule.getDefRemediationBaseEffort()); + changed = true; + } + if (!Objects.equals(customRule.getGapDescription(), templateRule.getGapDescription())) { + customRule.setGapDescription(templateRule.getGapDescription()); + changed = true; + } + if (customRule.getStatus() != templateRule.getStatus()) { + customRule.setStatus(templateRule.getStatus()); + changed = true; + } + if (!Objects.equals(customRule.getSeverityString(), templateRule.getSeverityString())) { + customRule.setSeverity(templateRule.getSeverityString()); + changed = true; + } + if (!Objects.equals(customRule.getRepositoryKey(), templateRule.getRepositoryKey())) { + customRule.setRepositoryKey(templateRule.getRepositoryKey()); + changed = true; + } + return changed; + } + + /** + * SONAR-4642 + * <p/> + * Remove active rules on repositories that still exists. + * <p/> + * For instance, if the javascript repository do not provide anymore some rules, active rules related to this rules will be removed. + * But if the javascript repository do not exists anymore, then related active rules will not be removed. + * <p/> + * The side effect of this approach is that extended repositories will not be managed the same way. + * If an extended repository do not exists anymore, then related active rules will be removed. + */ + private List<ActiveRuleChange> removeActiveRulesOnStillExistingRepositories(DbSession dbSession, RulesRegistrationContext recorder, List<RulesDefinition.Repository> context) { + Set<String> existingAndRenamedRepositories = getExistingAndRenamedRepositories(recorder, context); + List<ActiveRuleChange> changes = new ArrayList<>(); + Profiler profiler = Profiler.create(LOG); + + recorder.getRemoved() + .filter(rule -> existingAndRenamedRepositories.contains(rule.getRepositoryKey())) + .forEach(rule -> { + // SONAR-4642 Remove active rules only when repository still exists + profiler.start(); + changes.addAll(qProfileRules.deleteRule(dbSession, rule)); + profiler.stopDebug(format("Remove active rule for rule %s", rule.getKey())); + }); + + return changes; + } + + private static Set<String> getExistingAndRenamedRepositories(RulesRegistrationContext recorder, Collection<RulesDefinition.Repository> context) { + return Stream.concat( + context.stream().map(RulesDefinition.ExtendedRepository::key), + recorder.getRenamed().map(Map.Entry::getValue).map(RuleKey::repository)) + .collect(Collectors.toSet()); + } + + private void update(DbSession session, RuleDto rule) { + rule.setUpdatedAt(system2.now()); + dbClient.ruleDao().update(session, rule); + } + +} diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesRegistrationContext.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesRegistrationContext.java new file mode 100644 index 00000000000..b3fd9ebce4e --- /dev/null +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/RulesRegistrationContext.java @@ -0,0 +1,212 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.registration; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.server.rule.RulesDefinition; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleParamDto; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableMap; + +class RulesRegistrationContext { + + private static final Logger LOG = Loggers.get(RulesRegistrationContext.class); + + // initial immutable data + private final Map<RuleKey, RuleDto> dbRules; + private final Set<RuleDto> known; + private final Map<String, Set<SingleDeprecatedRuleKey>> dbDeprecatedKeysByUuid; + private final Map<String, List<RuleParamDto>> ruleParamsByRuleUuid; + private final Map<RuleKey, RuleDto> dbRulesByDbDeprecatedKey; + // mutable data + private final Set<RuleDto> created = new HashSet<>(); + private final Map<RuleDto, RuleKey> renamed = new HashMap<>(); + private final Set<RuleDto> updated = new HashSet<>(); + private final Set<RuleDto> unchanged = new HashSet<>(); + private final Set<RuleDto> removed = new HashSet<>(); + + private RulesRegistrationContext(Map<RuleKey, RuleDto> dbRules, Map<String, Set<SingleDeprecatedRuleKey>> dbDeprecatedKeysByUuid, + Map<String, List<RuleParamDto>> ruleParamsByRuleUuid) { + this.dbRules = ImmutableMap.copyOf(dbRules); + this.known = ImmutableSet.copyOf(dbRules.values()); + this.dbDeprecatedKeysByUuid = dbDeprecatedKeysByUuid; + this.ruleParamsByRuleUuid = ruleParamsByRuleUuid; + this.dbRulesByDbDeprecatedKey = buildDbRulesByDbDeprecatedKey(dbDeprecatedKeysByUuid, dbRules); + } + + private static Map<RuleKey, RuleDto> buildDbRulesByDbDeprecatedKey(Map<String, Set<SingleDeprecatedRuleKey>> dbDeprecatedKeysByUuid, + Map<RuleKey, RuleDto> dbRules) { + Map<String, RuleDto> dbRulesByRuleUuid = dbRules.values().stream() + .collect(Collectors.toMap(RuleDto::getUuid, Function.identity())); + + Map<RuleKey, RuleDto> rulesByKey = new LinkedHashMap<>(); + for (Map.Entry<String, Set<SingleDeprecatedRuleKey>> entry : dbDeprecatedKeysByUuid.entrySet()) { + String ruleUuid = entry.getKey(); + RuleDto rule = dbRulesByRuleUuid.get(ruleUuid); + if (rule == null) { + LOG.warn("Could not retrieve rule with uuid %s referenced by a deprecated rule key. " + + "The following deprecated rule keys seem to be referencing a non-existing rule", + ruleUuid, entry.getValue()); + } else { + entry.getValue().forEach(d -> rulesByKey.put(d.getOldRuleKeyAsRuleKey(), rule)); + } + } + return unmodifiableMap(rulesByKey); + } + + boolean hasDbRules() { + return !dbRules.isEmpty(); + } + + Optional<RuleDto> getDbRuleFor(RulesDefinition.Rule ruleDef) { + RuleKey ruleKey = RuleKey.of(ruleDef.repository().key(), ruleDef.key()); + Optional<RuleDto> res = Stream.concat(Stream.of(ruleKey), ruleDef.deprecatedRuleKeys().stream()) + .map(dbRules::get) + .filter(Objects::nonNull) + .findFirst(); + // may occur in case of plugin downgrade + if (res.isEmpty()) { + return Optional.ofNullable(dbRulesByDbDeprecatedKey.get(ruleKey)); + } + return res; + } + + Map<RuleKey, SingleDeprecatedRuleKey> getDbDeprecatedKeysByOldRuleKey() { + return dbDeprecatedKeysByUuid.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toMap(SingleDeprecatedRuleKey::getOldRuleKeyAsRuleKey, Function.identity())); + } + + Set<SingleDeprecatedRuleKey> getDBDeprecatedKeysFor(RuleDto rule) { + return dbDeprecatedKeysByUuid.getOrDefault(rule.getUuid(), emptySet()); + } + + List<RuleParamDto> getRuleParametersFor(String ruleUuid) { + return ruleParamsByRuleUuid.getOrDefault(ruleUuid, emptyList()); + } + + Stream<RuleDto> getRemaining() { + Set<RuleDto> res = new HashSet<>(dbRules.values()); + res.removeAll(unchanged); + res.removeAll(renamed.keySet()); + res.removeAll(updated); + res.removeAll(removed); + return res.stream(); + } + + Stream<RuleDto> getRemoved() { + return removed.stream(); + } + + public Stream<Map.Entry<RuleDto, RuleKey>> getRenamed() { + return renamed.entrySet().stream(); + } + + Stream<RuleDto> getAllModified() { + return Stream.of( + created.stream(), + updated.stream(), + removed.stream(), + renamed.keySet().stream()) + .flatMap(s -> s); + } + + boolean isCreated(RuleDto ruleDto) { + return created.contains(ruleDto); + } + + boolean isRenamed(RuleDto ruleDto) { + return renamed.containsKey(ruleDto); + } + + boolean isUpdated(RuleDto ruleDto) { + return updated.contains(ruleDto); + } + + void created(RuleDto ruleDto) { + checkState(!known.contains(ruleDto), "known RuleDto can't be created"); + created.add(ruleDto); + } + + void renamed(RuleDto ruleDto) { + ensureKnown(ruleDto); + renamed.put(ruleDto, ruleDto.getKey()); + } + + void updated(RuleDto ruleDto) { + ensureKnown(ruleDto); + updated.add(ruleDto); + } + + void removed(RuleDto ruleDto) { + ensureKnown(ruleDto); + removed.add(ruleDto); + } + + void unchanged(RuleDto ruleDto) { + ensureKnown(ruleDto); + unchanged.add(ruleDto); + } + + private void ensureKnown(RuleDto ruleDto) { + checkState(known.contains(ruleDto), "unknown RuleDto"); + } + + static RulesRegistrationContext create(DbClient dbClient, DbSession dbSession) { + Map<RuleKey, RuleDto> allRules = dbClient.ruleDao().selectAll(dbSession).stream() + .collect(Collectors.toMap(RuleDto::getKey, Function.identity())); + Map<String, Set<SingleDeprecatedRuleKey>> existingDeprecatedKeysById = loadDeprecatedRuleKeys(dbClient, dbSession); + Map<String, List<RuleParamDto>> ruleParamsByRuleUuid = loadAllRuleParameters(dbClient, dbSession); + return new RulesRegistrationContext(allRules, existingDeprecatedKeysById, ruleParamsByRuleUuid); + } + + private static Map<String, List<RuleParamDto>> loadAllRuleParameters(DbClient dbClient, DbSession dbSession) { + return dbClient.ruleDao().selectAllRuleParams(dbSession).stream() + .collect(Collectors.groupingBy(RuleParamDto::getRuleUuid)); + } + + private static Map<String, Set<SingleDeprecatedRuleKey>> loadDeprecatedRuleKeys(DbClient dbClient, DbSession dbSession) { + return dbClient.ruleDao().selectAllDeprecatedRuleKeys(dbSession).stream() + .map(SingleDeprecatedRuleKey::from) + .collect(Collectors.groupingBy(SingleDeprecatedRuleKey::getRuleUuid, Collectors.toSet())); + } +} diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/SingleDeprecatedRuleKey.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/SingleDeprecatedRuleKey.java index ff72d360f2b..68cd19fc332 100644 --- a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/SingleDeprecatedRuleKey.java +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/SingleDeprecatedRuleKey.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.server.rule; +package org.sonar.server.rule.registration; import java.util.Objects; import java.util.Set; diff --git a/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/StartupRuleUpdater.java b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/StartupRuleUpdater.java new file mode 100644 index 00000000000..be40beb9ec6 --- /dev/null +++ b/server/sonar-webserver-core/src/main/java/org/sonar/server/rule/registration/StartupRuleUpdater.java @@ -0,0 +1,335 @@ +/* + * SonarQube + * Copyright (C) 2009-2023 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.rule.registration; + +import com.google.common.collect.Sets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import javax.annotation.Nullable; +import org.apache.commons.lang.StringUtils; +import org.sonar.api.rule.RuleStatus; +import org.sonar.api.rules.RuleType; +import org.sonar.api.server.debt.DebtRemediationFunction; +import org.sonar.api.server.rule.RulesDefinition; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.api.utils.log.Profiler; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.qualityprofile.ActiveRuleDto; +import org.sonar.db.qualityprofile.ActiveRuleParamDto; +import org.sonar.db.rule.DeprecatedRuleKeyDto; +import org.sonar.db.rule.RuleDescriptionSectionDto; +import org.sonar.db.rule.RuleDto; +import org.sonar.db.rule.RuleParamDto; +import org.sonar.server.rule.RuleDescriptionSectionsGeneratorResolver; + +import static com.google.common.collect.Sets.difference; +import static java.lang.String.format; +import static java.util.Collections.emptySet; +import static org.apache.commons.lang.StringUtils.isNotEmpty; + +/** + * The class detects changes between the rule definition coming from plugins during startup and rule from database. + * In case any changes are detected the rule is updated with the new information from plugin. + */ +public class StartupRuleUpdater { + + private static final Logger LOG = Loggers.get(StartupRuleUpdater.class); + + private final DbClient dbClient; + private final System2 system2; + private final UuidFactory uuidFactory; + private final RuleDescriptionSectionsGeneratorResolver sectionsGeneratorResolver; + + public StartupRuleUpdater(DbClient dbClient, System2 system2, UuidFactory uuidFactory, + RuleDescriptionSectionsGeneratorResolver sectionsGeneratorResolver) { + this.dbClient = dbClient; + this.system2 = system2; + this.uuidFactory = uuidFactory; + this.sectionsGeneratorResolver = sectionsGeneratorResolver; + } + + /** + * Returns true in case there was any change detected between rule in the database and rule from the plugin. + */ + boolean findChangesAndUpdateRule(RulesDefinition.Rule ruleDef, RuleDto ruleDto) { + boolean ruleMerged = mergeRule(ruleDef, ruleDto); + boolean debtDefinitionsMerged = mergeDebtDefinitions(ruleDef, ruleDto); + boolean tagsMerged = mergeTags(ruleDef, ruleDto); + boolean securityStandardsMerged = mergeSecurityStandards(ruleDef, ruleDto); + boolean educationPrinciplesMerged = mergeEducationPrinciples(ruleDef, ruleDto); + return ruleMerged || debtDefinitionsMerged || tagsMerged || securityStandardsMerged || educationPrinciplesMerged; + } + + void updateDeprecatedKeys(RulesRegistrationContext context, RulesDefinition.Rule ruleDef, RuleDto rule, DbSession dbSession) { + Set<SingleDeprecatedRuleKey> deprecatedRuleKeysFromDefinition = SingleDeprecatedRuleKey.from(ruleDef); + Set<SingleDeprecatedRuleKey> deprecatedRuleKeysFromDB = context.getDBDeprecatedKeysFor(rule); + + // DeprecatedKeys that must be deleted + List<String> uuidsToBeDeleted = difference(deprecatedRuleKeysFromDB, deprecatedRuleKeysFromDefinition).stream() + .map(SingleDeprecatedRuleKey::getUuid) + .toList(); + + dbClient.ruleDao().deleteDeprecatedRuleKeys(dbSession, uuidsToBeDeleted); + + // DeprecatedKeys that must be created + Sets.SetView<SingleDeprecatedRuleKey> deprecatedRuleKeysToBeCreated = difference(deprecatedRuleKeysFromDefinition, deprecatedRuleKeysFromDB); + + deprecatedRuleKeysToBeCreated + .forEach(r -> dbClient.ruleDao().insert(dbSession, new DeprecatedRuleKeyDto() + .setUuid(uuidFactory.create()) + .setRuleUuid(rule.getUuid()) + .setOldRepositoryKey(r.getOldRepositoryKey()) + .setOldRuleKey(r.getOldRuleKey()) + .setCreatedAt(system2.now()))); + } + + private boolean mergeRule(RulesDefinition.Rule def, RuleDto dto) { + boolean changed = false; + if (!Objects.equals(dto.getName(), def.name())) { + dto.setName(def.name()); + changed = true; + } + if (mergeDescription(def, dto)) { + changed = true; + } + if (!Objects.equals(dto.getPluginKey(), def.pluginKey())) { + dto.setPluginKey(def.pluginKey()); + changed = true; + } + if (!Objects.equals(dto.getConfigKey(), def.internalKey())) { + dto.setConfigKey(def.internalKey()); + changed = true; + } + String severity = def.severity(); + if (!Objects.equals(dto.getSeverityString(), severity)) { + dto.setSeverity(severity); + changed = true; + } + boolean isTemplate = def.template(); + if (isTemplate != dto.isTemplate()) { + dto.setIsTemplate(isTemplate); + changed = true; + } + if (def.status() != dto.getStatus()) { + dto.setStatus(def.status()); + changed = true; + } + if (!Objects.equals(dto.getScope().name(), def.scope().name())) { + dto.setScope(RuleDto.Scope.valueOf(def.scope().name())); + changed = true; + } + if (!Objects.equals(dto.getLanguage(), def.repository().language())) { + dto.setLanguage(def.repository().language()); + changed = true; + } + RuleType type = RuleType.valueOf(def.type().name()); + if (!Objects.equals(dto.getType(), type.getDbConstant())) { + dto.setType(type); + changed = true; + } + if (dto.isAdHoc()) { + dto.setIsAdHoc(false); + changed = true; + } + return changed; + } + + private static boolean mergeEducationPrinciples(RulesDefinition.Rule ruleDef, RuleDto dto) { + boolean changed = false; + if (dto.getEducationPrinciples().size() != ruleDef.educationPrincipleKeys().size() || + !dto.getEducationPrinciples().containsAll(ruleDef.educationPrincipleKeys())) { + dto.setEducationPrinciples(ruleDef.educationPrincipleKeys()); + changed = true; + } + return changed; + } + + private static boolean mergeTags(RulesDefinition.Rule ruleDef, RuleDto dto) { + boolean changed = false; + + if (RuleStatus.REMOVED == ruleDef.status()) { + dto.setSystemTags(emptySet()); + changed = true; + } else if (dto.getSystemTags().size() != ruleDef.tags().size() || + !dto.getSystemTags().containsAll(ruleDef.tags())) { + dto.setSystemTags(ruleDef.tags()); + changed = true; + } + return changed; + } + + private static boolean mergeSecurityStandards(RulesDefinition.Rule ruleDef, RuleDto dto) { + boolean changed = false; + + if (RuleStatus.REMOVED == ruleDef.status()) { + dto.setSecurityStandards(emptySet()); + changed = true; + } else if (dto.getSecurityStandards().size() != ruleDef.securityStandards().size() || + !dto.getSecurityStandards().containsAll(ruleDef.securityStandards())) { + dto.setSecurityStandards(ruleDef.securityStandards()); + changed = true; + } + return changed; + } + + private static boolean containsHtmlDescription(RulesDefinition.Rule rule) { + return isNotEmpty(rule.htmlDescription()) || !rule.ruleDescriptionSections().isEmpty(); + } + + private static boolean ruleDescriptionSectionsUnchanged(RuleDto ruleDto, Set<RuleDescriptionSectionDto> newRuleDescriptionSectionDtos) { + if (ruleDto.getRuleDescriptionSectionDtos().size() != newRuleDescriptionSectionDtos.size()) { + return false; + } + return ruleDto.getRuleDescriptionSectionDtos().stream() + .allMatch(sectionDto -> contains(newRuleDescriptionSectionDtos, sectionDto)); + } + + private static boolean contains(Set<RuleDescriptionSectionDto> sectionDtos, RuleDescriptionSectionDto sectionDto) { + return sectionDtos.stream() + .filter(s -> s.getKey().equals(sectionDto.getKey()) && s.getContent().equals(sectionDto.getContent())) + .anyMatch(s -> Objects.equals(s.getContext(), sectionDto.getContext())); + } + + private static boolean mergeDebtDefinitions(RuleDto dto, @Nullable String remediationFunction, + @Nullable String remediationCoefficient, @Nullable String remediationOffset, @Nullable String gapDescription) { + boolean changed = false; + + if (!Objects.equals(dto.getDefRemediationFunction(), remediationFunction)) { + dto.setDefRemediationFunction(remediationFunction); + changed = true; + } + if (!Objects.equals(dto.getDefRemediationGapMultiplier(), remediationCoefficient)) { + dto.setDefRemediationGapMultiplier(remediationCoefficient); + changed = true; + } + if (!Objects.equals(dto.getDefRemediationBaseEffort(), remediationOffset)) { + dto.setDefRemediationBaseEffort(remediationOffset); + changed = true; + } + if (!Objects.equals(dto.getGapDescription(), gapDescription)) { + dto.setGapDescription(gapDescription); + changed = true; + } + return changed; + } + + + private static boolean mergeDebtDefinitions(RulesDefinition.Rule def, RuleDto dto) { + // Debt definitions are set to null if the sub-characteristic and the remediation function are null + DebtRemediationFunction debtRemediationFunction = def.debtRemediationFunction(); + boolean hasDebt = debtRemediationFunction != null; + if (hasDebt) { + return mergeDebtDefinitions(dto, + debtRemediationFunction.type().name(), + debtRemediationFunction.gapMultiplier(), + debtRemediationFunction.baseEffort(), + def.gapDescription()); + } + return mergeDebtDefinitions(dto, null, null, null, null); + } + + + private boolean mergeDescription(RulesDefinition.Rule rule, RuleDto ruleDto) { + Set<RuleDescriptionSectionDto> newRuleDescriptionSectionDtos = sectionsGeneratorResolver.generateFor(rule); + if (ruleDescriptionSectionsUnchanged(ruleDto, newRuleDescriptionSectionDtos)) { + return false; + } + ruleDto.replaceRuleDescriptionSectionDtos(newRuleDescriptionSectionDtos); + if (containsHtmlDescription(rule)) { + ruleDto.setDescriptionFormat(RuleDto.Format.HTML); + return true; + } else if (isNotEmpty(rule.markdownDescription())) { + ruleDto.setDescriptionFormat(RuleDto.Format.MARKDOWN); + return true; + } + return false; + } + + + void mergeParams(RulesRegistrationContext context, RulesDefinition.Rule ruleDef, RuleDto rule, DbSession session) { + List<RuleParamDto> paramDtos = context.getRuleParametersFor(rule.getUuid()); + Map<String, RuleParamDto> existingParamsByName = new HashMap<>(); + + Profiler profiler = Profiler.create(LOG); + for (RuleParamDto paramDto : paramDtos) { + RulesDefinition.Param paramDef = ruleDef.param(paramDto.getName()); + if (paramDef == null) { + profiler.start(); + dbClient.activeRuleDao().deleteParamsByRuleParam(session, paramDto); + profiler.stopDebug(format("Propagate deleted param with name %s to active rules of rule %s", paramDto.getName(), rule.getKey())); + dbClient.ruleDao().deleteRuleParam(session, paramDto.getUuid()); + } else { + if (mergeParam(paramDto, paramDef)) { + dbClient.ruleDao().updateRuleParam(session, rule, paramDto); + } + existingParamsByName.put(paramDto.getName(), paramDto); + } + } + + // Create newly parameters + for (RulesDefinition.Param param : ruleDef.params()) { + RuleParamDto paramDto = existingParamsByName.get(param.key()); + if (paramDto != null) { + continue; + } + paramDto = RuleParamDto.createFor(rule) + .setName(param.key()) + .setDescription(param.description()) + .setDefaultValue(param.defaultValue()) + .setType(param.type().toString()); + dbClient.ruleDao().insertRuleParam(session, rule, paramDto); + if (StringUtils.isEmpty(param.defaultValue())) { + continue; + } + // Propagate the default value to existing active rule parameters + profiler.start(); + for (ActiveRuleDto activeRule : dbClient.activeRuleDao().selectByRuleUuid(session, rule.getUuid())) { + ActiveRuleParamDto activeParam = ActiveRuleParamDto.createFor(paramDto).setValue(param.defaultValue()); + dbClient.activeRuleDao().insertParam(session, activeRule, activeParam); + } + profiler.stopDebug(format("Propagate new param with name %s to active rules of rule %s", paramDto.getName(), rule.getKey())); + } + } + + private static boolean mergeParam(RuleParamDto paramDto, RulesDefinition.Param paramDef) { + boolean changed = false; + if (!Objects.equals(paramDto.getType(), paramDef.type().toString())) { + paramDto.setType(paramDef.type().toString()); + changed = true; + } + if (!Objects.equals(paramDto.getDefaultValue(), paramDef.defaultValue())) { + paramDto.setDefaultValue(paramDef.defaultValue()); + changed = true; + } + if (!Objects.equals(paramDto.getDescription(), paramDef.description())) { + paramDto.setDescription(paramDef.description()); + changed = true; + } + return changed; + } + +} diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/RegisterRulesTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/registration/RulesRegistrantIT.java index 89a8a76de27..050c2a00855 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/RegisterRulesTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/registration/RulesRegistrantIT.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.server.rule; +package org.sonar.server.rule.registration; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; @@ -73,6 +73,10 @@ import org.sonar.server.plugins.ServerPluginRepository; import org.sonar.server.qualityprofile.ActiveRuleChange; import org.sonar.server.qualityprofile.QProfileRules; import org.sonar.server.qualityprofile.index.ActiveRuleIndexer; +import org.sonar.server.rule.RuleDefinitionsLoader; +import org.sonar.server.rule.RuleDescriptionSectionsGenerator; +import org.sonar.server.rule.RuleDescriptionSectionsGeneratorResolver; +import org.sonar.server.rule.WebServerRuleFinder; import org.sonar.server.rule.index.RuleIndex; import org.sonar.server.rule.index.RuleIndexDefinition; import org.sonar.server.rule.index.RuleIndexer; @@ -112,7 +116,7 @@ import static org.sonar.db.rule.RuleDescriptionSectionDto.createDefaultRuleDescr import static org.sonar.server.qualityprofile.ActiveRuleChange.Type.DEACTIVATED; @RunWith(DataProviderRunner.class) -public class RegisterRulesTest { +public class RulesRegistrantIT { private static final String FAKE_PLUGIN_KEY = "unittest"; private static final Date DATE1 = DateUtils.parseDateTime("2014-01-01T19:10:03+0100"); @@ -148,17 +152,19 @@ public class RegisterRulesTest { private final RuleDescriptionSectionsGenerator ruleDescriptionSectionsGenerator = mock(RuleDescriptionSectionsGenerator.class); private final RuleDescriptionSectionsGeneratorResolver resolver = mock(RuleDescriptionSectionsGeneratorResolver.class); + private final RulesKeyVerifier rulesKeyVerifier = new RulesKeyVerifier(); + private final StartupRuleUpdater startupRuleUpdater = new StartupRuleUpdater(dbClient, system, uuidFactory, resolver); + @Before public void before() { ruleIndexer = new RuleIndexer(es.client(), dbClient); ruleIndex = new RuleIndex(es.client(), system); activeRuleIndexer = new ActiveRuleIndexer(dbClient, es.client()); - when(resolver.getRuleDescriptionSectionsGenerator(any())).thenReturn(ruleDescriptionSectionsGenerator); - when(ruleDescriptionSectionsGenerator.generateSections(any())).thenAnswer(answer -> { + when(resolver.generateFor(any())).thenAnswer(answer -> { RulesDefinition.Rule rule = answer.getArgument(0, RulesDefinition.Rule.class); String description = rule.htmlDescription() == null ? rule.markdownDescription() : rule.htmlDescription(); - Set<RuleDescriptionSectionDto> ruleDescriptionSectionDtos = rule.ruleDescriptionSections().stream() // + Set<RuleDescriptionSectionDto> ruleDescriptionSectionDtos = rule.ruleDescriptionSections().stream() .map(s -> builder() .uuid(UuidFactoryFast.getInstance().create()) .key(s.getKey()) @@ -207,8 +213,8 @@ public class RegisterRulesTest { private void verifyHotspot(RuleDto hotspotRule) { assertThat(hotspotRule.getName()).isEqualTo("Hotspot"); assertThat(hotspotRule.getDefaultRuleDescriptionSection().getContent()).isEqualTo("Minimal hotspot"); - assertThat(hotspotRule.getCreatedAt()).isEqualTo(RegisterRulesTest.DATE1.getTime()); - assertThat(hotspotRule.getUpdatedAt()).isEqualTo(RegisterRulesTest.DATE1.getTime()); + assertThat(hotspotRule.getCreatedAt()).isEqualTo(RulesRegistrantIT.DATE1.getTime()); + assertThat(hotspotRule.getUpdatedAt()).isEqualTo(RulesRegistrantIT.DATE1.getTime()); assertThat(hotspotRule.getType()).isEqualTo(RuleType.SECURITY_HOTSPOT.getDbConstant()); assertThat(hotspotRule.getSecurityStandards()).containsExactly("cwe:1", "cwe:123", "cwe:863", "owaspTop10-2021:a1", "owaspTop10-2021:a3"); } @@ -1142,8 +1148,8 @@ public class RegisterRulesTest { when(languages.get(any())).thenReturn(mock(Language.class)); reset(webServerRuleFinder); - RegisterRules task = new RegisterRules(loader, qProfileRules, dbClient, ruleIndexer, activeRuleIndexer, languages, system, webServerRuleFinder, uuidFactory, metadataIndex, - resolver); + RulesRegistrant task = new RulesRegistrant(loader, qProfileRules, dbClient, ruleIndexer, activeRuleIndexer, languages, system, webServerRuleFinder, uuidFactory, metadataIndex, + resolver, rulesKeyVerifier, startupRuleUpdater); task.start(); // Execute a commit to refresh session state as the task is using its own session db.getSession().commit(); diff --git a/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/SingleDeprecatedRuleKeyTest.java b/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/registration/SingleDeprecatedRuleKeyTest.java index 1ab9b1ef5a5..77c8e1e267d 100644 --- a/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/SingleDeprecatedRuleKeyTest.java +++ b/server/sonar-webserver-core/src/test/java/org/sonar/server/rule/registration/SingleDeprecatedRuleKeyTest.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.server.rule; +package org.sonar.server.rule.registration; import com.google.common.collect.ImmutableSet; import java.util.Set; @@ -26,6 +26,7 @@ import org.junit.Test; import org.sonar.api.rule.RuleKey; import org.sonar.api.server.rule.RulesDefinition; import org.sonar.db.rule.DeprecatedRuleKeyDto; +import org.sonar.server.rule.registration.SingleDeprecatedRuleKey; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; diff --git a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java index f25523ff456..9e6c5f13f8c 100644 --- a/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java +++ b/server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevelStartup.java @@ -39,9 +39,11 @@ import org.sonar.server.qualityprofile.builtin.BuiltInQualityProfilesUpdateListe import org.sonar.server.rule.AdvancedRuleDescriptionSectionsGenerator; import org.sonar.server.rule.LegacyHotspotRuleDescriptionSectionsGenerator; import org.sonar.server.rule.LegacyIssueRuleDescriptionSectionsGenerator; -import org.sonar.server.rule.RegisterRules; +import org.sonar.server.rule.registration.RulesKeyVerifier; +import org.sonar.server.rule.registration.RulesRegistrant; import org.sonar.server.rule.RuleDescriptionSectionsGeneratorResolver; import org.sonar.server.rule.WebServerRuleFinder; +import org.sonar.server.rule.registration.StartupRuleUpdater; import org.sonar.server.startup.GeneratePluginIndex; import org.sonar.server.startup.RegisterMetrics; import org.sonar.server.startup.RegisterPermissionTemplates; @@ -72,7 +74,9 @@ public class PlatformLevelStartup extends PlatformLevel { AdvancedRuleDescriptionSectionsGenerator.class, LegacyHotspotRuleDescriptionSectionsGenerator.class, LegacyIssueRuleDescriptionSectionsGenerator.class, - RegisterRules.class, + RulesRegistrant.class, + RulesKeyVerifier.class, + StartupRuleUpdater.class, BuiltInQProfileLoader.class); addIfStartupLeader( BuiltInQualityProfilesUpdateListener.class, |