source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2024 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.ce.task.projectanalysis.issue;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashMap;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Optional;
  28 import javax.annotation.Nullable;
  29 import org.jetbrains.annotations.NotNull;
  30 import org.slf4j.LoggerFactory;
  31 import org.sonar.api.issue.impact.SoftwareQuality;
  32 import org.sonar.api.rule.RuleKey;
  33 import org.sonar.api.rules.RuleType;
  34 import org.sonar.api.server.rule.internal.ImpactMapper;
  35 import org.sonar.api.utils.Duration;
  36 import org.sonar.ce.task.projectanalysis.batch.BatchReportReader;
  37 import org.sonar.ce.task.projectanalysis.component.Component;
  38 import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
  39 import org.sonar.ce.task.projectanalysis.issue.filter.IssueFilter;
  40 import org.sonar.ce.task.projectanalysis.qualityprofile.ActiveRulesHolder;
  41 import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepository;
  42 import org.sonar.core.issue.DefaultIssue;
  43 import org.sonar.core.issue.tracking.Input;
  44 import org.sonar.core.issue.tracking.LazyInput;
  45 import org.sonar.core.issue.tracking.LineHashSequence;
  46 import org.sonar.core.util.CloseableIterator;
  47 import org.sonar.db.protobuf.DbCommons;
  48 import org.sonar.db.protobuf.DbIssues;
  49 import org.sonar.scanner.protocol.Constants;
  50 import org.sonar.scanner.protocol.Constants.Severity;
  51 import org.sonar.scanner.protocol.output.ScannerReport;
  52 import org.sonar.scanner.protocol.output.ScannerReport.Impact;
  53 import org.sonar.scanner.protocol.output.ScannerReport.IssueType;
  54 import org.sonar.server.rule.CommonRuleKeys;
  55
  56 import static java.util.stream.Collectors.toMap;
  57 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
  58 import static org.sonar.api.issue.Issue.STATUS_OPEN;
  59 import static org.sonar.api.issue.Issue.STATUS_TO_REVIEW;
  60
  61 public class TrackerRawInputFactory {
  62   private static final long DEFAULT_EXTERNAL_ISSUE_EFFORT = 0L;
  63   private final TreeRootHolder treeRootHolder;
  64   private final BatchReportReader reportReader;
  65   private final IssueFilter issueFilter;
  66   private final SourceLinesHashRepository sourceLinesHash;
  67   private final RuleRepository ruleRepository;
  68   private final ActiveRulesHolder activeRulesHolder;
  69
  70   public TrackerRawInputFactory(TreeRootHolder treeRootHolder, BatchReportReader reportReader, SourceLinesHashRepository sourceLinesHash,
  71     IssueFilter issueFilter, RuleRepository ruleRepository, ActiveRulesHolder activeRulesHolder) {
  72     this.treeRootHolder = treeRootHolder;
  73     this.reportReader = reportReader;
  74     this.sourceLinesHash = sourceLinesHash;
  75     this.issueFilter = issueFilter;
  76     this.ruleRepository = ruleRepository;
  77     this.activeRulesHolder = activeRulesHolder;
  78   }
  79
  80   public Input<DefaultIssue> create(Component component) {
  81     return new RawLazyInput(component);
  82   }
  83
  84   private class RawLazyInput extends LazyInput<DefaultIssue> {
  85     private final Component component;
  86
  87     private RawLazyInput(Component component) {
  88       this.component = component;
  89     }
  90
  91     @Override
  92     protected LineHashSequence loadLineHashSequence() {
  93       if (component.getType() == Component.Type.FILE) {
  94         return new LineHashSequence(sourceLinesHash.getLineHashesMatchingDBVersion(component));
  95       } else {
  96         return new LineHashSequence(Collections.emptyList());
  97       }
  98     }
  99
 100     @Override
 101     protected List<DefaultIssue> loadIssues() {
 102       List<DefaultIssue> result = new ArrayList<>();
 103
 104       if (component.getReportAttributes().getRef() == null) {
 105         return result;
 106       }
 107
 108       try (CloseableIterator<ScannerReport.Issue> reportIssues = reportReader.readComponentIssues(component.getReportAttributes().getRef())) {
 109         // optimization - do not load line hashes if there are no issues -> getLineHashSequence() is executed
 110         // as late as possible
 111         while (reportIssues.hasNext()) {
 112           ScannerReport.Issue reportIssue = reportIssues.next();
 113           if (isOnInactiveRule(reportIssue)) {
 114             continue;
 115           }
 116           if (!isIssueOnUnsupportedCommonRule(reportIssue)) {
 117             LoggerFactory.getLogger(getClass()).debug("Ignored issue from analysis report on rule {}:{}", reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 118             continue;
 119           }
 120           DefaultIssue issue = toIssue(getLineHashSequence(), reportIssue);
 121           if (issueFilter.accept(issue, component)) {
 122             result.add(issue);
 123           }
 124         }
 125       }
 126
 127       Map<RuleKey, ScannerReport.AdHocRule> adHocRuleMap = new HashMap<>();
 128       try (CloseableIterator<ScannerReport.AdHocRule> reportAdHocRule = reportReader.readAdHocRules()) {
 129         while (reportAdHocRule.hasNext()) {
 130           ScannerReport.AdHocRule adHocRule = reportAdHocRule.next();
 131           adHocRuleMap.put(RuleKey.of(RuleKey.EXTERNAL_RULE_REPO_PREFIX + adHocRule.getEngineId(), adHocRule.getRuleId()), adHocRule);
 132         }
 133       }
 134
 135       try (CloseableIterator<ScannerReport.ExternalIssue> reportExternalIssues = reportReader.readComponentExternalIssues(component.getReportAttributes().getRef())) {
 136         // optimization - do not load line hashes if there are no issues -> getLineHashSequence() is executed
 137         // as late as possible
 138         while (reportExternalIssues.hasNext()) {
 139           ScannerReport.ExternalIssue reportExternalIssue = reportExternalIssues.next();
 140           result.add(toExternalIssue(getLineHashSequence(), reportExternalIssue, adHocRuleMap));
 141         }
 142       }
 143
 144       return result;
 145     }
 146
 147     private boolean isOnInactiveRule(ScannerReport.Issue reportIssue) {
 148       RuleKey ruleKey = RuleKey.of(reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 149       return !activeRulesHolder.get(ruleKey).isPresent();
 150     }
 151
 152     private boolean isIssueOnUnsupportedCommonRule(ScannerReport.Issue issue) {
 153       // issues on batch common rules are ignored. This feature
 154       // is natively supported by compute engine since 5.2.
 155       return !issue.getRuleRepository().startsWith(CommonRuleKeys.REPOSITORY_PREFIX);
 156     }
 157
 158     private DefaultIssue toIssue(LineHashSequence lineHashSeq, ScannerReport.Issue reportIssue) {
 159       DefaultIssue issue = new DefaultIssue();
 160       init(issue, STATUS_OPEN);
 161       RuleKey ruleKey = RuleKey.of(reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 162       issue.setRuleKey(ruleKey);
 163       if (reportIssue.hasTextRange()) {
 164         int startLine = reportIssue.getTextRange().getStartLine();
 165         issue.setLine(startLine);
 166         issue.setChecksum(lineHashSeq.getHashForLine(startLine));
 167       } else {
 168         issue.setChecksum("");
 169       }
 170       if (isNotEmpty(reportIssue.getMsg())) {
 171         issue.setMessage(reportIssue.getMsg());
 172         if (!reportIssue.getMsgFormattingList().isEmpty()) {
 173           issue.setMessageFormattings(convertMessageFormattings(reportIssue.getMsgFormattingList()));
 174         }
 175       } else {
 176         Rule rule = ruleRepository.getByKey(ruleKey);
 177         issue.setMessage(rule.getName());
 178       }
 179       if (reportIssue.getSeverity() != Severity.UNSET_SEVERITY) {
 180         issue.setSeverity(reportIssue.getSeverity().name());
 181       }
 182       if (Double.compare(reportIssue.getGap(), 0D) != 0) {
 183         issue.setGap(reportIssue.getGap());
 184       }
 185       DbIssues.Locations.Builder dbLocationsBuilder = DbIssues.Locations.newBuilder();
 186       if (reportIssue.hasTextRange()) {
 187         dbLocationsBuilder.setTextRange(convertTextRange(reportIssue.getTextRange()));
 188       }
 189       for (ScannerReport.Flow flow : reportIssue.getFlowList()) {
 190         if (flow.getLocationCount() > 0) {
 191           DbIssues.Flow.Builder dbFlowBuilder = convertLocations(flow);
 192           dbLocationsBuilder.addFlow(dbFlowBuilder);
 193         }
 194       }
 195       issue.setIsFromExternalRuleEngine(false);
 196       issue.setLocations(dbLocationsBuilder.build());
 197       issue.setQuickFixAvailable(reportIssue.getQuickFixAvailable());
 198       issue.setRuleDescriptionContextKey(reportIssue.hasRuleDescriptionContextKey() ? reportIssue.getRuleDescriptionContextKey() : null);
 199       issue.setCodeVariants(reportIssue.getCodeVariantsList());
 200
 201       issue.replaceImpacts(replaceDefaultWithOverridenImpacts(issue.ruleKey(), reportIssue.getOverridenImpactsList()));
 202       return issue;
 203     }
 204
 205     private Map<SoftwareQuality, org.sonar.api.issue.impact.Severity> replaceDefaultWithOverridenImpacts(RuleKey ruleKey, List<Impact> overridenImpactsList) {
 206       Map<SoftwareQuality, org.sonar.api.issue.impact.Severity> overridenImpactMap = getOverridenImpactMap(overridenImpactsList);
 207       return ruleRepository.getByKey(ruleKey).getDefaultImpacts().entrySet()
 208         .stream()
 209         .collect(toMap(
 210           Map.Entry::getKey,
 211           entry -> overridenImpactMap.containsKey(entry.getKey()) ? overridenImpactMap.get(entry.getKey()) : entry.getValue()));
 212     }
 213
 214     private static Map<SoftwareQuality, org.sonar.api.issue.impact.Severity> getOverridenImpactMap(List<Impact> overridenImpactsList) {
 215       return overridenImpactsList.stream()
 216         .collect(toMap(
 217           impact -> SoftwareQuality.valueOf(impact.getSoftwareQuality()),
 218           impact -> org.sonar.api.issue.impact.Severity.valueOf(impact.getSeverity())));
 219     }
 220
 221     private DbIssues.Flow.Builder convertLocations(ScannerReport.Flow flow) {
 222       DbIssues.Flow.Builder dbFlowBuilder = DbIssues.Flow.newBuilder();
 223       for (ScannerReport.IssueLocation location : flow.getLocationList()) {
 224         convertLocation(location).ifPresent(dbFlowBuilder::addLocation);
 225       }
 226       if (isNotEmpty(flow.getDescription())) {
 227         dbFlowBuilder.setDescription(flow.getDescription());
 228       }
 229       toFlowType(flow.getType()).ifPresent(dbFlowBuilder::setType);
 230       return dbFlowBuilder;
 231     }
 232
 233
 234     private DefaultIssue toExternalIssue(LineHashSequence lineHashSeq, ScannerReport.ExternalIssue reportExternalIssue, Map<RuleKey, ScannerReport.AdHocRule> adHocRuleMap) {
 235       DefaultIssue issue = new DefaultIssue();
 236       RuleKey ruleKey = RuleKey.of(RuleKey.EXTERNAL_RULE_REPO_PREFIX + reportExternalIssue.getEngineId(), reportExternalIssue.getRuleId());
 237       issue.setRuleKey(ruleKey);
 238       ruleRepository.addOrUpdateAddHocRuleIfNeeded(ruleKey, () -> toAdHocRule(reportExternalIssue, adHocRuleMap.get(issue.ruleKey())));
 239
 240       Rule existingRule = ruleRepository.getByKey(ruleKey);
 241       issue.setSeverity(determineDeprecatedSeverity(reportExternalIssue, existingRule));
 242       issue.setType(determineDeprecatedType(reportExternalIssue, existingRule));
 243       issue.replaceImpacts(replaceDefaultWithOverridenImpacts(issue.ruleKey(), reportExternalIssue.getImpactsList()));
 244
 245       init(issue, issue.type() == RuleType.SECURITY_HOTSPOT ? STATUS_TO_REVIEW : STATUS_OPEN);
 246
 247       if (reportExternalIssue.hasTextRange()) {
 248         int startLine = reportExternalIssue.getTextRange().getStartLine();
 249         issue.setLine(startLine);
 250         issue.setChecksum(lineHashSeq.getHashForLine(startLine));
 251       } else {
 252         issue.setChecksum("");
 253       }
 254       if (isNotEmpty(reportExternalIssue.getMsg())) {
 255         issue.setMessage(reportExternalIssue.getMsg());
 256         if (!reportExternalIssue.getMsgFormattingList().isEmpty()) {
 257           issue.setMessageFormattings(convertMessageFormattings(reportExternalIssue.getMsgFormattingList()));
 258         }
 259       }
 260       issue.setEffort(Duration.create(reportExternalIssue.getEffort() != 0 ? reportExternalIssue.getEffort() : DEFAULT_EXTERNAL_ISSUE_EFFORT));
 261       DbIssues.Locations.Builder dbLocationsBuilder = DbIssues.Locations.newBuilder();
 262       if (reportExternalIssue.hasTextRange()) {
 263         dbLocationsBuilder.setTextRange(convertTextRange(reportExternalIssue.getTextRange()));
 264       }
 265       for (ScannerReport.Flow flow : reportExternalIssue.getFlowList()) {
 266         if (flow.getLocationCount() > 0) {
 267           DbIssues.Flow.Builder dbFlowBuilder = convertLocations(flow);
 268           dbLocationsBuilder.addFlow(dbFlowBuilder);
 269         }
 270       }
 271       issue.setIsFromExternalRuleEngine(true);
 272       issue.setLocations(dbLocationsBuilder.build());
 273
 274       return issue;
 275     }
 276
 277     private NewAdHocRule toAdHocRule(ScannerReport.ExternalIssue reportIssue, @Nullable ScannerReport.AdHocRule adHocRule) {
 278       if (adHocRule != null) {
 279         return new NewAdHocRule(adHocRule);
 280       }
 281       return new NewAdHocRule(reportIssue);
 282     }
 283
 284     private Optional<DbIssues.FlowType> toFlowType(ScannerReport.FlowType flowType) {
 285       switch (flowType) {
 286         case DATA:
 287           return Optional.of(DbIssues.FlowType.DATA);
 288         case EXECUTION:
 289           return Optional.of(DbIssues.FlowType.EXECUTION);
 290         case UNDEFINED:
 291           return Optional.empty();
 292         default:
 293           throw new IllegalArgumentException("Unrecognized type: " + flowType);
 294       }
 295     }
 296
 297     private RuleType toRuleType(IssueType type) {
 298       switch (type) {
 299         case BUG:
 300           return RuleType.BUG;
 301         case CODE_SMELL:
 302           return RuleType.CODE_SMELL;
 303         case VULNERABILITY:
 304           return RuleType.VULNERABILITY;
 305         case SECURITY_HOTSPOT:
 306           return RuleType.SECURITY_HOTSPOT;
 307         case UNRECOGNIZED:
 308         default:
 309           throw new IllegalStateException("Invalid issue type: " + type);
 310       }
 311     }
 312
 313     private DefaultIssue init(DefaultIssue issue, String initialStatus) {
 314       issue.setStatus(initialStatus);
 315       issue.setResolution(null);
 316       issue.setComponentUuid(component.getUuid());
 317       issue.setComponentKey(component.getKey());
 318       issue.setProjectUuid(treeRootHolder.getRoot().getUuid());
 319       issue.setProjectKey(treeRootHolder.getRoot().getKey());
 320       return issue;
 321     }
 322
 323     private Optional<DbIssues.Location> convertLocation(ScannerReport.IssueLocation source) {
 324       DbIssues.Location.Builder target = DbIssues.Location.newBuilder();
 325       if (source.getComponentRef() != 0 && source.getComponentRef() != component.getReportAttributes().getRef()) {
 326         // SONAR-10781 Component might not exist because on PR, only changed components are included in the report
 327         Optional<Component> optionalComponent = treeRootHolder.getOptionalComponentByRef(source.getComponentRef());
 328         if (!optionalComponent.isPresent()) {
 329           return Optional.empty();
 330         }
 331         target.setComponentId(optionalComponent.get().getUuid());
 332       }
 333       if (isNotEmpty(source.getMsg())) {
 334         target.setMsg(source.getMsg());
 335         source.getMsgFormattingList()
 336           .forEach(m -> target.addMsgFormatting(convertMessageFormatting(m)));
 337       }
 338       if (source.hasTextRange()) {
 339         ScannerReport.TextRange sourceRange = source.getTextRange();
 340         DbCommons.TextRange.Builder targetRange = convertTextRange(sourceRange);
 341         target.setTextRange(targetRange);
 342       }
 343       return Optional.of(target.build());
 344     }
 345
 346     private DbCommons.TextRange.Builder convertTextRange(ScannerReport.TextRange sourceRange) {
 347       DbCommons.TextRange.Builder targetRange = DbCommons.TextRange.newBuilder();
 348       targetRange.setStartLine(sourceRange.getStartLine());
 349       targetRange.setStartOffset(sourceRange.getStartOffset());
 350       targetRange.setEndLine(sourceRange.getEndLine());
 351       targetRange.setEndOffset(sourceRange.getEndOffset());
 352       return targetRange;
 353     }
 354
 355     private RuleType determineDeprecatedType(ScannerReport.ExternalIssue reportExternalIssue, Rule rule) {
 356       if (reportExternalIssue.getType() != ScannerReport.IssueType.UNSET) {
 357         return toRuleType(reportExternalIssue.getType());
 358       } else if (rule.getType() != null) {
 359         return rule.getType();
 360       } else if (!rule.getDefaultImpacts().isEmpty()) {
 361         SoftwareQuality impactSoftwareQuality = ImpactMapper.getBestImpactForBackmapping(rule.getDefaultImpacts()).getKey();
 362         return ImpactMapper.convertToRuleType(impactSoftwareQuality);
 363       } else {
 364         throw new IllegalArgumentException("Cannot determine the type for issue of rule %s".formatted(reportExternalIssue.getRuleId()));
 365       }
 366     }
 367
 368     private static String determineDeprecatedSeverity(ScannerReport.ExternalIssue reportExternalIssue, Rule rule) {
 369       if (reportExternalIssue.getSeverity() != Constants.Severity.UNSET_SEVERITY) {
 370         return reportExternalIssue.getSeverity().name();
 371       } else if (rule.getSeverity() != null) {
 372         return rule.getSeverity();
 373       } else if (!rule.getDefaultImpacts().isEmpty()) {
 374         org.sonar.api.issue.impact.Severity impactSeverity = ImpactMapper.getBestImpactForBackmapping(rule.getDefaultImpacts()).getValue();
 375         return ImpactMapper.convertToDeprecatedSeverity(impactSeverity);
 376       } else {
 377         throw new IllegalArgumentException("Cannot determine the severity for issue of rule %s".formatted(reportExternalIssue.getRuleId()));
 378       }
 379     }
 380
 381   }
 382
 383
 384   private static DbIssues.MessageFormattings convertMessageFormattings(List<ScannerReport.MessageFormatting> msgFormattings) {
 385     DbIssues.MessageFormattings.Builder builder = DbIssues.MessageFormattings.newBuilder();
 386     msgFormattings.stream()
 387       .forEach(m -> builder.addMessageFormatting(TrackerRawInputFactory.convertMessageFormatting(m)));
 388     return builder.build();
 389   }
 390
 391   @NotNull
 392   private static DbIssues.MessageFormatting convertMessageFormatting(ScannerReport.MessageFormatting m) {
 393     DbIssues.MessageFormatting.Builder msgFormattingBuilder = DbIssues.MessageFormatting.newBuilder();
 394     return msgFormattingBuilder
 395       .setStart(m.getStart())
 396       .setEnd(m.getEnd())
 397       .setType(DbIssues.MessageFormattingType.valueOf(m.getType().name())).build();
 398   }
 399
 400 }