source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2023 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.ce.task.projectanalysis.issue;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashMap;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Optional;
  28 import javax.annotation.Nullable;
  29 import org.jetbrains.annotations.NotNull;
  30 import org.sonar.api.rule.RuleKey;
  31 import org.sonar.api.rules.RuleType;
  32 import org.sonar.api.utils.Duration;
  33 import org.sonar.api.utils.log.Loggers;
  34 import org.sonar.ce.task.projectanalysis.batch.BatchReportReader;
  35 import org.sonar.ce.task.projectanalysis.component.Component;
  36 import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
  37 import org.sonar.ce.task.projectanalysis.issue.commonrule.CommonRuleEngine;
  38 import org.sonar.ce.task.projectanalysis.issue.filter.IssueFilter;
  39 import org.sonar.ce.task.projectanalysis.qualityprofile.ActiveRulesHolder;
  40 import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepository;
  41 import org.sonar.core.issue.DefaultIssue;
  42 import org.sonar.core.issue.tracking.Input;
  43 import org.sonar.core.issue.tracking.LazyInput;
  44 import org.sonar.core.issue.tracking.LineHashSequence;
  45 import org.sonar.core.util.CloseableIterator;
  46 import org.sonar.db.protobuf.DbCommons;
  47 import org.sonar.db.protobuf.DbIssues;
  48 import org.sonar.scanner.protocol.Constants.Severity;
  49 import org.sonar.scanner.protocol.output.ScannerReport;
  50 import org.sonar.scanner.protocol.output.ScannerReport.IssueType;
  51 import org.sonar.server.rule.CommonRuleKeys;
  52
  53 import static org.apache.commons.lang.StringUtils.isNotEmpty;
  54 import static org.sonar.api.issue.Issue.STATUS_OPEN;
  55 import static org.sonar.api.issue.Issue.STATUS_TO_REVIEW;
  56
  57 public class TrackerRawInputFactory {
  58   private static final long DEFAULT_EXTERNAL_ISSUE_EFFORT = 0L;
  59   private final TreeRootHolder treeRootHolder;
  60   private final BatchReportReader reportReader;
  61   private final CommonRuleEngine commonRuleEngine;
  62   private final IssueFilter issueFilter;
  63   private final SourceLinesHashRepository sourceLinesHash;
  64   private final RuleRepository ruleRepository;
  65   private final ActiveRulesHolder activeRulesHolder;
  66
  67   public TrackerRawInputFactory(TreeRootHolder treeRootHolder, BatchReportReader reportReader, SourceLinesHashRepository sourceLinesHash, CommonRuleEngine commonRuleEngine,
  68     IssueFilter issueFilter, RuleRepository ruleRepository, ActiveRulesHolder activeRulesHolder) {
  69     this.treeRootHolder = treeRootHolder;
  70     this.reportReader = reportReader;
  71     this.sourceLinesHash = sourceLinesHash;
  72     this.commonRuleEngine = commonRuleEngine;
  73     this.issueFilter = issueFilter;
  74     this.ruleRepository = ruleRepository;
  75     this.activeRulesHolder = activeRulesHolder;
  76   }
  77
  78   public Input<DefaultIssue> create(Component component) {
  79     return new RawLazyInput(component);
  80   }
  81
  82   private class RawLazyInput extends LazyInput<DefaultIssue> {
  83     private final Component component;
  84
  85     private RawLazyInput(Component component) {
  86       this.component = component;
  87     }
  88
  89     @Override
  90     protected LineHashSequence loadLineHashSequence() {
  91       if (component.getType() == Component.Type.FILE) {
  92         return new LineHashSequence(sourceLinesHash.getLineHashesMatchingDBVersion(component));
  93       } else {
  94         return new LineHashSequence(Collections.emptyList());
  95       }
  96     }
  97
  98     @Override
  99     protected List<DefaultIssue> loadIssues() {
 100       List<DefaultIssue> result = new ArrayList<>();
 101
 102       for (DefaultIssue commonRuleIssue : commonRuleEngine.process(component)) {
 103         if (issueFilter.accept(commonRuleIssue, component)) {
 104           result.add(init(commonRuleIssue, STATUS_OPEN));
 105         }
 106       }
 107
 108       if (component.getReportAttributes().getRef() == null) {
 109         return result;
 110       }
 111
 112       try (CloseableIterator<ScannerReport.Issue> reportIssues = reportReader.readComponentIssues(component.getReportAttributes().getRef())) {
 113         // optimization - do not load line hashes if there are no issues -> getLineHashSequence() is executed
 114         // as late as possible
 115         while (reportIssues.hasNext()) {
 116           ScannerReport.Issue reportIssue = reportIssues.next();
 117           if (isOnInactiveRule(reportIssue)) {
 118             continue;
 119           }
 120           if (!isIssueOnUnsupportedCommonRule(reportIssue)) {
 121             Loggers.get(getClass()).debug("Ignored issue from analysis report on rule {}:{}", reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 122             continue;
 123           }
 124           DefaultIssue issue = toIssue(getLineHashSequence(), reportIssue);
 125           if (issueFilter.accept(issue, component)) {
 126             result.add(issue);
 127           }
 128         }
 129       }
 130
 131       Map<RuleKey, ScannerReport.AdHocRule> adHocRuleMap = new HashMap<>();
 132       try (CloseableIterator<ScannerReport.AdHocRule> reportAdHocRule = reportReader.readAdHocRules()) {
 133         while (reportAdHocRule.hasNext()) {
 134           ScannerReport.AdHocRule adHocRule = reportAdHocRule.next();
 135           adHocRuleMap.put(RuleKey.of(RuleKey.EXTERNAL_RULE_REPO_PREFIX + adHocRule.getEngineId(), adHocRule.getRuleId()), adHocRule);
 136         }
 137       }
 138
 139       try (CloseableIterator<ScannerReport.ExternalIssue> reportExternalIssues = reportReader.readComponentExternalIssues(component.getReportAttributes().getRef())) {
 140         // optimization - do not load line hashes if there are no issues -> getLineHashSequence() is executed
 141         // as late as possible
 142         while (reportExternalIssues.hasNext()) {
 143           ScannerReport.ExternalIssue reportExternalIssue = reportExternalIssues.next();
 144           result.add(toExternalIssue(getLineHashSequence(), reportExternalIssue, adHocRuleMap));
 145         }
 146       }
 147
 148       return result;
 149     }
 150
 151     private boolean isOnInactiveRule(ScannerReport.Issue reportIssue) {
 152       RuleKey ruleKey = RuleKey.of(reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 153       return !activeRulesHolder.get(ruleKey).isPresent();
 154     }
 155
 156     private boolean isIssueOnUnsupportedCommonRule(ScannerReport.Issue issue) {
 157       // issues on batch common rules are ignored. This feature
 158       // is natively supported by compute engine since 5.2.
 159       return !issue.getRuleRepository().startsWith(CommonRuleKeys.REPOSITORY_PREFIX);
 160     }
 161
 162     private DefaultIssue toIssue(LineHashSequence lineHashSeq, ScannerReport.Issue reportIssue) {
 163       DefaultIssue issue = new DefaultIssue();
 164       init(issue, STATUS_OPEN);
 165       RuleKey ruleKey = RuleKey.of(reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 166       issue.setRuleKey(ruleKey);
 167       if (reportIssue.hasTextRange()) {
 168         int startLine = reportIssue.getTextRange().getStartLine();
 169         issue.setLine(startLine);
 170         issue.setChecksum(lineHashSeq.getHashForLine(startLine));
 171       } else {
 172         issue.setChecksum("");
 173       }
 174       if (isNotEmpty(reportIssue.getMsg())) {
 175         issue.setMessage(reportIssue.getMsg());
 176         if (!reportIssue.getMsgFormattingList().isEmpty()) {
 177           issue.setMessageFormattings(convertMessageFormattings(reportIssue.getMsgFormattingList()));
 178         }
 179       } else {
 180         Rule rule = ruleRepository.getByKey(ruleKey);
 181         issue.setMessage(rule.getName());
 182       }
 183       if (reportIssue.getSeverity() != Severity.UNSET_SEVERITY) {
 184         issue.setSeverity(reportIssue.getSeverity().name());
 185       }
 186       if (Double.compare(reportIssue.getGap(), 0D) != 0) {
 187         issue.setGap(reportIssue.getGap());
 188       }
 189       DbIssues.Locations.Builder dbLocationsBuilder = DbIssues.Locations.newBuilder();
 190       if (reportIssue.hasTextRange()) {
 191         dbLocationsBuilder.setTextRange(convertTextRange(reportIssue.getTextRange()));
 192       }
 193       for (ScannerReport.Flow flow : reportIssue.getFlowList()) {
 194         if (flow.getLocationCount() > 0) {
 195           DbIssues.Flow.Builder dbFlowBuilder = convertLocations(flow);
 196           dbLocationsBuilder.addFlow(dbFlowBuilder);
 197         }
 198       }
 199       issue.setIsFromExternalRuleEngine(false);
 200       issue.setLocations(dbLocationsBuilder.build());
 201       issue.setQuickFixAvailable(reportIssue.getQuickFixAvailable());
 202       issue.setRuleDescriptionContextKey(reportIssue.hasRuleDescriptionContextKey() ? reportIssue.getRuleDescriptionContextKey() : null);
 203       return issue;
 204     }
 205
 206     private DbIssues.Flow.Builder convertLocations(ScannerReport.Flow flow) {
 207       DbIssues.Flow.Builder dbFlowBuilder = DbIssues.Flow.newBuilder();
 208       for (ScannerReport.IssueLocation location : flow.getLocationList()) {
 209         convertLocation(location).ifPresent(dbFlowBuilder::addLocation);
 210       }
 211       if (isNotEmpty(flow.getDescription())) {
 212         dbFlowBuilder.setDescription(flow.getDescription());
 213       }
 214       toFlowType(flow.getType()).ifPresent(dbFlowBuilder::setType);
 215       return dbFlowBuilder;
 216     }
 217
 218     private DefaultIssue toExternalIssue(LineHashSequence lineHashSeq, ScannerReport.ExternalIssue reportExternalIssue, Map<RuleKey, ScannerReport.AdHocRule> adHocRuleMap) {
 219       DefaultIssue issue = new DefaultIssue();
 220       RuleType type = toRuleType(reportExternalIssue.getType());
 221       init(issue, type == RuleType.SECURITY_HOTSPOT ? STATUS_TO_REVIEW : STATUS_OPEN);
 222
 223       RuleKey ruleKey = RuleKey.of(RuleKey.EXTERNAL_RULE_REPO_PREFIX + reportExternalIssue.getEngineId(), reportExternalIssue.getRuleId());
 224       issue.setRuleKey(ruleKey);
 225       if (reportExternalIssue.hasTextRange()) {
 226         int startLine = reportExternalIssue.getTextRange().getStartLine();
 227         issue.setLine(startLine);
 228         issue.setChecksum(lineHashSeq.getHashForLine(startLine));
 229       } else {
 230         issue.setChecksum("");
 231       }
 232       if (isNotEmpty(reportExternalIssue.getMsg())) {
 233         issue.setMessage(reportExternalIssue.getMsg());
 234         if (!reportExternalIssue.getMsgFormattingList().isEmpty()) {
 235           issue.setMessageFormattings(convertMessageFormattings(reportExternalIssue.getMsgFormattingList()));
 236         }
 237       }
 238       if (reportExternalIssue.getSeverity() != Severity.UNSET_SEVERITY) {
 239         issue.setSeverity(reportExternalIssue.getSeverity().name());
 240       }
 241       issue.setEffort(Duration.create(reportExternalIssue.getEffort() != 0 ? reportExternalIssue.getEffort() : DEFAULT_EXTERNAL_ISSUE_EFFORT));
 242       DbIssues.Locations.Builder dbLocationsBuilder = DbIssues.Locations.newBuilder();
 243       if (reportExternalIssue.hasTextRange()) {
 244         dbLocationsBuilder.setTextRange(convertTextRange(reportExternalIssue.getTextRange()));
 245       }
 246       for (ScannerReport.Flow flow : reportExternalIssue.getFlowList()) {
 247         if (flow.getLocationCount() > 0) {
 248           DbIssues.Flow.Builder dbFlowBuilder = convertLocations(flow);
 249           dbLocationsBuilder.addFlow(dbFlowBuilder);
 250         }
 251       }
 252       issue.setIsFromExternalRuleEngine(true);
 253       issue.setLocations(dbLocationsBuilder.build());
 254       issue.setType(type);
 255
 256       ruleRepository.addOrUpdateAddHocRuleIfNeeded(ruleKey, () -> toAdHocRule(reportExternalIssue, adHocRuleMap.get(issue.ruleKey())));
 257       return issue;
 258     }
 259
 260     private NewAdHocRule toAdHocRule(ScannerReport.ExternalIssue reportIssue, @Nullable ScannerReport.AdHocRule adHocRule) {
 261       if (adHocRule != null) {
 262         return new NewAdHocRule(adHocRule);
 263       }
 264       return new NewAdHocRule(reportIssue);
 265     }
 266
 267     private Optional<DbIssues.FlowType> toFlowType(ScannerReport.FlowType flowType) {
 268       switch (flowType) {
 269         case DATA:
 270           return Optional.of(DbIssues.FlowType.DATA);
 271         case EXECUTION:
 272           return Optional.of(DbIssues.FlowType.EXECUTION);
 273         case UNDEFINED:
 274           return Optional.empty();
 275         default:
 276           throw new IllegalArgumentException("Unrecognized type: " + flowType);
 277       }
 278     }
 279
 280     private RuleType toRuleType(IssueType type) {
 281       switch (type) {
 282         case BUG:
 283           return RuleType.BUG;
 284         case CODE_SMELL:
 285           return RuleType.CODE_SMELL;
 286         case VULNERABILITY:
 287           return RuleType.VULNERABILITY;
 288         case SECURITY_HOTSPOT:
 289           return RuleType.SECURITY_HOTSPOT;
 290         case UNRECOGNIZED:
 291         default:
 292           throw new IllegalStateException("Invalid issue type: " + type);
 293       }
 294     }
 295
 296     private DefaultIssue init(DefaultIssue issue, String initialStatus) {
 297       issue.setStatus(initialStatus);
 298       issue.setResolution(null);
 299       issue.setComponentUuid(component.getUuid());
 300       issue.setComponentKey(component.getKey());
 301       issue.setProjectUuid(treeRootHolder.getRoot().getUuid());
 302       issue.setProjectKey(treeRootHolder.getRoot().getKey());
 303       return issue;
 304     }
 305
 306     private Optional<DbIssues.Location> convertLocation(ScannerReport.IssueLocation source) {
 307       DbIssues.Location.Builder target = DbIssues.Location.newBuilder();
 308       if (source.getComponentRef() != 0 && source.getComponentRef() != component.getReportAttributes().getRef()) {
 309         // SONAR-10781 Component might not exist because on PR, only changed components are included in the report
 310         Optional<Component> optionalComponent = treeRootHolder.getOptionalComponentByRef(source.getComponentRef());
 311         if (!optionalComponent.isPresent()) {
 312           return Optional.empty();
 313         }
 314         target.setComponentId(optionalComponent.get().getUuid());
 315       }
 316       if (isNotEmpty(source.getMsg())) {
 317         target.setMsg(source.getMsg());
 318         source.getMsgFormattingList()
 319           .forEach(m -> target.addMsgFormatting(convertMessageFormatting(m)));
 320       }
 321       if (source.hasTextRange()) {
 322         ScannerReport.TextRange sourceRange = source.getTextRange();
 323         DbCommons.TextRange.Builder targetRange = convertTextRange(sourceRange);
 324         target.setTextRange(targetRange);
 325       }
 326       return Optional.of(target.build());
 327     }
 328
 329     private DbCommons.TextRange.Builder convertTextRange(ScannerReport.TextRange sourceRange) {
 330       DbCommons.TextRange.Builder targetRange = DbCommons.TextRange.newBuilder();
 331       targetRange.setStartLine(sourceRange.getStartLine());
 332       targetRange.setStartOffset(sourceRange.getStartOffset());
 333       targetRange.setEndLine(sourceRange.getEndLine());
 334       targetRange.setEndOffset(sourceRange.getEndOffset());
 335       return targetRange;
 336     }
 337   }
 338
 339   private static DbIssues.MessageFormattings convertMessageFormattings(List<ScannerReport.MessageFormatting> msgFormattings) {
 340     DbIssues.MessageFormattings.Builder builder = DbIssues.MessageFormattings.newBuilder();
 341     msgFormattings.stream()
 342       .forEach(m -> builder.addMessageFormatting(TrackerRawInputFactory.convertMessageFormatting(m)));
 343     return builder.build();
 344   }
 345
 346   @NotNull
 347   private static DbIssues.MessageFormatting convertMessageFormatting(ScannerReport.MessageFormatting m) {
 348     DbIssues.MessageFormatting.Builder msgFormattingBuilder = DbIssues.MessageFormatting.newBuilder();
 349     return msgFormattingBuilder
 350       .setStart(m.getStart())
 351       .setEnd(m.getEnd())
 352       .setType(DbIssues.MessageFormattingType.valueOf(m.getType().name())).build();
 353   }
 354
 355 }