source.dussan.org Git - sonarqube.git/blob

   1 /*
   2  * SonarQube
   3  * Copyright (C) 2009-2022 SonarSource SA
   4  * mailto:info AT sonarsource DOT com
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 3 of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  19  */
  20 package org.sonar.ce.task.projectanalysis.issue;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collections;
  24 import java.util.HashMap;
  25 import java.util.List;
  26 import java.util.Map;
  27 import java.util.Optional;
  28 import java.util.regex.Pattern;
  29 import javax.annotation.Nullable;
  30 import org.apache.commons.codec.digest.DigestUtils;
  31 import org.sonar.api.rule.RuleKey;
  32 import org.sonar.api.rules.RuleType;
  33 import org.sonar.api.utils.Duration;
  34 import org.sonar.api.utils.log.Loggers;
  35 import org.sonar.ce.task.projectanalysis.batch.BatchReportReader;
  36 import org.sonar.ce.task.projectanalysis.component.Component;
  37 import org.sonar.ce.task.projectanalysis.component.TreeRootHolder;
  38 import org.sonar.ce.task.projectanalysis.issue.commonrule.CommonRuleEngine;
  39 import org.sonar.ce.task.projectanalysis.issue.filter.IssueFilter;
  40 import org.sonar.ce.task.projectanalysis.qualityprofile.ActiveRulesHolder;
  41 import org.sonar.ce.task.projectanalysis.source.SourceLinesHashRepository;
  42 import org.sonar.ce.task.projectanalysis.source.SourceLinesRepository;
  43 import org.sonar.core.issue.DefaultIssue;
  44 import org.sonar.core.issue.tracking.Input;
  45 import org.sonar.core.issue.tracking.LazyInput;
  46 import org.sonar.core.issue.tracking.LineHashSequence;
  47 import org.sonar.core.util.CloseableIterator;
  48 import org.sonar.db.protobuf.DbCommons;
  49 import org.sonar.db.protobuf.DbIssues;
  50 import org.sonar.scanner.protocol.Constants.Severity;
  51 import org.sonar.scanner.protocol.output.ScannerReport;
  52 import org.sonar.scanner.protocol.output.ScannerReport.IssueType;
  53 import org.sonar.server.rule.CommonRuleKeys;
  54
  55 import static org.apache.commons.lang.StringUtils.isNotEmpty;
  56 import static org.sonar.api.issue.Issue.STATUS_OPEN;
  57 import static org.sonar.api.issue.Issue.STATUS_TO_REVIEW;
  58
  59 public class TrackerRawInputFactory {
  60   private static final Pattern MATCH_ALL_WHITESPACES = Pattern.compile("\\s");
  61   private static final long DEFAULT_EXTERNAL_ISSUE_EFFORT = 0L;
  62   private final TreeRootHolder treeRootHolder;
  63   private final BatchReportReader reportReader;
  64   private final CommonRuleEngine commonRuleEngine;
  65   private final IssueFilter issueFilter;
  66   private final SourceLinesHashRepository sourceLinesHash;
  67   private final SourceLinesRepository sourceLinesRepository;
  68   private final RuleRepository ruleRepository;
  69   private final ActiveRulesHolder activeRulesHolder;
  70
  71   public TrackerRawInputFactory(TreeRootHolder treeRootHolder, BatchReportReader reportReader, SourceLinesHashRepository sourceLinesHash,
  72     SourceLinesRepository sourceLinesRepository, CommonRuleEngine commonRuleEngine, IssueFilter issueFilter, RuleRepository ruleRepository,
  73     ActiveRulesHolder activeRulesHolder) {
  74     this.treeRootHolder = treeRootHolder;
  75     this.reportReader = reportReader;
  76     this.sourceLinesHash = sourceLinesHash;
  77     this.sourceLinesRepository = sourceLinesRepository;
  78     this.commonRuleEngine = commonRuleEngine;
  79     this.issueFilter = issueFilter;
  80     this.ruleRepository = ruleRepository;
  81     this.activeRulesHolder = activeRulesHolder;
  82   }
  83
  84   public Input<DefaultIssue> create(Component component) {
  85     return new RawLazyInput(component);
  86   }
  87
  88   private class RawLazyInput extends LazyInput<DefaultIssue> {
  89     private final Component component;
  90
  91     private RawLazyInput(Component component) {
  92       this.component = component;
  93     }
  94
  95     @Override
  96     protected LineHashSequence loadLineHashSequence() {
  97       if (component.getType() == Component.Type.FILE) {
  98         return new LineHashSequence(sourceLinesHash.getLineHashesMatchingDBVersion(component));
  99       } else {
 100         return new LineHashSequence(Collections.emptyList());
 101       }
 102     }
 103
 104     @Override
 105     protected List<DefaultIssue> loadIssues() {
 106       List<DefaultIssue> result = new ArrayList<>();
 107
 108       for (DefaultIssue commonRuleIssue : commonRuleEngine.process(component)) {
 109         if (issueFilter.accept(commonRuleIssue, component)) {
 110           result.add(init(commonRuleIssue, STATUS_OPEN));
 111         }
 112       }
 113
 114       if (component.getReportAttributes().getRef() == null) {
 115         return result;
 116       }
 117
 118       try (CloseableIterator<ScannerReport.Issue> reportIssues = reportReader.readComponentIssues(component.getReportAttributes().getRef())) {
 119         // optimization - do not load line hashes if there are no issues -> getLineHashSequence() is executed
 120         // as late as possible
 121         while (reportIssues.hasNext()) {
 122           ScannerReport.Issue reportIssue = reportIssues.next();
 123           if (isOnInactiveRule(reportIssue)) {
 124             continue;
 125           }
 126           if (!isIssueOnUnsupportedCommonRule(reportIssue)) {
 127             Loggers.get(getClass()).debug("Ignored issue from analysis report on rule {}:{}", reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 128             continue;
 129           }
 130           DefaultIssue issue = toIssue(getLineHashSequence(), reportIssue);
 131           if (issueFilter.accept(issue, component)) {
 132             result.add(issue);
 133           }
 134         }
 135       }
 136
 137       Map<RuleKey, ScannerReport.AdHocRule> adHocRuleMap = new HashMap<>();
 138       try (CloseableIterator<ScannerReport.AdHocRule> reportAdHocRule = reportReader.readAdHocRules()) {
 139         while (reportAdHocRule.hasNext()) {
 140           ScannerReport.AdHocRule adHocRule = reportAdHocRule.next();
 141           adHocRuleMap.put(RuleKey.of(RuleKey.EXTERNAL_RULE_REPO_PREFIX + adHocRule.getEngineId(), adHocRule.getRuleId()), adHocRule);
 142         }
 143       }
 144
 145       try (CloseableIterator<ScannerReport.ExternalIssue> reportExternalIssues = reportReader.readComponentExternalIssues(component.getReportAttributes().getRef())) {
 146         // optimization - do not load line hashes if there are no issues -> getLineHashSequence() is executed
 147         // as late as possible
 148         while (reportExternalIssues.hasNext()) {
 149           ScannerReport.ExternalIssue reportExternalIssue = reportExternalIssues.next();
 150           result.add(toExternalIssue(getLineHashSequence(), reportExternalIssue, adHocRuleMap));
 151         }
 152       }
 153
 154       return result;
 155     }
 156
 157     private boolean isOnInactiveRule(ScannerReport.Issue reportIssue) {
 158       RuleKey ruleKey = RuleKey.of(reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 159       return !activeRulesHolder.get(ruleKey).isPresent();
 160     }
 161
 162     private boolean isIssueOnUnsupportedCommonRule(ScannerReport.Issue issue) {
 163       // issues on batch common rules are ignored. This feature
 164       // is natively supported by compute engine since 5.2.
 165       return !issue.getRuleRepository().startsWith(CommonRuleKeys.REPOSITORY_PREFIX);
 166     }
 167
 168     private DefaultIssue toIssue(LineHashSequence lineHashSeq, ScannerReport.Issue reportIssue) {
 169       DefaultIssue issue = new DefaultIssue();
 170       init(issue, STATUS_OPEN);
 171       RuleKey ruleKey = RuleKey.of(reportIssue.getRuleRepository(), reportIssue.getRuleKey());
 172       issue.setRuleKey(ruleKey);
 173       if (reportIssue.hasTextRange()) {
 174         int startLine = reportIssue.getTextRange().getStartLine();
 175         issue.setLine(startLine);
 176         issue.setChecksum(lineHashSeq.getHashForLine(startLine));
 177       } else {
 178         issue.setChecksum("");
 179       }
 180       if (isNotEmpty(reportIssue.getMsg())) {
 181         issue.setMessage(reportIssue.getMsg());
 182       } else {
 183         Rule rule = ruleRepository.getByKey(ruleKey);
 184         issue.setMessage(rule.getName());
 185       }
 186       if (reportIssue.getSeverity() != Severity.UNSET_SEVERITY) {
 187         issue.setSeverity(reportIssue.getSeverity().name());
 188       }
 189       if (Double.compare(reportIssue.getGap(), 0D) != 0) {
 190         issue.setGap(reportIssue.getGap());
 191       }
 192       DbIssues.Locations.Builder dbLocationsBuilder = DbIssues.Locations.newBuilder();
 193       if (reportIssue.hasTextRange()) {
 194         DbCommons.TextRange.Builder textRange = convertTextRange(reportIssue.getTextRange());
 195         dbLocationsBuilder.setTextRange(textRange);
 196         dbLocationsBuilder.setChecksum(calculateLocationHash(textRange));
 197       }
 198       for (ScannerReport.Flow flow : reportIssue.getFlowList()) {
 199         if (flow.getLocationCount() > 0) {
 200           DbIssues.Flow.Builder dbFlowBuilder = DbIssues.Flow.newBuilder();
 201           for (ScannerReport.IssueLocation location : flow.getLocationList()) {
 202             convertLocation(location).ifPresent(dbFlowBuilder::addLocation);
 203           }
 204           dbLocationsBuilder.addFlow(dbFlowBuilder);
 205         }
 206       }
 207       issue.setIsFromExternalRuleEngine(false);
 208       issue.setLocations(dbLocationsBuilder.build());
 209       issue.setQuickFixAvailable(reportIssue.getQuickFixAvailable());
 210       return issue;
 211     }
 212
 213     private DefaultIssue toExternalIssue(LineHashSequence lineHashSeq, ScannerReport.ExternalIssue reportExternalIssue, Map<RuleKey, ScannerReport.AdHocRule> adHocRuleMap) {
 214       DefaultIssue issue = new DefaultIssue();
 215       RuleType type = toRuleType(reportExternalIssue.getType());
 216       init(issue, type == RuleType.SECURITY_HOTSPOT ? STATUS_TO_REVIEW : STATUS_OPEN);
 217
 218       RuleKey ruleKey = RuleKey.of(RuleKey.EXTERNAL_RULE_REPO_PREFIX + reportExternalIssue.getEngineId(), reportExternalIssue.getRuleId());
 219       issue.setRuleKey(ruleKey);
 220       if (reportExternalIssue.hasTextRange()) {
 221         int startLine = reportExternalIssue.getTextRange().getStartLine();
 222         issue.setLine(startLine);
 223         issue.setChecksum(lineHashSeq.getHashForLine(startLine));
 224       } else {
 225         issue.setChecksum("");
 226       }
 227       if (isNotEmpty(reportExternalIssue.getMsg())) {
 228         issue.setMessage(reportExternalIssue.getMsg());
 229       }
 230       if (reportExternalIssue.getSeverity() != Severity.UNSET_SEVERITY) {
 231         issue.setSeverity(reportExternalIssue.getSeverity().name());
 232       }
 233       issue.setEffort(Duration.create(reportExternalIssue.getEffort() != 0 ? reportExternalIssue.getEffort() : DEFAULT_EXTERNAL_ISSUE_EFFORT));
 234       DbIssues.Locations.Builder dbLocationsBuilder = DbIssues.Locations.newBuilder();
 235       if (reportExternalIssue.hasTextRange()) {
 236         dbLocationsBuilder.setTextRange(convertTextRange(reportExternalIssue.getTextRange()));
 237       }
 238       for (ScannerReport.Flow flow : reportExternalIssue.getFlowList()) {
 239         if (flow.getLocationCount() > 0) {
 240           DbIssues.Flow.Builder dbFlowBuilder = DbIssues.Flow.newBuilder();
 241           for (ScannerReport.IssueLocation location : flow.getLocationList()) {
 242             convertLocation(location).ifPresent(dbFlowBuilder::addLocation);
 243           }
 244           dbLocationsBuilder.addFlow(dbFlowBuilder);
 245         }
 246       }
 247       issue.setIsFromExternalRuleEngine(true);
 248       issue.setLocations(dbLocationsBuilder.build());
 249       issue.setType(type);
 250
 251       ruleRepository.addOrUpdateAddHocRuleIfNeeded(ruleKey, () -> toAdHocRule(reportExternalIssue, adHocRuleMap.get(issue.ruleKey())));
 252       return issue;
 253     }
 254
 255     private NewAdHocRule toAdHocRule(ScannerReport.ExternalIssue reportIssue, @Nullable ScannerReport.AdHocRule adHocRule) {
 256       if (adHocRule != null) {
 257         return new NewAdHocRule(adHocRule);
 258       }
 259       return new NewAdHocRule(reportIssue);
 260     }
 261
 262     private RuleType toRuleType(IssueType type) {
 263       switch (type) {
 264         case BUG:
 265           return RuleType.BUG;
 266         case CODE_SMELL:
 267           return RuleType.CODE_SMELL;
 268         case VULNERABILITY:
 269           return RuleType.VULNERABILITY;
 270         case SECURITY_HOTSPOT:
 271           return RuleType.SECURITY_HOTSPOT;
 272         case UNRECOGNIZED:
 273         default:
 274           throw new IllegalStateException("Invalid issue type: " + type);
 275       }
 276     }
 277
 278     private DefaultIssue init(DefaultIssue issue, String initialStatus) {
 279       issue.setStatus(initialStatus);
 280       issue.setResolution(null);
 281       issue.setComponentUuid(component.getUuid());
 282       issue.setComponentKey(component.getKey());
 283       issue.setProjectUuid(treeRootHolder.getRoot().getUuid());
 284       issue.setProjectKey(treeRootHolder.getRoot().getKey());
 285       return issue;
 286     }
 287
 288     private Optional<DbIssues.Location> convertLocation(ScannerReport.IssueLocation source) {
 289       DbIssues.Location.Builder target = DbIssues.Location.newBuilder();
 290       if (source.getComponentRef() != 0 && source.getComponentRef() != component.getReportAttributes().getRef()) {
 291         // SONAR-10781 Component might not exist because on PR, only changed components are included in the report
 292         Optional<Component> optionalComponent = treeRootHolder.getOptionalComponentByRef(source.getComponentRef());
 293         if (!optionalComponent.isPresent()) {
 294           return Optional.empty();
 295         }
 296         target.setComponentId(optionalComponent.get().getUuid());
 297       }
 298       if (isNotEmpty(source.getMsg())) {
 299         target.setMsg(source.getMsg());
 300       }
 301       if (source.hasTextRange()) {
 302         ScannerReport.TextRange sourceRange = source.getTextRange();
 303         DbCommons.TextRange.Builder targetRange = convertTextRange(sourceRange);
 304         target.setTextRange(targetRange);
 305         target.setChecksum(calculateLocationHash(targetRange));
 306       }
 307       return Optional.of(target.build());
 308     }
 309
 310     private String calculateLocationHash(DbCommons.TextRange.Builder textRange) {
 311       try (CloseableIterator<String> linesIterator = sourceLinesRepository.readLines(component)) {
 312         StringBuilder toHash = new StringBuilder();
 313         int lineNo = 1;
 314         while (linesIterator.hasNext()) {
 315           String line = linesIterator.next();
 316           if (lineNo == textRange.getStartLine() && lineNo == textRange.getEndLine()) {
 317             toHash.append(line, textRange.getStartOffset(), textRange.getEndOffset());
 318           } else if (lineNo == textRange.getStartLine()) {
 319             toHash.append(line, textRange.getStartOffset(), line.length());
 320           } else if (lineNo > textRange.getStartLine() && lineNo < textRange.getEndLine()) {
 321             toHash.append(line);
 322           } else if (lineNo == textRange.getEndLine()) {
 323             toHash.append(line, 0, textRange.getEndOffset());
 324           } else if (lineNo > textRange.getEndLine()) {
 325             break;
 326           }
 327           lineNo++;
 328         }
 329         String issueContentWithoutWhitespaces = MATCH_ALL_WHITESPACES.matcher(toHash.toString()).replaceAll("");
 330         return DigestUtils.md5Hex(issueContentWithoutWhitespaces);
 331       }
 332
 333     }
 334
 335     private DbCommons.TextRange.Builder convertTextRange(ScannerReport.TextRange sourceRange) {
 336       DbCommons.TextRange.Builder targetRange = DbCommons.TextRange.newBuilder();
 337       targetRange.setStartLine(sourceRange.getStartLine());
 338       targetRange.setStartOffset(sourceRange.getStartOffset());
 339       targetRange.setEndLine(sourceRange.getEndLine());
 340       targetRange.setEndOffset(sourceRange.getEndOffset());
 341       return targetRange;
 342     }
 343   }
 344 }