Browse Source

SONAR-7003 Refactor batch issue tracking

tags/5.3-RC1
Duarte Meneses 8 years ago
parent
commit
3e019c0afb
28 changed files with 533 additions and 717 deletions
  1. 23
    13
      it/it-tests/src/test/java/it/analysis/IssueJsonReportTest.java
  2. 7
    2
      sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchComponents.java
  3. 5
    5
      sonar-batch/src/main/java/org/sonar/batch/issue/DefaultIssueCallback.java
  4. 27
    11
      sonar-batch/src/main/java/org/sonar/batch/issue/IssueTransformer.java
  5. 2
    2
      sonar-batch/src/main/java/org/sonar/batch/issue/TrackedIssueAdapter.java
  6. 4
    0
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/FileHashes.java
  7. 0
    340
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTracking.java
  8. 0
    69
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTrackingBlocksRecognizer.java
  9. 58
    0
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTrackingInput.java
  10. 0
    109
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTrackingResult.java
  11. 16
    21
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTransition.java
  12. 78
    42
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java
  13. 11
    7
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/ServerIssueFromWs.java
  14. 40
    3
      sonar-batch/src/main/java/org/sonar/batch/issue/tracking/TrackedIssue.java
  15. 2
    2
      sonar-batch/src/main/java/org/sonar/batch/postjob/DefaultPostJobContext.java
  16. 1
    1
      sonar-batch/src/main/java/org/sonar/batch/scan/report/IssuesReportBuilder.java
  17. 3
    3
      sonar-batch/src/main/java/org/sonar/batch/scan/report/JSONReport.java
  18. 9
    9
      sonar-batch/src/main/resources/org/sonar/batch/scan/report/issuesreport.ftl
  19. 25
    25
      sonar-batch/src/test/java/org/sonar/batch/issue/tracking/TrackedIssueTest.java
  20. 1
    1
      sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/EmptyFileTest.java
  21. 14
    2
      sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/IssueModeAndReportsMediumTest.java
  22. 86
    0
      sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/NoPreviousAnalysisTest.java
  23. 1
    1
      sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/ScanOnlyChangedTest.java
  24. 15
    26
      sonar-core/src/main/java/org/sonar/core/issue/tracking/LineHashSequence.java
  25. 12
    5
      sonar-core/src/main/java/org/sonar/core/issue/tracking/Tracker.java
  26. 75
    0
      sonar-core/src/main/java/org/sonar/core/util/UuidFactoryFast.java
  27. 4
    0
      sonar-core/src/main/java/org/sonar/core/util/Uuids.java
  28. 14
    18
      sonar-core/src/test/java/org/sonar/core/util/UuidFactoryFastTest.java

+ 23
- 13
it/it-tests/src/test/java/it/analysis/IssueJsonReportTest.java View File

@@ -64,8 +64,7 @@ public class IssueJsonReportTest {
for (Object issue : issues) {
JSONObject jsonIssue = (JSONObject) issue;
assertThat(jsonIssue.get("startLine")).isNotNull();
assertThat(jsonIssue.get("endLine")).isNotNull();

assertThat(jsonIssue.get("line")).isEqualTo(jsonIssue.get("startLine"));
assertThat(jsonIssue.get("endLine")).isEqualTo(jsonIssue.get("startLine"));

assertThat(jsonIssue.get("endOffset")).isNotNull();
@@ -97,14 +96,20 @@ public class IssueJsonReportTest {
JSONObject obj = ItUtils.getJSONReport(result);
JSONArray issues = (JSONArray) obj.get("issues");

for (Object i : issues) {
JSONObject issue = (JSONObject) i;
assertThat(issue.get("startLine")).isIn(6L, 9L);
assertThat(issue.get("line")).isIn(6L, 9L);
assertThat(issue.get("endLine")).isIn(6L, 15L);
assertThat(issue.get("startOffset")).isIn(27L, 20L);
assertThat(issue.get("endOffset")).isIn(32L, 2L);
}
JSONObject issue1 = (JSONObject) issues.get(0);
JSONObject issue2 = (JSONObject) issues.get(1);

assertThat(issue1.get("startLine")).isIn(6L);
assertThat(issue1.get("line")).isIn(6L);
assertThat(issue1.get("endLine")).isIn(6L);
assertThat(issue1.get("startOffset")).isIn(27L);
assertThat(issue1.get("endOffset")).isIn(32L);

assertThat(issue2.get("startLine")).isIn(9L);
assertThat(issue2.get("line")).isIn(9L);
assertThat(issue2.get("endLine")).isIn(15L);
assertThat(issue2.get("startOffset")).isIn(20L);
assertThat(issue2.get("endOffset")).isIn(2L);

}

@@ -260,13 +265,18 @@ public class IssueJsonReportTest {
assertThat(sanitize("5.0.0-5868-SILVER-SNAPSHOT")).isEqualTo("<SONAR_VERSION>");
}

private static String sanitize(String s) {
// sanitize issue uuid keys
s = s.replaceAll("\"[a-zA-Z_0-9\\-]{20}\"", "<ISSUE_KEY>");
@Test
public void issueSanityCheck() {
assertThat(sanitize("s\"0150F1EBDB8E000003\"f")).isEqualTo("s<ISSUE_KEY>f");
}

private static String sanitize(String s) {
// sanitize sonar version. Note that "-SILVER-SNAPSHOT" is used by Goldeneye jobs
s = s.replaceAll("\\d\\.\\d(.\\d)?(\\-.*)?\\-SNAPSHOT", "<SONAR_VERSION>");

// sanitize issue uuid keys
s = s.replaceAll("\"[a-zA-Z_0-9\\-]{15,20}\"", "<ISSUE_KEY>");

return ItUtils.sanitizeTimezones(s);
}


+ 7
- 2
sonar-batch/src/main/java/org/sonar/batch/bootstrap/BatchComponents.java View File

@@ -19,11 +19,16 @@
*/
package org.sonar.batch.bootstrap;

import org.sonar.batch.issue.tracking.TrackedIssue;

import org.sonar.batch.issue.tracking.ServerIssueFromWs;
import org.sonar.core.issue.tracking.Tracker;
import com.google.common.collect.Lists;

import java.util.Collection;
import java.util.List;

import org.sonar.batch.cpd.CpdComponents;
import org.sonar.batch.issue.tracking.IssueTracking;
import org.sonar.batch.scan.report.ConsoleReport;
import org.sonar.batch.scan.report.HtmlReport;
import org.sonar.batch.scan.report.IssuesReportBuilder;
@@ -53,7 +58,7 @@ public class BatchComponents {
CodeColorizerSensor.class,

// Issues tracking
IssueTracking.class,
new Tracker<TrackedIssue, ServerIssueFromWs>(),

// Reports
ConsoleReport.class,

+ 5
- 5
sonar-batch/src/main/java/org/sonar/batch/issue/DefaultIssueCallback.java View File

@@ -77,11 +77,11 @@ public class DefaultIssueCallback implements IssueCallback {
newIssue.setAssigneeName(getAssigneeName(issue.assignee()));
newIssue.setComponentKey(issue.componentKey());
newIssue.setKey(issue.key());
newIssue.setMessage(issue.message());
newIssue.setMessage(issue.getMessage());
newIssue.setNew(issue.isNew());
newIssue.setResolution(issue.resolution());
newIssue.setRuleKey(issue.ruleKey().toString());
newIssue.setRuleName(getRuleName(issue.ruleKey()));
newIssue.setRuleKey(issue.getRuleKey().toString());
newIssue.setRuleName(getRuleName(issue.getRuleKey()));
newIssue.setSeverity(issue.severity());
newIssue.setStatus(issue.status());
newIssue.setStartLine(issue.startLine());
@@ -97,8 +97,8 @@ public class DefaultIssueCallback implements IssueCallback {
if (!StringUtils.isEmpty(issue.assignee())) {
userLoginNames.add(issue.assignee());
}
if (issue.ruleKey() != null) {
ruleKeys.add(issue.ruleKey());
if (issue.getRuleKey() != null) {
ruleKeys.add(issue.getRuleKey());
}
}


+ 27
- 11
sonar-batch/src/main/java/org/sonar/batch/issue/IssueTransformer.java View File

@@ -19,26 +19,32 @@
*/
package org.sonar.batch.issue;

import com.google.common.base.Preconditions;
import org.sonar.batch.issue.tracking.SourceHashHolder;

import org.sonar.core.util.Uuids;
import org.sonar.batch.protocol.input.BatchInput.ServerIssue;
import com.google.common.base.Preconditions;
import org.sonar.api.issue.Issue;
import org.sonar.api.rule.RuleKey;
import org.sonar.batch.index.BatchComponent;
import org.sonar.batch.issue.tracking.TrackedIssue;
import org.sonar.batch.protocol.output.BatchReport;
import org.sonar.batch.protocol.output.BatchReport.TextRange;
import org.sonar.core.component.ComponentKeys;
import org.sonar.core.util.Uuids;

import java.util.Date;
import javax.annotation.Nullable;

import org.sonar.batch.issue.tracking.TrackedIssue;
import org.sonar.api.rule.RuleKey;
import org.sonar.batch.index.BatchComponent;
import org.sonar.batch.protocol.output.BatchReport;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;

public class IssueTransformer {
private IssueTransformer() {
// static only
}

public static TrackedIssue toTrackedIssue(org.sonar.batch.protocol.input.BatchInput.ServerIssue serverIssue) {
public static TrackedIssue toTrackedIssue(ServerIssue serverIssue) {
TrackedIssue issue = new TrackedIssue();
issue.setKey(serverIssue.getKey());
issue.setStatus(serverIssue.getStatus());
@@ -69,15 +75,25 @@ public class IssueTransformer {
issue.setResolution(Issue.RESOLUTION_REMOVED);
}

public static TrackedIssue toTrackedIssue(BatchComponent component, BatchReport.Issue rawIssue) {
public static Collection<TrackedIssue> toTrackedIssue(BatchComponent component, Collection<BatchReport.Issue> rawIssues, @Nullable SourceHashHolder hashes) {
List<TrackedIssue> issues = new ArrayList<>(rawIssues.size());

for (BatchReport.Issue issue : rawIssues) {
issues.add(toTrackedIssue(component, issue, hashes));
}

return issues;
}

public static TrackedIssue toTrackedIssue(BatchComponent component, BatchReport.Issue rawIssue, @Nullable SourceHashHolder hashes) {
RuleKey ruleKey = RuleKey.of(rawIssue.getRuleRepository(), rawIssue.getRuleKey());

Preconditions.checkNotNull(component.key(), "Component key must be set");
Preconditions.checkNotNull(ruleKey, "Rule key must be set");

TrackedIssue issue = new TrackedIssue();
TrackedIssue issue = new TrackedIssue(hashes != null ? hashes.getHashedSource() : null);

issue.setKey(Uuids.create());
issue.setKey(Uuids.createFast());
issue.setComponentKey(component.key());
issue.setRuleKey(ruleKey);
issue.setEffortToFix(rawIssue.hasEffortToFix() ? rawIssue.getEffortToFix() : null);

+ 2
- 2
sonar-batch/src/main/java/org/sonar/batch/issue/TrackedIssueAdapter.java View File

@@ -52,7 +52,7 @@ public class TrackedIssueAdapter implements Issue {

@Override
public RuleKey ruleKey() {
return issue.ruleKey();
return issue.getRuleKey();
}

@Override
@@ -62,7 +62,7 @@ public class TrackedIssueAdapter implements Issue {

@Override
public String message() {
return issue.message();
return issue.getMessage();
}

@Override

+ 4
- 0
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/FileHashes.java View File

@@ -84,6 +84,10 @@ public final class FileHashes {
public Collection<Integer> getLinesForHash(String hash) {
return linesByHash.get(hash);
}
public String[] hashes() {
return hashes;
}

public String getHash(int line) {
// indices in array are shifted one line before

+ 0
- 340
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTracking.java View File

@@ -1,340 +0,0 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

package org.sonar.batch.issue.tracking;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.sonar.api.batch.BatchSide;
import org.sonar.api.batch.InstantiationStrategy;
import org.sonar.api.rule.RuleKey;
import org.sonar.batch.protocol.output.BatchReport;

@InstantiationStrategy(InstantiationStrategy.PER_BATCH)
@BatchSide
public class IssueTracking {

private SourceHashHolder sourceHashHolder;

/**
* @param sourceHashHolder Null when working on resource that is not a file (directory/project)
*/
public IssueTrackingResult track(@Nullable SourceHashHolder sourceHashHolder, Collection<ServerIssue> previousIssues, Collection<BatchReport.Issue> rawIssues) {
this.sourceHashHolder = sourceHashHolder;
IssueTrackingResult result = new IssueTrackingResult();

// Map new issues with old ones
mapIssues(rawIssues, previousIssues, sourceHashHolder, result);
return result;
}

private String checksum(BatchReport.Issue rawIssue) {
if (sourceHashHolder != null && rawIssue.hasLine()) {
FileHashes hashedSource = sourceHashHolder.getHashedSource();
// Extra verification if some plugin managed to create issue on a wrong line
Preconditions.checkState(rawIssue.getLine() <= hashedSource.length(), "Invalid line number for issue %s. File has only %s line(s)", rawIssue, hashedSource.length());
return hashedSource.getHash(rawIssue.getLine());
}
return null;
}

@VisibleForTesting
void mapIssues(Collection<BatchReport.Issue> rawIssues, @Nullable Collection<ServerIssue> previousIssues, @Nullable SourceHashHolder sourceHashHolder,
IssueTrackingResult result) {
boolean hasLastScan = false;

if (previousIssues != null) {
hasLastScan = true;
mapLastIssues(rawIssues, previousIssues, result);
}

// If each new issue matches an old one we can stop the matching mechanism
if (result.matched().size() != rawIssues.size()) {
if (sourceHashHolder != null && hasLastScan) {
FileHashes hashedReference = sourceHashHolder.getHashedReference();
if (hashedReference != null) {
mapNewissues(hashedReference, sourceHashHolder.getHashedSource(), rawIssues, result);
}
}
mapIssuesOnSameRule(rawIssues, result);
}
}

private void mapLastIssues(Collection<BatchReport.Issue> rawIssues, Collection<ServerIssue> previousIssues, IssueTrackingResult result) {
for (ServerIssue lastIssue : previousIssues) {
result.addUnmatched(lastIssue);
}

// Try first to match issues on same rule with same line and with same checksum (but not necessarily with same message)
for (BatchReport.Issue rawIssue : rawIssues) {
if (isNotAlreadyMapped(rawIssue, result)) {
mapIssue(
rawIssue,
findLastIssueWithSameLineAndChecksum(rawIssue, result),
result);
}
}
}

private void mapNewissues(FileHashes hashedReference, FileHashes hashedSource, Collection<BatchReport.Issue> rawIssues, IssueTrackingResult result) {

IssueTrackingBlocksRecognizer rec = new IssueTrackingBlocksRecognizer(hashedReference, hashedSource);

RollingFileHashes a = RollingFileHashes.create(hashedReference, 5);
RollingFileHashes b = RollingFileHashes.create(hashedSource, 5);

Multimap<Integer, BatchReport.Issue> rawIssuesByLines = rawIssuesByLines(rawIssues, rec, result);
Multimap<Integer, ServerIssue> lastIssuesByLines = lastIssuesByLines(result.unmatched(), rec);

Map<Integer, HashOccurrence> map = Maps.newHashMap();

for (Integer line : lastIssuesByLines.keySet()) {
int hash = a.getHash(line);
HashOccurrence hashOccurrence = map.get(hash);
if (hashOccurrence == null) {
// first occurrence in A
hashOccurrence = new HashOccurrence();
hashOccurrence.lineA = line;
hashOccurrence.countA = 1;
map.put(hash, hashOccurrence);
} else {
hashOccurrence.countA++;
}
}

for (Integer line : rawIssuesByLines.keySet()) {
int hash = b.getHash(line);
HashOccurrence hashOccurrence = map.get(hash);
if (hashOccurrence != null) {
hashOccurrence.lineB = line;
hashOccurrence.countB++;
}
}

for (HashOccurrence hashOccurrence : map.values()) {
if (hashOccurrence.countA == 1 && hashOccurrence.countB == 1) {
// Guaranteed that lineA has been moved to lineB, so we can map all issues on lineA to all issues on lineB
map(rawIssuesByLines.get(hashOccurrence.lineB), lastIssuesByLines.get(hashOccurrence.lineA), result);
lastIssuesByLines.removeAll(hashOccurrence.lineA);
rawIssuesByLines.removeAll(hashOccurrence.lineB);
}
}

// Check if remaining number of lines exceeds threshold
if (lastIssuesByLines.keySet().size() * rawIssuesByLines.keySet().size() < 250000) {
List<LinePair> possibleLinePairs = Lists.newArrayList();
for (Integer oldLine : lastIssuesByLines.keySet()) {
for (Integer newLine : rawIssuesByLines.keySet()) {
int weight = rec.computeLengthOfMaximalBlock(oldLine, newLine);
possibleLinePairs.add(new LinePair(oldLine, newLine, weight));
}
}
Collections.sort(possibleLinePairs, LINE_PAIR_COMPARATOR);
for (LinePair linePair : possibleLinePairs) {
// High probability that lineA has been moved to lineB, so we can map all Issues on lineA to all Issues on lineB
map(rawIssuesByLines.get(linePair.lineB), lastIssuesByLines.get(linePair.lineA), result);
}
}
}

private void mapIssuesOnSameRule(Collection<BatchReport.Issue> rawIssues, IssueTrackingResult result) {
// Try then to match issues on same rule with same message and with same checksum
for (BatchReport.Issue rawIssue : rawIssues) {
if (isNotAlreadyMapped(rawIssue, result)) {
mapIssue(
rawIssue,
findLastIssueWithSameChecksumAndMessage(rawIssue, result.unmatchedByKeyForRule(ruleKey(rawIssue)).values()),
result);
}
// Try then to match issues on same rule with same line and with same message
if (isNotAlreadyMapped(rawIssue, result)) {
mapIssue(
rawIssue,
findLastIssueWithSameLineAndMessage(rawIssue, result.unmatchedByKeyForRule(ruleKey(rawIssue)).values()),
result);
}
// Last check: match issue if same rule and same checksum but different line and different message
// See SONAR-2812
if (isNotAlreadyMapped(rawIssue, result)) {
mapIssue(
rawIssue,
findLastIssueWithSameChecksum(rawIssue, result.unmatchedByKeyForRule(ruleKey(rawIssue)).values()),
result);
}
}
}

private static RuleKey ruleKey(BatchReport.Issue rawIssue) {
return RuleKey.of(rawIssue.getRuleRepository(), rawIssue.getRuleKey());
}

private void map(Collection<BatchReport.Issue> rawIssues, Collection<ServerIssue> previousIssues, IssueTrackingResult result) {
for (BatchReport.Issue rawIssue : rawIssues) {
if (isNotAlreadyMapped(rawIssue, result)) {
for (ServerIssue previousIssue : previousIssues) {
if (isNotAlreadyMapped(previousIssue, result) && Objects.equal(ruleKey(rawIssue), previousIssue.ruleKey())) {
mapIssue(rawIssue, previousIssue, result);
break;
}
}
}
}
}

private Multimap<Integer, BatchReport.Issue> rawIssuesByLines(Collection<BatchReport.Issue> rawIssues, IssueTrackingBlocksRecognizer rec, IssueTrackingResult result) {
Multimap<Integer, BatchReport.Issue> rawIssuesByLines = LinkedHashMultimap.create();
for (BatchReport.Issue rawIssue : rawIssues) {
if (isNotAlreadyMapped(rawIssue, result) && rawIssue.hasLine() && rec.isValidLineInSource(rawIssue.getLine())) {
rawIssuesByLines.put(rawIssue.getLine(), rawIssue);
}
}
return rawIssuesByLines;
}

private static Multimap<Integer, ServerIssue> lastIssuesByLines(Collection<ServerIssue> previousIssues, IssueTrackingBlocksRecognizer rec) {
Multimap<Integer, ServerIssue> previousIssuesByLines = LinkedHashMultimap.create();
for (ServerIssue previousIssue : previousIssues) {
if (rec.isValidLineInReference(previousIssue.line())) {
previousIssuesByLines.put(previousIssue.line(), previousIssue);
}
}
return previousIssuesByLines;
}

private ServerIssue findLastIssueWithSameChecksum(BatchReport.Issue rawIssue, Collection<ServerIssue> previousIssues) {
for (ServerIssue previousIssue : previousIssues) {
if (isSameChecksum(rawIssue, previousIssue)) {
return previousIssue;
}
}
return null;
}

private ServerIssue findLastIssueWithSameLineAndMessage(BatchReport.Issue rawIssue, Collection<ServerIssue> previousIssues) {
for (ServerIssue previousIssue : previousIssues) {
if (isSameLine(rawIssue, previousIssue) && isSameMessage(rawIssue, previousIssue)) {
return previousIssue;
}
}
return null;
}

private ServerIssue findLastIssueWithSameChecksumAndMessage(BatchReport.Issue rawIssue, Collection<ServerIssue> previousIssues) {
for (ServerIssue previousIssue : previousIssues) {
if (isSameChecksum(rawIssue, previousIssue) && isSameMessage(rawIssue, previousIssue)) {
return previousIssue;
}
}
return null;
}

private ServerIssue findLastIssueWithSameLineAndChecksum(BatchReport.Issue rawIssue, IssueTrackingResult result) {
Collection<ServerIssue> sameRuleAndSameLineAndSameChecksum = result.unmatchedForRuleAndForLineAndForChecksum(ruleKey(rawIssue), line(rawIssue), checksum(rawIssue));
if (!sameRuleAndSameLineAndSameChecksum.isEmpty()) {
return sameRuleAndSameLineAndSameChecksum.iterator().next();
}
return null;
}

@CheckForNull
private static Integer line(BatchReport.Issue rawIssue) {
return rawIssue.hasLine() ? rawIssue.getLine() : null;
}

private static boolean isNotAlreadyMapped(ServerIssue previousIssue, IssueTrackingResult result) {
return result.unmatched().contains(previousIssue);
}

private static boolean isNotAlreadyMapped(BatchReport.Issue rawIssue, IssueTrackingResult result) {
return !result.isMatched(rawIssue);
}

private boolean isSameChecksum(BatchReport.Issue rawIssue, ServerIssue previousIssue) {
return Objects.equal(previousIssue.checksum(), checksum(rawIssue));
}

private boolean isSameLine(BatchReport.Issue rawIssue, ServerIssue previousIssue) {
return Objects.equal(previousIssue.line(), line(rawIssue));
}

private boolean isSameMessage(BatchReport.Issue rawIssue, ServerIssue previousIssue) {
return Objects.equal(message(rawIssue), previousIssue.message());
}

@CheckForNull
private static String message(BatchReport.Issue rawIssue) {
return rawIssue.hasMsg() ? rawIssue.getMsg() : null;
}

private static void mapIssue(BatchReport.Issue rawIssue, @Nullable ServerIssue ref, IssueTrackingResult result) {
if (ref != null) {
result.setMatch(rawIssue, ref);
}
}

@Override
public String toString() {
return getClass().getSimpleName();
}

private static class LinePair {
int lineA;
int lineB;
int weight;

public LinePair(int lineA, int lineB, int weight) {
this.lineA = lineA;
this.lineB = lineB;
this.weight = weight;
}
}

private static class HashOccurrence {
int lineA;
int lineB;
int countA;
int countB;
}

private static final Comparator<LinePair> LINE_PAIR_COMPARATOR = new Comparator<LinePair>() {
@Override
public int compare(LinePair o1, LinePair o2) {
int weightDiff = o2.weight - o1.weight;
if (weightDiff != 0) {
return weightDiff;
} else {
return Math.abs(o1.lineA - o1.lineB) - Math.abs(o2.lineA - o2.lineB);
}
}
};

}

+ 0
- 69
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTrackingBlocksRecognizer.java View File

@@ -1,69 +0,0 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.batch.issue.tracking;

import javax.annotation.Nullable;

public class IssueTrackingBlocksRecognizer {

private final FileHashes a;
private final FileHashes b;

public IssueTrackingBlocksRecognizer(FileHashes a, FileHashes b) {
this.a = a;
this.b = b;
}

public boolean isValidLineInReference(@Nullable Integer line) {
return (line != null) && (0 <= line - 1) && (line - 1 < a.length());
}

public boolean isValidLineInSource(@Nullable Integer line) {
return (line != null) && (0 <= line - 1) && (line - 1 < b.length());
}

/**
* @param startA number of line from first version of text (numbering starts from 1)
* @param startB number of line from second version of text (numbering starts from 1)
*/
public int computeLengthOfMaximalBlock(int startA, int startB) {
if (!a.getHash(startA).equals(b.getHash(startB))) {
return 0;
}
int length = 0;
int ai = startA;
int bi = startB;
while (ai <= a.length() && bi <= b.length() && a.getHash(ai).equals(b.getHash(bi))) {
ai++;
bi++;
length++;
}
ai = startA;
bi = startB;
while (ai > 0 && bi > 0 && a.getHash(ai).equals(b.getHash(bi))) {
ai--;
bi--;
length++;
}
// Note that position (startA, startB) was counted twice
return length - 1;
}

}

+ 58
- 0
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTrackingInput.java View File

@@ -0,0 +1,58 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.batch.issue.tracking;

import org.sonar.core.issue.tracking.Trackable;
import org.sonar.core.issue.tracking.BlockHashSequence;
import org.sonar.core.issue.tracking.LineHashSequence;

import java.util.Collection;
import java.util.List;

import org.sonar.core.issue.tracking.Input;

public class IssueTrackingInput<T extends Trackable> implements Input<T> {

private final Collection<T> issues;
private final LineHashSequence lineHashes;
private final BlockHashSequence blockHashes;

public IssueTrackingInput(Collection<T> issues, List<String> hashes) {
this.issues = issues;
this.lineHashes = new LineHashSequence(hashes);
this.blockHashes = BlockHashSequence.create(lineHashes);
}

@Override
public LineHashSequence getLineHashSequence() {
return lineHashes;
}

@Override
public BlockHashSequence getBlockHashSequence() {
return blockHashes;
}

@Override
public Collection<T> getIssues() {
return issues;
}

}

+ 0
- 109
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTrackingResult.java View File

@@ -1,109 +0,0 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.batch.issue.tracking;

import com.google.common.collect.HashMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import javax.annotation.Nullable;
import org.apache.commons.lang.StringUtils;
import org.sonar.api.rule.RuleKey;
import org.sonar.batch.protocol.output.BatchReport;

class IssueTrackingResult {
private final Map<String, ServerIssue> unmatchedByKey = new HashMap<>();
private final Map<RuleKey, Map<String, ServerIssue>> unmatchedByRuleAndKey = new HashMap<>();
private final Map<RuleKey, Map<Integer, Multimap<String, ServerIssue>>> unmatchedByRuleAndLineAndChecksum = new HashMap<>();
private final Map<BatchReport.Issue, ServerIssue> matched = Maps.newIdentityHashMap();

Collection<ServerIssue> unmatched() {
return unmatchedByKey.values();
}

Map<String, ServerIssue> unmatchedByKeyForRule(RuleKey ruleKey) {
return unmatchedByRuleAndKey.containsKey(ruleKey) ? unmatchedByRuleAndKey.get(ruleKey) : Collections.<String, ServerIssue>emptyMap();
}

Collection<ServerIssue> unmatchedForRuleAndForLineAndForChecksum(RuleKey ruleKey, @Nullable Integer line, @Nullable String checksum) {
if (!unmatchedByRuleAndLineAndChecksum.containsKey(ruleKey)) {
return Collections.emptyList();
}
Map<Integer, Multimap<String, ServerIssue>> unmatchedForRule = unmatchedByRuleAndLineAndChecksum.get(ruleKey);
Integer lineNotNull = line != null ? line : 0;
if (!unmatchedForRule.containsKey(lineNotNull)) {
return Collections.emptyList();
}
Multimap<String, ServerIssue> unmatchedForRuleAndLine = unmatchedForRule.get(lineNotNull);
String checksumNotNull = StringUtils.defaultString(checksum, "");
if (!unmatchedForRuleAndLine.containsKey(checksumNotNull)) {
return Collections.emptyList();
}
return unmatchedForRuleAndLine.get(checksumNotNull);
}

Collection<BatchReport.Issue> matched() {
return matched.keySet();
}

boolean isMatched(BatchReport.Issue issue) {
return matched.containsKey(issue);
}

ServerIssue matching(BatchReport.Issue issue) {
return matched.get(issue);
}

void addUnmatched(ServerIssue i) {
unmatchedByKey.put(i.key(), i);
RuleKey ruleKey = i.ruleKey();
if (!unmatchedByRuleAndKey.containsKey(ruleKey)) {
unmatchedByRuleAndKey.put(ruleKey, new HashMap<String, ServerIssue>());
unmatchedByRuleAndLineAndChecksum.put(ruleKey, new HashMap<Integer, Multimap<String, ServerIssue>>());
}
unmatchedByRuleAndKey.get(ruleKey).put(i.key(), i);
Map<Integer, Multimap<String, ServerIssue>> unmatchedForRule = unmatchedByRuleAndLineAndChecksum.get(ruleKey);
Integer lineNotNull = lineNotNull(i);
if (!unmatchedForRule.containsKey(lineNotNull)) {
unmatchedForRule.put(lineNotNull, HashMultimap.<String, ServerIssue>create());
}
Multimap<String, ServerIssue> unmatchedForRuleAndLine = unmatchedForRule.get(lineNotNull);
String checksumNotNull = StringUtils.defaultString(i.checksum(), "");
unmatchedForRuleAndLine.put(checksumNotNull, i);
}

private static Integer lineNotNull(ServerIssue i) {
Integer line = i.line();
return line != null ? line : 0;
}

void setMatch(BatchReport.Issue issue, ServerIssue matching) {
matched.put(issue, matching);
RuleKey ruleKey = matching.ruleKey();
unmatchedByRuleAndKey.get(ruleKey).remove(matching.key());
unmatchedByKey.remove(matching.key());
Integer lineNotNull = lineNotNull(matching);
String checksumNotNull = StringUtils.defaultString(matching.checksum(), "");
unmatchedByRuleAndLineAndChecksum.get(ruleKey).get(lineNotNull).get(checksumNotNull).remove(matching);
}
}

+ 16
- 21
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/IssueTransition.java View File

@@ -21,15 +21,6 @@ package org.sonar.batch.issue.tracking;

import org.sonar.batch.issue.IssueTransformer;

import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

import java.util.Date;
import java.util.List;
import java.util.Set;

import javax.annotation.Nullable;

import org.sonar.api.batch.BatchSide;
import org.sonar.api.resources.Project;
import org.sonar.batch.index.BatchComponent;
@@ -40,6 +31,13 @@ import org.sonar.batch.protocol.output.BatchReportReader;
import org.sonar.batch.report.ReportPublisher;
import org.sonar.core.util.CloseableIterator;

import javax.annotation.Nullable;

import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;

@BatchSide
public class IssueTransition {
private final IssueCache issueCache;
@@ -76,7 +74,7 @@ public class IssueTransition {

public void trackIssues(BatchReportReader reader, BatchComponent component) {
// raw issues = all the issues created by rule engines during this module scan and not excluded by filters
Set<BatchReport.Issue> rawIssues = Sets.newIdentityHashSet();
List<BatchReport.Issue> rawIssues = new LinkedList<>();
try (CloseableIterator<BatchReport.Issue> it = reader.readComponentIssues(component.batchId())) {
while (it.hasNext()) {
rawIssues.add(it.next());
@@ -87,27 +85,24 @@ public class IssueTransition {

List<TrackedIssue> trackedIssues;
if (localIssueTracking != null) {
trackedIssues = localIssueTracking.trackIssues(component, rawIssues);
trackedIssues = localIssueTracking.trackIssues(component, rawIssues, analysisDate);
} else {
trackedIssues = Lists.newArrayList();
trackedIssues = doTransition(rawIssues, component);
}

// Unmatched raw issues = new issues
addUnmatchedRawIssues(component, rawIssues, trackedIssues);

for (TrackedIssue issue : trackedIssues) {
issueCache.put(issue);
}
}

private void addUnmatchedRawIssues(BatchComponent component, Set<org.sonar.batch.protocol.output.BatchReport.Issue> rawIssues, List<TrackedIssue> trackedIssues) {
for (BatchReport.Issue rawIssue : rawIssues) {

TrackedIssue tracked = IssueTransformer.toTrackedIssue(component, rawIssue);
tracked.setCreationDate(analysisDate);
private static List<TrackedIssue> doTransition(List<BatchReport.Issue> rawIssues, BatchComponent component) {
List<TrackedIssue> issues = new ArrayList<>(rawIssues.size());

trackedIssues.add(tracked);
for (BatchReport.Issue issue : rawIssues) {
issues.add(IssueTransformer.toTrackedIssue(component, issue, null));
}

return issues;
}

}

+ 78
- 42
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/LocalIssueTracking.java View File

@@ -19,20 +19,24 @@
*/
package org.sonar.batch.issue.tracking;

import org.sonar.core.issue.tracking.Tracking;
import org.sonar.core.issue.tracking.Input;
import org.sonar.core.issue.tracking.Tracker;
import org.sonar.batch.issue.IssueTransformer;

import org.sonar.api.batch.fs.InputFile.Status;
import org.sonar.batch.analysis.DefaultAnalysisMode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.Map;

import javax.annotation.CheckForNull;
import javax.annotation.Nullable;

import org.sonar.api.batch.BatchSide;
import org.sonar.api.batch.fs.internal.DefaultInputFile;
@@ -46,7 +50,7 @@ import org.sonar.batch.repository.ProjectRepositories;

@BatchSide
public class LocalIssueTracking {
private final IssueTracking tracking;
private final Tracker<TrackedIssue, ServerIssueFromWs> tracker;
private final ServerLineHashesLoader lastLineHashes;
private final ActiveRules activeRules;
private final ServerIssueRepository serverIssueRepository;
@@ -54,9 +58,9 @@ public class LocalIssueTracking {

private boolean hasServerAnalysis;

public LocalIssueTracking(IssueTracking tracking, ServerLineHashesLoader lastLineHashes,
public LocalIssueTracking(Tracker<TrackedIssue, ServerIssueFromWs> tracker, ServerLineHashesLoader lastLineHashes,
ActiveRules activeRules, ServerIssueRepository serverIssueRepository, ProjectRepositories projectRepositories, DefaultAnalysisMode mode) {
this.tracking = tracking;
this.tracker = tracker;
this.lastLineHashes = lastLineHashes;
this.serverIssueRepository = serverIssueRepository;
this.mode = mode;
@@ -70,11 +74,11 @@ public class LocalIssueTracking {
}
}

public List<TrackedIssue> trackIssues(BatchComponent component, Set<BatchReport.Issue> rawIssues) {
List<TrackedIssue> trackedIssues = Lists.newArrayList();
public List<TrackedIssue> trackIssues(BatchComponent component, Collection<BatchReport.Issue> reportIssues, Date analysisDate) {
List<TrackedIssue> trackedIssues = new LinkedList<>();
if (hasServerAnalysis) {
// all the issues that are not closed in db before starting this module scan, including manual issues
Collection<ServerIssue> serverIssues = loadServerIssues(component);
Collection<ServerIssueFromWs> serverIssues = loadServerIssues(component);

if (shouldCopyServerIssues(component)) {
// raw issues should be empty, we just need to deal with server issues (SONAR-6931)
@@ -82,13 +86,17 @@ public class LocalIssueTracking {
} else {

SourceHashHolder sourceHashHolder = loadSourceHashes(component);
Collection<TrackedIssue> rIssues = IssueTransformer.toTrackedIssue(component, reportIssues, sourceHashHolder);

IssueTrackingResult trackingResult = tracking.track(sourceHashHolder, serverIssues, rawIssues);
Input<ServerIssueFromWs> baseIssues = createBaseInput(serverIssues, sourceHashHolder);
Input<TrackedIssue> rawIssues = createRawInput(rIssues, sourceHashHolder);

// unmatched from server = issues that have been resolved + issues on disabled/removed rules + manual issues
addUnmatchedFromServer(trackingResult.unmatched(), sourceHashHolder, trackedIssues);
Tracking<TrackedIssue, ServerIssueFromWs> track = tracker.track(rawIssues, baseIssues);

mergeMatched(component, trackingResult, trackedIssues, rawIssues);
addUnmatchedFromServer(track.getUnmatchedBases(), sourceHashHolder, trackedIssues);
addUnmatchedFromServer(track.getOpenManualIssuesByLine().values(), sourceHashHolder, trackedIssues);
mergeMatched(track, trackedIssues, rIssues);
addUnmatchedFromReport(track.getUnmatchedRaws(), trackedIssues, analysisDate);
}
}

@@ -100,6 +108,29 @@ public class LocalIssueTracking {
return trackedIssues;
}

private static Input<ServerIssueFromWs> createBaseInput(Collection<ServerIssueFromWs> serverIssues, @Nullable SourceHashHolder sourceHashHolder) {
List<String> refHashes;

if (sourceHashHolder != null && sourceHashHolder.getHashedReference() != null) {
refHashes = Arrays.asList(sourceHashHolder.getHashedReference().hashes());
} else {
refHashes = new ArrayList<>(0);
}

return new IssueTrackingInput<>(serverIssues, refHashes);
}

private static Input<TrackedIssue> createRawInput(Collection<TrackedIssue> rIssues, @Nullable SourceHashHolder sourceHashHolder) {
List<String> baseHashes;
if (sourceHashHolder != null && sourceHashHolder.getHashedSource() != null) {
baseHashes = Arrays.asList(sourceHashHolder.getHashedSource().hashes());
} else {
baseHashes = new ArrayList<>(0);
}

return new IssueTrackingInput<>(rIssues, baseHashes);
}

private boolean shouldCopyServerIssues(BatchComponent component) {
if (!mode.scanAllFiles() && component.isFile()) {
DefaultInputFile inputFile = (DefaultInputFile) component.inputComponent();
@@ -110,12 +141,12 @@ public class LocalIssueTracking {
return false;
}

private void copyServerIssues(Collection<ServerIssue> serverIssues, List<TrackedIssue> trackedIssues) {
for (ServerIssue serverIssue : serverIssues) {
org.sonar.batch.protocol.input.BatchInput.ServerIssue unmatchedPreviousIssue = ((ServerIssueFromWs) serverIssue).getDto();
private void copyServerIssues(Collection<ServerIssueFromWs> serverIssues, List<TrackedIssue> trackedIssues) {
for (ServerIssueFromWs serverIssue : serverIssues) {
org.sonar.batch.protocol.input.BatchInput.ServerIssue unmatchedPreviousIssue = serverIssue.getDto();
TrackedIssue unmatched = IssueTransformer.toTrackedIssue(unmatchedPreviousIssue);

ActiveRule activeRule = activeRules.find(unmatched.ruleKey());
ActiveRule activeRule = activeRules.find(unmatched.getRuleKey());
unmatched.setNew(false);

if (activeRule == null) {
@@ -140,8 +171,8 @@ public class LocalIssueTracking {
return sourceHashHolder;
}

private Collection<ServerIssue> loadServerIssues(BatchComponent component) {
Collection<ServerIssue> serverIssues = new ArrayList<>();
private Collection<ServerIssueFromWs> loadServerIssues(BatchComponent component) {
Collection<ServerIssueFromWs> serverIssues = new ArrayList<>();
for (org.sonar.batch.protocol.input.BatchInput.ServerIssue previousIssue : serverIssueRepository.byComponent(component)) {
serverIssues.add(new ServerIssueFromWs(previousIssue));
}
@@ -149,42 +180,47 @@ public class LocalIssueTracking {
}

@VisibleForTesting
protected void mergeMatched(BatchComponent component, IssueTrackingResult result, List<TrackedIssue> trackedIssues, Collection<BatchReport.Issue> rawIssues) {
for (BatchReport.Issue rawIssue : result.matched()) {
rawIssues.remove(rawIssue);
org.sonar.batch.protocol.input.BatchInput.ServerIssue ref = ((ServerIssueFromWs) result.matching(rawIssue)).getDto();

TrackedIssue tracked = IssueTransformer.toTrackedIssue(component, rawIssue);
protected void mergeMatched(Tracking<TrackedIssue, ServerIssueFromWs> result, Collection<TrackedIssue> mergeTo, Collection<TrackedIssue> rawIssues) {
for (Map.Entry<TrackedIssue, ServerIssueFromWs> e : result.getMatchedRaws().entrySet()) {
org.sonar.batch.protocol.input.BatchInput.ServerIssue dto = e.getValue().getDto();
TrackedIssue tracked = e.getKey();

// invariant fields
tracked.setKey(ref.getKey());
tracked.setKey(dto.getKey());

// non-persisted fields
tracked.setNew(false);

// fields to update with old values
tracked.setResolution(ref.hasResolution() ? ref.getResolution() : null);
tracked.setStatus(ref.getStatus());
tracked.setAssignee(ref.hasAssigneeLogin() ? ref.getAssigneeLogin() : null);
tracked.setCreationDate(new Date(ref.getCreationDate()));
tracked.setResolution(dto.hasResolution() ? dto.getResolution() : null);
tracked.setStatus(dto.getStatus());
tracked.setAssignee(dto.hasAssigneeLogin() ? dto.getAssigneeLogin() : null);
tracked.setCreationDate(new Date(dto.getCreationDate()));

if (ref.getManualSeverity()) {
if (dto.getManualSeverity()) {
// Severity overriden by user
tracked.setSeverity(ref.getSeverity().name());
tracked.setSeverity(dto.getSeverity().name());
}
trackedIssues.add(tracked);
mergeTo.add(tracked);
}
}

private void addUnmatchedFromServer(Collection<ServerIssue> unmatchedIssues, SourceHashHolder sourceHashHolder, Collection<TrackedIssue> issues) {
for (ServerIssue unmatchedIssue : unmatchedIssues) {
org.sonar.batch.protocol.input.BatchInput.ServerIssue unmatchedPreviousIssue = ((ServerIssueFromWs) unmatchedIssue).getDto();
private void addUnmatchedFromServer(Iterable<ServerIssueFromWs> unmatchedIssues, SourceHashHolder sourceHashHolder, Collection<TrackedIssue> mergeTo) {
for (ServerIssueFromWs unmatchedIssue : unmatchedIssues) {
org.sonar.batch.protocol.input.BatchInput.ServerIssue unmatchedPreviousIssue = unmatchedIssue.getDto();
TrackedIssue unmatched = IssueTransformer.toTrackedIssue(unmatchedPreviousIssue);
if (unmatchedIssue.ruleKey().isManual() && !Issue.STATUS_CLOSED.equals(unmatchedPreviousIssue.getStatus())) {
if (unmatchedIssue.getRuleKey().isManual() && !Issue.STATUS_CLOSED.equals(unmatchedPreviousIssue.getStatus())) {
relocateManualIssue(unmatched, unmatchedIssue, sourceHashHolder);
}
updateUnmatchedIssue(unmatched, false /* manual issues can be kept open */);
issues.add(unmatched);
mergeTo.add(unmatched);
}
}

private static void addUnmatchedFromReport(Iterable<TrackedIssue> rawIssues, Collection<TrackedIssue> trackedIssues, Date analysisDate) {
for (TrackedIssue rawIssue : rawIssues) {
rawIssue.setCreationDate(analysisDate);
trackedIssues.add(rawIssue);
}
}

@@ -197,10 +233,10 @@ public class LocalIssueTracking {
}

private void updateUnmatchedIssue(TrackedIssue issue, boolean forceEndOfLife) {
ActiveRule activeRule = activeRules.find(issue.ruleKey());
ActiveRule activeRule = activeRules.find(issue.getRuleKey());
issue.setNew(false);

boolean manualIssue = issue.ruleKey().isManual();
boolean manualIssue = issue.getRuleKey().isManual();
boolean isRemovedRule = activeRule == null;

if (isRemovedRule) {
@@ -210,8 +246,8 @@ public class LocalIssueTracking {
}
}

private static void relocateManualIssue(TrackedIssue newIssue, ServerIssue oldIssue, SourceHashHolder sourceHashHolder) {
Integer previousLine = oldIssue.line();
private static void relocateManualIssue(TrackedIssue newIssue, ServerIssueFromWs oldIssue, SourceHashHolder sourceHashHolder) {
Integer previousLine = oldIssue.getLine();
if (previousLine == null) {
return;
}

+ 11
- 7
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/ServerIssueFromWs.java View File

@@ -19,9 +19,12 @@
*/
package org.sonar.batch.issue.tracking;

import javax.annotation.CheckForNull;

import org.sonar.core.issue.tracking.Trackable;
import org.sonar.api.rule.RuleKey;

public class ServerIssueFromWs implements ServerIssue {
public class ServerIssueFromWs implements Trackable {

private org.sonar.batch.protocol.input.BatchInput.ServerIssue dto;

@@ -33,29 +36,30 @@ public class ServerIssueFromWs implements ServerIssue {
return dto;
}

@Override
public String key() {
return dto.getKey();
}

@Override
public RuleKey ruleKey() {
public RuleKey getRuleKey() {
return RuleKey.of(dto.getRuleRepository(), dto.getRuleKey());
}

@Override
public String checksum() {
@CheckForNull
public String getLineHash() {
return dto.hasChecksum() ? dto.getChecksum() : null;
}

@Override
public Integer line() {
@CheckForNull
public Integer getLine() {
return dto.hasLine() ? dto.getLine() : null;
}

@Override
public String message() {
return dto.hasMsg() ? dto.getMsg() : null;
public String getMessage() {
return dto.hasMsg() ? dto.getMsg() : "";
}

}

+ 40
- 3
sonar-batch/src/main/java/org/sonar/batch/issue/tracking/TrackedIssue.java View File

@@ -19,12 +19,19 @@
*/
package org.sonar.batch.issue.tracking;

import com.google.common.base.Preconditions;

import javax.annotation.CheckForNull;
import javax.annotation.Nullable;

import org.sonar.core.issue.tracking.Trackable;

import java.io.Serializable;
import java.util.Date;

import org.sonar.api.rule.RuleKey;

public class TrackedIssue implements Serializable {
public class TrackedIssue implements Trackable, Serializable {
private static final long serialVersionUID = -1755017079070964287L;

private RuleKey ruleKey;
@@ -44,7 +51,31 @@ public class TrackedIssue implements Serializable {
private String componentKey;
private String message;

public String message() {
private transient FileHashes hashes;

public TrackedIssue() {
hashes = null;
}

public TrackedIssue(@Nullable FileHashes hashes) {
this.hashes = hashes;
}

@Override
@CheckForNull
public String getLineHash() {
if (getLine() == null || hashes == null) {
return null;
}

int line = getLine();
Preconditions.checkState(line <= hashes.length(), "Invalid line number for issue %s. File has only %s line(s)", this, hashes.length());

return hashes.getHash(line);
}

@Override
public String getMessage() {
return message;
}

@@ -68,6 +99,11 @@ public class TrackedIssue implements Serializable {
return startLine;
}

@Override
public Integer getLine() {
return startLine;
}

public void setStartLine(Integer startLine) {
this.startLine = startLine;
}
@@ -132,7 +168,8 @@ public class TrackedIssue implements Serializable {
this.status = status;
}

public RuleKey ruleKey() {
@Override
public RuleKey getRuleKey() {
return ruleKey;
}


+ 2
- 2
sonar-batch/src/main/java/org/sonar/batch/postjob/DefaultPostJobContext.java View File

@@ -87,7 +87,7 @@ public class DefaultPostJobContext implements PostJobContext {

@Override
public RuleKey ruleKey() {
return wrapped.ruleKey();
return wrapped.getRuleKey();
}

@Override
@@ -113,7 +113,7 @@ public class DefaultPostJobContext implements PostJobContext {

@Override
public String message() {
return wrapped.message();
return wrapped.getMessage();
}

@Override

+ 1
- 1
sonar-batch/src/main/java/org/sonar/batch/scan/report/IssuesReportBuilder.java View File

@@ -97,7 +97,7 @@ public class IssuesReportBuilder {

@CheckForNull
private Rule findRule(TrackedIssue issue) {
return rules.find(issue.ruleKey());
return rules.find(issue.getRuleKey());
}

}

+ 3
- 3
sonar-batch/src/main/java/org/sonar/batch/scan/report/JSONReport.java View File

@@ -144,9 +144,9 @@ public class JSONReport implements Reporter {
.prop("startOffset", issue.startLineOffset())
.prop("endLine", issue.endLine())
.prop("endOffset", issue.endLineOffset())
.prop("message", issue.message())
.prop("message", issue.getMessage())
.prop("severity", issue.severity())
.prop("rule", issue.ruleKey().toString())
.prop("rule", issue.getRuleKey().toString())
.prop("status", issue.status())
.prop("resolution", issue.resolution())
.prop("isNew", issue.isNew())
@@ -160,7 +160,7 @@ public class JSONReport implements Reporter {
logins.add(issue.assignee());
}
json.endObject();
ruleKeys.add(issue.ruleKey());
ruleKeys.add(issue.getRuleKey());
}
}
json.endArray();

+ 9
- 9
sonar-batch/src/main/resources/org/sonar/batch/scan/report/issuesreport.ftl View File

@@ -13,7 +13,7 @@
<#assign issues=resourceReport.getIssues()>
<#list issues as issue>
<#if complete || issue.isNew()>
{'k': '${issue.key()}', 'r': 'R${issue.ruleKey()}', 'l': ${(issue.startLine()!0)?c}, 'new': ${issue.isNew()?string}, 's': '${issue.severity()?lower_case}'}<#if issue_has_next>,</#if>
{'k': '${issue.key()}', 'r': 'R${issue.getRuleKey()}', 'l': ${(issue.startLine()!0)?c}, 'new': ${issue.isNew()?string}, 's': '${issue.severity()?lower_case}'}<#if issue_has_next>,</#if>
</#if>
</#list>
]
@@ -365,10 +365,10 @@
<div class="issue" id="${issue.key()}">
<div class="vtitle">
<i class="icon-severity-${issue.severity()?lower_case}"></i>
<#if issue.message()??>
<span class="rulename">${issue.message()?html}</span>
<#if issue.getMessage()?has_content>
<span class="rulename">${issue.getMessage()?html}</span>
<#else>
<span class="rulename">${ruleNameProvider.nameForHTML(issue.ruleKey())}</span>
<span class="rulename">${ruleNameProvider.nameForHTML(issue.getRuleKey())}</span>
</#if>
&nbsp;
<img src="issuesreport_files/sep12.png">&nbsp;
@@ -382,7 +382,7 @@
</span>
</div>
<div class="discussionComment">
${ruleNameProvider.nameForHTML(issue.ruleKey())}
${ruleNameProvider.nameForHTML(issue.getRuleKey())}
</div>
</div>
<#assign issueId = issueId + 1>
@@ -414,10 +414,10 @@
<div class="issue" id="${issue.key()}">
<div class="vtitle">
<i class="icon-severity-${issue.severity()?lower_case}"></i>
<#if issue.message()??>
<span class="rulename">${issue.message()?html}</span>
<#if issue.getMessage()?has_content>
<span class="rulename">${issue.getMessage()?html}</span>
<#else>
<span class="rulename">${ruleNameProvider.nameForHTML(issue.ruleKey())}</span>
<span class="rulename">${ruleNameProvider.nameForHTML(issue.getRuleKey())}</span>
</#if>
&nbsp;
<img src="issuesreport_files/sep12.png">&nbsp;
@@ -433,7 +433,7 @@

</div>
<div class="discussionComment">
${ruleNameProvider.nameForHTML(issue.ruleKey())}
${ruleNameProvider.nameForHTML(issue.getRuleKey())}
</div>
</div>
<#assign issueId = issueId + 1>

sonar-batch/src/main/java/org/sonar/batch/issue/tracking/ServerIssue.java → sonar-batch/src/test/java/org/sonar/batch/issue/tracking/TrackedIssueTest.java View File

@@ -19,29 +19,29 @@
*/
package org.sonar.batch.issue.tracking;

import org.sonar.api.rule.RuleKey;
import javax.annotation.CheckForNull;
public interface ServerIssue {
String key();
RuleKey ruleKey();
/**
* Null for issue with no line
*/
@CheckForNull
String checksum();
/**
* Global issues have no line
*/
@CheckForNull
Integer line();
@CheckForNull
String message();
import static org.assertj.core.api.Assertions.assertThat;
import org.junit.Test;
public class TrackedIssueTest {
@Test
public void round_trip() {
TrackedIssue issue = new TrackedIssue();
issue.setStartLine(15);
assertThat(issue.getLine()).isEqualTo(15);
assertThat(issue.startLine()).isEqualTo(15);
}
@Test
public void hashes() {
String[] hashArr = new String[] {
"hash1", "hash2", "hash3"
};
FileHashes hashes = FileHashes.create(hashArr);
TrackedIssue issue = new TrackedIssue(hashes);
issue.setStartLine(1);
assertThat(issue.getLineHash()).isEqualTo("hash1");
}
}

+ 1
- 1
sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/EmptyFileTest.java View File

@@ -78,7 +78,7 @@ public class EmptyFileTest {
.start();

for(TrackedIssue i : result.trackedIssues()) {
System.out.println(i.startLine() + " " + i.message());
System.out.println(i.startLine() + " " + i.getMessage());
}
assertThat(result.trackedIssues()).hasSize(11);

+ 14
- 2
sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/IssueModeAndReportsMediumTest.java View File

@@ -19,8 +19,9 @@
*/
package org.sonar.batch.mediumtest.issuesmode;

import org.sonar.batch.issue.tracking.TrackedIssue;
import org.assertj.core.api.Condition;

import org.sonar.batch.issue.tracking.TrackedIssue;
import com.google.common.collect.ImmutableMap;

import java.io.File;
@@ -157,7 +158,7 @@ public class IssueModeAndReportsMediumTest {
int resolvedIssue = 0;
for (TrackedIssue issue : result.trackedIssues()) {
System.out
.println(issue.message() + " " + issue.key() + " " + issue.ruleKey() + " " + issue.isNew() + " " + issue.resolution() + " " + issue.componentKey() + " "
.println(issue.getMessage() + " " + issue.key() + " " + issue.getRuleKey() + " " + issue.isNew() + " " + issue.resolution() + " " + issue.componentKey() + " "
+ issue.startLine());
if (issue.isNew()) {
newIssues++;
@@ -171,6 +172,17 @@ public class IssueModeAndReportsMediumTest {
assertThat(newIssues).isEqualTo(16);
assertThat(openIssues).isEqualTo(3);
assertThat(resolvedIssue).isEqualTo(1);

// assert that original fields of a matched issue are kept
assertThat(result.trackedIssues()).haveExactly(1, new Condition<TrackedIssue>() {
@Override
public boolean matches(TrackedIssue value) {
return value.isNew() == false
&& "resolved-on-project".equals(value.key())
&& "OPEN".equals(value.status())
&& new Date(date("14/03/2004")).equals(value.creationDate());
}
});
}

@Test

+ 86
- 0
sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/NoPreviousAnalysisTest.java View File

@@ -0,0 +1,86 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.batch.mediumtest.issuesmode;

import org.sonar.batch.mediumtest.TaskResult;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;

import java.io.File;

import static org.assertj.core.api.Assertions.assertThat;

import com.google.common.collect.ImmutableMap;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.sonar.api.CoreProperties;
import org.sonar.api.utils.log.LogTester;
import org.sonar.batch.mediumtest.BatchMediumTester;
import org.sonar.xoo.XooPlugin;
import org.sonar.xoo.rule.XooRulesDefinition;

public class NoPreviousAnalysisTest {
@Rule
public TemporaryFolder temp = new TemporaryFolder();

@Rule
public LogTester logTester = new LogTester();

public BatchMediumTester tester = BatchMediumTester.builder()
.bootstrapProperties(ImmutableMap.of(CoreProperties.ANALYSIS_MODE, CoreProperties.ANALYSIS_MODE_ISSUES))
.registerPlugin("xoo", new XooPlugin())
.addRules(new XooRulesDefinition())
.addDefaultQProfile("xoo", "Sonar Way")
.addActiveRule("xoo", "OneIssuePerLine", null, "One issue per line", "MAJOR", "my/internal/key", "xoo")
.setPreviousAnalysisDate(null)
.build();

@Before
public void prepare() {
tester.start();
}

@After
public void stop() {
tester.stop();
}

@Test
public void testIssueTrackingWithIssueOnEmptyFile() throws Exception {
File projectDir = copyProject("/mediumtest/xoo/sample");

TaskResult result = tester
.newScanTask(new File(projectDir, "sonar-project.properties"))
.start();
assertThat(result.trackedIssues()).hasSize(14);
}
private File copyProject(String path) throws Exception {
File projectDir = temp.newFolder();
File originalProjectDir = new File(IssueModeAndReportsMediumTest.class.getResource(path).toURI());
FileUtils.copyDirectory(originalProjectDir, projectDir, FileFilterUtils.notFileFilter(FileFilterUtils.nameFileFilter(".sonar")));
return projectDir;
}
}

+ 1
- 1
sonar-batch/src/test/java/org/sonar/batch/mediumtest/issuesmode/ScanOnlyChangedTest.java View File

@@ -193,7 +193,7 @@ public class ScanOnlyChangedTest {
int resolvedIssue = 0;
for (TrackedIssue issue : result.trackedIssues()) {
System.out
.println(issue.message() + " " + issue.key() + " " + issue.ruleKey() + " " + issue.isNew() + " " + issue.resolution() + " " + issue.componentKey() + " "
.println(issue.getMessage() + " " + issue.key() + " " + issue.getRuleKey() + " " + issue.isNew() + " " + issue.resolution() + " " + issue.componentKey() + " "
+ issue.startLine());
if (issue.isNew()) {
newIssues++;

+ 15
- 26
sonar-core/src/main/java/org/sonar/core/issue/tracking/LineHashSequence.java View File

@@ -19,11 +19,13 @@
*/
package org.sonar.core.issue.tracking;

import com.google.common.collect.SetMultimap;
import com.google.common.collect.HashMultimap;
import com.google.common.base.Strings;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import java.util.Set;
import org.sonar.core.hash.SourceLinesHashesComputer;

/**
@@ -31,21 +33,21 @@ import org.sonar.core.hash.SourceLinesHashesComputer;
*/
public class LineHashSequence {

private static final int[] EMPTY_INTS = new int[0];

/**
* Hashes of lines. Line 1 is at index 0. No null elements.
*/
private final List<String> hashes;
private final Map<String, int[]> linesByHash;
private final SetMultimap<String, Integer> lineByHash;

public LineHashSequence(List<String> hashes) {
this.hashes = hashes;
this.linesByHash = new HashMap<>(hashes.size());
for (int line = 1; line <= hashes.size(); line++) {
String hash = hashes.get(line - 1);
int[] lines = linesByHash.get(hash);
linesByHash.put(hash, appendLineTo(line, lines));
this.lineByHash = HashMultimap.create();
int lineNo = 1;
for (String hash : hashes) {
lineByHash.put(hash, lineNo);
lineNo++;
}
}

@@ -66,9 +68,8 @@ public class LineHashSequence {
/**
* The lines, starting with 1, that matches the given hash.
*/
public int[] getLinesForHash(String hash) {
int[] lines = linesByHash.get(hash);
return lines == null ? EMPTY_INTS : lines;
public Set<Integer> getLinesForHash(String hash) {
return lineByHash.get(hash);
}

/**
@@ -86,18 +87,6 @@ public class LineHashSequence {
return hashes;
}

private static int[] appendLineTo(int line, @Nullable int[] to) {
int[] result;
if (to == null) {
result = new int[] {line};
} else {
result = new int[to.length + 1];
System.arraycopy(to, 0, result, 0, to.length);
result[result.length - 1] = line;
}
return result;
}

public static LineHashSequence createForLines(List<String> lines) {
SourceLinesHashesComputer hashesComputer = new SourceLinesHashesComputer(lines.size());
for (String line : lines) {

+ 12
- 5
sonar-core/src/main/java/org/sonar/core/issue/tracking/Tracker.java View File

@@ -19,18 +19,25 @@
*/
package org.sonar.core.issue.tracking;

import org.sonar.api.batch.BatchSide;
import org.sonar.api.batch.InstantiationStrategy;
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;

import java.util.Collection;
import java.util.Objects;
import java.util.Set;

import javax.annotation.Nonnull;

import org.apache.commons.lang.StringUtils;
import org.sonar.api.rule.RuleKey;

import static com.google.common.collect.FluentIterable.from;

@InstantiationStrategy(InstantiationStrategy.PER_BATCH)
@BatchSide
public class Tracker<RAW extends Trackable, BASE extends Trackable> {

public Tracking<RAW, BASE> track(Input<RAW> rawInput, Input<BASE> baseInput) {
@@ -98,10 +105,10 @@ public class Tracker<RAW extends Trackable, BASE extends Trackable> {
baseHash = baseInput.getLineHashSequence().getHashForLine(base.getLine());
}
if (!Strings.isNullOrEmpty(baseHash)) {
int[] rawLines = rawInput.getLineHashSequence().getLinesForHash(baseHash);
if (rawLines.length == 1) {
tracking.keepManualIssueOpen(base, rawLines[0]);
} else if (rawLines.length == 0 && rawInput.getLineHashSequence().hasLine(base.getLine())) {
Set<Integer> rawLines = rawInput.getLineHashSequence().getLinesForHash(baseHash);
if (rawLines.size() == 1) {
tracking.keepManualIssueOpen(base, rawLines.iterator().next());
} else if (rawLines.isEmpty() && rawInput.getLineHashSequence().hasLine(base.getLine())) {
// still valid (???). We didn't manage to correctly detect code move, so the
// issue is kept at the same location, even if code changes
tracking.keepManualIssueOpen(base, base.getLine());

+ 75
- 0
sonar-core/src/main/java/org/sonar/core/util/UuidFactoryFast.java View File

@@ -0,0 +1,75 @@
/*
* SonarQube, open source software quality management tool.
* Copyright (C) 2008-2014 SonarSource
* mailto:contact AT sonarsource DOT com
*
* SonarQube is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* SonarQube is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.core.util;

/**
* NOT thread safe
* About 10x faster than {@link UuidFactoryImpl}
*/
public class UuidFactoryFast implements UuidFactory {
private static UuidFactoryFast instance = new UuidFactoryFast();
private static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
private static int sequenceNumber = 0;

private UuidFactoryFast() {
//
}

@Override
public String create() {
long timestamp = System.currentTimeMillis();

byte[] uuidBytes = new byte[9];

// Only use lower 6 bytes of the timestamp (this will suffice beyond the year 10000):
putLong(uuidBytes, timestamp, 0, 6);

// Sequence number adds 3 bytes:
putLong(uuidBytes, getSequenceNumber(), 6, 3);

return byteArrayToHex(uuidBytes);
}

public static UuidFactoryFast getInstance() {
return instance;
}
private static int getSequenceNumber() {
return sequenceNumber++;
}

/** Puts the lower numberOfLongBytes from l into the array, starting index pos. */
private static void putLong(byte[] array, long l, int pos, int numberOfLongBytes) {
for (int i = 0; i < numberOfLongBytes; ++i) {
array[pos + numberOfLongBytes - i - 1] = (byte) (l >>> (i * 8));
}
}

public static String byteArrayToHex(byte[] bytes) {
char[] hexChars = new char[bytes.length * 2];
for (int j = 0; j < bytes.length; j++) {
int v = bytes[j] & 0xFF;
hexChars[j * 2] = HEX_ARRAY[v >>> 4];
hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F];
}
return new String(hexChars);
}

}

+ 4
- 0
sonar-core/src/main/java/org/sonar/core/util/Uuids.java View File

@@ -44,4 +44,8 @@ public class Uuids {
public static String create() {
return UuidFactoryImpl.INSTANCE.create();
}

public static String createFast() {
return UuidFactoryFast.getInstance().create();
}
}

sonar-batch/src/test/java/org/sonar/batch/issue/tracking/IssueTrackingBlocksRecognizerTest.java → sonar-core/src/test/java/org/sonar/core/util/UuidFactoryFastTest.java View File

@@ -17,33 +17,29 @@
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.batch.issue.tracking;
package org.sonar.core.util;

import org.junit.Test;

import static org.assertj.core.api.Assertions.assertThat;

public class IssueTrackingBlocksRecognizerTest {
public class UuidFactoryFastTest {
UuidFactory underTest = UuidFactoryFast.getInstance();

@Test
public void test() {
assertThat(compute(t("abcde"), t("abcde"), 4, 4)).isEqualTo(5);
assertThat(compute(t("abcde"), t("abcd"), 4, 4)).isEqualTo(4);
assertThat(compute(t("bcde"), t("abcde"), 4, 4)).isEqualTo(0);
assertThat(compute(t("bcde"), t("abcde"), 3, 4)).isEqualTo(4);
public void create_different_uuids() {
// this test is not enough to ensure that generated strings are unique,
// but it still does a simple and stupid verification
assertThat(underTest.create()).isNotEqualTo(underTest.create());
}

private static int compute(FileHashes a, FileHashes b, int ai, int bi) {
IssueTrackingBlocksRecognizer rec = new IssueTrackingBlocksRecognizer(a, b);
return rec.computeLengthOfMaximalBlock(ai, bi);
}
@Test
public void test_format_of_uuid() throws Exception {
String uuid = underTest.create();

private static FileHashes t(String text) {
String[] array = new String[text.length()];
for (int i = 0; i < text.length(); i++) {
array[i] = "" + text.charAt(i);
}
return FileHashes.create(array);
}
assertThat(uuid.length()).isGreaterThan(10).isLessThan(40);

// URL-safe: only letters, digits, dash and underscore.
assertThat(uuid).matches("^[\\w\\-_]+$");
}
}

Loading…
Cancel
Save