SONAR-5871 Migration of duplications into file_sources

5 files changed, 144 insertions, 8 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java
index 06c4a2a56e7..9ceecdd1730 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java
@@ -22,9 +22,11 @@ package org.sonar.server.db.migrations.v50;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.utils.System2;
 import org.sonar.core.persistence.Database;
-import org.sonar.server.db.migrations.*;
+import org.sonar.server.db.migrations.BaseDataChange;
+import org.sonar.server.db.migrations.MassUpdate;
 import org.sonar.server.db.migrations.Select.Row;
 import org.sonar.server.db.migrations.Select.RowReader;
+import org.sonar.server.db.migrations.SqlStatement;
 
 import javax.annotation.Nullable;
 
@@ -92,7 +94,11 @@ public class FeedFileSources extends BaseDataChange {
 
     // overall_cover_cond_by_line
     "m12.text_value, " +
-    "m12.measure_data  " +
+    "m12.measure_data,  " +
+
+    // duplication_data
+    "m13.text_value, " +
+    "m13.measure_data  " +
 
     "FROM snapshots s " +
     "JOIN snapshot_sources ss " +
@@ -125,6 +131,8 @@ public class FeedFileSources extends BaseDataChange {
     "ON m11.snapshot_id = s.id AND m11.metric_id = ? " +
     "LEFT JOIN project_measures m12 " +
     "ON m12.snapshot_id = s.id AND m12.metric_id = ? " +
+    "LEFT JOIN project_measures m13 " +
+    "ON m13.snapshot_id = s.id AND m13.metric_id = ? " +
     "WHERE " +
     "f.enabled = ? " +
     "AND f.scope = 'FIL' " +
@@ -167,6 +175,8 @@ public class FeedFileSources extends BaseDataChange {
       byte[] longOverallCond = row.getBytes(26);
       byte[] shortOverallCovCond = row.getBytes(27);
       byte[] longOverallCovCond = row.getBytes(28);
+      byte[] shortDuplicationData = row.getBytes(29);
+      byte[] longDuplicationData = row.getBytes(30);
 
       String[] sourceData = new FileSourceDto(source,
         ofNullableBytes(shortRevisions, longRevisions),
@@ -180,7 +190,8 @@ public class FeedFileSources extends BaseDataChange {
         ofNullableBytes(shortItCovCond, longItCovCond),
         ofNullableBytes(shortOverallHits, longOverallHits),
         ofNullableBytes(shortOverallCond, longOverallCond),
-        ofNullableBytes(shortOverallCovCond, longOverallCovCond)
+        ofNullableBytes(shortOverallCovCond, longOverallCovCond),
+        ofNullableBytes(shortDuplicationData, longDuplicationData)
         ).getSourceData();
 
       update.setString(1, projectUuid)
@@ -237,6 +248,7 @@ public class FeedFileSources extends BaseDataChange {
     Long overallCoverageHitsByLineMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'overall_coverage_line_hits_data'").get(simpleLongReader);
     Long overallConditionsByLineMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'overall_conditions_by_line'").get(simpleLongReader);
     Long overallCoveredConditionsByLineMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'overall_covered_conditions_by_line'").get(simpleLongReader);
+    Long duplicationDataMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'duplications_data'").get(simpleLongReader);
 
     MassUpdate massUpdate = context.prepareMassUpdate();
     massUpdate.select(SELECT_FILES_AND_MEASURES_SQL)
@@ -253,7 +265,8 @@ public class FeedFileSources extends BaseDataChange {
       .setLong(11, zeroIfNull(overallCoverageHitsByLineMetricId))
       .setLong(12, zeroIfNull(overallConditionsByLineMetricId))
       .setLong(13, zeroIfNull(overallCoveredConditionsByLineMetricId))
-      .setBoolean(14, true);
+      .setLong(14, zeroIfNull(duplicationDataMetricId))
+      .setBoolean(15, true);
 
     massUpdate.update("INSERT INTO file_sources" +
       "(project_uuid, file_uuid, created_at, updated_at, data, line_hashes, data_hash)" +
diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java
index 4114d2f1ea6..f1bbcb00af0 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java
@@ -20,14 +20,28 @@
 package org.sonar.server.db.migrations.v50;
 
 import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang.StringUtils;
+import org.codehaus.stax2.XMLInputFactory2;
+import org.codehaus.staxmate.SMInputFactory;
+import org.codehaus.staxmate.in.SMHierarchicCursor;
+import org.codehaus.staxmate.in.SMInputCursor;
+import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.utils.KeyValueFormat;
+import org.sonar.api.utils.SonarException;
 import org.sonar.api.utils.text.CsvWriter;
 
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamException;
+
 import java.io.ByteArrayOutputStream;
 import java.io.OutputStreamWriter;
+import java.io.StringReader;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 import static com.google.common.base.Charsets.UTF_8;
@@ -50,11 +64,12 @@ class FileSourceDto {
   private final Map<Integer, String> overallHits;
   private final Map<Integer, String> overallConditions;
   private final Map<Integer, String> overallCoveredConditions;
+  private final List<List<Block>> duplicationGroups;
 
   FileSourceDto(String source, String revisions, String authors, String dates,
     String utHits, String utConditions, String utCoveredConditions,
     String itHits, String itConditions, String itCoveredConditions,
-    String overallHits, String overallConditions, String overallCoveredConditions) {
+    String overallHits, String overallConditions, String overallCoveredConditions, String duplicationData) {
     sourceSplitter = Splitter.onPattern("\r?\n|\r").split(source).iterator();
     this.revisions = KeyValueFormat.parseIntString(revisions);
     this.authors = KeyValueFormat.parseIntString(authors);
@@ -68,12 +83,13 @@ class FileSourceDto {
     this.overallHits = KeyValueFormat.parseIntString(overallHits);
     this.overallConditions = KeyValueFormat.parseIntString(overallConditions);
     this.overallCoveredConditions = KeyValueFormat.parseIntString(overallCoveredConditions);
+    this.duplicationGroups = StringUtils.isNotBlank(duplicationData) ? parseDuplicationData(duplicationData) : Collections.<List<Block>>emptyList();
   }
 
   String[] getSourceData() {
     String highlighting = "";
     String symbolRefs = "";
-    String duplications = "";
+    Map<Integer, String> duplicationsPerLine = computeDuplicationsPerLine(duplicationGroups);
     ByteArrayOutputStream output = new ByteArrayOutputStream();
     int line = 0;
     String sourceLine = null;
@@ -87,7 +103,7 @@ class FileSourceDto {
         utHits.get(line), utConditions.get(line), utCoveredConditions.get(line),
         itHits.get(line), itConditions.get(line), itCoveredConditions.get(line),
         overallHits.get(line), overallConditions.get(line), overallCoveredConditions.get(line),
-        highlighting, symbolRefs, duplications, sourceLine);
+        highlighting, symbolRefs, duplicationsPerLine.get(line), sourceLine);
     }
     csv.close();
     return new String[] {new String(output.toByteArray(), UTF_8), lineHashes.toString()};
@@ -101,4 +117,98 @@ class FileSourceDto {
     return DigestUtils.md5Hex(reducedLine);
   }
 
+  private Map<Integer, String> computeDuplicationsPerLine(List<List<Block>> duplicationGroups) {
+    Map<Integer, String> result = new HashMap<Integer, String>();
+    if (duplicationGroups.isEmpty()) {
+      return result;
+    }
+    Map<Integer, StringBuilder> dupPerLine = new HashMap<Integer, StringBuilder>();
+    int blockId = 1;
+    for (List<Block> group : duplicationGroups) {
+      Block originBlock = group.get(0);
+      addBlock(blockId, originBlock, dupPerLine);
+      blockId++;
+      for (int i = 1; i < group.size(); i++) {
+        Block duplicate = group.get(i);
+        if (duplicate.resourceKey.equals(originBlock.resourceKey)) {
+          addBlock(blockId, duplicate, dupPerLine);
+          blockId++;
+        }
+      }
+    }
+    for (Map.Entry<Integer, StringBuilder> entry : dupPerLine.entrySet()) {
+      result.put(entry.getKey(), entry.getValue().toString());
+    }
+    return result;
+  }
+
+  private void addBlock(int blockId, Block block, Map<Integer, StringBuilder> dupPerLine) {
+    int currentLine = block.start;
+    for (int i = 0; i < block.length; i++) {
+      if (dupPerLine.get(currentLine) == null) {
+        dupPerLine.put(currentLine, new StringBuilder());
+      }
+      if (dupPerLine.get(currentLine).length() > 0) {
+        dupPerLine.get(currentLine).append(',');
+      }
+      dupPerLine.get(currentLine).append(blockId);
+      currentLine++;
+    }
+
+  }
+
+  /**
+   * Parses data of {@link CoreMetrics#DUPLICATIONS_DATA}.
+   */
+  private static List<List<Block>> parseDuplicationData(String data) {
+    try {
+      ImmutableList.Builder<List<Block>> groups = ImmutableList.builder();
+      StringReader reader = new StringReader(data);
+      SMInputFactory inputFactory = initStax();
+      SMHierarchicCursor rootC = inputFactory.rootElementCursor(reader);
+      // <duplications>
+      rootC.advance();
+      SMInputCursor groupsCursor = rootC.childElementCursor("g");
+      while (groupsCursor.getNext() != null) {
+        // <g>
+        SMInputCursor blocksCursor = groupsCursor.childElementCursor("b");
+        ImmutableList.Builder<Block> group = ImmutableList.builder();
+        while (blocksCursor.getNext() != null) {
+          // <b>
+          String resourceKey = blocksCursor.getAttrValue("r");
+          int firstLine = getAttrIntValue(blocksCursor, "s");
+          int numberOfLines = getAttrIntValue(blocksCursor, "l");
+
+          group.add(new Block(resourceKey, firstLine, numberOfLines));
+        }
+        groups.add(group.build());
+      }
+      return groups.build();
+    } catch (XMLStreamException e) {
+      throw new SonarException(e.getMessage(), e);
+    }
+  }
+
+  private static int getAttrIntValue(SMInputCursor cursor, String attrName) throws XMLStreamException {
+    return cursor.getAttrIntValue(cursor.findAttrIndex(null, attrName));
+  }
+
+  private static SMInputFactory initStax() {
+    XMLInputFactory xmlFactory = XMLInputFactory2.newInstance();
+    return new SMInputFactory(xmlFactory);
+  }
+
+  private static class Block {
+
+    final String resourceKey;
+    final int start;
+    final int length;
+
+    public Block(String resourceKey, int s, int l) {
+      this.resourceKey = resourceKey;
+      this.start = s;
+      this.length = l;
+    }
+  }
+
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java
index cfe9bdcedc6..82d45e5b814 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java
@@ -185,6 +185,17 @@ public class FeedFileSourcesTest {
         "(12, 6, ?)");
       overallCoveredCondStmt.setBytes(1, "1=4".getBytes(Charsets.UTF_8));
       overallCoveredCondStmt.executeUpdate();
+
+      PreparedStatement duplicationDataStmt = connection.prepareStatement("insert into project_measures " +
+        "(metric_id, snapshot_id, " + columnName + ") " +
+        "values " +
+        "(13, 6, ?)");
+      duplicationDataStmt
+        .setBytes(
+          1,
+          "<duplications><g><b s=\"1\" l=\"1\" r=\"MyProject:src/main/xoo/prj/MyFile.xoo\"/><b s=\"2\" l=\"1\" r=\"MyProject:src/main/xoo/prj/MyFile.xoo\"/><b s=\"3\" l=\"1\" r=\"MyProject:src/main/xoo/prj/AnotherFile.xoo\"/></g></duplications>"
+            .getBytes(Charsets.UTF_8));
+      duplicationDataStmt.executeUpdate();
     } finally {
       DbUtils.commitAndCloseQuietly(connection);
     }
diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml
index e4ae2ea1c28..954a51fa83d 100644
--- a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml
+++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml
@@ -1,7 +1,7 @@
 <dataset>
 
   <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="1416238020000" updated_at="1414770242000"
-    data="aef12a,alice,2014-04-25T12:34:56+0100,1,4,2,2,5,3,3,6,4,,,,class Foo {&#13;&#10;abe465,bob,2014-07-25T12:34:56+0100,,,,,,,,,,,,,  // Empty&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,0,,,0,,,0,,,,,,}&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,,,,,,,,,,,,,&#13;&#10;"
+    data="aef12a,alice,2014-04-25T12:34:56+0100,1,4,2,2,5,3,3,6,4,,,1,class Foo {&#13;&#10;abe465,bob,2014-07-25T12:34:56+0100,,,,,,,,,,,,2,  // Empty&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,0,,,0,,,0,,,,,,}&#13;&#10;afb789,carol,2014-03-23T12:34:56+0100,,,,,,,,,,,,,&#13;&#10;"
     line_hashes="6a19ce786467960a3a9b0d26383a464a&#10;aab2dbc5fdeaa80b050b1d049ede357c&#10;cbb184dd8e05c9709e5dcaedaa0495cf&#10;&#10;"
     data_hash="" />
 
diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml
index 60f9ea2effb..f977bd54963 100644
--- a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml
+++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml
@@ -24,6 +24,8 @@
     user_managed="false" enabled="true" origin="JAV" worst_value="[null]" best_value="[null]" optimized_best_value="[null]" hidden="[false]" delete_historical_data="false" />
   <metrics id="12" name="overall_covered_conditions_by_line" description="[null]" direction="0" domain="Tests" short_name="Covered conditions by line" qualitative="false" val_type="DATA"
     user_managed="false" enabled="true" origin="JAV" worst_value="[null]" best_value="[null]" optimized_best_value="[null]" hidden="[false]" delete_historical_data="false" />
+  <metrics id="13" name="duplications_data" description="[null]" direction="0" domain="Duplications" short_name="Duplication data" qualitative="false" val_type="DATA"
+    user_managed="false" enabled="true" origin="JAV" worst_value="[null]" best_value="[null]" optimized_best_value="[null]" hidden="[false]" delete_historical_data="false" />
 
   <projects id="1" uuid="uuid-MyProject" kee="MyProject" scope="PRJ" qualifier="TRK" />
   <projects id="2" uuid="uuid-prj" kee="MyProject:src/main/xoo/prj" scope="DIR" qualifier="DIR" />