From 000da09563cf669ea0f3bbe81d7101c5ffe9736e Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Tue, 2 Dec 2014 10:54:28 +0100 Subject: [PATCH] SONAR-5871 Migration of duplications into file_sources --- .../db/migrations/v50/FeedFileSources.java | 21 +++- .../db/migrations/v50/FileSourceDto.java | 116 +++++++++++++++++- .../migrations/v50/FeedFileSourcesTest.java | 11 ++ .../FeedFileSourcesTest/after-with-scm.xml | 2 +- .../v50/FeedFileSourcesTest/before.xml | 2 + 5 files changed, 144 insertions(+), 8 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java index 06c4a2a56e7..9ceecdd1730 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FeedFileSources.java @@ -22,9 +22,11 @@ package org.sonar.server.db.migrations.v50; import org.apache.commons.lang.StringUtils; import org.sonar.api.utils.System2; import org.sonar.core.persistence.Database; -import org.sonar.server.db.migrations.*; +import org.sonar.server.db.migrations.BaseDataChange; +import org.sonar.server.db.migrations.MassUpdate; import org.sonar.server.db.migrations.Select.Row; import org.sonar.server.db.migrations.Select.RowReader; +import org.sonar.server.db.migrations.SqlStatement; import javax.annotation.Nullable; @@ -92,7 +94,11 @@ public class FeedFileSources extends BaseDataChange { // overall_cover_cond_by_line "m12.text_value, " + - "m12.measure_data " + + "m12.measure_data, " + + + // duplication_data + "m13.text_value, " + + "m13.measure_data " + "FROM snapshots s " + "JOIN snapshot_sources ss " + @@ -125,6 +131,8 @@ public class FeedFileSources extends BaseDataChange { "ON m11.snapshot_id = s.id AND m11.metric_id = ? " + "LEFT JOIN project_measures m12 " + "ON m12.snapshot_id = s.id AND m12.metric_id = ? " + + "LEFT JOIN project_measures m13 " + + "ON m13.snapshot_id = s.id AND m13.metric_id = ? " + "WHERE " + "f.enabled = ? " + "AND f.scope = 'FIL' " + @@ -167,6 +175,8 @@ public class FeedFileSources extends BaseDataChange { byte[] longOverallCond = row.getBytes(26); byte[] shortOverallCovCond = row.getBytes(27); byte[] longOverallCovCond = row.getBytes(28); + byte[] shortDuplicationData = row.getBytes(29); + byte[] longDuplicationData = row.getBytes(30); String[] sourceData = new FileSourceDto(source, ofNullableBytes(shortRevisions, longRevisions), @@ -180,7 +190,8 @@ public class FeedFileSources extends BaseDataChange { ofNullableBytes(shortItCovCond, longItCovCond), ofNullableBytes(shortOverallHits, longOverallHits), ofNullableBytes(shortOverallCond, longOverallCond), - ofNullableBytes(shortOverallCovCond, longOverallCovCond) + ofNullableBytes(shortOverallCovCond, longOverallCovCond), + ofNullableBytes(shortDuplicationData, longDuplicationData) ).getSourceData(); update.setString(1, projectUuid) @@ -237,6 +248,7 @@ public class FeedFileSources extends BaseDataChange { Long overallCoverageHitsByLineMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'overall_coverage_line_hits_data'").get(simpleLongReader); Long overallConditionsByLineMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'overall_conditions_by_line'").get(simpleLongReader); Long overallCoveredConditionsByLineMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'overall_covered_conditions_by_line'").get(simpleLongReader); + Long duplicationDataMetricId = context.prepareSelect("SELECT id FROM metrics WHERE name = 'duplications_data'").get(simpleLongReader); MassUpdate massUpdate = context.prepareMassUpdate(); massUpdate.select(SELECT_FILES_AND_MEASURES_SQL) @@ -253,7 +265,8 @@ public class FeedFileSources extends BaseDataChange { .setLong(11, zeroIfNull(overallCoverageHitsByLineMetricId)) .setLong(12, zeroIfNull(overallConditionsByLineMetricId)) .setLong(13, zeroIfNull(overallCoveredConditionsByLineMetricId)) - .setBoolean(14, true); + .setLong(14, zeroIfNull(duplicationDataMetricId)) + .setBoolean(15, true); massUpdate.update("INSERT INTO file_sources" + "(project_uuid, file_uuid, created_at, updated_at, data, line_hashes, data_hash)" + diff --git a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java index 4114d2f1ea6..f1bbcb00af0 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java +++ b/server/sonar-server/src/main/java/org/sonar/server/db/migrations/v50/FileSourceDto.java @@ -20,14 +20,28 @@ package org.sonar.server.db.migrations.v50; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; +import org.codehaus.stax2.XMLInputFactory2; +import org.codehaus.staxmate.SMInputFactory; +import org.codehaus.staxmate.in.SMHierarchicCursor; +import org.codehaus.staxmate.in.SMInputCursor; +import org.sonar.api.measures.CoreMetrics; import org.sonar.api.utils.KeyValueFormat; +import org.sonar.api.utils.SonarException; import org.sonar.api.utils.text.CsvWriter; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; + import java.io.ByteArrayOutputStream; import java.io.OutputStreamWriter; +import java.io.StringReader; +import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import static com.google.common.base.Charsets.UTF_8; @@ -50,11 +64,12 @@ class FileSourceDto { private final Map overallHits; private final Map overallConditions; private final Map overallCoveredConditions; + private final List> duplicationGroups; FileSourceDto(String source, String revisions, String authors, String dates, String utHits, String utConditions, String utCoveredConditions, String itHits, String itConditions, String itCoveredConditions, - String overallHits, String overallConditions, String overallCoveredConditions) { + String overallHits, String overallConditions, String overallCoveredConditions, String duplicationData) { sourceSplitter = Splitter.onPattern("\r?\n|\r").split(source).iterator(); this.revisions = KeyValueFormat.parseIntString(revisions); this.authors = KeyValueFormat.parseIntString(authors); @@ -68,12 +83,13 @@ class FileSourceDto { this.overallHits = KeyValueFormat.parseIntString(overallHits); this.overallConditions = KeyValueFormat.parseIntString(overallConditions); this.overallCoveredConditions = KeyValueFormat.parseIntString(overallCoveredConditions); + this.duplicationGroups = StringUtils.isNotBlank(duplicationData) ? parseDuplicationData(duplicationData) : Collections.>emptyList(); } String[] getSourceData() { String highlighting = ""; String symbolRefs = ""; - String duplications = ""; + Map duplicationsPerLine = computeDuplicationsPerLine(duplicationGroups); ByteArrayOutputStream output = new ByteArrayOutputStream(); int line = 0; String sourceLine = null; @@ -87,7 +103,7 @@ class FileSourceDto { utHits.get(line), utConditions.get(line), utCoveredConditions.get(line), itHits.get(line), itConditions.get(line), itCoveredConditions.get(line), overallHits.get(line), overallConditions.get(line), overallCoveredConditions.get(line), - highlighting, symbolRefs, duplications, sourceLine); + highlighting, symbolRefs, duplicationsPerLine.get(line), sourceLine); } csv.close(); return new String[] {new String(output.toByteArray(), UTF_8), lineHashes.toString()}; @@ -101,4 +117,98 @@ class FileSourceDto { return DigestUtils.md5Hex(reducedLine); } + private Map computeDuplicationsPerLine(List> duplicationGroups) { + Map result = new HashMap(); + if (duplicationGroups.isEmpty()) { + return result; + } + Map dupPerLine = new HashMap(); + int blockId = 1; + for (List group : duplicationGroups) { + Block originBlock = group.get(0); + addBlock(blockId, originBlock, dupPerLine); + blockId++; + for (int i = 1; i < group.size(); i++) { + Block duplicate = group.get(i); + if (duplicate.resourceKey.equals(originBlock.resourceKey)) { + addBlock(blockId, duplicate, dupPerLine); + blockId++; + } + } + } + for (Map.Entry entry : dupPerLine.entrySet()) { + result.put(entry.getKey(), entry.getValue().toString()); + } + return result; + } + + private void addBlock(int blockId, Block block, Map dupPerLine) { + int currentLine = block.start; + for (int i = 0; i < block.length; i++) { + if (dupPerLine.get(currentLine) == null) { + dupPerLine.put(currentLine, new StringBuilder()); + } + if (dupPerLine.get(currentLine).length() > 0) { + dupPerLine.get(currentLine).append(','); + } + dupPerLine.get(currentLine).append(blockId); + currentLine++; + } + + } + + /** + * Parses data of {@link CoreMetrics#DUPLICATIONS_DATA}. + */ + private static List> parseDuplicationData(String data) { + try { + ImmutableList.Builder> groups = ImmutableList.builder(); + StringReader reader = new StringReader(data); + SMInputFactory inputFactory = initStax(); + SMHierarchicCursor rootC = inputFactory.rootElementCursor(reader); + // + rootC.advance(); + SMInputCursor groupsCursor = rootC.childElementCursor("g"); + while (groupsCursor.getNext() != null) { + // + SMInputCursor blocksCursor = groupsCursor.childElementCursor("b"); + ImmutableList.Builder group = ImmutableList.builder(); + while (blocksCursor.getNext() != null) { + // + String resourceKey = blocksCursor.getAttrValue("r"); + int firstLine = getAttrIntValue(blocksCursor, "s"); + int numberOfLines = getAttrIntValue(blocksCursor, "l"); + + group.add(new Block(resourceKey, firstLine, numberOfLines)); + } + groups.add(group.build()); + } + return groups.build(); + } catch (XMLStreamException e) { + throw new SonarException(e.getMessage(), e); + } + } + + private static int getAttrIntValue(SMInputCursor cursor, String attrName) throws XMLStreamException { + return cursor.getAttrIntValue(cursor.findAttrIndex(null, attrName)); + } + + private static SMInputFactory initStax() { + XMLInputFactory xmlFactory = XMLInputFactory2.newInstance(); + return new SMInputFactory(xmlFactory); + } + + private static class Block { + + final String resourceKey; + final int start; + final int length; + + public Block(String resourceKey, int s, int l) { + this.resourceKey = resourceKey; + this.start = s; + this.length = l; + } + } + } diff --git a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java index cfe9bdcedc6..82d45e5b814 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/db/migrations/v50/FeedFileSourcesTest.java @@ -185,6 +185,17 @@ public class FeedFileSourcesTest { "(12, 6, ?)"); overallCoveredCondStmt.setBytes(1, "1=4".getBytes(Charsets.UTF_8)); overallCoveredCondStmt.executeUpdate(); + + PreparedStatement duplicationDataStmt = connection.prepareStatement("insert into project_measures " + + "(metric_id, snapshot_id, " + columnName + ") " + + "values " + + "(13, 6, ?)"); + duplicationDataStmt + .setBytes( + 1, + "" + .getBytes(Charsets.UTF_8)); + duplicationDataStmt.executeUpdate(); } finally { DbUtils.commitAndCloseQuietly(connection); } diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml index e4ae2ea1c28..954a51fa83d 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/after-with-scm.xml @@ -1,7 +1,7 @@ diff --git a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml index 60f9ea2effb..f977bd54963 100644 --- a/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml +++ b/server/sonar-server/src/test/resources/org/sonar/server/db/migrations/v50/FeedFileSourcesTest/before.xml @@ -24,6 +24,8 @@ user_managed="false" enabled="true" origin="JAV" worst_value="[null]" best_value="[null]" optimized_best_value="[null]" hidden="[false]" delete_historical_data="false" />