@@ -150,6 +150,10 @@ public interface Measure { | |||
*/ | |||
double getVariation(); | |||
default boolean isEmpty() { | |||
return getValueType() == ValueType.NO_VALUE && !hasVariation() && getData() == null; | |||
} | |||
static NewMeasureBuilder newMeasureBuilder() { | |||
return new NewMeasureBuilder(); | |||
} |
@@ -55,6 +55,8 @@ public interface Metric { | |||
*/ | |||
int getDecimalScale(); | |||
boolean isDeleteHistoricalData(); | |||
enum MetricType { | |||
INT(Measure.ValueType.INT), | |||
MILLISEC(Measure.ValueType.LONG), |
@@ -44,6 +44,7 @@ public enum MetricDtoToMetric implements Function<MetricDto, Metric> { | |||
return new MetricImpl( | |||
metricDto.getId(), metricDto.getKey(), metricDto.getShortName(), metricType, | |||
decimalScale, | |||
DoubleCache.intern(metricDto.getBestValue()), metricDto.isOptimizedBestValue()); | |||
DoubleCache.intern(metricDto.getBestValue()), metricDto.isOptimizedBestValue(), | |||
metricDto.isDeleteHistoricalData()); | |||
} | |||
} |
@@ -41,13 +41,14 @@ public final class MetricImpl implements Metric { | |||
private final Integer decimalScale; | |||
private final Double bestValue; | |||
private final boolean bestValueOptimized; | |||
private boolean deleteHistoricalData; | |||
public MetricImpl(int id, String key, String name, MetricType type) { | |||
this(id, key, name, type, null, null, false); | |||
this(id, key, name, type, null, null, false, false); | |||
} | |||
public MetricImpl(int id, String key, String name, MetricType type, @Nullable Integer decimalScale, | |||
@Nullable Double bestValue, boolean bestValueOptimized) { | |||
@Nullable Double bestValue, boolean bestValueOptimized, boolean deleteHistoricalData) { | |||
checkArgument(!bestValueOptimized || bestValue != null, "A BestValue must be specified if Metric is bestValueOptimized"); | |||
this.id = id; | |||
this.key = checkNotNull(key); | |||
@@ -60,6 +61,7 @@ public final class MetricImpl implements Metric { | |||
} | |||
this.bestValueOptimized = bestValueOptimized; | |||
this.bestValue = bestValue; | |||
this.deleteHistoricalData = deleteHistoricalData; | |||
} | |||
@Override | |||
@@ -99,6 +101,11 @@ public final class MetricImpl implements Metric { | |||
return bestValueOptimized; | |||
} | |||
@Override | |||
public boolean isDeleteHistoricalData() { | |||
return deleteHistoricalData; | |||
} | |||
@Override | |||
public boolean equals(@Nullable Object o) { | |||
if (this == o) { | |||
@@ -125,6 +132,7 @@ public final class MetricImpl implements Metric { | |||
.add("type", type) | |||
.add("bestValue", bestValue) | |||
.add("bestValueOptimized", bestValueOptimized) | |||
.add("deleteHistoricalData", deleteHistoricalData) | |||
.toString(); | |||
} | |||
} |
@@ -20,8 +20,6 @@ | |||
package org.sonar.ce.task.projectanalysis.step; | |||
import java.util.Map; | |||
import java.util.function.Predicate; | |||
import javax.annotation.Nonnull; | |||
import org.sonar.ce.task.projectanalysis.component.Component; | |||
import org.sonar.ce.task.projectanalysis.component.CrawlerDepthLimit; | |||
import org.sonar.ce.task.projectanalysis.component.DepthTraversalTypeAwareCrawler; | |||
@@ -86,11 +84,6 @@ public class PersistMeasuresStep implements ComputationStep { | |||
persistMeasures(project); | |||
} | |||
@Override | |||
public void visitDirectory(Component directory) { | |||
// measures of directories are never read. No need to persist them. | |||
} | |||
@Override | |||
public void visitView(Component view) { | |||
persistMeasures(view); | |||
@@ -112,27 +105,17 @@ public class PersistMeasuresStep implements ComputationStep { | |||
for (Map.Entry<String, Measure> e : measures.entrySet()) { | |||
Measure measure = e.getValue(); | |||
if (!NonEmptyMeasure.INSTANCE.test(measure)) { | |||
if (measure.isEmpty()) { | |||
continue; | |||
} | |||
String metricKey = e.getKey(); | |||
Metric metric = metricRepository.getByKey(metricKey); | |||
MeasureDto measureDto = measureToMeasureDto.toMeasureDto(measure, metric, component); | |||
measureDao.insert(session, measureDto); | |||
inserts++; | |||
if (!metric.isDeleteHistoricalData()) { | |||
MeasureDto measureDto = measureToMeasureDto.toMeasureDto(measure, metric, component); | |||
measureDao.insert(session, measureDto); | |||
inserts++; | |||
} | |||
} | |||
} | |||
} | |||
private enum NonEmptyMeasure implements Predicate<Measure> { | |||
INSTANCE; | |||
@Override | |||
public boolean test(@Nonnull Measure input) { | |||
return input.getValueType() != Measure.ValueType.NO_VALUE || input.hasVariation() || input.getData() != null; | |||
} | |||
} | |||
} |
@@ -175,6 +175,7 @@ public class BestValueOptimizationTest { | |||
} | |||
private static MetricImpl createMetric(Metric.MetricType metricType, double bestValue) { | |||
return new MetricImpl(metricType.hashCode() + (int) bestValue, "key" + metricType + bestValue, "name" + metricType + bestValue, metricType, null, bestValue, true); | |||
return new MetricImpl(metricType.hashCode() + (int) bestValue, "key" + metricType + bestValue, "name" + metricType + bestValue, metricType, null, | |||
bestValue, true, false); | |||
} | |||
} |
@@ -228,7 +228,7 @@ public class LiveMeasureDtoToMeasureTest { | |||
@Test | |||
public void toMeasure_should_not_loose_decimals_of_float_values() { | |||
MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false); | |||
MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false, false); | |||
LiveMeasureDto LiveMeasureDto = new LiveMeasureDto() | |||
.setValue(0.12345); | |||
@@ -321,7 +321,7 @@ public class MeasureDtoToMeasureTest { | |||
@Test | |||
public void toMeasure_should_not_loose_decimals_of_float_values() { | |||
MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false); | |||
MetricImpl metric = new MetricImpl(42, "double", "name", Metric.MetricType.FLOAT, 5, null, false, false); | |||
MeasureDto measureDto = new MeasureDto() | |||
.setValue(0.12345); | |||
@@ -47,6 +47,8 @@ public class MetricDtoToMetricTest { | |||
assertThat(metric.getType()).isEqualTo(metricType); | |||
assertThat(metric.isBestValueOptimized()).isFalse(); | |||
assertThat(metric.getBestValue()).isEqualTo(SOME_BEST_VALUE); | |||
assertThat(metric.isDeleteHistoricalData()).isTrue(); | |||
} | |||
} | |||
@@ -67,6 +69,7 @@ public class MetricDtoToMetricTest { | |||
.setShortName(metricType.name() + "_name") | |||
.setValueType(metricType.name()) | |||
.setBestValue(SOME_BEST_VALUE) | |||
.setDeleteHistoricalData(true) | |||
.setEnabled(true); | |||
} | |||
} |
@@ -53,7 +53,7 @@ public class MetricImplTest { | |||
expectedException.expect(IllegalArgumentException.class); | |||
expectedException.expectMessage("A BestValue must be specified if Metric is bestValueOptimized"); | |||
new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.INT, 1, null, true); | |||
new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.INT, 1, null, true, false); | |||
} | |||
@Test | |||
@@ -72,8 +72,8 @@ public class MetricImplTest { | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.FLOAT)).isEqualTo(expected); | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING)).isEqualTo(expected); | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, 0d, true)).isEqualTo(expected); | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, null, false)).isEqualTo(expected); | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, 0d, true, true)).isEqualTo(expected); | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.STRING, null, null, false, false)).isEqualTo(expected); | |||
assertThat(new MetricImpl(SOME_ID, "some other key", SOME_NAME, Metric.MetricType.FLOAT)).isNotEqualTo(expected); | |||
} | |||
@@ -87,8 +87,8 @@ public class MetricImplTest { | |||
@Test | |||
public void all_fields_are_displayed_in_toString() { | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.FLOAT, 1, 951d, true).toString()) | |||
.isEqualTo("MetricImpl{id=42, key=key, name=name, type=FLOAT, bestValue=951.0, bestValueOptimized=true}"); | |||
assertThat(new MetricImpl(SOME_ID, SOME_KEY, SOME_NAME, Metric.MetricType.FLOAT, 1, 951d, true, false).toString()) | |||
.isEqualTo("MetricImpl{id=42, key=key, name=name, type=FLOAT, bestValue=951.0, bestValueOptimized=true, deleteHistoricalData=false}"); | |||
} | |||
} |
@@ -54,6 +54,7 @@ public class PersistMeasuresStepTest extends BaseStepTest { | |||
private static final Metric STRING_METRIC = new Metric.Builder("string-metric", "String metric", Metric.ValueType.STRING).create(); | |||
private static final Metric INT_METRIC = new Metric.Builder("int-metric", "int metric", Metric.ValueType.INT).create(); | |||
private static final Metric NON_HISTORICAL_METRIC = new Metric.Builder("nh-metric", "nh metric", Metric.ValueType.INT).setDeleteHistoricalData(true).create(); | |||
private static final String ANALYSIS_UUID = "a1"; | |||
@@ -80,8 +81,24 @@ public class PersistMeasuresStepTest extends BaseStepTest { | |||
analysisMetadataHolder.setUuid(ANALYSIS_UUID); | |||
MetricDto stringMetricDto = db.measures().insertMetric(m -> m.setKey(STRING_METRIC.getKey()).setValueType(Metric.ValueType.STRING.name())); | |||
MetricDto intMetricDto = db.measures().insertMetric(m -> m.setKey(INT_METRIC.getKey()).setValueType(Metric.ValueType.INT.name())); | |||
MetricDto nhMetricDto = db.measures().insertMetric(m -> m.setKey(NON_HISTORICAL_METRIC.getKey()).setValueType(Metric.ValueType.INT.name())); | |||
metricRepository.add(stringMetricDto.getId(), STRING_METRIC); | |||
metricRepository.add(intMetricDto.getId(), INT_METRIC); | |||
metricRepository.add(nhMetricDto.getId(), NON_HISTORICAL_METRIC); | |||
} | |||
@Test | |||
public void measures_on_non_historical_metrics_are_not_persisted() { | |||
prepareProject(); | |||
measureRepository.addRawMeasure(REF_1, NON_HISTORICAL_METRIC.getKey(), newMeasureBuilder().create(1)); | |||
measureRepository.addRawMeasure(REF_1, INT_METRIC.getKey(), newMeasureBuilder().create(2)); | |||
TestComputationStepContext context = execute(); | |||
assertThatMeasureIsNotPersisted("project-uuid", NON_HISTORICAL_METRIC); | |||
MeasureDto persistedMeasure = selectMeasure("project-uuid", INT_METRIC).get(); | |||
assertThat(persistedMeasure.getValue()).isEqualTo(2); | |||
assertNbOfInserts(context, 1); | |||
} | |||
@Test |
@@ -64,7 +64,7 @@ public class MetricRepositoryRule extends ExternalResource implements MetricRepo | |||
id, coreMetric.getKey(), coreMetric.getName(), | |||
convert(coreMetric.getType()), | |||
coreMetric.getDecimalScale(), | |||
coreMetric.getBestValue(), coreMetric.isOptimizedBestValue()); | |||
coreMetric.getBestValue(), coreMetric.isOptimizedBestValue(), coreMetric.getDeleteHistoricalData()); | |||
} | |||
private static Metric.MetricType convert(org.sonar.api.measures.Metric.ValueType coreMetricType) { |
@@ -74,15 +74,6 @@ public class MeasureDao implements Dao { | |||
return mapper(dbSession).selectByQueryOnSingleComponent(query); | |||
} | |||
public List<PastMeasureDto> selectPastMeasures(DbSession dbSession, String componentUuid, String analysisUuid, Collection<Integer> metricIds) { | |||
if (metricIds.isEmpty()) { | |||
return emptyList(); | |||
} | |||
return executeLargeInputs( | |||
metricIds, | |||
ids -> mapper(dbSession).selectPastMeasuresOnSingleAnalysis(componentUuid, analysisUuid, ids)); | |||
} | |||
/** | |||
* Select measures of: | |||
* - one component |
@@ -44,9 +44,6 @@ public interface MeasureMapper { | |||
List<MeasureDto> selectByQueryOnSingleComponent(@Param("query") MeasureQuery query); | |||
List<PastMeasureDto> selectPastMeasuresOnSingleAnalysis(@Param("componentUuid") String componentUuid, @Param("analysisUuid") String analysisUuid, | |||
@Param("metricIds") List<Integer> metricIds); | |||
List<MeasureDto> selectPastMeasuresOnSeveralAnalyses(@Param("query") PastMeasureQuery query); | |||
void insert(MeasureDto measureDto); |
@@ -65,6 +65,7 @@ public class MeasureQuery { | |||
this.metricKeys = metricKeys; | |||
} | |||
@CheckForNull | |||
public String getAnalysisUuid() { | |||
return analysisUuid; | |||
} |
@@ -134,15 +134,6 @@ class PurgeCommands { | |||
deleteAnalysisDuplications(analysisUuidsPartitions); | |||
profiler.start("deleteSnapshotWastedMeasures (project_measures)"); | |||
List<Long> metricIdsWithoutHistoricalData = purgeMapper.selectMetricIdsWithoutHistoricalData(); | |||
if (!metricIdsWithoutHistoricalData.isEmpty()) { | |||
analysisUuidsPartitions | |||
.forEach(analysisUuidsPartition -> purgeMapper.deleteAnalysisWastedMeasures(analysisUuidsPartition, metricIdsWithoutHistoricalData)); | |||
session.commit(); | |||
} | |||
profiler.stop(); | |||
profiler.start("updatePurgeStatusToOne (snapshots)"); | |||
analysisUuidsPartitions.forEach(purgeMapper::updatePurgeStatusToOne); | |||
session.commit(); |
@@ -54,10 +54,6 @@ public interface PurgeMapper { | |||
void fullDeleteComponentMeasures(@Param("componentUuids") List<String> componentUuids); | |||
List<Long> selectMetricIdsWithoutHistoricalData(); | |||
void deleteAnalysisWastedMeasures(@Param("analysisUuids") List<String> analysisUuids, @Param("metricIds") List<Long> metricIds); | |||
/** | |||
* Purge status flag is used to not attempt to remove duplications & historical data of analyses | |||
* for which we already removed them. |
@@ -97,16 +97,6 @@ | |||
</if> | |||
</sql> | |||
<select id="selectPastMeasuresOnSingleAnalysis" parameterType="map" resultType="org.sonar.db.measure.PastMeasureDto"> | |||
select pm.id as id, pm.metric_id as metricId, pm.value as value | |||
from project_measures pm | |||
inner join snapshots analysis on analysis.uuid = pm.analysis_uuid | |||
where | |||
pm.component_uuid = #{componentUuid,jdbcType=VARCHAR} | |||
and analysis.uuid = #{analysisUuid,jdbcType=VARCHAR} | |||
and pm.metric_id in <foreach item="metricId" collection="metricIds" open="(" separator="," close=")">#{metricId}</foreach> | |||
</select> | |||
<select id="selectPastMeasuresOnSeveralAnalyses" parameterType="map" resultType="Measure"> | |||
select <include refid="measureColumns"/> | |||
from project_measures pm |
@@ -58,10 +58,6 @@ | |||
and pb.updated_at < #{toDate} | |||
</select> | |||
<select id="selectMetricIdsWithoutHistoricalData" resultType="long"> | |||
select id from metrics where delete_historical_data=${_true} | |||
</select> | |||
<select id="selectRootAndModulesOrSubviewsByProjectUuid" resultType="IdUuidPair" parameterType="String"> | |||
select | |||
p.id, p.uuid | |||
@@ -173,20 +169,6 @@ | |||
</foreach> | |||
</delete> | |||
<delete id="deleteAnalysisWastedMeasures" parameterType="map"> | |||
delete from project_measures | |||
<where> | |||
analysis_uuid in | |||
<foreach collection="analysisUuids" open="(" close=")" item="analysisUuid" separator=","> | |||
#{analysisUuid} | |||
</foreach> | |||
and metric_id in | |||
<foreach collection="metricIds" open="(" item="metricId" separator="," close=")"> | |||
#{metricId,jdbcType=INTEGER} | |||
</foreach> | |||
</where> | |||
</delete> | |||
<update id="updatePurgeStatusToOne" parameterType="map"> | |||
update | |||
snapshots |
@@ -116,44 +116,6 @@ public class PurgeCommandsTest { | |||
assertThat(countDuplications(analysis4)).isZero(); | |||
} | |||
@Test | |||
public void purgeAnalyses_deletes_measures_of_metrics_without_historical_data() { | |||
MetricDto noHistoryMetric = dbTester.measures().insertMetric(t -> t.setDeleteHistoricalData(true)); | |||
MetricDto withHistoryMetric = dbTester.measures().insertMetric(t -> t.setDeleteHistoricalData(false)); | |||
ComponentDto project = dbTester.components().insertPrivateProject(); | |||
SnapshotDto analysis1 = dbTester.components().insertSnapshot(project); | |||
SnapshotDto analysis2 = dbTester.components().insertSnapshot(project); | |||
SnapshotDto analysis3 = dbTester.components().insertSnapshot(project); | |||
SnapshotDto analysis4 = dbTester.components().insertSnapshot(project); | |||
int count = 1 + random.nextInt(12); | |||
for (SnapshotDto analysis : Arrays.asList(analysis1, analysis2, analysis3, analysis4)) { | |||
IntStream.range(0, count).forEach(i -> { | |||
dbTester.measures().insertMeasure(project, analysis, noHistoryMetric); | |||
dbTester.measures().insertMeasure(project, analysis, withHistoryMetric); | |||
}); | |||
} | |||
underTest.purgeAnalyses(toIdUuidPairs(analysis1)); | |||
assertThat(countMeasures(analysis1, noHistoryMetric)).isZero(); | |||
assertThat(countMeasures(analysis1, withHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis2, noHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis2, withHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis3, noHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis3, withHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis4, noHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis4, withHistoryMetric)).isEqualTo(count); | |||
underTest.purgeAnalyses(toIdUuidPairs(analysis1, analysis3, analysis4)); | |||
assertThat(countMeasures(analysis1, withHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis2, noHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis2, noHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis2, withHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis3, noHistoryMetric)).isZero(); | |||
assertThat(countMeasures(analysis3, withHistoryMetric)).isEqualTo(count); | |||
assertThat(countMeasures(analysis4, noHistoryMetric)).isZero(); | |||
assertThat(countMeasures(analysis4, withHistoryMetric)).isEqualTo(count); | |||
} | |||
/** | |||
* Test that SQL queries execution do not fail with a huge number of parameter | |||
*/ |
@@ -88,7 +88,6 @@ import static java.util.Collections.emptySet; | |||
import static java.util.Collections.singletonList; | |||
import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; | |||
import static org.assertj.core.api.Assertions.assertThat; | |||
import static org.assertj.core.api.Assertions.tuple; | |||
import static org.mockito.Mockito.mock; | |||
import static org.mockito.Mockito.verify; | |||
import static org.mockito.Mockito.verifyZeroInteractions; | |||
@@ -213,43 +212,6 @@ public class PurgeDaoTest { | |||
assertThat(uuidsIn("projects")).containsOnly(project.uuid()); | |||
} | |||
@Test | |||
public void shouldDeleteNonHistoricalData() { | |||
MetricDto metricWithHistory = db.measures().insertMetric(t -> t.setDeleteHistoricalData(false)); | |||
MetricDto metricWithoutHistory = db.measures().insertMetric(t -> t.setDeleteHistoricalData(true)); | |||
ComponentDto project = db.components().insertPrivateProject(); | |||
SnapshotDto lastAnalysis = db.components().insertSnapshot(project, t -> t.setLast(true)); | |||
SnapshotDto oldAnalysis = db.components().insertSnapshot(project, t -> t.setLast(false)); | |||
db.measures().insertMeasure(project, lastAnalysis, metricWithHistory); | |||
db.measures().insertMeasure(project, lastAnalysis, metricWithoutHistory); | |||
db.measures().insertMeasure(project, oldAnalysis, metricWithHistory); | |||
db.measures().insertMeasure(project, oldAnalysis, metricWithoutHistory); | |||
ComponentDto otherProject = db.components().insertPrivateProject(); | |||
SnapshotDto otherLastAnalysis = db.components().insertSnapshot(otherProject, t -> t.setLast(true)); | |||
SnapshotDto otherOldAnalysis = db.components().insertSnapshot(otherProject, t -> t.setLast(false)); | |||
db.measures().insertMeasure(otherProject, otherLastAnalysis, metricWithHistory); | |||
db.measures().insertMeasure(otherProject, otherLastAnalysis, metricWithoutHistory); | |||
db.measures().insertMeasure(otherProject, otherOldAnalysis, metricWithHistory); | |||
db.measures().insertMeasure(otherProject, otherOldAnalysis, metricWithoutHistory); | |||
PurgeConfiguration conf = new PurgeConfiguration(project.uuid(), project.uuid(), 30, Optional.of(30), System2.INSTANCE, emptySet()); | |||
underTest.purge(dbSession, conf, PurgeListener.EMPTY, new PurgeProfiler()); | |||
dbSession.commit(); | |||
assertThat(db.select("select metric_id as \"METRIC\",analysis_uuid as \"ANALYSIS\" from project_measures")) | |||
.extracting(t -> ((Long) t.get("METRIC")).intValue(), t -> t.get("ANALYSIS")) | |||
.containsOnly( | |||
tuple(metricWithHistory.getId(), lastAnalysis.getUuid()), | |||
tuple(metricWithoutHistory.getId(), lastAnalysis.getUuid()), | |||
tuple(metricWithHistory.getId(), oldAnalysis.getUuid()), | |||
tuple(metricWithHistory.getId(), otherLastAnalysis.getUuid()), | |||
tuple(metricWithoutHistory.getId(), otherLastAnalysis.getUuid()), | |||
tuple(metricWithHistory.getId(), otherOldAnalysis.getUuid()), | |||
tuple(metricWithoutHistory.getId(), otherOldAnalysis.getUuid())); | |||
} | |||
@Test | |||
public void close_issues_clean_index_and_file_sources_of_disabled_components_specified_by_uuid_in_configuration() { | |||
RuleDefinitionDto rule = db.rules().insert(); |