diff options
author | Julien HENRY <julien.henry@sonarsource.com> | 2014-04-23 14:41:32 +0200 |
---|---|---|
committer | Julien HENRY <julien.henry@sonarsource.com> | 2014-04-30 17:27:21 +0200 |
commit | 171cd79c0b5152461434951ed5d6e5e58849a7b7 (patch) | |
tree | 4b6443640045992fd3eb06de6cba382ab0621628 /sonar-batch | |
parent | ca2bf42757a1e6471dab4c3fe8f4d98bfd2267d4 (diff) | |
download | sonarqube-171cd79c0b5152461434951ed5d6e5e58849a7b7.tar.gz sonarqube-171cd79c0b5152461434951ed5d6e5e58849a7b7.zip |
SONAR-3437, SONAR-5189 Store measures in a persistit cache
Diffstat (limited to 'sonar-batch')
21 files changed, 608 insertions, 424 deletions
diff --git a/sonar-batch/pom.xml b/sonar-batch/pom.xml index 8e527478c01..dae636cb3e7 100644 --- a/sonar-batch/pom.xml +++ b/sonar-batch/pom.xml @@ -25,6 +25,10 @@ <artifactId>akiban-persistit</artifactId> </dependency> <dependency> + <groupId>com.esotericsoftware.kryo</groupId> + <artifactId>kryo</artifactId> + </dependency> + <dependency> <groupId>org.codehaus.sonar</groupId> <artifactId>sonar-core</artifactId> </dependency> diff --git a/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java b/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java index 466be91dc79..565417efad0 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/DefaultTimeMachine.java @@ -38,7 +38,12 @@ import org.sonar.batch.index.DefaultIndex; import javax.annotation.Nullable; import javax.persistence.Query; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; public class DefaultTimeMachine implements TimeMachine { @@ -155,7 +160,6 @@ public class DefaultTimeMachine implements TimeMachine { static Measure toMeasure(MeasureModel model, Metric metric, @Nullable Characteristic characteristic) { // NOTE: measures on rule are not supported Measure measure = new Measure(metric); - measure.setId(model.getId()); measure.setDescription(model.getDescription()); measure.setValue(model.getValue()); measure.setData(model.getData(metric)); diff --git a/sonar-batch/src/main/java/org/sonar/batch/debt/DebtDecorator.java b/sonar-batch/src/main/java/org/sonar/batch/debt/DebtDecorator.java index 71e9cc36a1c..17475936308 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/debt/DebtDecorator.java +++ b/sonar-batch/src/main/java/org/sonar/batch/debt/DebtDecorator.java @@ -48,6 +48,7 @@ import org.sonar.api.technicaldebt.batch.TechnicalDebtModel; import javax.annotation.CheckForNull; import javax.annotation.Nullable; + import java.util.Arrays; import java.util.List; import java.util.Map; @@ -110,7 +111,7 @@ public final class DebtDecorator implements Decorator { for (Measure measure : context.getChildrenMeasures(MeasuresFilters.rules(CoreMetrics.TECHNICAL_DEBT))) { Long debt = measure.getValue().longValue(); RuleMeasure ruleMeasure = (RuleMeasure) measure; - total += computeDebt(debt, ruleMeasure.getRule().ruleKey(), ruleDebts, characteristicDebts); + total += computeDebt(debt, ruleMeasure.ruleKey(), ruleDebts, characteristicDebts); } context.saveMeasure(CoreMetrics.TECHNICAL_DEBT, total.doubleValue()); diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/Bucket.java b/sonar-batch/src/main/java/org/sonar/batch/index/Bucket.java index af6a87ae8a6..ea4ed3b11a0 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/Bucket.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/Bucket.java @@ -19,24 +19,15 @@ */ package org.sonar.batch.index; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; -import org.sonar.api.measures.CoreMetrics; -import org.sonar.api.measures.Measure; -import org.sonar.api.measures.MeasuresFilter; -import org.sonar.api.measures.MeasuresFilters; import org.sonar.api.resources.Resource; -import org.sonar.api.utils.SonarException; -import java.util.Collection; import java.util.Collections; import java.util.List; public final class Bucket { private Resource resource; - private ListMultimap<String, Measure> measuresByMetric = ArrayListMultimap.create(); private Bucket parent; private List<Bucket> children; @@ -79,30 +70,7 @@ public final class Bucket { return parent; } - public void addMeasure(Measure measure) { - List<Measure> metricMeasures = measuresByMetric.get(measure.getMetric().getKey()); - - boolean add = true; - if (metricMeasures != null) { - int index = metricMeasures.indexOf(measure); - if (index > -1) { - if (metricMeasures.get(index) == measure) { - add = false; - } else if (measure.getMetric().equals(CoreMetrics.TESTS)) { - // Hack for SONAR-5212 - measuresByMetric.remove(measure.getMetric().getKey(), metricMeasures.get(index)); - } else { - throw new SonarException("Can not add twice the same measure on " + resource + ": " + measure); - } - } - } - if (add) { - measuresByMetric.put(measure.getMetric().getKey(), measure); - } - } - public void clear() { - measuresByMetric = null; children = null; if (parent != null) { parent.removeChild(this); @@ -110,16 +78,6 @@ public final class Bucket { } } - public <M> M getMeasures(final MeasuresFilter<M> filter) { - Collection<Measure> unfiltered; - if (filter instanceof MeasuresFilters.MetricFilter) { - unfiltered = measuresByMetric.get(((MeasuresFilters.MetricFilter) filter).filterOnMetricKey()); - } else { - unfiltered = measuresByMetric.values(); - } - return filter.filter(unfiltered); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/Cache.java b/sonar-batch/src/main/java/org/sonar/batch/index/Cache.java index 46f6bbf0a84..39815dcfcb3 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/Cache.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/Cache.java @@ -26,6 +26,7 @@ import com.persistit.exception.PersistitException; import org.apache.commons.lang.builder.ToStringBuilder; import javax.annotation.CheckForNull; + import java.io.Serializable; import java.util.Iterator; import java.util.Set; @@ -219,7 +220,6 @@ public class Cache<V extends Serializable> { } } - /** * Clears the default as well as all group caches. */ @@ -294,6 +294,20 @@ public class Cache<V extends Serializable> { } /** + * Lazy-loading values for given keys + */ + public Iterable<V> values(Object firstKey, Object secondKey) { + try { + exchange.clear(); + exchange.append(firstKey).append(secondKey).append(Key.BEFORE); + Exchange iteratorExchange = new Exchange(exchange); + return new ValueIterable<V>(iteratorExchange, false); + } catch (Exception e) { + throw new IllegalStateException("Fail to get values from cache " + name, e); + } + } + + /** * Lazy-loading values for a given key */ public Iterable<V> values(Object key) { @@ -352,7 +366,6 @@ public class Cache<V extends Serializable> { } } - // // LAZY ITERATORS AND ITERABLES // diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/Caches.java b/sonar-batch/src/main/java/org/sonar/batch/index/Caches.java index f440db4da24..55e2de06495 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/Caches.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/Caches.java @@ -19,23 +19,35 @@ */ package org.sonar.batch.index; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import com.persistit.Exchange; import com.persistit.Persistit; +import com.persistit.Value; import com.persistit.Volume; +import com.persistit.encoding.CoderContext; +import com.persistit.encoding.ValueCoder; import com.persistit.exception.PersistitException; import com.persistit.logging.Slf4jAdapter; import org.apache.commons.io.FileUtils; +import org.objenesis.strategy.SerializingInstantiatorStrategy; import org.picocontainer.Startable; import org.slf4j.LoggerFactory; import org.sonar.api.BatchComponent; +import org.sonar.api.measures.Measure; import org.sonar.api.utils.TempFolder; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.Serializable; import java.util.Properties; import java.util.Set; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.InflaterInputStream; /** * Factory of caches @@ -49,6 +61,7 @@ public class Caches implements BatchComponent, Startable { private Persistit persistit; private Volume volume; private final TempFolder tempFolder; + private Kryo kryo; public Caches(TempFolder tempFolder) { this.tempFolder = tempFolder; @@ -60,6 +73,8 @@ public class Caches implements BatchComponent, Startable { tempDir = tempFolder.newDir("caches"); persistit = new Persistit(); persistit.setPersistitLogger(new Slf4jAdapter(LoggerFactory.getLogger("PERSISTIT"))); + kryo = new Kryo(); + kryo.setInstantiatorStrategy(new SerializingInstantiatorStrategy()); Properties props = new Properties(); props.setProperty("datapath", tempDir.getAbsolutePath()); props.setProperty("logpath", "${datapath}/log"); @@ -70,6 +85,7 @@ public class Caches implements BatchComponent, Startable { props.setProperty("volume.1", "${datapath}/persistit,create,pageSize:8192,initialPages:10,extensionPages:100,maximumPages:25000"); persistit.setProperties(props); persistit.initialize(); + persistit.getCoderManager().registerValueCoder(Measure.class, new MeasureValueCoder()); volume = persistit.createTemporaryVolume(); } catch (Exception e) { @@ -77,6 +93,28 @@ public class Caches implements BatchComponent, Startable { } } + /** + * Special value coder for measures because measure_data field may be too big for persitit. We use Kryo + * with deflate compression as workaround. + * + */ + private class MeasureValueCoder implements ValueCoder { + public void put(Value value, Object object, CoderContext context) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + Output output = new Output(new DeflaterOutputStream(baos)); + kryo.writeObject(output, object); + output.close(); + value.putByteArray(baos.toByteArray()); + } + + public Object get(Value value, Class clazz, CoderContext context) { + Input input = new Input(new InflaterInputStream(new ByteArrayInputStream(value.getByteArray()))); + Object someObject = kryo.readObject(input, clazz); + input.close(); + return someObject; + } + } + public <V extends Serializable> Cache<V> createCache(String cacheName) { Preconditions.checkState(volume != null && volume.isOpened(), "Caches are not initialized"); Preconditions.checkState(!cacheNames.contains(cacheName), "Cache is already created: " + cacheName); diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java index d5e42948fc5..dc0bac9be7e 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java @@ -31,6 +31,7 @@ import org.sonar.api.batch.SonarIndex; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.database.model.Snapshot; import org.sonar.api.design.Dependency; +import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.Measure; import org.sonar.api.measures.MeasuresFilter; import org.sonar.api.measures.MeasuresFilters; @@ -52,12 +53,15 @@ import org.sonar.api.violations.ViolationQuery; import org.sonar.batch.ProjectTree; import org.sonar.batch.issue.DeprecatedViolations; import org.sonar.batch.issue.ModuleIssues; +import org.sonar.batch.qualitygate.QualityGateVerifier; +import org.sonar.batch.scan.measure.MeasureCache; import org.sonar.core.component.ComponentKeys; import org.sonar.core.component.ScanGraph; import javax.annotation.CheckForNull; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -85,17 +89,19 @@ public class DefaultIndex extends SonarIndex { private ProjectTree projectTree; private final DeprecatedViolations deprecatedViolations; private ModuleIssues moduleIssues; + private final MeasureCache measureCache; private ResourceKeyMigration migration; public DefaultIndex(PersistenceManager persistence, ProjectTree projectTree, MetricFinder metricFinder, - ScanGraph graph, DeprecatedViolations deprecatedViolations, ResourceKeyMigration migration) { + ScanGraph graph, DeprecatedViolations deprecatedViolations, ResourceKeyMigration migration, MeasureCache measureCache) { this.persistence = persistence; this.projectTree = projectTree; this.metricFinder = metricFinder; this.graph = graph; this.deprecatedViolations = deprecatedViolations; this.migration = migration; + this.measureCache = measureCache; } public void start() { @@ -174,26 +180,23 @@ public class DefaultIndex extends SonarIndex { @Override public Measure getMeasure(Resource resource, Metric metric) { - Bucket bucket = buckets.get(resource); - if (bucket != null) { - return bucket.getMeasures(MeasuresFilters.metric(metric)); - } - return null; + return getMeasures(resource, MeasuresFilters.metric(metric)); } @Override public <M> M getMeasures(Resource resource, MeasuresFilter<M> filter) { - Bucket bucket = buckets.get(resource); - if (bucket != null) { - // TODO the data measures which are not kept in memory are not reloaded yet. Use getMeasure(). - return bucket.getMeasures(filter); + // Reload resource so that effective key is populated + Resource indexedResource = getResource(resource); + Iterable<Measure> unfiltered = measureCache.byResource(indexedResource); + Collection<Measure> all = new ArrayList<Measure>(); + if (unfiltered != null) { + for (Measure measure : unfiltered) { + all.add(measure); + } } - return null; + return filter.filter(all); } - /** - * the measure is updated if it's already registered. - */ @Override public Measure addMeasure(Resource resource, Measure measure) { Bucket bucket = getBucket(resource); @@ -203,15 +206,27 @@ public class DefaultIndex extends SonarIndex { throw new SonarException("Unknown metric: " + measure.getMetricKey()); } measure.setMetric(metric); - bucket.addMeasure(measure); - - if (measure.getPersistenceMode().useDatabase()) { - persistence.saveMeasure(bucket.getResource(), measure); + if (measureCache.contains(resource, measure) + // Hack for SONAR-5212 + && !measure.getMetric().equals(CoreMetrics.TESTS)) { + throw new SonarException("Can not add twice the same measure on " + resource + ": " + measure); } + measureCache.put(resource, measure); } return measure; } + /** + * Used by some core features like TendencyDecorator, {@link QualityGateVerifier}, VariationDecorator + * that need to update some existing measures + */ + public void updateMeasure(Resource resource, Measure measure) { + if (!measureCache.contains(resource, measure)) { + throw new SonarException("Can't update measure on " + resource + ": " + measure); + } + measureCache.put(resource, measure); + } + // // // diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java index 9dd3460e0ff..39b9d8d472a 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java @@ -22,7 +22,6 @@ package org.sonar.batch.index; import org.sonar.api.batch.Event; import org.sonar.api.database.model.Snapshot; import org.sonar.api.design.Dependency; -import org.sonar.api.measures.Measure; import org.sonar.api.resources.Project; import org.sonar.api.resources.ProjectLink; import org.sonar.api.resources.Resource; @@ -34,17 +33,14 @@ public final class DefaultPersistenceManager implements PersistenceManager { private ResourcePersister resourcePersister; private SourcePersister sourcePersister; - private MeasurePersister measurePersister; private DependencyPersister dependencyPersister; private LinkPersister linkPersister; private EventPersister eventPersister; public DefaultPersistenceManager(ResourcePersister resourcePersister, SourcePersister sourcePersister, - MeasurePersister measurePersister, DependencyPersister dependencyPersister, - LinkPersister linkPersister, EventPersister eventPersister) { + DependencyPersister dependencyPersister, LinkPersister linkPersister, EventPersister eventPersister) { this.resourcePersister = resourcePersister; this.sourcePersister = sourcePersister; - this.measurePersister = measurePersister; this.dependencyPersister = dependencyPersister; this.linkPersister = linkPersister; this.eventPersister = eventPersister; @@ -55,14 +51,6 @@ public final class DefaultPersistenceManager implements PersistenceManager { sourcePersister.clear(); } - public void setDelayedMode(boolean b) { - measurePersister.setDelayedMode(b); - } - - public void dump() { - measurePersister.dump(); - } - public void saveProject(Project project, Project parent) { resourcePersister.saveProject(project, parent); } @@ -82,12 +70,6 @@ public final class DefaultPersistenceManager implements PersistenceManager { return sourcePersister.getSource(resource); } - public void saveMeasure(Resource resource, Measure measure) { - if (ResourceUtils.isPersistable(resource)) { - measurePersister.saveMeasure(resource, measure); - } - } - public void saveDependency(Project project, Dependency dependency, Dependency parentDependency) { if (ResourceUtils.isPersistable(dependency.getFrom()) && ResourceUtils.isPersistable(dependency.getTo())) { dependencyPersister.saveDependency(project, dependency, parentDependency); diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java index e394f3199ee..09dfd493df2 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java @@ -20,84 +20,66 @@ package org.sonar.batch.index; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.LinkedHashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.SetMultimap; import org.apache.ibatis.session.SqlSession; -import org.slf4j.LoggerFactory; import org.sonar.api.database.model.MeasureMapper; import org.sonar.api.database.model.MeasureModel; import org.sonar.api.database.model.Snapshot; -import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.Measure; import org.sonar.api.measures.RuleMeasure; import org.sonar.api.resources.Resource; import org.sonar.api.resources.ResourceUtils; +import org.sonar.api.rule.RuleKey; import org.sonar.api.rules.Rule; import org.sonar.api.rules.RuleFinder; import org.sonar.api.technicaldebt.batch.Characteristic; import org.sonar.api.utils.SonarException; +import org.sonar.batch.index.Cache.Entry; +import org.sonar.batch.scan.measure.MeasureCache; import org.sonar.core.persistence.MyBatis; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -public final class MeasurePersister { +public final class MeasurePersister implements ScanPersister { private final MyBatis mybatis; - private final ResourcePersister resourcePersister; private final RuleFinder ruleFinder; - private final SetMultimap<Resource, Measure> unsavedMeasuresByResource = LinkedHashMultimap.create(); - private boolean delayedMode = false; + private final MeasureCache measureCache; + private final SnapshotCache snapshotCache; + private final ResourceCache resourceCache; - public MeasurePersister(MyBatis mybatis, ResourcePersister resourcePersister, RuleFinder ruleFinder) { + public MeasurePersister(MyBatis mybatis, RuleFinder ruleFinder, + MeasureCache measureCache, SnapshotCache snapshotCache, ResourceCache resourceCache) { this.mybatis = mybatis; - this.resourcePersister = resourcePersister; this.ruleFinder = ruleFinder; + this.measureCache = measureCache; + this.snapshotCache = snapshotCache; + this.resourceCache = resourceCache; } - public void setDelayedMode(boolean delayedMode) { - this.delayedMode = delayedMode; - } + @Override + public void persist() { + SqlSession session = mybatis.openSession(); + try { + MeasureMapper mapper = session.getMapper(MeasureMapper.class); - public void dump() { - LoggerFactory.getLogger(getClass()).debug("{} measures to dump", unsavedMeasuresByResource.size()); + for (Entry<Measure> entry : measureCache.entries()) { + String effectiveKey = entry.key()[0].toString(); + Measure measure = entry.value(); + Resource resource = resourceCache.get(effectiveKey); - insert(getMeasuresToSave()); - } - - public void saveMeasure(Resource resource, Measure measure) { - if (shouldSaveLater(measure)) { - if (measure.getMetric().equals(CoreMetrics.TESTS) && unsavedMeasuresByResource.get(resource).contains(measure)) { - // Hack for SONAR-5212 - unsavedMeasuresByResource.remove(resource, measure); + if (shouldPersistMeasure(resource, measure)) { + Snapshot snapshot = snapshotCache.get(effectiveKey); + MeasureModel measureModel = model(measure).setSnapshotId(snapshot.getId()); + try { + mapper.insert(measureModel); + } catch (Exception e) { + // SONAR-4066 + throw new SonarException(String.format("Unable to save measure for metric [%s] on component [%s]", measure.getMetricKey(), resource.getKey()), e); + } + } } - unsavedMeasuresByResource.put(resource, measure); - return; - } - try { - insertOrUpdate(resource, measure); - } catch (Exception e) { - // SONAR-4066 - throw new SonarException(String.format("Unable to save measure for metric [%s] on component [%s]", measure.getMetricKey(), resource.getKey()), e); - } - } - private MeasureModel insertOrUpdate(Resource resource, Measure measure) { - Snapshot snapshot = resourcePersister.getSnapshotOrFail(resource); - if (measure.getId() != null) { - return update(measure, snapshot); - } - if (shouldPersistMeasure(resource, measure)) { - MeasureModel insert = insert(measure, snapshot); - measure.setId(insert.getId()); - return insert; + session.commit(); + } finally { + MyBatis.closeQuietly(session); } - return null; - } - - private boolean shouldSaveLater(Measure measure) { - return delayedMode && measure.getPersistenceMode().useMemory(); } @VisibleForTesting @@ -115,24 +97,6 @@ public final class MeasurePersister { || isNotEmpty; } - private List<MeasureModelAndDetails> getMeasuresToSave() { - List<MeasureModelAndDetails> measures = Lists.newArrayList(); - - Map<Resource, Collection<Measure>> map = unsavedMeasuresByResource.asMap(); - for (Map.Entry<Resource, Collection<Measure>> entry : map.entrySet()) { - Resource resource = entry.getKey(); - Snapshot snapshot = resourcePersister.getSnapshot(entry.getKey()); - for (Measure measure : entry.getValue()) { - if (shouldPersistMeasure(resource, measure)) { - measures.add(new MeasureModelAndDetails(model(measure).setSnapshotId(snapshot.getId()), resource.getKey(), measure.getMetricKey())); - } - } - } - - unsavedMeasuresByResource.clear(); - return measures; - } - private MeasureModel model(Measure measure) { MeasureModel model = new MeasureModel(); // we assume that the index has updated the metric @@ -162,9 +126,9 @@ public final class MeasurePersister { if (measure instanceof RuleMeasure) { RuleMeasure ruleMeasure = (RuleMeasure) measure; model.setRulePriority(ruleMeasure.getSeverity()); - Rule rule = ruleMeasure.getRule(); - if (rule != null) { - Rule ruleWithId = ruleFinder.findByKey(rule.getRepositoryKey(), rule.getKey()); + RuleKey ruleKey = ruleMeasure.ruleKey(); + if (ruleKey != null) { + Rule ruleWithId = ruleFinder.findByKey(ruleKey); if (ruleWithId == null) { throw new SonarException("Can not save a measure with unknown rule " + ruleMeasure); } @@ -173,86 +137,4 @@ public final class MeasurePersister { } return model; } - - private void insert(Iterable<MeasureModelAndDetails> values) { - SqlSession session = mybatis.openSession(false); - try { - MeasureMapper mapper = session.getMapper(MeasureMapper.class); - - for (MeasureModelAndDetails value : values) { - try { - mapper.insert(value.getMeasureModel()); - } catch (Exception e) { - // SONAR-4066 - throw new SonarException(String.format("Unable to save measure for metric [%s] on component [%s]", value.getMetricKey(), value.getResourceKey()), e); - } - } - - session.commit(); - } finally { - MyBatis.closeQuietly(session); - } - } - - private MeasureModel insert(Measure measure, Snapshot snapshot) { - MeasureModel value = model(measure); - value.setSnapshotId(snapshot.getId()); - - SqlSession session = mybatis.openSession(false); - try { - MeasureMapper mapper = session.getMapper(MeasureMapper.class); - - mapper.insert(value); - - session.commit(); - } finally { - MyBatis.closeQuietly(session); - } - - return value; - } - - private MeasureModel update(Measure measure, Snapshot snapshot) { - MeasureModel value = model(measure); - value.setId(measure.getId()); - value.setSnapshotId(snapshot.getId()); - - SqlSession session = mybatis.openSession(false); - try { - MeasureMapper mapper = session.getMapper(MeasureMapper.class); - - mapper.update(value); - - session.commit(); - } finally { - MyBatis.closeQuietly(session); - } - - return value; - } - - // SONAR-4066 - private static class MeasureModelAndDetails { - private final MeasureModel measureModel; - private final String resourceKey; - private final String metricKey; - - public MeasureModelAndDetails(MeasureModel measureModel, String resourceKey, String metricKey) { - this.measureModel = measureModel; - this.resourceKey = resourceKey; - this.metricKey = metricKey; - } - - public MeasureModel getMeasureModel() { - return measureModel; - } - - public String getResourceKey() { - return resourceKey; - } - - public String getMetricKey() { - return metricKey; - } - } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java b/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java index dec88e49d15..d90f63d8b66 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java @@ -22,7 +22,6 @@ package org.sonar.batch.index; import org.sonar.api.batch.Event; import org.sonar.api.database.model.Snapshot; import org.sonar.api.design.Dependency; -import org.sonar.api.measures.Measure; import org.sonar.api.resources.Project; import org.sonar.api.resources.ProjectLink; import org.sonar.api.resources.Resource; @@ -32,10 +31,6 @@ import java.util.List; public interface PersistenceManager { void clear(); - void setDelayedMode(boolean b); - - void dump(); - void saveProject(Project project, Project parent); Snapshot saveResource(Project project, Resource resource, Resource parent); @@ -44,8 +39,6 @@ public interface PersistenceManager { String getSource(Resource resource); - void saveMeasure(Resource resource, Measure measure); - void saveDependency(Project project, Dependency dependency, Dependency parentDependency); void saveLink(Project project, ProjectLink link); diff --git a/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java b/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java index 0ba16ea66d7..90bafa7dc3e 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java +++ b/sonar-batch/src/main/java/org/sonar/batch/phases/PhaseExecutor.java @@ -116,8 +116,6 @@ public final class PhaseExecutor { executeInitializersPhase(); - persistenceManager.setDelayedMode(true); - if (phases.isEnabled(Phases.Phase.SENSOR)) { // Index and lock the filesystem fs.index(); @@ -135,12 +133,6 @@ public final class PhaseExecutor { decoratorsExecutor.execute(); } - String saveMeasures = "Save measures"; - eventBus.fireEvent(new BatchStepEvent(saveMeasures, true)); - persistenceManager.dump(); - eventBus.fireEvent(new BatchStepEvent(saveMeasures, false)); - persistenceManager.setDelayedMode(false); - if (module.isRoot()) { jsonReport.execute(); @@ -162,6 +154,7 @@ public final class PhaseExecutor { LOGGER.debug("Execute {}", persister.getClass().getName()); persister.persist(); } + eventBus.fireEvent(new BatchStepEvent(persistersStep, false)); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/qualitygate/QualityGateVerifier.java b/sonar-batch/src/main/java/org/sonar/batch/qualitygate/QualityGateVerifier.java index ddf1553e42f..ac18daadf95 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/qualitygate/QualityGateVerifier.java +++ b/sonar-batch/src/main/java/org/sonar/batch/qualitygate/QualityGateVerifier.java @@ -23,7 +23,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; -import org.sonar.api.batch.*; +import org.sonar.api.batch.Decorator; +import org.sonar.api.batch.DecoratorBarriers; +import org.sonar.api.batch.DecoratorContext; +import org.sonar.api.batch.DependedUpon; +import org.sonar.api.batch.DependsUpon; import org.sonar.api.database.model.Snapshot; import org.sonar.api.i18n.I18n; import org.sonar.api.measures.CoreMetrics; @@ -34,10 +38,15 @@ import org.sonar.api.resources.Resource; import org.sonar.api.resources.ResourceUtils; import org.sonar.api.utils.Duration; import org.sonar.api.utils.Durations; +import org.sonar.batch.index.DefaultIndex; import org.sonar.core.qualitygate.db.QualityGateConditionDto; import org.sonar.core.timemachine.Periods; -import java.util.*; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; public class QualityGateVerifier implements Decorator { @@ -55,13 +64,15 @@ public class QualityGateVerifier implements Decorator { private Periods periods; private I18n i18n; private Durations durations; + private final DefaultIndex index; - public QualityGateVerifier(QualityGate qualityGate, Snapshot snapshot, Periods periods, I18n i18n, Durations durations) { + public QualityGateVerifier(QualityGate qualityGate, Snapshot snapshot, Periods periods, I18n i18n, Durations durations, DefaultIndex index) { this.qualityGate = qualityGate; this.snapshot = snapshot; this.periods = periods; this.i18n = i18n; this.durations = durations; + this.index = index; } @DependedUpon @@ -77,7 +88,7 @@ public class QualityGateVerifier implements Decorator { @DependsUpon public Collection<Metric> dependsUponMetrics() { Set<Metric> metrics = Sets.newHashSet(); - for (ResolvedCondition condition: qualityGate.conditions()) { + for (ResolvedCondition condition : qualityGate.conditions()) { metrics.add(condition.metric()); } return metrics; @@ -91,16 +102,16 @@ public class QualityGateVerifier implements Decorator { @Override public void decorate(Resource resource, DecoratorContext context) { if (ResourceUtils.isRootProject(resource)) { - checkProjectConditions(context); + checkProjectConditions(resource, context); } } - private void checkProjectConditions(DecoratorContext context) { + private void checkProjectConditions(Resource resource, DecoratorContext context) { Metric.Level globalLevel = Metric.Level.OK; QualityGateDetails details = new QualityGateDetails(); List<String> labels = Lists.newArrayList(); - for (ResolvedCondition condition: qualityGate.conditions()) { + for (ResolvedCondition condition : qualityGate.conditions()) { Measure measure = context.getMeasure(condition.metric()); if (measure != null) { Metric.Level level = ConditionUtils.getLevel(condition, measure); @@ -112,7 +123,7 @@ public class QualityGateVerifier implements Decorator { labels.add(text); } - context.saveMeasure(measure); + index.updateMeasure(resource, measure); if (Metric.Level.WARN == level && globalLevel != Metric.Level.ERROR) { globalLevel = Metric.Level.WARN; diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java index 59e68e6078c..335884d7f87 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java @@ -68,6 +68,7 @@ import org.sonar.batch.rule.RulesProvider; import org.sonar.batch.scan.filesystem.InputFileCache; import org.sonar.batch.scan.maven.FakeMavenPluginExecutor; import org.sonar.batch.scan.maven.MavenPluginExecutor; +import org.sonar.batch.scan.measure.MeasureCache; import org.sonar.batch.source.HighlightableBuilder; import org.sonar.batch.source.SymbolizableBuilder; import org.sonar.core.component.ScanGraph; @@ -188,6 +189,9 @@ public class ProjectScanContainer extends ComponentContainer { // Differential periods PeriodsDefinition.class, + // Measures + MeasureCache.class, + ProjectSettingsReady.class); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/measure/MeasureCache.java b/sonar-batch/src/main/java/org/sonar/batch/scan/measure/MeasureCache.java new file mode 100644 index 00000000000..b29b6c83ee0 --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/measure/MeasureCache.java @@ -0,0 +1,82 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.scan.measure; + +import com.google.common.base.Preconditions; +import org.sonar.api.BatchComponent; +import org.sonar.api.measures.Measure; +import org.sonar.api.measures.RuleMeasure; +import org.sonar.api.resources.Resource; +import org.sonar.batch.index.Cache; +import org.sonar.batch.index.Cache.Entry; +import org.sonar.batch.index.Caches; + +/** + * Cache of all measures. This cache is shared amongst all project modules. + */ +public class MeasureCache implements BatchComponent { + + private final Cache<Measure> cache; + + public MeasureCache(Caches caches) { + cache = caches.createCache("measures"); + } + + public Iterable<Entry<Measure>> entries() { + return cache.entries(); + } + + public Iterable<Measure> byResource(Resource r) { + return cache.values(r.getEffectiveKey()); + } + + public MeasureCache put(Resource resource, Measure measure) { + Preconditions.checkNotNull(resource.getEffectiveKey()); + Preconditions.checkNotNull(measure.getMetricKey()); + cache.put(resource.getEffectiveKey(), computeMeasureKey(measure), measure); + return this; + } + + public boolean contains(Resource resource, Measure measure) { + Preconditions.checkNotNull(resource.getEffectiveKey()); + Preconditions.checkNotNull(measure.getMetricKey()); + return cache.containsKey(resource.getEffectiveKey(), computeMeasureKey(measure)); + } + + private static String computeMeasureKey(Measure m) { + StringBuilder sb = new StringBuilder(); + if (m.getMetricKey() != null) { + sb.append(m.getMetricKey()); + } + sb.append("|"); + if (m.getCharacteristic() != null) { + sb.append(m.getCharacteristic().key()); + } + sb.append("|"); + if (m.getPersonId() != null) { + sb.append(m.getPersonId()); + } + if (m instanceof RuleMeasure) { + sb.append("|"); + sb.append(((RuleMeasure) m).ruleKey()); + } + return sb.toString(); + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/measure/package-info.java b/sonar-batch/src/main/java/org/sonar/batch/scan/measure/package-info.java new file mode 100644 index 00000000000..64f9c876250 --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/measure/package-info.java @@ -0,0 +1,27 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +/** + * This package is a part of bootstrap process, so we should take care about backward compatibility. + */ +@ParametersAreNonnullByDefault +package org.sonar.batch.scan.measure; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/sonar-batch/src/test/java/org/sonar/batch/debt/DebtDecoratorTest.java b/sonar-batch/src/test/java/org/sonar/batch/debt/DebtDecoratorTest.java index fc82ece746c..6edbbab8fbf 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/debt/DebtDecoratorTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/debt/DebtDecoratorTest.java @@ -59,7 +59,7 @@ import java.util.Collections; import static com.google.common.collect.Lists.newArrayList; import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Matchers.any; -import static org.mockito.Mockito.argThat; +import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -204,7 +204,7 @@ public class DebtDecoratorTest { new RuleMeasure(CoreMetrics.TECHNICAL_DEBT, org.sonar.api.rules.Rule.create(ruleKey1.repository(), ruleKey1.rule()), null, null) .setValue(5d * ONE_DAY_IN_MINUTES) - )); + )); decorator.decorate(resource, context); verify(context).saveMeasure(CoreMetrics.TECHNICAL_DEBT, 7d * ONE_DAY_IN_MINUTES); @@ -225,7 +225,7 @@ public class DebtDecoratorTest { new RuleMeasure(CoreMetrics.TECHNICAL_DEBT, org.sonar.api.rules.Rule.create(ruleKey2.repository(), ruleKey2.rule()) , null, null).setValue(10d * ONE_DAY_IN_MINUTES) - )); + )); decorator.decorate(resource, context); verify(context).saveMeasure(CoreMetrics.TECHNICAL_DEBT, 15d * ONE_DAY_IN_MINUTES); @@ -319,8 +319,7 @@ public class DebtDecoratorTest { description.appendText(new StringBuilder() .append("value=").append(value).append(",") .append("characteristic=").append(characteristic.key()).append(",") - .append("metric=").append(metric.getKey()).toString()) - ; + .append("metric=").append(metric.getKey()).toString()); } } @@ -342,8 +341,7 @@ public class DebtDecoratorTest { } RuleMeasure m = (RuleMeasure) o; return ObjectUtils.equals(metric, m.getMetric()) && - ObjectUtils.equals(ruleKey.repository(), m.getRule().getRepositoryKey()) && - ObjectUtils.equals(ruleKey.rule(), m.getRule().getKey()) && + ObjectUtils.equals(ruleKey, m.ruleKey()) && ObjectUtils.equals(value, m.getValue()); } diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/BucketTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/BucketTest.java index 2fc521a64d5..cb20f15cde6 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/index/BucketTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/index/BucketTest.java @@ -20,12 +20,9 @@ package org.sonar.batch.index; import org.junit.Test; -import org.sonar.api.measures.Measure; -import org.sonar.api.measures.MeasuresFilters; import org.sonar.api.measures.Metric; import org.sonar.api.resources.Directory; import org.sonar.api.resources.File; -import org.sonar.api.utils.SonarException; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.core.IsNot.not; @@ -52,42 +49,6 @@ public class BucketTest { } @Test - public void shouldAddNewMeasure() { - Bucket fileBucket = new Bucket(javaFile); - Measure measure = new Measure(ncloc).setValue(1200.0); - fileBucket.addMeasure(measure); - - assertThat(fileBucket.getMeasures(MeasuresFilters.all()).size(), is(1)); - assertThat(fileBucket.getMeasures(MeasuresFilters.metric(ncloc)), is(measure)); - } - - @Test - public void shouldUpdateMeasure() { - Bucket fileBucket = new Bucket(javaFile); - Measure measure = new Measure(ncloc).setValue(1200.0); - fileBucket.addMeasure(measure); - - assertThat(fileBucket.getMeasures(MeasuresFilters.all()).size(), is(1)); - assertThat(fileBucket.getMeasures(MeasuresFilters.metric(ncloc)).getValue(), is(1200.0)); - - measure.setValue(500.0); - fileBucket.addMeasure(measure); - - assertThat(fileBucket.getMeasures(MeasuresFilters.all()).size(), is(1)); - assertThat(fileBucket.getMeasures(MeasuresFilters.metric(ncloc)).getValue(), is(500.0)); - } - - @Test(expected = SonarException.class) - public void shouldFailIfAddingSameMeasures() { - Bucket fileBucket = new Bucket(javaFile); - Measure measure = new Measure(ncloc).setValue(1200.0); - fileBucket.addMeasure(measure); - - measure = new Measure(ncloc).setValue(500.0); - fileBucket.addMeasure(measure); - } - - @Test public void shouldBeEquals() { assertEquals(new Bucket(directory), new Bucket(directory)); assertEquals(new Bucket(directory).hashCode(), new Bucket(directory).hashCode()); diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/DefaultIndexTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/DefaultIndexTest.java index 4abc71694cf..9a162fbf425 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/index/DefaultIndexTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/index/DefaultIndexTest.java @@ -28,7 +28,13 @@ import org.sonar.api.measures.Measure; import org.sonar.api.measures.MeasuresFilters; import org.sonar.api.measures.MetricFinder; import org.sonar.api.profiles.RulesProfile; -import org.sonar.api.resources.*; +import org.sonar.api.resources.Directory; +import org.sonar.api.resources.File; +import org.sonar.api.resources.Java; +import org.sonar.api.resources.Library; +import org.sonar.api.resources.Project; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.resources.Resource; import org.sonar.api.rules.Rule; import org.sonar.api.rules.RuleFinder; import org.sonar.api.rules.Violation; @@ -36,6 +42,7 @@ import org.sonar.api.violations.ViolationQuery; import org.sonar.batch.ProjectTree; import org.sonar.batch.issue.DeprecatedViolations; import org.sonar.batch.issue.ModuleIssues; +import org.sonar.batch.scan.measure.MeasureCache; import org.sonar.core.component.ScanGraph; import java.io.IOException; @@ -68,7 +75,8 @@ public class DefaultIndexTest { ruleFinder = mock(RuleFinder.class); ProjectTree projectTree = mock(ProjectTree.class); - index = new DefaultIndex(mock(PersistenceManager.class), projectTree, metricFinder, mock(ScanGraph.class), deprecatedViolations, mock(ResourceKeyMigration.class)); + index = new DefaultIndex(mock(PersistenceManager.class), projectTree, metricFinder, mock(ScanGraph.class), deprecatedViolations, mock(ResourceKeyMigration.class), + mock(MeasureCache.class)); java.io.File baseDir = temp.newFolder(); project = new Project("project"); diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java index 0b3363527da..f0a2bb666aa 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java @@ -36,8 +36,11 @@ import org.sonar.api.rules.Rule; import org.sonar.api.rules.RuleFinder; import org.sonar.api.rules.RulePriority; import org.sonar.api.utils.SonarException; +import org.sonar.batch.scan.measure.MeasureCache; import org.sonar.core.persistence.AbstractDaoTestCase; +import java.util.Arrays; + import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -58,21 +61,28 @@ public class MeasurePersisterTest extends AbstractDaoTestCase { MeasurePersister measurePersister; RuleFinder ruleFinder = mock(RuleFinder.class); - ResourcePersister resourcePersister = mock(ResourcePersister.class); Project project = new Project("foo"); Directory aDirectory = new Directory("org/foo"); File aFile = new File("org/foo/Bar.java"); Snapshot projectSnapshot = snapshot(PROJECT_SNAPSHOT_ID); Snapshot packageSnapshot = snapshot(PACKAGE_SNAPSHOT_ID); + private SnapshotCache snapshotCache; + + private MeasureCache measureCache; + @Before public void mockResourcePersister() { - when(resourcePersister.getSnapshotOrFail(project)).thenReturn(projectSnapshot); - when(resourcePersister.getSnapshotOrFail(aDirectory)).thenReturn(packageSnapshot); - when(resourcePersister.getSnapshot(project)).thenReturn(projectSnapshot); - when(resourcePersister.getSnapshot(aDirectory)).thenReturn(packageSnapshot); + snapshotCache = mock(SnapshotCache.class); + measureCache = mock(MeasureCache.class); + ResourceCache resourceCache = mock(ResourceCache.class); + when(snapshotCache.get("foo")).thenReturn(projectSnapshot); + when(snapshotCache.get("foo:org/foo")).thenReturn(packageSnapshot); + when(resourceCache.get("foo")).thenReturn(project); + when(resourceCache.get("foo:org/foo/Bar.java")).thenReturn(aFile); + when(resourceCache.get("foo:org/foo")).thenReturn(aDirectory); - measurePersister = new MeasurePersister(getMyBatis(), resourcePersister, ruleFinder); + measurePersister = new MeasurePersister(getMyBatis(), ruleFinder, measureCache, snapshotCache, resourceCache); } @Test @@ -80,10 +90,10 @@ public class MeasurePersisterTest extends AbstractDaoTestCase { setupData("empty"); Measure measure = new Measure(ncloc()).setValue(1234.0); - measurePersister.saveMeasure(project, measure); + when(measureCache.entries()).thenReturn(Arrays.asList(new Cache.Entry<Measure>(new String[] {"foo", "ncloc"}, measure))); + measurePersister.persist(); checkTables("shouldInsertMeasure", "project_measures"); - assertThat(measure.getId()).isNotNull(); } @Test @@ -91,11 +101,12 @@ public class MeasurePersisterTest extends AbstractDaoTestCase { setupData("empty"); Measure measure = new Measure(ncloc()).setValue(1234.0).setAlertText(TOO_LONG); + when(measureCache.entries()).thenReturn(Arrays.asList(new Cache.Entry<Measure>(new String[] {"foo", "ncloc"}, measure))); thrown.expect(SonarException.class); thrown.expectMessage("Unable to save measure for metric [ncloc] on component [foo]"); - measurePersister.saveMeasure(project, measure); + measurePersister.persist(); } @Test @@ -103,13 +114,14 @@ public class MeasurePersisterTest extends AbstractDaoTestCase { setupData("empty"); Rule rule = Rule.create("pmd", "key"); - when(ruleFinder.findByKey("pmd", "key")).thenReturn(rule); + when(ruleFinder.findByKey(rule.ruleKey())).thenReturn(rule); Measure measure = new RuleMeasure(ncloc(), rule, RulePriority.MAJOR, 1).setValue(1234.0); - measurePersister.saveMeasure(project, measure); + when(measureCache.entries()).thenReturn(Arrays.asList(new Cache.Entry<Measure>(new String[] {"foo", "ncloc"}, measure))); + + measurePersister.persist(); checkTables("shouldInsertRuleMeasure", "project_measures"); - assertThat(measure.getId()).isNotNull(); } @Test @@ -117,117 +129,50 @@ public class MeasurePersisterTest extends AbstractDaoTestCase { setupData("empty"); Measure withLargeData = new Measure(ncloc()).setData(LONG); - measurePersister.saveMeasure(project, withLargeData); + when(measureCache.entries()).thenReturn(Arrays.asList(new Cache.Entry<Measure>(new String[] {"foo", "ncloc"}, withLargeData))); - checkTables("shouldInsertMeasureWithLargeData", "project_measures"); + measurePersister.persist(); - assertThat(withLargeData.getId()).isNotNull(); + checkTables("shouldInsertMeasureWithLargeData", "project_measures"); } @Test public void should_not_save_best_values() { setupData("empty"); - measurePersister.saveMeasure(aFile, new Measure(coverage()).setValue(100.0)); - - assertEmptyTables("project_measures"); - } + Measure measure = new Measure(coverage()).setValue(100.0); + when(measureCache.entries()).thenReturn(Arrays.asList(new Cache.Entry<Measure>(new String[] {"foo:org/foo/Bar.java", "coverage"}, measure))); - @Test - public void should_not_save_memory_only_measures() { - setupData("empty"); - - measurePersister.saveMeasure(aFile, new Measure("ncloc").setPersistenceMode(PersistenceMode.MEMORY)); + measurePersister.persist(); assertEmptyTables("project_measures"); } @Test - public void should_always_save_non_file_measures() { - setupData("empty"); - - measurePersister.saveMeasure(project, new Measure(ncloc()).setValue(200.0)); - measurePersister.saveMeasure(aDirectory, new Measure(ncloc()).setValue(300.0)); - - checkTables("shouldAlwaysPersistNonFileMeasures", "project_measures"); - } - - @Test - public void should_update_measure() { - setupData("data"); - - measurePersister.saveMeasure(project, new Measure(coverage()).setValue(12.5).setId(1L)); - measurePersister.saveMeasure(project, new Measure(coverage()).setData(SHORT).setId(2L)); - measurePersister.saveMeasure(aDirectory, new Measure(coverage()).setData(LONG).setId(3L)); - - checkTables("shouldUpdateMeasure", "project_measures"); - } - - @Test - public void should_add_delayed_measure_several_times() { + public void should_not_save_memory_only_measures() { setupData("empty"); - Measure measure = new Measure(ncloc()); - - measurePersister.setDelayedMode(true); - measurePersister.saveMeasure(project, measure.setValue(200.0)); - measurePersister.saveMeasure(project, measure.setValue(300.0)); - measurePersister.dump(); + Measure measure = new Measure("ncloc").setPersistenceMode(PersistenceMode.MEMORY); + when(measureCache.entries()).thenReturn(Arrays.asList(new Cache.Entry<Measure>(new String[] {"foo:org/foo/Bar.java", "ncloc"}, measure))); - checkTables("shouldAddDelayedMeasureSeveralTimes", "project_measures"); - } - - @Test - public void should_delay_saving() { - setupData("empty"); - - measurePersister.setDelayedMode(true); - measurePersister.saveMeasure(project, new Measure(ncloc()).setValue(1234.0).setData(SHORT)); - measurePersister.saveMeasure(aDirectory, new Measure(ncloc()).setValue(50.0).setData(LONG)); + measurePersister.persist(); assertEmptyTables("project_measures"); - - measurePersister.dump(); - checkTables("shouldDelaySaving", "project_measures"); - } - - @Test - public void should_display_contextual_info_when_error_during_delay_saving() { - setupData("empty"); - - measurePersister.setDelayedMode(true); - - measurePersister.saveMeasure(project, new Measure(ncloc()).setValue(1234.0).setData(SHORT).setAlertText(TOO_LONG)); - - thrown.expect(SonarException.class); - thrown.expectMessage("Unable to save measure for metric [ncloc] on component [foo]"); - - measurePersister.dump(); - } - - @Test - public void should_not_delay_saving_with_database_only_measure() { - setupData("empty"); - - measurePersister.setDelayedMode(true); - measurePersister.saveMeasure(project, new Measure(ncloc()).setValue(1234.0).setPersistenceMode(PersistenceMode.DATABASE)); - measurePersister.saveMeasure(aDirectory, new Measure(ncloc()).setValue(50.0)); - - checkTables("shouldInsertMeasure", "project_measures"); } @Test - public void should_not_save_best_value_measures_in_delayed_mode() { + public void should_always_save_non_file_measures() { setupData("empty"); - measurePersister.setDelayedMode(true); - measurePersister.saveMeasure(aFile, new Measure(coverage()).setValue(100.0)); + Measure measure1 = new Measure(ncloc()).setValue(200.0); + Measure measure2 = new Measure(ncloc()).setValue(300.0); + when(measureCache.entries()).thenReturn(Arrays.asList( + new Cache.Entry<Measure>(new String[] {"foo", "ncloc"}, measure1), + new Cache.Entry<Measure>(new String[] {"foo:org/foo", "ncloc"}, measure2))); - assertEmptyTables("project_measures"); + measurePersister.persist(); - measurePersister.dump(); - - assertEmptyTables("project_measures"); + checkTables("shouldAlwaysPersistNonFileMeasures", "project_measures"); } @Test diff --git a/sonar-batch/src/test/java/org/sonar/batch/qualitygate/QualityGateVerifierTest.java b/sonar-batch/src/test/java/org/sonar/batch/qualitygate/QualityGateVerifierTest.java index 9dd6a731f12..28339d61c9c 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/qualitygate/QualityGateVerifierTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/qualitygate/QualityGateVerifierTest.java @@ -39,6 +39,7 @@ import org.sonar.api.resources.Resource; import org.sonar.api.test.IsMeasure; import org.sonar.api.utils.Duration; import org.sonar.api.utils.Durations; +import org.sonar.batch.index.DefaultIndex; import org.sonar.core.qualitygate.db.QualityGateConditionDto; import org.sonar.core.timemachine.Periods; @@ -69,6 +70,7 @@ public class QualityGateVerifierTest { Periods periods; I18n i18n; Durations durations; + private DefaultIndex index; @Before public void before() { @@ -89,7 +91,8 @@ public class QualityGateVerifierTest { snapshot = mock(Snapshot.class); qualityGate = mock(QualityGate.class); when(qualityGate.isEnabled()).thenReturn(true); - verifier = new QualityGateVerifier(qualityGate, snapshot, periods, i18n, durations); + index = mock(DefaultIndex.class); + verifier = new QualityGateVerifier(qualityGate, snapshot, periods, i18n, durations, index); project = new Project("foo"); } @@ -130,8 +133,8 @@ public class QualityGateVerifierTest { verifier.decorate(project, context); - verify(context).saveMeasure(argThat(hasLevel(measureClasses, Metric.Level.OK))); - verify(context).saveMeasure(argThat(hasLevel(measureCoverage, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureClasses, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureCoverage, Metric.Level.OK))); verify(context).saveMeasure(argThat(new IsMeasure(CoreMetrics.ALERT_STATUS, Metric.Level.OK.toString()))); verify(context).saveMeasure(argThat(new IsMeasure(CoreMetrics.QUALITY_GATE_DETAILS, "{\"level\":\"OK\"," + "\"conditions\":" @@ -166,8 +169,8 @@ public class QualityGateVerifierTest { verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.WARN, null))); - verify(context).saveMeasure(argThat(hasLevel(measureClasses, Metric.Level.OK))); - verify(context).saveMeasure(argThat(hasLevel(measureCoverage, Metric.Level.WARN))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureClasses, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureCoverage, Metric.Level.WARN))); } @@ -184,8 +187,8 @@ public class QualityGateVerifierTest { verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.ERROR, null))); - verify(context).saveMeasure(argThat(hasLevel(measureClasses, Metric.Level.WARN))); - verify(context).saveMeasure(argThat(hasLevel(measureCoverage, Metric.Level.ERROR))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureClasses, Metric.Level.WARN))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureCoverage, Metric.Level.ERROR))); } @Test @@ -255,9 +258,9 @@ public class QualityGateVerifierTest { verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.OK, null))); - verify(context).saveMeasure(argThat(hasLevel(measureClasses, Metric.Level.OK))); - verify(context).saveMeasure(argThat(hasLevel(measureCoverage, Metric.Level.OK))); - verify(context).saveMeasure(argThat(hasLevel(measureComplexity, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureClasses, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureCoverage, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureComplexity, Metric.Level.OK))); } @Test @@ -283,9 +286,9 @@ public class QualityGateVerifierTest { verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.WARN, null))); - verify(context).saveMeasure(argThat(hasLevel(measureClasses, Metric.Level.WARN))); - verify(context).saveMeasure(argThat(hasLevel(measureCoverage, Metric.Level.WARN))); - verify(context).saveMeasure(argThat(hasLevel(measureComplexity, Metric.Level.WARN))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureClasses, Metric.Level.WARN))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureCoverage, Metric.Level.WARN))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureComplexity, Metric.Level.WARN))); } @Test @@ -299,7 +302,7 @@ public class QualityGateVerifierTest { verifier.decorate(project, context); verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.OK, null))); - verify(context).saveMeasure(argThat(hasLevel(measureClasses, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureClasses, Metric.Level.OK))); } @Test @@ -317,7 +320,7 @@ public class QualityGateVerifierTest { verifier.decorate(project, context); verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.OK, null))); - verify(context).saveMeasure(argThat(hasLevel(measureRatingMetric, Metric.Level.OK))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureRatingMetric, Metric.Level.OK))); } @Test @@ -332,7 +335,7 @@ public class QualityGateVerifierTest { verifier.decorate(project, context); verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.WARN, null))); - verify(context).saveMeasure(argThat(hasLevel(measureClasses, Metric.Level.WARN))); + verify(index).updateMeasure(eq(project), argThat(hasLevel(measureClasses, Metric.Level.WARN))); } @Test(expected = NotImplementedException.class) @@ -405,7 +408,7 @@ public class QualityGateVerifierTest { verifier.decorate(project, context); // First call to saveMeasure is for the update of debt - verify(context).saveMeasure(argThat(matchesMetric(metric, Level.ERROR, "The Debt > 1h"))); + verify(index).updateMeasure(eq(project), argThat(matchesMetric(metric, Level.ERROR, "The Debt > 1h"))); verify(context).saveMeasure(argThat(matchesMetric(CoreMetrics.ALERT_STATUS, Metric.Level.ERROR, "The Debt > 1h"))); verify(context).saveMeasure(argThat(new IsMeasure(CoreMetrics.QUALITY_GATE_DETAILS, "{\"level\":\"ERROR\"," + "\"conditions\":" diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/measure/MeasureCacheTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/measure/MeasureCacheTest.java new file mode 100644 index 00000000000..a8f42fde075 --- /dev/null +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/measure/MeasureCacheTest.java @@ -0,0 +1,262 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.scan.measure; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.measures.CoreMetrics; +import org.sonar.api.measures.Measure; +import org.sonar.api.measures.RuleMeasure; +import org.sonar.api.resources.Directory; +import org.sonar.api.resources.File; +import org.sonar.api.resources.Project; +import org.sonar.api.resources.Resource; +import org.sonar.api.rule.RuleKey; +import org.sonar.api.rules.RulePriority; +import org.sonar.api.technicaldebt.batch.internal.DefaultCharacteristic; +import org.sonar.batch.index.Cache.Entry; +import org.sonar.batch.index.Caches; +import org.sonar.batch.index.CachesTest; + +import java.util.Date; +import java.util.Iterator; + +import static org.fest.assertions.Assertions.assertThat; + +public class MeasureCacheTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + Caches caches; + + @Before + public void start() throws Exception { + caches = CachesTest.createCacheOnTemp(temp); + caches.start(); + } + + @After + public void stop() { + caches.stop(); + } + + @Test + public void should_add_measure() throws Exception { + MeasureCache cache = new MeasureCache(caches); + Project p = new Project("struts"); + + assertThat(cache.entries()).hasSize(0); + + assertThat(cache.byResource(p)).hasSize(0); + + Measure m = new Measure(CoreMetrics.NCLOC, 1.0); + cache.put(p, m); + + assertThat(cache.contains(p, m)).isTrue(); + assertThat(cache.entries()).hasSize(1); + Iterator<Entry<Measure>> iterator = cache.entries().iterator(); + iterator.hasNext(); + Entry<Measure> next = iterator.next(); + assertThat(next.value()).isEqualTo(m); + assertThat(next.key()[0]).isEqualTo("struts"); + + assertThat(cache.byResource(p)).hasSize(1); + assertThat(cache.byResource(p).iterator().next()).isEqualTo(m); + + Measure mRule = RuleMeasure.createForPriority(CoreMetrics.CRITICAL_VIOLATIONS, RulePriority.BLOCKER, 1.0); + cache.put(p, mRule); + + assertThat(cache.entries()).hasSize(2); + + assertThat(cache.byResource(p)).hasSize(2); + } + + /** + * This test fails when compression is not enabled for measures. PersistIt seems to be ok with + * put but fail when reading value. + */ + @Test + public void should_add_measure_with_big_data() throws Exception { + MeasureCache cache = new MeasureCache(caches); + Project p = new Project("struts"); + + assertThat(cache.entries()).hasSize(0); + + assertThat(cache.byResource(p)).hasSize(0); + + Measure m = new Measure(CoreMetrics.NCLOC, 1.0).setDate(new Date()); + m.setAlertText("foooooooooooooooooooooooooooooooooooo"); + StringBuilder data = new StringBuilder(); + for (int i = 0; i < 1048575; i++) { + data.append("a"); + } + m.setData(data.toString()); + + cache.put(p, m); + + assertThat(cache.contains(p, m)).isTrue(); + assertThat(cache.entries()).hasSize(1); + Iterator<Entry<Measure>> iterator = cache.entries().iterator(); + iterator.hasNext(); + Entry<Measure> next = iterator.next(); + assertThat(next.value()).isEqualTo(m); + assertThat(next.key()[0]).isEqualTo("struts"); + + assertThat(cache.byResource(p)).hasSize(1); + assertThat(cache.byResource(p).iterator().next()).isEqualTo(m); + + RuleMeasure mRule = RuleMeasure.createForPriority(CoreMetrics.CRITICAL_VIOLATIONS, RulePriority.BLOCKER, 1.0); + mRule.setRuleKey(RuleKey.of("repo", "rule")); + cache.put(p, mRule); + + assertThat(cache.entries()).hasSize(2); + } + + /** + * This test fails when compression is not enabled for measures. Size exceed PersistIt max value. + */ + @Test + public void should_add_measure_with_too_big_data_for_persistit() throws Exception { + MeasureCache cache = new MeasureCache(caches); + Project p = new Project("struts"); + + assertThat(cache.entries()).hasSize(0); + + assertThat(cache.byResource(p)).hasSize(0); + + Measure m = new Measure(CoreMetrics.NCLOC, 1.0).setDate(new Date()); + StringBuilder data = new StringBuilder(); + for (int i = 0; i < 500000; i++) { + data.append("some data"); + } + m.setData(data.toString()); + + cache.put(p, m); + + assertThat(cache.contains(p, m)).isTrue(); + assertThat(cache.entries()).hasSize(1); + Iterator<Entry<Measure>> iterator = cache.entries().iterator(); + iterator.hasNext(); + Entry<Measure> next = iterator.next(); + assertThat(next.value()).isEqualTo(m); + assertThat(next.key()[0]).isEqualTo("struts"); + + assertThat(cache.byResource(p)).hasSize(1); + assertThat(cache.byResource(p).iterator().next()).isEqualTo(m); + + RuleMeasure mRule = RuleMeasure.createForPriority(CoreMetrics.CRITICAL_VIOLATIONS, RulePriority.BLOCKER, 1.0); + mRule.setRuleKey(RuleKey.of("repo", "rule")); + cache.put(p, mRule); + + assertThat(cache.entries()).hasSize(2); + } + + @Test + public void should_add_measure_with_too_big_data_for_persistit_with_compression() throws Exception { + MeasureCache cache = new MeasureCache(caches); + Project p = new Project("struts"); + + assertThat(cache.entries()).hasSize(0); + + assertThat(cache.byResource(p)).hasSize(0); + + Measure m = new Measure(CoreMetrics.NCLOC, 1.0).setDate(new Date()); + StringBuilder data = new StringBuilder(); + for (int i = 0; i < 50000000; i++) { + data.append((char) ('z' * Math.random())); + } + m.setData(data.toString()); + + thrown.expect(IllegalStateException.class); + thrown.expectMessage("Fail to put element in the cache measures"); + + cache.put(p, m); + } + + @Test + public void should_add_measure_with_same_metric() throws Exception { + MeasureCache cache = new MeasureCache(caches); + Project p = new Project("struts"); + + assertThat(cache.entries()).hasSize(0); + assertThat(cache.byResource(p)).hasSize(0); + + Measure m1 = new Measure(CoreMetrics.NCLOC, 1.0); + Measure m2 = new Measure(CoreMetrics.NCLOC, 1.0).setCharacteristic(new DefaultCharacteristic().setKey("charac")); + Measure m3 = new Measure(CoreMetrics.NCLOC, 1.0).setPersonId(2); + Measure m4 = new RuleMeasure(CoreMetrics.NCLOC, RuleKey.of("repo", "rule"), RulePriority.BLOCKER, null); + cache.put(p, m1); + cache.put(p, m2); + cache.put(p, m3); + cache.put(p, m4); + + assertThat(cache.entries()).hasSize(4); + + assertThat(cache.byResource(p)).hasSize(4); + } + + @Test + public void should_get_measures() throws Exception { + MeasureCache cache = new MeasureCache(caches); + Project p = new Project("struts"); + Resource dir = new Directory("foo/bar").setEffectiveKey("struts:foo/bar"); + Resource file1 = new File("foo/bar/File1.txt").setEffectiveKey("struts:foo/bar/File1.txt"); + Resource file2 = new File("foo/bar/File2.txt").setEffectiveKey("struts:foo/bar/File2.txt"); + + assertThat(cache.entries()).hasSize(0); + + assertThat(cache.byResource(p)).hasSize(0); + assertThat(cache.byResource(dir)).hasSize(0); + + Measure mFile1 = new Measure(CoreMetrics.NCLOC, 1.0); + cache.put(file1, mFile1); + Measure mFile2 = new Measure(CoreMetrics.NCLOC, 3.0); + cache.put(file2, mFile2); + + assertThat(cache.entries()).hasSize(2); + assertThat(cache.byResource(p)).hasSize(0); + assertThat(cache.byResource(dir)).hasSize(0); + + Measure mDir = new Measure(CoreMetrics.NCLOC, 4.0); + cache.put(dir, mDir); + + assertThat(cache.entries()).hasSize(3); + assertThat(cache.byResource(p)).hasSize(0); + assertThat(cache.byResource(dir)).hasSize(1); + assertThat(cache.byResource(dir).iterator().next()).isEqualTo(mDir); + + Measure mProj = new Measure(CoreMetrics.NCLOC, 4.0); + cache.put(p, mProj); + + assertThat(cache.entries()).hasSize(4); + assertThat(cache.byResource(p)).hasSize(1); + assertThat(cache.byResource(p).iterator().next()).isEqualTo(mProj); + assertThat(cache.byResource(dir)).hasSize(1); + assertThat(cache.byResource(dir).iterator().next()).isEqualTo(mDir); + } +} |