*/
package org.sonar.batch;
-import java.net.URLClassLoader;
-import java.util.Arrays;
-
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonar.jpa.session.DriverDatabaseConnector;
import org.sonar.jpa.session.ThreadLocalDatabaseSessionFactory;
+import java.net.URLClassLoader;
+import java.util.Arrays;
+
public class Batch {
private static final Logger LOG = LoggerFactory.getLogger(Batch.class);
addComponent(EventPersister.class);
addComponent(LinkPersister.class);
addComponent(MeasurePersister.class);
+ addComponent(MemoryOptimizer.class);
addComponent(DefaultResourcePersister.class);
addComponent(SourcePersister.class);
addComponent(ViolationPersister.class);
*/
package org.sonar.batch;
+import com.google.common.collect.Lists;
import org.sonar.api.batch.DecoratorContext;
import org.sonar.api.batch.Event;
import org.sonar.api.database.DatabaseSession;
import org.sonar.api.rules.Violation;
import org.sonar.batch.index.DefaultIndex;
-import java.util.*;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
public class DefaultDecoratorContext implements DecoratorContext {
}
public Measure getMeasure(Metric metric) {
- return index.getMeasures(resource, MeasuresFilters.metric(metric));
+ return index.getMeasure(resource, metric);
}
public Collection<Measure> getChildrenMeasures(MeasuresFilter filter) {
- List<Measure> result = new ArrayList<Measure>();
+ List<Measure> result = Lists.newArrayList();
for (DecoratorContext childContext : childrenContexts) {
Object childResult = childContext.getMeasures(filter);
if (childResult != null) {
*/
package org.sonar.batch.index;
-import java.util.*;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.sonar.batch.ResourceFilters;
import org.sonar.batch.ViolationFilters;
+import java.util.*;
+
public class DefaultIndex extends SonarIndex {
private static final Logger LOG = LoggerFactory.getLogger(DefaultIndex.class);
public Measure getMeasure(Resource resource, Metric metric) {
Bucket bucket = buckets.get(resource);
if (bucket != null) {
- return bucket.getMeasures(MeasuresFilters.metric(metric));
+ Measure measure = bucket.getMeasures(MeasuresFilters.metric(metric));
+ if (measure!=null) {
+ return persistence.reloadMeasure(measure);
+ }
}
return null;
}
public <M> M getMeasures(Resource resource, MeasuresFilter<M> filter) {
Bucket bucket = buckets.get(resource);
if (bucket != null) {
+ // TODO the data measures which are not kept in memory are not reloaded yet. Use getMeasure().
return bucket.getMeasures(filter);
}
return null;
throw new SonarException("Unknown metric: " + measure.getMetricKey());
}
measure.setMetric(metric);
- if (measure.getPersistenceMode().useMemory()) {
- bucket.addMeasure(measure);
- }
+ bucket.addMeasure(measure);
+
if (measure.getPersistenceMode().useDatabase()) {
persistence.saveMeasure(resource, measure);
}
- // TODO keep database measures in cache but remove data
}
return measure;
}
}
}
+ public Measure reloadMeasure(Measure measure) {
+ return measurePersister.reloadMeasure(measure);
+ }
+
public void saveDependency(Project project, Dependency dependency, Dependency parentDependency) {
if (ResourceUtils.isPersistable(dependency.getFrom()) && ResourceUtils.isPersistable(dependency.getTo())) {
dependencyPersister.saveDependency(project, dependency, parentDependency);
private DatabaseSession session;
private ResourcePersister resourcePersister;
private RuleFinder ruleFinder;
+ private MemoryOptimizer memoryOptimizer;
- public MeasurePersister(DatabaseSession session, ResourcePersister resourcePersister, RuleFinder ruleFinder) {
+
+ public MeasurePersister(DatabaseSession session, ResourcePersister resourcePersister, RuleFinder ruleFinder, MemoryOptimizer memoryOptimizer) {
this.session = session;
this.resourcePersister = resourcePersister;
this.ruleFinder = ruleFinder;
+ this.memoryOptimizer = memoryOptimizer;
}
public void setDelayedMode(boolean delayedMode) {
} else {
Snapshot snapshot = resourcePersister.getSnapshotOrFail(resource);
+ MeasureModel model = null;
if (measure.getId() != null) {
// update
- MeasureModel model = session.reattach(MeasureModel.class, measure.getId());
+ model = session.reattach(MeasureModel.class, measure.getId());
model = mergeModel(measure, model);
model.save(session);
} else if (shouldPersistMeasure(resource, measure)) {
// insert
- MeasureModel model = createModel(measure);
+ model = createModel(measure);
model.setSnapshotId(snapshot.getId());
model.save(session);
measure.setId(model.getId()); // could be removed
}
+ if (model != null) {
+ memoryOptimizer.evictDataMeasure(measure, model);
+ }
}
}
+ public Measure reloadMeasure(Measure measure) {
+ return memoryOptimizer.reloadMeasure(measure);
+ }
+
static boolean shouldPersistMeasure(Resource resource, Measure measure) {
Metric metric = measure.getMetric();
return measure.getPersistenceMode().useDatabase() &&
--- /dev/null
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2011 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02
+ */
+package org.sonar.batch.index;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.database.DatabaseSession;
+import org.sonar.api.database.model.MeasureData;
+import org.sonar.api.database.model.MeasureModel;
+import org.sonar.api.measures.Measure;
+import org.sonar.api.measures.PersistenceMode;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @since 2.7
+ */
+public class MemoryOptimizer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MemoryOptimizer.class);
+
+ private List<Measure> loadedMeasures = Lists.newArrayList();
+ private Map<Long, Integer> dataIdByMeasureId = Maps.newHashMap();
+ private DatabaseSession session;
+
+ public MemoryOptimizer(DatabaseSession session) {
+ this.session = session;
+ }
+
+ /**
+ * Remove data of a database measure from memory.
+ */
+ public void evictDataMeasure(Measure measure, MeasureModel model) {
+ if (PersistenceMode.DATABASE.equals(measure.getPersistenceMode())) {
+ MeasureData data = model.getMeasureData();
+ if (data != null && data.getId() != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Remove data measure from memory: " + measure.getMetricKey() + ", id=" + measure.getId());
+ }
+ measure.unsetData();
+ dataIdByMeasureId.put(measure.getId(), data.getId());
+ }
+ }
+ }
+
+ public Measure reloadMeasure(Measure measure) {
+ if (measure.getId() != null && dataIdByMeasureId.containsKey(measure.getId()) && !measure.hasData()) {
+ Integer dataId = dataIdByMeasureId.get(measure.getId());
+ MeasureData data = session.getSingleResult(MeasureData.class, "id", dataId);
+ if (data == null) {
+ LoggerFactory.getLogger(getClass()).error("The MEASURE_DATA row with id " + dataId + " is lost");
+
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reload the data measure: " + measure.getMetricKey() + ", id=" + measure.getId());
+ }
+ measure.setData(data.getText());
+ loadedMeasures.add(measure);
+ }
+ }
+ return measure;
+ }
+
+ public void flushMemory() {
+ if (LOG.isDebugEnabled() && loadedMeasures.size() > 0) {
+ LOG.debug("Flush " + loadedMeasures.size() + " data measures from memory: ");
+ }
+ for (Measure measure : loadedMeasures) {
+ measure.unsetData();
+ }
+ loadedMeasures.clear();
+ }
+
+ boolean isTracked(Long measureId) {
+ return dataIdByMeasureId.get(measureId)!=null;
+ }
+}
import org.sonar.api.database.model.Snapshot;
import org.sonar.api.design.Dependency;
import org.sonar.api.measures.Measure;
-import org.sonar.api.resources.File;
import org.sonar.api.resources.Project;
import org.sonar.api.resources.ProjectLink;
import org.sonar.api.resources.Resource;
void saveMeasure(Resource resource, Measure measure);
+ Measure reloadMeasure(Measure measure);
+
void saveDependency(Project project, Dependency dependency, Dependency parentDependency);
void saveLink(Project project, ProjectLink link);
import org.sonar.batch.DecoratorsSelector;
import org.sonar.batch.DefaultDecoratorContext;
import org.sonar.batch.index.DefaultIndex;
+import org.sonar.batch.index.MemoryOptimizer;
import java.util.Collection;
import java.util.IdentityHashMap;
private DatabaseSession session;
private static final Logger LOG = LoggerFactory.getLogger(DecoratorsExecutor.class);
private DefaultIndex index;
+ private MemoryOptimizer memoryOptimizer;
- public DecoratorsExecutor(BatchExtensionDictionnary extensionDictionnary, DefaultIndex index, DatabaseSession session) {
+ public DecoratorsExecutor(BatchExtensionDictionnary extensionDictionnary, DefaultIndex index, DatabaseSession session,
+ MemoryOptimizer memoryOptimizer) {
this.decoratorsSelector = new DecoratorsSelector(extensionDictionnary);
this.session = session;
this.index = index;
+ this.memoryOptimizer = memoryOptimizer;
}
for (Decorator decorator : decorators) {
profiler.start(decorator);
decorator.decorate(resource, context);
+ memoryOptimizer.flushMemory();
profiler.stop();
}
}
import org.sonar.api.resources.Project;
import org.sonar.api.utils.TimeProfiler;
import org.sonar.batch.MavenPluginExecutor;
+import org.sonar.batch.index.MemoryOptimizer;
import java.util.Collection;
private Collection<Sensor> sensors;
private DatabaseSession session;
private MavenPluginExecutor mavenExecutor;
+ private MemoryOptimizer memoryOptimizer;
- public SensorsExecutor(BatchExtensionDictionnary selector, Project project, DatabaseSession session, MavenPluginExecutor mavenExecutor) {
+ public SensorsExecutor(BatchExtensionDictionnary selector, Project project, DatabaseSession session, MavenPluginExecutor mavenExecutor,
+ MemoryOptimizer memoryOptimizer) {
this.sensors = selector.select(Sensor.class, project, true);
this.session = session;
this.mavenExecutor = mavenExecutor;
+ this.memoryOptimizer = memoryOptimizer;
}
public void execute(Project project, SensorContext context) {
TimeProfiler profiler = new TimeProfiler(logger).start("Sensor " + sensor);
sensor.analyse(project, context);
+ memoryOptimizer.flushMemory();
session.commit();
profiler.stop();
}
import java.util.List;
+import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertThat;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class MeasurePersisterTest extends AbstractDbUnitTestCase {
private JavaFile aFile = new JavaFile("org.foo.Bar");
private Snapshot projectSnapshot, packageSnapshot, fileSnapshot;
private Metric ncloc, coverage;
+ private MemoryOptimizer memoryOptimizer;
@Before
public void mockResourcePersister() {
when(resourcePersister.getSnapshot(project)).thenReturn(projectSnapshot);
when(resourcePersister.getSnapshot(aPackage)).thenReturn(packageSnapshot);
when(resourcePersister.getSnapshot(aFile)).thenReturn(fileSnapshot);
- measurePersister = new MeasurePersister(getSession(), resourcePersister, new DefaultRuleFinder(getSessionFactory()));
+ memoryOptimizer = mock(MemoryOptimizer.class);
+ measurePersister = new MeasurePersister(getSession(), resourcePersister, new DefaultRuleFinder(getSessionFactory()), memoryOptimizer);
}
@Test
checkTables("shouldInsertMeasure", "project_measures");
}
+ @Test
+ public void shouldRegisterPersistedMeasureToMemoryOptimizer() {
+ Measure measure = new Measure(ncloc).setValue(1234.0);
+
+ measurePersister.saveMeasure(project, measure);
+
+ verify(memoryOptimizer).evictDataMeasure(eq(measure), (MeasureModel)anyObject());
+ }
+
+
@Test
public void shouldUpdateMeasure() {
Measure measure = new Measure(coverage).setValue(12.5);
--- /dev/null
+/*
+ * Sonar, open source software quality management tool.
+ * Copyright (C) 2008-2011 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * Sonar is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * Sonar is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Sonar; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02
+ */
+package org.sonar.batch.index;
+
+import org.hamcrest.core.IsNull;
+import org.junit.Test;
+import org.sonar.api.database.model.MeasureData;
+import org.sonar.api.database.model.MeasureModel;
+import org.sonar.api.measures.CoreMetrics;
+import org.sonar.api.measures.Measure;
+import org.sonar.api.measures.PersistenceMode;
+import org.sonar.jpa.test.AbstractDbUnitTestCase;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.core.IsNull.nullValue;
+import static org.junit.Assert.assertThat;
+
+public class MemoryOptimizerTest extends AbstractDbUnitTestCase {
+
+ @Test
+ public void shouldEvictDatabaseOnlyMeasure() {
+ MemoryOptimizer optimizer = new MemoryOptimizer(getSession());
+ Measure measure = new Measure(CoreMetrics.CONDITIONS_BY_LINE)
+ .setData("10=23")
+ .setPersistenceMode(PersistenceMode.DATABASE)
+ .setId(12345L);
+ MeasureModel model = newPersistedModel();
+
+ optimizer.evictDataMeasure(measure, model);
+
+ assertThat(optimizer.isTracked(12345L),is(true));
+ assertThat(measure.getData(), nullValue());// data has been removed from memory
+ }
+
+ @Test
+ public void shouldNotEvictStandardMeasure() {
+ MemoryOptimizer optimizer = new MemoryOptimizer(getSession());
+ Measure measure = new Measure(CoreMetrics.PROFILE)
+ .setData("Sonar way")
+ .setId(12345L);
+ MeasureModel model = newPersistedModel();
+
+ optimizer.evictDataMeasure(measure, model);
+
+ assertThat(optimizer.isTracked(12345L),is(false));
+ assertThat(measure.getData(), is("Sonar way"));
+ }
+
+ @Test
+ public void shouldReloadEvictedMeasure() {
+ setupData("shouldReloadEvictedMeasure");
+ MemoryOptimizer optimizer = new MemoryOptimizer(getSession());
+ Measure measure = new Measure(CoreMetrics.CONDITIONS_BY_LINE)
+ .setData("initial")
+ .setPersistenceMode(PersistenceMode.DATABASE)
+ .setId(12345L);
+
+ optimizer.evictDataMeasure(measure, newPersistedModel());
+ assertThat(measure.getData(), nullValue());
+
+ optimizer.reloadMeasure(measure);
+
+ assertThat(measure.getData().length(), greaterThan(5));
+
+ optimizer.flushMemory();
+ assertThat(measure.getData(), nullValue());
+ }
+
+ private MeasureModel newPersistedModel() {
+ MeasureModel model = new MeasureModel();
+ model.setId(12345L);
+ MeasureData measureData = new MeasureData();
+ measureData.setId(500);
+ model.setMeasureData(measureData);
+ return model;
+ }
+}
--- /dev/null
+<dataset>
+
+
+ <measure_data id="2" measure_id="2" snapshot_id="2" data="[null]"/>
+ <measure_data id="500" measure_id="12345" snapshot_id="1" data="value from database"/>
+
+
+</dataset>
\ No newline at end of file
// SCM
+ // These metrics are computed by the SCM Activity plugin, since version 1.2.
public static final String SCM_COMMITS_KEY = "commits";
public static final Metric SCM_COMMITS = new Metric.Builder(SCM_COMMITS_KEY, Metric.ValueType.INT)
return this;
}
+ /**
+ * @since 2.7
+ */
+ public Measure unsetData() {
+ this.data=null;
+ return this;
+ }
+
/**
* @return the description of the measure
*/
* Creates a metric based on its key. Shortcut to Metric(key, ValueType.INT)
*
* @param key the metric key
+ * @deprecated since 2.7 use the Builder factory.
*/
+ @Deprecated
public Metric(String key) {
this(key, ValueType.INT);
}
*
* @param key the key
* @param type the type
+ * @deprecated since 2.7 use the Builder factory.
*/
+ @Deprecated
public Metric(String key, ValueType type) {
this(key, key, key, type, -1, Boolean.FALSE, null, false);
}
+ /**
+ * @deprecated since 2.7 use the Builder factory.
+ */
+ @Deprecated
public Metric(String key, String name, String description, ValueType type, Integer direction, Boolean qualitative, String domain) {
this(key, name, description, type, direction, qualitative, domain, false);
}
* @param qualitative whether the metric is qualitative
* @param domain the metric domain
* @param userManaged whether the metric is user managed
+ * @deprecated since 2.7 use the Builder factory.
*/
@Deprecated
public Metric(String key, String name, String description, ValueType type, Integer direction, Boolean qualitative, String domain, boolean userManaged) {
* @param qualitative whether the metric is qualitative
* @param domain the metric domain
* @param formula the metric formula
+ * @deprecated since 2.7 use the Builder factory.
*/
+ @Deprecated
public Metric(String key, String name, ValueType type, Integer direction, Boolean qualitative, String domain, Formula formula) {
this.key = key;
this.name = name;
assertFalse(measure.equals(ruleMeasure));
assertFalse(ruleMeasure.equals(measure));
}
+
+ @Test
+ public void shouldUnsetData() {
+ String data = "1=10;21=456";
+ Measure measure = new Measure(CoreMetrics.CONDITIONS_BY_LINE).setData( data);
+ assertThat(measure.hasData(), is(true));
+ assertThat(measure.getData(), is(data));
+
+ measure.unsetData();
+
+ assertThat(measure.hasData(), is(false));
+ assertThat(measure.getData(), nullValue());
+ }
}
projects / sonarqube.git / commitdiff