From: David Gageot Date: Mon, 9 Jul 2012 14:00:49 +0000 (+0200) Subject: SONAR-3437 Use MyBatis in batch mode X-Git-Tag: 3.2~161 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=9a1dfc90225636f980821a4c4be971035f0e6e05;p=sonarqube.git SONAR-3437 Use MyBatis in batch mode --- diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java index c270e2fd071..38b8e7ff3a3 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/MeasurePersister.java @@ -19,14 +19,20 @@ */ package org.sonar.batch.index; -import org.sonar.api.database.model.MeasureDto; +import javax.annotation.Nullable; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import com.google.common.collect.Iterables; import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Lists; import com.google.common.collect.SetMultimap; import org.apache.commons.lang.math.NumberUtils; import org.apache.ibatis.session.SqlSession; import org.slf4j.LoggerFactory; +import org.sonar.api.database.model.MeasureDataDto; +import org.sonar.api.database.model.MeasureDto; import org.sonar.api.database.model.MeasureModel; import org.sonar.api.database.model.MeasureModelMapper; import org.sonar.api.database.model.Snapshot; @@ -41,8 +47,13 @@ import org.sonar.api.utils.SonarException; import org.sonar.core.persistence.MyBatis; import java.util.Collection; +import java.util.List; import java.util.Map; +import static com.google.common.collect.Iterables.filter; + +import static com.google.common.base.Predicates.not; + public final class MeasurePersister { private final MyBatis mybatis; private final ResourcePersister resourcePersister; @@ -62,25 +73,38 @@ public final class MeasurePersister { this.delayedMode = delayedMode; } + public Measure reloadMeasure(Measure measure) { + return memoryOptimizer.reloadMeasure(measure); + } + + public void dump() { + LoggerFactory.getLogger(getClass()).debug("{} measures to dump", unsavedMeasuresByResource.size()); + + List measuresToSave = getMeasuresToSave(); + insert(filter(measuresToSave, HAS_LARGE_DATA)); + batchInsert(filter(measuresToSave, not(HAS_LARGE_DATA))); + } + public void saveMeasure(Resource resource, Measure measure) { if (shouldSaveLater(measure)) { unsavedMeasuresByResource.put(resource, measure); return; } - MeasureModel model = null; - if (measure.getId() != null) { - model = update(measure); - } else if (shouldPersistMeasure(resource, measure)) { - model = insert(measure, resourcePersister.getSnapshotOrFail(resource)); - } + MeasureModel model = insertOrUpdate(resource, measure); if (model != null) { memoryOptimizer.evictDataMeasure(measure, model); } } - public Measure reloadMeasure(Measure measure) { - return memoryOptimizer.reloadMeasure(measure); + private MeasureModel insertOrUpdate(Resource resource, Measure measure) { + if (measure.getId() != null) { + return update(measure); + } + if (shouldPersistMeasure(resource, measure)) { + return insert(measure, resourcePersister.getSnapshotOrFail(resource)); + } + return null; } private boolean shouldSaveLater(Measure measure) { @@ -111,29 +135,22 @@ public final class MeasurePersister { && (measure.getVariation5() == null || NumberUtils.compare(measure.getVariation5().doubleValue(), 0.0) == 0); } - public void dump() { - LoggerFactory.getLogger(getClass()).debug("{} measures to dump", unsavedMeasuresByResource.size()); - - SqlSession session = mybatis.openSession(); - try { - MeasureModelMapper mapper = session.getMapper(MeasureModelMapper.class); + private List getMeasuresToSave() { + List batch = Lists.newArrayList(); - Map> map = unsavedMeasuresByResource.asMap(); - for (Map.Entry> entry : map.entrySet()) { - Resource resource = entry.getKey(); - Snapshot snapshot = resourcePersister.getSnapshot(entry.getKey()); - for (Measure measure : entry.getValue()) { - if (shouldPersistMeasure(resource, measure)) { - mapper.insert(new MeasureDto(model(measure).setSnapshotId(snapshot.getId()))); - } + Map> map = unsavedMeasuresByResource.asMap(); + for (Map.Entry> entry : map.entrySet()) { + Resource resource = entry.getKey(); + Snapshot snapshot = resourcePersister.getSnapshot(entry.getKey()); + for (Measure measure : entry.getValue()) { + if (shouldPersistMeasure(resource, measure)) { + batch.add(new MeasureDto(model(measure).setSnapshotId(snapshot.getId()))); } } - session.commit(); - } finally { - MyBatis.closeQuietly(session); } unsavedMeasuresByResource.clear(); + return batch; } private MeasureModel model(Measure measure) { @@ -171,13 +188,44 @@ public final class MeasurePersister { return model; } + private void batchInsert(Iterable values) { + SqlSession session = mybatis.openBatchSession(); + try { + MeasureModelMapper mapper = session.getMapper(MeasureModelMapper.class); + for (MeasureDto value : values) { + mapper.insert(value); + } + session.commit(); + } finally { + MyBatis.closeQuietly(session); + } + } + + private void insert(Iterable values) { + SqlSession session = mybatis.openSession(); + try { + MeasureModelMapper mapper = session.getMapper(MeasureModelMapper.class); + for (MeasureDto value : values) { + mapper.insert(value); + mapper.insertData(new MeasureDataDto(value.getId(), value.getSnapshotId(), value.getMeasureData().getData())); + } + session.commit(); + } finally { + MyBatis.closeQuietly(session); + } + } + private MeasureModel insert(Measure measure, Snapshot snapshot) { MeasureModel model = model(measure).setSnapshotId(snapshot.getId()); SqlSession session = mybatis.openSession(); try { MeasureModelMapper mapper = session.getMapper(MeasureModelMapper.class); - mapper.insert(new MeasureDto(model)); + MeasureDto value = new MeasureDto(model); + mapper.insert(value); + if (value.getMeasureData() != null) { + mapper.insertData(new MeasureDataDto(value.getId(), value.getSnapshotId(), value.getMeasureData().getData())); + } session.commit(); } finally { MyBatis.closeQuietly(session); @@ -201,4 +249,10 @@ public final class MeasurePersister { return model; } + + private static final Predicate HAS_LARGE_DATA = new Predicate() { + public boolean apply(@Nullable MeasureDto measure) { + return (null != measure) && (measure.getMeasureData() != null); + } + }; } diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java index 3bb6707129a..f1ba1fb750a 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/index/MeasurePersisterTest.java @@ -19,6 +19,7 @@ */ package org.sonar.batch.index; +import org.apache.commons.lang.StringUtils; import org.junit.Before; import org.junit.Test; import org.sonar.api.database.model.MeasureModel; @@ -93,6 +94,16 @@ public class MeasurePersisterTest extends AbstractDaoTestCase { checkTables("shouldInsertRuleMeasure", "project_measures"); } + @Test + public void shouldInsertMeasureWithTextData() { + setupData("empty"); + + measurePersister.saveMeasure(project, new Measure(ncloc()).setData("SHORT")); + measurePersister.saveMeasure(project, new Measure(ncloc()).setData(StringUtils.repeat("0123456789", 10))); + + checkTables("shouldInsertMeasureWithLargeData", "project_measures", "measure_data"); + } + @Test public void shouldUpdateMeasure() { setupData("data"); diff --git a/sonar-batch/src/test/resources/org/sonar/batch/index/MeasurePersisterTest/shouldInsertMeasureWithLargeData-result.xml b/sonar-batch/src/test/resources/org/sonar/batch/index/MeasurePersisterTest/shouldInsertMeasureWithLargeData-result.xml new file mode 100644 index 00000000000..c90db3d3f14 --- /dev/null +++ b/sonar-batch/src/test/resources/org/sonar/batch/index/MeasurePersisterTest/shouldInsertMeasureWithLargeData-result.xml @@ -0,0 +1,18 @@ + + + + + + + + + diff --git a/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java b/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java index fc2c07a1821..73825c29e26 100644 --- a/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java +++ b/sonar-core/src/main/java/org/sonar/core/persistence/MyBatis.java @@ -19,36 +19,54 @@ */ package org.sonar.core.persistence; -import org.sonar.api.database.model.MeasureDto; - -import org.sonar.api.database.model.MeasureModel; - -import org.sonar.api.database.model.MeasureModelMapper; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.ibatis.builder.xml.XMLMapperBuilder; import org.apache.ibatis.logging.LogFactory; import org.apache.ibatis.mapping.Environment; -import org.apache.ibatis.session.*; +import org.apache.ibatis.session.Configuration; +import org.apache.ibatis.session.ExecutorType; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.SqlSessionFactory; +import org.apache.ibatis.session.SqlSessionFactoryBuilder; import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; import org.slf4j.LoggerFactory; import org.sonar.api.BatchComponent; import org.sonar.api.ServerComponent; -import org.sonar.core.dashboard.*; +import org.sonar.api.database.model.MeasureDataDto; +import org.sonar.api.database.model.MeasureDto; +import org.sonar.api.database.model.MeasureModelMapper; +import org.sonar.core.dashboard.ActiveDashboardDto; +import org.sonar.core.dashboard.ActiveDashboardMapper; +import org.sonar.core.dashboard.DashboardDto; +import org.sonar.core.dashboard.DashboardMapper; +import org.sonar.core.dashboard.WidgetDto; +import org.sonar.core.dashboard.WidgetMapper; +import org.sonar.core.dashboard.WidgetPropertyDto; +import org.sonar.core.dashboard.WidgetPropertyMapper; import org.sonar.core.dependency.DependencyDto; import org.sonar.core.dependency.DependencyMapper; import org.sonar.core.dependency.ResourceSnapshotDto; import org.sonar.core.dependency.ResourceSnapshotMapper; import org.sonar.core.duplication.DuplicationMapper; import org.sonar.core.duplication.DuplicationUnitDto; -import org.sonar.core.filter.*; +import org.sonar.core.filter.CriterionDto; +import org.sonar.core.filter.CriterionMapper; +import org.sonar.core.filter.FilterColumnDto; +import org.sonar.core.filter.FilterColumnMapper; +import org.sonar.core.filter.FilterDto; +import org.sonar.core.filter.FilterMapper; import org.sonar.core.properties.PropertiesMapper; import org.sonar.core.properties.PropertyDto; import org.sonar.core.purge.PurgeMapper; import org.sonar.core.purge.PurgeVendorMapper; import org.sonar.core.purge.PurgeableSnapshotDto; -import org.sonar.core.resource.*; +import org.sonar.core.resource.ResourceDto; +import org.sonar.core.resource.ResourceIndexDto; +import org.sonar.core.resource.ResourceIndexerMapper; +import org.sonar.core.resource.ResourceKeyUpdaterMapper; +import org.sonar.core.resource.ResourceMapper; +import org.sonar.core.resource.SnapshotDto; import org.sonar.core.review.ReviewCommentDto; import org.sonar.core.review.ReviewCommentMapper; import org.sonar.core.review.ReviewDto; @@ -57,7 +75,14 @@ import org.sonar.core.rule.RuleDto; import org.sonar.core.rule.RuleMapper; import org.sonar.core.template.LoadedTemplateDto; import org.sonar.core.template.LoadedTemplateMapper; -import org.sonar.core.user.*; +import org.sonar.core.user.AuthorDto; +import org.sonar.core.user.AuthorMapper; +import org.sonar.core.user.GroupDto; +import org.sonar.core.user.GroupRoleDto; +import org.sonar.core.user.RoleMapper; +import org.sonar.core.user.UserDto; +import org.sonar.core.user.UserMapper; +import org.sonar.core.user.UserRoleDto; import java.io.InputStream; @@ -105,6 +130,7 @@ public class MyBatis implements BatchComponent, ServerComponent { loadAlias(conf, "Widget", WidgetDto.class); loadAlias(conf, "WidgetProperty", WidgetPropertyDto.class); loadAlias(conf, "MeasureDto", MeasureDto.class); + loadAlias(conf, "MeasureDataDto", MeasureDataDto.class); loadMapper(conf, ActiveDashboardMapper.class); loadMapper(conf, AuthorMapper.class); @@ -175,7 +201,7 @@ public class MyBatis implements BatchComponent, ServerComponent { private InputStream getPathToMapper(Class mapperClass) { InputStream input = getClass().getResourceAsStream( - "/" + StringUtils.replace(mapperClass.getName(), ".", "/") + "-" + database.getDialect().getId() + ".xml"); + "/" + StringUtils.replace(mapperClass.getName(), ".", "/") + "-" + database.getDialect().getId() + ".xml"); if (input == null) { input = getClass().getResourceAsStream("/" + StringUtils.replace(mapperClass.getName(), ".", "/") + ".xml"); } diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureDataDto.java b/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureDataDto.java new file mode 100644 index 00000000000..cb9edd84f89 --- /dev/null +++ b/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureDataDto.java @@ -0,0 +1,44 @@ +/* + * Sonar, open source software quality management tool. + * Copyright (C) 2008-2012 SonarSource + * mailto:contact AT sonarsource DOT com + * + * Sonar is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * Sonar is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Sonar; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02 + */ +package org.sonar.api.database.model; + +public class MeasureDataDto { + private final Long measureId; + private final Integer snapshotId; + private final byte[] data; + + public MeasureDataDto(Long measureId, Integer snapshotId, byte[] data) { + this.measureId = measureId; + this.snapshotId = snapshotId; + this.data = data; + } + + public Long getMeasureId() { + return measureId; + } + + public Integer getSnapshotId() { + return snapshotId; + } + + public byte[] getData() { + return data; + } +} diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureDto.java b/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureDto.java index a0cb386407d..67e015ff5cb 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureDto.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureDto.java @@ -22,29 +22,28 @@ package org.sonar.api.database.model; import java.util.Date; public class MeasureDto { - private final Long id; - private final Double value; - private final String textValue; - private final Integer tendency; - private final Integer metricId; - private final Integer snapshotId; - private final Integer projectId; - private final String description; - private final Date measureDate; - private final Integer ruleId; + private final Long id; + private final Double value; + private final String textValue; + private final Integer tendency; + private final Integer metricId; + private final Integer snapshotId; + private final Integer projectId; + private final String description; + private final Date measureDate; + private final Integer ruleId; private final Integer rulePriority; private final String alertStatus; - private final String alertText; - private final Double variationValue1; - private final Double variationValue2; - private final Double variationValue3; - private final Double variationValue4; - private final Double variationValue5; - private final String url; - private final Integer characteristicId; - private final Integer personId; - - // private List measureData = new ArrayList(); + private final String alertText; + private final Double variationValue1; + private final Double variationValue2; + private final Double variationValue3; + private final Double variationValue4; + private final Double variationValue5; + private final String url; + private final Integer characteristicId; + private final Integer personId; + private final MeasureData measureData; public MeasureDto(MeasureModel model) { id = model.getId(); @@ -68,6 +67,7 @@ public class MeasureDto { url = model.getUrl(); characteristicId = (null == model.getCharacteristic()) ? null : model.getCharacteristic().getId(); personId = model.getPersonId(); + measureData = model.getMeasureData(); } public Long getId() { @@ -153,4 +153,8 @@ public class MeasureDto { public Integer getPersonId() { return personId; } + + public MeasureData getMeasureData() { + return measureData; + } } diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureModelMapper.java b/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureModelMapper.java index 615971f3ffe..77958804d76 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureModelMapper.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/database/model/MeasureModelMapper.java @@ -22,5 +22,7 @@ package org.sonar.api.database.model; public interface MeasureModelMapper { void insert(MeasureDto measure); + void insertData(MeasureDataDto data); + void update(MeasureDto measure); } diff --git a/sonar-plugin-api/src/main/resources/org/sonar/api/database/model/MeasureModelMapper.xml b/sonar-plugin-api/src/main/resources/org/sonar/api/database/model/MeasureModelMapper.xml index 594674a1b3c..b396b9c1d0a 100644 --- a/sonar-plugin-api/src/main/resources/org/sonar/api/database/model/MeasureModelMapper.xml +++ b/sonar-plugin-api/src/main/resources/org/sonar/api/database/model/MeasureModelMapper.xml @@ -16,7 +16,11 @@ ) - + + INSERT INTO measure_data (measure_id, snapshot_id, data) + VALUES (#{measureId}, #{snapshotId}, #{data}) + + UPDATE project_measures SET