From: Duarte Meneses Date: Fri, 28 Jul 2017 15:36:36 +0000 (+0200) Subject: Optimize copy of scanner measures X-Git-Tag: 6.6-RC1~732 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=16a8f3ec30a7adc2c04204d386ee8fe0ee40be21;p=sonarqube.git Optimize copy of scanner measures --- diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java index 419c4cd74e5..1656c3229a2 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/MeasureDao.java @@ -116,7 +116,7 @@ public class MeasureDao implements Dao { ids -> mapper(dbSession).selectProjectMeasuresOfDeveloper(developerId, metricIds)); } - public List selectByComponentsAndMetrics(DbSession dbSession, List componentUuids, List metricIds) { + public List selectByComponentsAndMetrics(DbSession dbSession, Collection componentUuids, Collection metricIds) { return executeLargeInputs(componentUuids, partitionComponentUuids -> mapper(dbSession).selectByComponentsAndMetrics(partitionComponentUuids, metricIds)); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MapBasedRawMeasureRepository.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MapBasedRawMeasureRepository.java index 0424109ec7e..7392ff616f8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MapBasedRawMeasureRepository.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MapBasedRawMeasureRepository.java @@ -19,24 +19,28 @@ */ package org.sonar.server.computation.task.projectanalysis.measure; -import com.google.common.base.Function; -import com.google.common.base.Optional; -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableSetMultimap; -import com.google.common.collect.SetMultimap; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.FluentIterable.from; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Set; + import javax.annotation.Nonnull; import javax.annotation.Nullable; + import org.sonar.server.computation.task.projectanalysis.component.Component; import org.sonar.server.computation.task.projectanalysis.metric.Metric; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.FluentIterable.from; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.SetMultimap; /** * Map based implementation of MeasureRepository which supports only raw measures. @@ -59,6 +63,11 @@ public final class MapBasedRawMeasureRepository implements MeasureRepository throw new UnsupportedOperationException("This implementation of MeasureRepository supports only raw measures"); } + @Override + public int loadAsRawMeasures(Collection components, Collection metrics) { + throw new UnsupportedOperationException("This implementation of MeasureRepository supports only raw measures"); + } + @Override public Optional getRawMeasure(final Component component, final Metric metric) { // fail fast diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepository.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepository.java index c5acb1d03c2..ecc86ce8c5e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepository.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepository.java @@ -19,13 +19,16 @@ */ package org.sonar.server.computation.task.projectanalysis.measure; -import com.google.common.base.Optional; -import com.google.common.collect.SetMultimap; +import java.util.Collection; import java.util.Set; + import org.sonar.server.computation.task.projectanalysis.component.Component; import org.sonar.server.computation.task.projectanalysis.metric.Metric; import org.sonar.server.computation.task.projectanalysis.metric.MetricImpl; +import com.google.common.base.Optional; +import com.google.common.collect.SetMultimap; + public interface MeasureRepository { /** @@ -40,6 +43,8 @@ public interface MeasureRepository { */ Optional getBaseMeasure(Component component, Metric metric); + int loadAsRawMeasures(Collection components, Collection metrics); + /** * Retrieves the measure created during the current analysis for the specified {@link Component} for the specified * {@link Metric} if it exists (ie. one created by the Compute Engine or the Batch) and which is not diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryImpl.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryImpl.java index 87d3a069265..9a0942003b3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryImpl.java @@ -19,10 +19,16 @@ */ package org.sonar.server.computation.task.projectanalysis.measure; -import com.google.common.base.Optional; -import com.google.common.collect.SetMultimap; +import static java.util.Objects.requireNonNull; +import static org.sonar.server.computation.task.projectanalysis.component.ComponentFunctions.toReportRef; + +import java.util.Collection; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; + import org.sonar.core.util.CloseableIterator; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -36,8 +42,8 @@ import org.sonar.server.computation.task.projectanalysis.metric.Metric; import org.sonar.server.computation.task.projectanalysis.metric.MetricRepository; import org.sonar.server.computation.task.projectanalysis.metric.ReportMetricValidator; -import static java.util.Objects.requireNonNull; -import static org.sonar.server.computation.task.projectanalysis.component.ComponentFunctions.toReportRef; +import com.google.common.base.Optional; +import com.google.common.collect.SetMultimap; public class MeasureRepositoryImpl implements MeasureRepository { private final MapBasedRawMeasureRepository delegate = new MapBasedRawMeasureRepository<>(toReportRef()); @@ -47,10 +53,10 @@ public class MeasureRepositoryImpl implements MeasureRepository { private final MetricRepository metricRepository; private final ReportMetricValidator reportMetricValidator; - private MeasureDtoToMeasure underTest = new MeasureDtoToMeasure(); + private MeasureDtoToMeasure measureTransformer = new MeasureDtoToMeasure(); private final Set loadedComponents = new HashSet<>(); - public MeasureRepositoryImpl(DbClient dbClient, BatchReportReader reportReader, MetricRepository metricRepository, + public MeasureRepositoryImpl(DbClient dbClient, BatchReportReader reportReader, MetricRepository metricRepository, ReportMetricValidator reportMetricValidator) { this.dbClient = dbClient; this.reportReader = reportReader; @@ -69,12 +75,38 @@ public class MeasureRepositoryImpl implements MeasureRepository { MeasureQuery query = MeasureQuery.builder().setComponentUuid(component.getUuid()).setMetricKey(metric.getKey()).build(); java.util.Optional measureDto = dbClient.measureDao().selectSingle(dbSession, query); if (measureDto.isPresent()) { - return underTest.toMeasure(measureDto.get(), metric); + return measureTransformer.toMeasure(measureDto.get(), metric); } return Optional.absent(); } } + @Override + public int loadAsRawMeasures(Collection components, Collection metrics) { + requireNonNull(components); + requireNonNull(metrics); + + Map componentsByUuid = components.stream() + .collect(Collectors.toMap(Component::getUuid, c -> c)); + Map metricsById = metrics.stream() + .collect(Collectors.toMap(Metric::getId, m -> m)); + + List measuresDto; + try (DbSession dbSession = dbClient.openSession(false)) { + measuresDto = dbClient.measureDao().selectByComponentsAndMetrics(dbSession, componentsByUuid.keySet(), metricsById.keySet()); + } + + for (MeasureDto dto : measuresDto) { + + Metric metric = metricsById.get(dto.getMetricId()); + Component component = componentsByUuid.get(dto.getComponentUuid()); + Measure measure = measureTransformer.toMeasure(dto, metric).get(); + + delegate.add(component, metric, measure); + } + return measuresDto.size(); + } + @Override public Optional getRawMeasure(Component component, Metric metric) { Optional local = delegate.getRawMeasure(component, metric); diff --git a/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/validation/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/validation/package-info.java new file mode 100644 index 00000000000..ad6b743f16a --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/computation/task/projectanalysis/validation/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.computation.task.projectanalysis.validation; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryRule.java b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryRule.java index c50ea46d20a..976ea71ae07 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryRule.java +++ b/server/sonar-server/src/test/java/org/sonar/server/computation/task/projectanalysis/measure/MeasureRepositoryRule.java @@ -19,18 +19,23 @@ */ package org.sonar.server.computation.task.projectanalysis.measure; -import com.google.common.base.Function; -import com.google.common.base.Optional; -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableSetMultimap; -import com.google.common.collect.SetMultimap; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.FluentIterable.from; +import static com.google.common.collect.Maps.filterKeys; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; + import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import javax.annotation.Nullable; + import org.junit.rules.ExternalResource; import org.sonar.server.computation.task.projectanalysis.component.Component; import org.sonar.server.computation.task.projectanalysis.component.ComponentProvider; @@ -43,12 +48,11 @@ import org.sonar.server.computation.task.projectanalysis.component.TreeRootHolde import org.sonar.server.computation.task.projectanalysis.metric.Metric; import org.sonar.server.computation.task.projectanalysis.metric.MetricRepositoryRule; -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.FluentIterable.from; -import static com.google.common.collect.Maps.filterKeys; -import static java.lang.String.format; -import static java.util.Objects.requireNonNull; +import com.google.common.base.Function; +import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableSetMultimap; +import com.google.common.collect.SetMultimap; /** * An implementation of MeasureRepository as a JUnit rule which provides add methods for raw measures and extra add @@ -62,6 +66,8 @@ public class MeasureRepositoryRule extends ExternalResource implements MeasureRe private final Map baseMeasures = new HashMap<>(); private final Map rawMeasures = new HashMap<>(); private final Map initialRawMeasures = new HashMap<>(); + private Collection loadedAsRawComponents; + private Collection loadedAsRawMetrics; private final Predicate> isAddedMeasure = new Predicate>() { @Override public boolean apply(@Nonnull Map.Entry input) { @@ -181,12 +187,26 @@ public class MeasureRepositoryRule extends ExternalResource implements MeasureRe return this; } - @Override public Optional getBaseMeasure(Component component, Metric metric) { return Optional.fromNullable(baseMeasures.get(new InternalKey(component, metric))); } + @Override + public int loadAsRawMeasures(Collection components, Collection metrics) { + this.loadedAsRawComponents = components; + this.loadedAsRawMetrics = metrics; + return 0; + } + + public Collection getComponentsLoadedAsRaw() { + return loadedAsRawComponents; + } + + public Collection getMetricsLoadedAsRaw() { + return loadedAsRawMetrics; + } + @Override public Optional getRawMeasure(Component component, Metric metric) { return Optional.fromNullable(rawMeasures.get(new InternalKey(component, metric)));