From 85877295a7db169742f56bc6c44b883267923cf7 Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Mon, 16 Feb 2015 18:31:16 +0100 Subject: SONAR-6000 Try to decrease size of duplications in persistit --- .../java/org/sonar/batch/cpd/DefaultCpdEngine.java | 37 +++++------ .../java/org/sonar/batch/cpd/JavaCpdEngine.java | 27 ++++---- .../batch/deprecated/DeprecatedSensorContext.java | 5 +- .../decorator/DefaultDecoratorContext.java | 4 +- .../org/sonar/batch/duplication/BlockCache.java | 56 ----------------- .../duplication/DefaultDuplicationValueCoder.java | 53 ++++++++++++++++ .../batch/duplication/DefaultTokenBuilder.java | 73 ---------------------- .../duplication/DuplicationBlockValueCoder.java | 6 +- .../sonar/batch/duplication/DuplicationCache.java | 32 +++++----- .../duplication/DuplicationGroupValueCoder.java | 56 ----------------- .../sonar/batch/duplication/DuplicationUtils.java | 11 ++-- .../sonar/batch/index/DuplicationPersister.java | 11 ++-- .../org/sonar/batch/index/SourceDataFactory.java | 33 +++++----- .../org/sonar/batch/mediumtest/TaskResult.java | 12 ++-- .../org/sonar/batch/scan/ProjectScanContainer.java | 9 +-- .../sonar/batch/sensor/DefaultSensorContext.java | 54 ++-------------- .../sonar/batch/sensor/DefaultSensorStorage.java | 12 +++- .../java/org/sonar/batch/cpd/CpdSensorTest.java | 11 +--- .../org/sonar/batch/cpd/DefaultCpdEngineTest.java | 6 +- .../org/sonar/batch/cpd/JavaCpdEngineTest.java | 63 ++++++++----------- .../batch/duplication/DuplicationCacheTest.java | 37 +++++------ .../batch/index/DuplicationPersisterTest.java | 15 ++--- .../sonar/batch/index/SourceDataFactoryTest.java | 26 ++++---- .../sonar/batch/mediumtest/cpd/CpdMediumTest.java | 14 ++--- .../batch/sensor/DefaultSensorContextTest.java | 4 +- .../batch/sensor/DefaultSensorStorageTest.java | 4 +- 26 files changed, 235 insertions(+), 436 deletions(-) delete mode 100644 sonar-batch/src/main/java/org/sonar/batch/duplication/BlockCache.java create mode 100644 sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultDuplicationValueCoder.java delete mode 100644 sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultTokenBuilder.java delete mode 100644 sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationGroupValueCoder.java (limited to 'sonar-batch/src') diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java index 6f95c3e47f3..33879ea8bc4 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/DefaultCpdEngine.java @@ -19,9 +19,6 @@ */ package org.sonar.batch.cpd; -import org.sonar.batch.cpd.index.IndexFactory; -import org.sonar.batch.cpd.index.SonarDuplicationsIndex; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; @@ -38,10 +35,10 @@ import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.config.Settings; import org.sonar.api.resources.Project; import org.sonar.api.utils.SonarException; -import org.sonar.batch.duplication.BlockCache; +import org.sonar.batch.cpd.index.IndexFactory; +import org.sonar.batch.cpd.index.SonarDuplicationsIndex; import org.sonar.duplications.DuplicationPredicates; import org.sonar.duplications.block.Block; -import org.sonar.duplications.block.FileBlocks; import org.sonar.duplications.index.CloneGroup; import org.sonar.duplications.internal.pmd.TokenizerBridge; @@ -68,20 +65,18 @@ public class DefaultCpdEngine extends CpdEngine { private final CpdMappings mappings; private final FileSystem fs; private final Settings settings; - private final BlockCache blockCache; private final Project project; - public DefaultCpdEngine(@Nullable Project project, IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings, BlockCache duplicationCache) { + public DefaultCpdEngine(@Nullable Project project, IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings) { this.project = project; this.indexFactory = indexFactory; this.mappings = mappings; this.fs = fs; this.settings = settings; - this.blockCache = duplicationCache; } - public DefaultCpdEngine(IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings, BlockCache duplicationCache) { - this(null, indexFactory, mappings, fs, settings, duplicationCache); + public DefaultCpdEngine(IndexFactory indexFactory, CpdMappings mappings, FileSystem fs, Settings settings) { + this(null, indexFactory, mappings, fs, settings); } @Override @@ -91,6 +86,12 @@ public class DefaultCpdEngine extends CpdEngine { @Override public void analyse(String languageKey, SensorContext context) { + CpdMapping mapping = mappings.getMapping(languageKey); + if (mapping == null) { + LOG.debug("No CpdMapping for language " + languageKey); + return; + } + String[] cpdExclusions = settings.getStringArray(CoreProperties.CPD_EXCLUSIONS); logExclusions(cpdExclusions, LOG); FilePredicates p = fs.predicates(); @@ -103,8 +104,6 @@ public class DefaultCpdEngine extends CpdEngine { return; } - CpdMapping mapping = mappings.getMapping(languageKey); - // Create index SonarDuplicationsIndex index = indexFactory.create(project, languageKey); populateIndex(languageKey, sourceFiles, mapping, index); @@ -140,20 +139,12 @@ public class DefaultCpdEngine extends CpdEngine { } private void populateIndex(String languageKey, List sourceFiles, CpdMapping mapping, SonarDuplicationsIndex index) { - TokenizerBridge bridge = null; - if (mapping != null) { - bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey)); - } + TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey)); for (InputFile inputFile : sourceFiles) { LOG.debug("Populating index from {}", inputFile); String resourceEffectiveKey = ((DeprecatedDefaultInputFile) inputFile).key(); - FileBlocks fileBlocks = blockCache.byComponent(resourceEffectiveKey); - if (fileBlocks != null) { - index.insert(inputFile, fileBlocks.blocks()); - } else if (bridge != null) { - List blocks2 = bridge.chunk(resourceEffectiveKey, inputFile.file()); - index.insert(inputFile, blocks2); - } + List blocks2 = bridge.chunk(resourceEffectiveKey, inputFile.file()); + index.insert(inputFile, blocks2); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java b/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java index 37bae21c422..2205227cd31 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java +++ b/sonar-batch/src/main/java/org/sonar/batch/cpd/JavaCpdEngine.java @@ -20,9 +20,6 @@ package org.sonar.batch.cpd; -import org.sonar.batch.cpd.index.IndexFactory; -import org.sonar.batch.cpd.index.SonarDuplicationsIndex; - import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import org.apache.commons.io.IOUtils; @@ -34,14 +31,16 @@ import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile; import org.sonar.api.batch.sensor.SensorContext; -import org.sonar.api.batch.sensor.duplication.DuplicationBuilder; -import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplicationBuilder; +import org.sonar.api.batch.sensor.duplication.NewDuplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; import org.sonar.api.config.Settings; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.resources.Project; import org.sonar.api.utils.KeyValueFormat; import org.sonar.api.utils.SonarException; +import org.sonar.batch.cpd.index.IndexFactory; +import org.sonar.batch.cpd.index.SonarDuplicationsIndex; import org.sonar.duplications.block.Block; import org.sonar.duplications.block.BlockChunker; import org.sonar.duplications.detector.suffixtree.SuffixTreeCloneDetectionAlgorithm; @@ -66,7 +65,12 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.*; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; public class JavaCpdEngine extends CpdEngine { @@ -226,16 +230,17 @@ public class JavaCpdEngine extends CpdEngine { .setFromCore() .save(); - DuplicationBuilder builder = context.duplicationBuilder(inputFile); for (CloneGroup duplication : duplications) { - builder.originBlock(duplication.getOriginPart().getStartLine(), duplication.getOriginPart().getEndLine()); + NewDuplication builder = context.newDuplication(); + ClonePart originPart = duplication.getOriginPart(); + builder.originBlock(inputFile, originPart.getStartLine(), originPart.getEndLine()); for (ClonePart part : duplication.getCloneParts()) { - if (!part.equals(duplication.getOriginPart())) { - ((DefaultDuplicationBuilder) builder).isDuplicatedBy(part.getResourceId(), part.getStartLine(), part.getEndLine()); + if (!part.equals(originPart)) { + ((DefaultDuplication) builder).isDuplicatedBy(part.getResourceId(), part.getStartLine(), part.getEndLine()); } } + builder.save(); } - context.saveDuplications(inputFile, builder.build()); } private static int computeBlockAndLineCount(Iterable duplications, Set duplicatedLines) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/deprecated/DeprecatedSensorContext.java b/sonar-batch/src/main/java/org/sonar/batch/deprecated/DeprecatedSensorContext.java index e02cadce624..2b309da1100 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/deprecated/DeprecatedSensorContext.java +++ b/sonar-batch/src/main/java/org/sonar/batch/deprecated/DeprecatedSensorContext.java @@ -45,7 +45,6 @@ import org.sonar.api.resources.Qualifiers; import org.sonar.api.resources.Resource; import org.sonar.api.rules.Violation; import org.sonar.api.utils.SonarException; -import org.sonar.batch.duplication.BlockCache; import org.sonar.batch.duplication.DuplicationCache; import org.sonar.batch.index.ComponentDataCache; import org.sonar.batch.sensor.DefaultSensorContext; @@ -67,8 +66,8 @@ public class DeprecatedSensorContext extends DefaultSensorContext implements Sen public DeprecatedSensorContext(SonarIndex index, Project project, Settings settings, FileSystem fs, ActiveRules activeRules, AnalysisMode analysisMode, ComponentDataCache componentDataCache, CoverageExclusions coverageFilter, - BlockCache blockCache, DuplicationCache duplicationCache, SensorStorage sensorStorage) { - super(settings, fs, activeRules, analysisMode, componentDataCache, blockCache, duplicationCache, sensorStorage); + DuplicationCache duplicationCache, SensorStorage sensorStorage) { + super(settings, fs, activeRules, analysisMode, componentDataCache, duplicationCache, sensorStorage); this.index = index; this.project = project; this.coverageFilter = coverageFilter; diff --git a/sonar-batch/src/main/java/org/sonar/batch/deprecated/decorator/DefaultDecoratorContext.java b/sonar-batch/src/main/java/org/sonar/batch/deprecated/decorator/DefaultDecoratorContext.java index 1740a1aebd9..a74f91fa138 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/deprecated/decorator/DefaultDecoratorContext.java +++ b/sonar-batch/src/main/java/org/sonar/batch/deprecated/decorator/DefaultDecoratorContext.java @@ -25,7 +25,7 @@ import com.google.common.collect.Lists; import org.sonar.api.batch.DecoratorContext; import org.sonar.api.batch.Event; import org.sonar.api.batch.SonarIndex; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.design.Dependency; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.Measure; @@ -126,7 +126,7 @@ public class DefaultDecoratorContext implements DecoratorContext { String metricKey = ((MeasuresFilters.MetricFilter) filter).filterOnMetricKey(); if (CoreMetrics.DUPLICATIONS_DATA_KEY.equals(metricKey)) { // Hack for SONAR-5765 - List group = duplicationCache.byComponent(resource.getEffectiveKey()); + Iterable group = duplicationCache.byComponent(resource.getEffectiveKey()); if (group != null) { unfiltered = Arrays.asList(new Measure(CoreMetrics.DUPLICATIONS_DATA, DuplicationUtils.toXml(group))); } else { diff --git a/sonar-batch/src/main/java/org/sonar/batch/duplication/BlockCache.java b/sonar-batch/src/main/java/org/sonar/batch/duplication/BlockCache.java deleted file mode 100644 index f6a4e3d18bf..00000000000 --- a/sonar-batch/src/main/java/org/sonar/batch/duplication/BlockCache.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.batch.duplication; - -import org.sonar.api.BatchComponent; -import org.sonar.batch.index.Cache; -import org.sonar.batch.index.Cache.Entry; -import org.sonar.batch.index.Caches; -import org.sonar.duplications.block.FileBlocks; - -import javax.annotation.CheckForNull; - -/** - * Cache of duplication blocks. This cache is shared amongst all project modules. - */ -public class BlockCache implements BatchComponent { - - private final Cache cache; - - public BlockCache(Caches caches) { - caches.registerValueCoder(FileBlocks.class, new FileBlocksValueCoder()); - cache = caches.createCache("blocks"); - } - - public Iterable> entries() { - return cache.entries(); - } - - @CheckForNull - public FileBlocks byComponent(String effectiveKey) { - return cache.get(effectiveKey); - } - - public BlockCache put(String effectiveKey, FileBlocks blocks) { - cache.put(effectiveKey, blocks); - return this; - } - -} diff --git a/sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultDuplicationValueCoder.java b/sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultDuplicationValueCoder.java new file mode 100644 index 00000000000..1ab206e8edf --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultDuplicationValueCoder.java @@ -0,0 +1,53 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.duplication; + +import com.persistit.Value; +import com.persistit.encoding.CoderContext; +import com.persistit.encoding.ValueCoder; +import org.sonar.api.batch.sensor.duplication.Duplication; +import org.sonar.api.batch.sensor.duplication.Duplication.Block; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; + +class DefaultDuplicationValueCoder implements ValueCoder { + + private DuplicationBlockValueCoder blockCoder = new DuplicationBlockValueCoder(); + + @Override + public void put(Value value, Object object, CoderContext context) { + DefaultDuplication c = (DefaultDuplication) object; + blockCoder.put(value, c.originBlock(), context); + value.put(c.duplicates().size()); + for (Duplication.Block block : c.duplicates()) { + blockCoder.put(value, block, context); + } + } + + @Override + public Object get(Value value, Class clazz, CoderContext context) { + DefaultDuplication g = new DefaultDuplication(); + g.setOriginBlock((Block) blockCoder.get(value, Duplication.Block.class, context)); + int count = value.getInt(); + for (int i = 0; i < count; i++) { + g.duplicates().add((Block) blockCoder.get(value, Duplication.Block.class, context)); + } + return g; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultTokenBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultTokenBuilder.java deleted file mode 100644 index 6add5331875..00000000000 --- a/sonar-batch/src/main/java/org/sonar/batch/duplication/DefaultTokenBuilder.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.batch.duplication; - -import com.google.common.base.Preconditions; -import net.sourceforge.pmd.cpd.TokenEntry; -import org.sonar.api.batch.fs.InputFile; -import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder; -import org.sonar.duplications.block.Block; -import org.sonar.duplications.block.FileBlocks; -import org.sonar.duplications.internal.pmd.PmdBlockChunker; -import org.sonar.duplications.internal.pmd.TokenizerBridge; -import org.sonar.duplications.internal.pmd.TokensLine; - -import java.util.ArrayList; -import java.util.List; - -public class DefaultTokenBuilder implements DuplicationTokenBuilder { - - private final BlockCache cache; - private final InputFile inputFile; - private final List tokens = new ArrayList(); - private final PmdBlockChunker blockChunker; - private boolean done = false; - private int previousLine = 0; - - public DefaultTokenBuilder(InputFile inputFile, BlockCache cache, PmdBlockChunker blockChunker) { - this.inputFile = inputFile; - this.cache = cache; - this.blockChunker = blockChunker; - TokenEntry.clearImages(); - } - - @Override - public DefaultTokenBuilder addToken(int line, String image) { - Preconditions.checkState(!done, "done() already called"); - Preconditions.checkState(line >= previousLine, "Token should be created in order. Previous line was " + previousLine + " and you tried to create a token at line " + line); - TokenEntry cpdToken = new TokenEntry(image, inputFile.absolutePath(), line); - tokens.add(cpdToken); - previousLine = line; - return this; - } - - @Override - public void done() { - Preconditions.checkState(!done, "done() already called"); - tokens.add(TokenEntry.getEOF()); - TokenEntry.clearImages(); - List tokensLines = TokenizerBridge.convert(tokens); - List blocks = blockChunker.chunk(((DefaultInputFile) inputFile).key(), tokensLines); - - cache.put(((DefaultInputFile) inputFile).key(), new FileBlocks(((DefaultInputFile) inputFile).key(), blocks)); - tokens.clear(); - } -} diff --git a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationBlockValueCoder.java b/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationBlockValueCoder.java index 26c857a5d23..50dbd316bb7 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationBlockValueCoder.java +++ b/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationBlockValueCoder.java @@ -22,13 +22,13 @@ package org.sonar.batch.duplication; import com.persistit.Value; import com.persistit.encoding.CoderContext; import com.persistit.encoding.ValueCoder; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; class DuplicationBlockValueCoder implements ValueCoder { @Override public void put(Value value, Object object, CoderContext context) { - DuplicationGroup.Block b = (DuplicationGroup.Block) object; + Duplication.Block b = (Duplication.Block) object; value.putUTF(b.resourceKey()); value.put(b.startLine()); value.put(b.length()); @@ -39,6 +39,6 @@ class DuplicationBlockValueCoder implements ValueCoder { String resourceKey = value.getString(); int startLine = value.getInt(); int length = value.getInt(); - return new DuplicationGroup.Block(resourceKey, startLine, length); + return new Duplication.Block(resourceKey, startLine, length); } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationCache.java b/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationCache.java index 9652308ef82..08c102fdf8a 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationCache.java +++ b/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationCache.java @@ -19,39 +19,41 @@ */ package org.sonar.batch.duplication; +import com.google.common.base.Function; +import com.google.common.collect.Iterables; import org.sonar.api.BatchComponent; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.batch.index.Cache; -import org.sonar.batch.index.Cache.Entry; import org.sonar.batch.index.Caches; -import javax.annotation.CheckForNull; - -import java.util.List; - /** * Cache of duplication blocks. This cache is shared amongst all project modules. */ public class DuplicationCache implements BatchComponent { - private final Cache> cache; + private final Cache cache; + private int sequence = 1; public DuplicationCache(Caches caches) { - caches.registerValueCoder(DuplicationGroup.class, new DuplicationGroupValueCoder()); + caches.registerValueCoder(DefaultDuplication.class, new DefaultDuplicationValueCoder()); cache = caches.createCache("duplications"); } - public Iterable>> entries() { - return cache.entries(); + public Iterable componentKeys() { + return Iterables.transform(cache.keySet(), new Function() { + @Override + public String apply(Object input) { + return input.toString(); + } + }); } - @CheckForNull - public List byComponent(String effectiveKey) { - return cache.get(effectiveKey); + public Iterable byComponent(String effectiveKey) { + return cache.values(effectiveKey); } - public DuplicationCache put(String effectiveKey, List blocks) { - cache.put(effectiveKey, blocks); + public DuplicationCache put(String effectiveKey, DefaultDuplication duplication) { + cache.put(effectiveKey, sequence++, duplication); return this; } diff --git a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationGroupValueCoder.java b/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationGroupValueCoder.java deleted file mode 100644 index f53dfb4a569..00000000000 --- a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationGroupValueCoder.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.batch.duplication; - -import com.persistit.Value; -import com.persistit.encoding.CoderContext; -import com.persistit.encoding.ValueCoder; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup.Block; - -import java.util.ArrayList; -import java.util.List; - -class DuplicationGroupValueCoder implements ValueCoder { - - private DuplicationBlockValueCoder blockCoder = new DuplicationBlockValueCoder(); - - @Override - public void put(Value value, Object object, CoderContext context) { - DuplicationGroup c = (DuplicationGroup) object; - blockCoder.put(value, c.originBlock(), context); - value.put(c.duplicates().size()); - for (DuplicationGroup.Block block : c.duplicates()) { - blockCoder.put(value, block, context); - } - } - - @Override - public Object get(Value value, Class clazz, CoderContext context) { - DuplicationGroup g = new DuplicationGroup((Block) blockCoder.get(value, DuplicationGroup.Block.class, context)); - int count = value.getInt(); - List blocks = new ArrayList(count); - for (int i = 0; i < count; i++) { - blocks.add((Block) blockCoder.get(value, DuplicationGroup.Block.class, context)); - } - g.setDuplicates(blocks); - return g; - } -} diff --git a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationUtils.java b/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationUtils.java index b068ee0be8a..5f2b95ed511 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationUtils.java +++ b/sonar-batch/src/main/java/org/sonar/batch/duplication/DuplicationUtils.java @@ -20,7 +20,8 @@ package org.sonar.batch.duplication; import org.apache.commons.lang.StringEscapeUtils; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; public class DuplicationUtils { @@ -28,13 +29,13 @@ public class DuplicationUtils { // Utility class } - public static String toXml(Iterable duplications) { + public static String toXml(Iterable duplications) { StringBuilder xml = new StringBuilder(); xml.append(""); - for (DuplicationGroup duplication : duplications) { + for (Duplication duplication : duplications) { xml.append(""); toXml(xml, duplication.originBlock()); - for (DuplicationGroup.Block part : duplication.duplicates()) { + for (Duplication.Block part : duplication.duplicates()) { toXml(xml, part); } xml.append(""); @@ -43,7 +44,7 @@ public class DuplicationUtils { return xml.toString(); } - private static void toXml(StringBuilder xml, DuplicationGroup.Block part) { + private static void toXml(StringBuilder xml, Duplication.Block part) { xml.append("> entry : duplicationCache.entries()) { - String effectiveKey = entry.key()[0].toString(); - Measure measure = new Measure(duplicationMetricWithId, DuplicationUtils.toXml(entry.value())).setPersistenceMode(PersistenceMode.DATABASE); + for (String effectiveKey : duplicationCache.componentKeys()) { + Iterable dups = duplicationCache.byComponent(effectiveKey); + Measure measure = new Measure(duplicationMetricWithId, DuplicationUtils.toXml(dups)).setPersistenceMode(PersistenceMode.DATABASE); BatchResource batchResource = resourceCache.get(effectiveKey); if (MeasurePersister.shouldPersistMeasure(batchResource.resource(), measure)) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java index aa61c83c77e..6d1cfc1e7f1 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java +++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourceDataFactory.java @@ -24,7 +24,8 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.sonar.api.BatchComponent; import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.batch.sensor.symbol.Symbol; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.Measure; @@ -341,29 +342,23 @@ public class SourceDataFactory implements BatchComponent { } void applyDuplications(String inputFileKey, FileSourceDb.Data.Builder to) { - List groups = duplicationCache.byComponent(inputFileKey); - if (groups != null) { - int blockId = 1; - for (Iterator it = groups.iterator(); it.hasNext();) { - DuplicationGroup group = it.next(); - addBlock(blockId, group.originBlock(), to); - blockId++; - for (Iterator dupsIt = group.duplicates().iterator(); dupsIt.hasNext();) { - DuplicationGroup.Block dups = dupsIt.next(); - if (inputFileKey.equals(dups.resourceKey())) { - addBlock(blockId, dups, to); - blockId++; - } - // Save memory - dupsIt.remove(); + Iterable groups = duplicationCache.byComponent(inputFileKey); + int blockId = 1; + for (Iterator it = groups.iterator(); it.hasNext();) { + Duplication group = it.next(); + addBlock(blockId, group.originBlock(), to); + blockId++; + for (Iterator dupsIt = group.duplicates().iterator(); dupsIt.hasNext();) { + Duplication.Block dups = dupsIt.next(); + if (inputFileKey.equals(dups.resourceKey())) { + addBlock(blockId, dups, to); + blockId++; } - // Save memory - it.remove(); } } } - private void addBlock(int blockId, DuplicationGroup.Block block, FileSourceDb.Data.Builder to) { + private void addBlock(int blockId, Duplication.Block block, FileSourceDb.Data.Builder to) { int currentLine = block.startLine(); for (int i = 0; i < block.length(); i++) { if (currentLine <= to.getLinesCount()) { diff --git a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java index 4d89b9e4b3e..8013dd54d9d 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java +++ b/sonar-batch/src/main/java/org/sonar/batch/mediumtest/TaskResult.java @@ -19,6 +19,7 @@ */ package org.sonar.batch.mediumtest; +import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,7 +27,7 @@ import org.sonar.api.batch.fs.InputDir; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.sensor.dependency.Dependency; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; import org.sonar.api.batch.sensor.highlighting.TypeOfText; import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; import org.sonar.api.batch.sensor.symbol.Symbol; @@ -67,7 +68,7 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver { private List issues = new ArrayList<>(); private List measures = new ArrayList<>(); - private Map> duplications = new HashMap<>(); + private Map> duplications = new HashMap<>(); private Map inputFiles = new HashMap<>(); private Map inputDirs = new HashMap<>(); private Map highlightingPerFile = new HashMap<>(); @@ -142,9 +143,8 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver { private void storeDuplication(ProjectScanContainer container) { DuplicationCache duplicationCache = container.getComponentByType(DuplicationCache.class); - for (Entry> entry : duplicationCache.entries()) { - String effectiveKey = entry.key()[0].toString(); - duplications.put(effectiveKey, entry.value()); + for (String effectiveKey : duplicationCache.componentKeys()) { + duplications.put(effectiveKey, Lists.newArrayList(duplicationCache.byComponent(effectiveKey))); } } @@ -210,7 +210,7 @@ public class TaskResult implements org.sonar.batch.mediumtest.ScanTaskObserver { return inputDirs.get(relativePath); } - public List duplicationsFor(InputFile inputFile) { + public List duplicationsFor(InputFile inputFile) { return duplications.get(((DefaultInputFile) inputFile).key()); } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java index 98c46431093..850b6e984bf 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java @@ -19,10 +19,6 @@ */ package org.sonar.batch.scan; -import org.sonar.batch.deprecated.components.DefaultResourceCreationLock; - -import org.sonar.batch.deprecated.components.PeriodsDefinition; -import org.sonar.batch.repository.language.DefaultLanguagesRepository; import com.google.common.annotations.VisibleForTesting; import org.sonar.api.BatchComponent; import org.sonar.api.CoreProperties; @@ -45,7 +41,8 @@ import org.sonar.batch.bootstrap.ExtensionUtils; import org.sonar.batch.bootstrap.MetricProvider; import org.sonar.batch.debt.DebtModelProvider; import org.sonar.batch.debt.IssueChangelogDebtCalculator; -import org.sonar.batch.duplication.BlockCache; +import org.sonar.batch.deprecated.components.DefaultResourceCreationLock; +import org.sonar.batch.deprecated.components.PeriodsDefinition; import org.sonar.batch.duplication.DuplicationCache; import org.sonar.batch.index.Caches; import org.sonar.batch.index.ComponentDataCache; @@ -70,6 +67,7 @@ import org.sonar.batch.phases.GraphPersister; import org.sonar.batch.profiling.PhasesSumUpTimeProfiler; import org.sonar.batch.repository.ProjectRepositoriesProvider; import org.sonar.batch.repository.ProjectScmRepositoryLoader; +import org.sonar.batch.repository.language.DefaultLanguagesRepository; import org.sonar.batch.rule.ActiveRulesProvider; import org.sonar.batch.rule.RulesProvider; import org.sonar.batch.scan.filesystem.InputPathCache; @@ -189,7 +187,6 @@ public class ProjectScanContainer extends ComponentContainer { MeasureCache.class, // Duplications - BlockCache.class, DuplicationCache.class, ProjectSettings.class, diff --git a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java index 9e6884f0a8d..654134b0519 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java +++ b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java @@ -19,7 +19,6 @@ */ package org.sonar.batch.sensor; -import com.google.common.base.Preconditions; import org.sonar.api.batch.AnalysisMode; import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputFile; @@ -29,10 +28,8 @@ import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.SensorStorage; import org.sonar.api.batch.sensor.dependency.Dependency; import org.sonar.api.batch.sensor.dependency.internal.DefaultDependency; -import org.sonar.api.batch.sensor.duplication.DuplicationBuilder; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; -import org.sonar.api.batch.sensor.duplication.DuplicationTokenBuilder; -import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplicationBuilder; +import org.sonar.api.batch.sensor.duplication.NewDuplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.batch.sensor.highlighting.HighlightingBuilder; import org.sonar.api.batch.sensor.issue.Issue; import org.sonar.api.batch.sensor.issue.internal.DefaultIssue; @@ -46,16 +43,12 @@ import org.sonar.api.batch.sensor.test.internal.DefaultCoverage; import org.sonar.api.batch.sensor.test.internal.DefaultTestCaseCoverage; import org.sonar.api.batch.sensor.test.internal.DefaultTestCaseExecution; import org.sonar.api.config.Settings; -import org.sonar.batch.duplication.BlockCache; -import org.sonar.batch.duplication.DefaultTokenBuilder; import org.sonar.batch.duplication.DuplicationCache; import org.sonar.batch.highlighting.DefaultHighlightingBuilder; import org.sonar.batch.index.ComponentDataCache; import org.sonar.batch.symbol.DefaultSymbolTableBuilder; -import org.sonar.duplications.internal.pmd.PmdBlockChunker; import java.io.Serializable; -import java.util.List; public class DefaultSensorContext implements SensorContext { @@ -63,19 +56,17 @@ public class DefaultSensorContext implements SensorContext { private final FileSystem fs; private final ActiveRules activeRules; private final ComponentDataCache componentDataCache; - private final BlockCache blockCache; private final DuplicationCache duplicationCache; private final SensorStorage sensorStorage; private final AnalysisMode analysisMode; public DefaultSensorContext(Settings settings, FileSystem fs, ActiveRules activeRules, AnalysisMode analysisMode, ComponentDataCache componentDataCache, - BlockCache blockCache, DuplicationCache duplicationCache, SensorStorage sensorStorage) { + DuplicationCache duplicationCache, SensorStorage sensorStorage) { this.settings = settings; this.fs = fs; this.activeRules = activeRules; this.analysisMode = analysisMode; this.componentDataCache = componentDataCache; - this.blockCache = blockCache; this.duplicationCache = duplicationCache; this.sensorStorage = sensorStorage; } @@ -121,43 +112,8 @@ public class DefaultSensorContext implements SensorContext { } @Override - public DuplicationTokenBuilder duplicationTokenBuilder(InputFile inputFile) { - PmdBlockChunker blockChunker = new PmdBlockChunker(getBlockSize(inputFile.language())); - - return new DefaultTokenBuilder(inputFile, blockCache, blockChunker); - } - - @Override - public DuplicationBuilder duplicationBuilder(InputFile inputFile) { - return new DefaultDuplicationBuilder(inputFile); - } - - @Override - public void saveDuplications(InputFile inputFile, List duplications) { - Preconditions.checkState(!duplications.isEmpty(), "Empty duplications"); - String effectiveKey = ((DefaultInputFile) inputFile).key(); - for (DuplicationGroup duplicationGroup : duplications) { - Preconditions.checkState(effectiveKey.equals(duplicationGroup.originBlock().resourceKey()), "Invalid duplication group"); - } - duplicationCache.put(effectiveKey, duplications); - } - - private int getBlockSize(String languageKey) { - int blockSize = settings.getInt("sonar.cpd." + languageKey + ".minimumLines"); - if (blockSize == 0) { - blockSize = getDefaultBlockSize(languageKey); - } - return blockSize; - } - - private static int getDefaultBlockSize(String languageKey) { - if ("cobol".equals(languageKey)) { - return 30; - } else if ("abap".equals(languageKey) || "natur".equals(languageKey)) { - return 20; - } else { - return 10; - } + public NewDuplication newDuplication() { + return new DefaultDuplication(sensorStorage); } @Override diff --git a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java index 30639a31553..300fe651380 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java +++ b/sonar-batch/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java @@ -26,6 +26,8 @@ import org.sonar.api.batch.fs.InputPath; import org.sonar.api.batch.measure.MetricFinder; import org.sonar.api.batch.rule.ActiveRules; import org.sonar.api.batch.sensor.SensorStorage; +import org.sonar.api.batch.sensor.duplication.Duplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.batch.sensor.issue.Issue; import org.sonar.api.batch.sensor.issue.Issue.Severity; import org.sonar.api.batch.sensor.measure.Measure; @@ -53,7 +55,6 @@ import org.sonar.api.test.MutableTestCase; import org.sonar.api.test.MutableTestPlan; import org.sonar.api.test.MutableTestable; import org.sonar.api.test.Testable; -import org.sonar.batch.duplication.BlockCache; import org.sonar.batch.duplication.DuplicationCache; import org.sonar.batch.index.ComponentDataCache; import org.sonar.batch.index.DefaultIndex; @@ -68,16 +69,18 @@ public class DefaultSensorStorage implements SensorStorage { private final ResourcePerspectives perspectives; private final DefaultIndex sonarIndex; private final CoverageExclusions coverageExclusions; + private final DuplicationCache duplicationCache; public DefaultSensorStorage(MetricFinder metricFinder, Project project, ResourcePerspectives perspectives, - Settings settings, FileSystem fs, ActiveRules activeRules, ComponentDataCache componentDataCache, BlockCache blockCache, + Settings settings, FileSystem fs, ActiveRules activeRules, ComponentDataCache componentDataCache, DuplicationCache duplicationCache, DefaultIndex sonarIndex, CoverageExclusions coverageExclusions) { this.metricFinder = metricFinder; this.project = project; this.perspectives = perspectives; this.sonarIndex = sonarIndex; this.coverageExclusions = coverageExclusions; + this.duplicationCache = duplicationCache; } private Metric findMetricOrFail(String metricKey) { @@ -260,4 +263,9 @@ public class DefaultSensorStorage implements SensorStorage { .setWeight(dep.weight()) .setParent(parentDep)); } + + @Override + public void store(Duplication duplication) { + duplicationCache.put(duplication.originBlock().resourceKey(), (DefaultDuplication) duplication); + } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java index 2c6f195b880..a99d356c87b 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java @@ -19,13 +19,6 @@ */ package org.sonar.batch.cpd; -import org.sonar.batch.cpd.CpdMappings; -import org.sonar.batch.cpd.CpdComponents; -import org.sonar.batch.cpd.CpdSensor; -import org.sonar.batch.cpd.DefaultCpdEngine; -import org.sonar.batch.cpd.JavaCpdEngine; -import org.sonar.batch.cpd.index.IndexFactory; - import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,7 +27,7 @@ import org.sonar.api.batch.fs.internal.DefaultFileSystem; import org.sonar.api.config.PropertyDefinitions; import org.sonar.api.config.Settings; import org.sonar.api.resources.Java; -import org.sonar.batch.duplication.BlockCache; +import org.sonar.batch.cpd.index.IndexFactory; import java.io.IOException; @@ -55,7 +48,7 @@ public class CpdSensorTest { public void setUp() throws IOException { IndexFactory indexFactory = mock(IndexFactory.class); sonarEngine = new JavaCpdEngine(indexFactory, null, null); - sonarBridgeEngine = new DefaultCpdEngine(indexFactory, new CpdMappings(), null, null, mock(BlockCache.class)); + sonarBridgeEngine = new DefaultCpdEngine(indexFactory, new CpdMappings(), null, null); settings = new Settings(new PropertyDefinitions(CpdComponents.class)); DefaultFileSystem fs = new DefaultFileSystem(temp.newFolder().toPath()); diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java index 9ce86ce3ee4..fb2a632e9c9 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/DefaultCpdEngineTest.java @@ -19,14 +19,12 @@ */ package org.sonar.batch.cpd; -import org.sonar.batch.cpd.DefaultCpdEngine; - import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.sonar.api.config.Settings; import org.sonar.api.resources.Project; -import org.sonar.batch.duplication.BlockCache; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; @@ -42,7 +40,7 @@ public class DefaultCpdEngineTest { @Before public void init() { settings = new Settings(); - engine = new DefaultCpdEngine(null, null, null, settings, mock(BlockCache.class)); + engine = new DefaultCpdEngine(null, null, null, settings); } @Test diff --git a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java index 711f70f9be4..2de7d5952cf 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/cpd/JavaCpdEngineTest.java @@ -19,22 +19,17 @@ */ package org.sonar.batch.cpd; -import org.sonar.batch.cpd.JavaCpdEngine; - import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.mockito.InOrder; -import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile; import org.sonar.api.batch.sensor.SensorContext; import org.sonar.api.batch.sensor.SensorStorage; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; -import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplicationBuilder; +import org.sonar.api.batch.sensor.duplication.NewDuplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.batch.sensor.measure.Measure; import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; import org.sonar.api.measures.CoreMetrics; @@ -46,9 +41,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; @@ -60,7 +53,6 @@ public class JavaCpdEngineTest { SensorContext context = mock(SensorContext.class); DeprecatedDefaultInputFile inputFile; - private DefaultDuplicationBuilder duplicationBuilder; private SensorStorage storage = mock(SensorStorage.class); @Before @@ -71,9 +63,13 @@ public class JavaCpdEngineTest { return new DefaultMeasure(storage); } }); - inputFile = new DeprecatedDefaultInputFile("foo", "src/main/java/Foo.java"); - duplicationBuilder = spy(new DefaultDuplicationBuilder(inputFile)); - when(context.duplicationBuilder(any(InputFile.class))).thenReturn(duplicationBuilder); + when(context.newDuplication()).then(new Answer() { + @Override + public NewDuplication answer(InvocationOnMock invocation) throws Throwable { + return new DefaultDuplication(storage); + } + }); + inputFile = (DeprecatedDefaultInputFile) new DeprecatedDefaultInputFile("foo", "src/main/java/Foo.java").setLines(300); inputFile.setModuleBaseDir(temp.newFolder().toPath()); } @@ -97,10 +93,9 @@ public class JavaCpdEngineTest { verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(3)); verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATION_LINES_DATA).onFile(inputFile).withValue("1=0;2=1;3=1;4=1;5=0")); - InOrder inOrder = Mockito.inOrder(duplicationBuilder); - inOrder.verify(duplicationBuilder).originBlock(2, 4); - inOrder.verify(duplicationBuilder).isDuplicatedBy("key2", 15, 17); - inOrder.verify(duplicationBuilder).build(); + verify(storage).store(new DefaultDuplication() + .originBlock(inputFile, 2, 4) + .isDuplicatedBy("key2", 15, 17)); } @Test @@ -112,10 +107,9 @@ public class JavaCpdEngineTest { verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_BLOCKS).onFile(inputFile).withValue(2)); verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(400)); - InOrder inOrder = Mockito.inOrder(duplicationBuilder); - inOrder.verify(duplicationBuilder).originBlock(5, 204); - inOrder.verify(duplicationBuilder).isDuplicatedBy("key1", 215, 414); - inOrder.verify(duplicationBuilder).build(); + verify(storage).store(new DefaultDuplication() + .originBlock(inputFile, 5, 204) + .isDuplicatedBy("key1", 215, 414)); } @Test @@ -127,17 +121,10 @@ public class JavaCpdEngineTest { verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_BLOCKS).onFile(inputFile).withValue(1)); verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(200)); - InOrder inOrder = Mockito.inOrder(duplicationBuilder); - inOrder.verify(duplicationBuilder).originBlock(5, 204); - inOrder.verify(duplicationBuilder).isDuplicatedBy("key2", 15, 214); - inOrder.verify(duplicationBuilder).isDuplicatedBy("key3", 25, 224); - inOrder.verify(duplicationBuilder).build(); - - verify(context).saveDuplications(inputFile, Arrays.asList( - new DuplicationGroup(new DuplicationGroup.Block("foo:src/main/java/Foo.java", 5, 200)) - .addDuplicate(new DuplicationGroup.Block("key2", 15, 200)) - .addDuplicate(new DuplicationGroup.Block("key3", 25, 200)) - )); + verify(storage).store(new DefaultDuplication() + .originBlock(inputFile, 5, 204) + .isDuplicatedBy("key2", 15, 214) + .isDuplicatedBy("key3", 25, 224)); } @Test @@ -151,12 +138,12 @@ public class JavaCpdEngineTest { verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_BLOCKS).onFile(inputFile).withValue(2)); verify(storage).store(new DefaultMeasure().forMetric(CoreMetrics.DUPLICATED_LINES).onFile(inputFile).withValue(210)); - InOrder inOrder = Mockito.inOrder(duplicationBuilder); - inOrder.verify(duplicationBuilder).originBlock(5, 204); - inOrder.verify(duplicationBuilder).isDuplicatedBy("key2", 15, 214); - inOrder.verify(duplicationBuilder).originBlock(15, 214); - inOrder.verify(duplicationBuilder).isDuplicatedBy("key3", 15, 214); - inOrder.verify(duplicationBuilder).build(); + verify(storage).store(new DefaultDuplication() + .originBlock(inputFile, 5, 204) + .isDuplicatedBy("key2", 15, 214)); + verify(storage).store(new DefaultDuplication() + .originBlock(inputFile, 15, 214) + .isDuplicatedBy("key3", 15, 214)); } private CloneGroup newCloneGroup(ClonePart... parts) { diff --git a/sonar-batch/src/test/java/org/sonar/batch/duplication/DuplicationCacheTest.java b/sonar-batch/src/test/java/org/sonar/batch/duplication/DuplicationCacheTest.java index 32503c21ef9..514860bc011 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/duplication/DuplicationCacheTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/duplication/DuplicationCacheTest.java @@ -25,14 +25,11 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.batch.index.Caches; import org.sonar.batch.index.CachesTest; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - import static org.assertj.core.api.Assertions.assertThat; public class DuplicationCacheTest { @@ -60,24 +57,28 @@ public class DuplicationCacheTest { public void should_add_clone_groups() throws Exception { DuplicationCache cache = new DuplicationCache(caches); - DuplicationGroup group1 = new DuplicationGroup(new DuplicationGroup.Block("foo", 1, 2)) - .addDuplicate(new DuplicationGroup.Block("foo", 1, 2)) - .addDuplicate(new DuplicationGroup.Block("foo2", 12, 22)) - .addDuplicate(new DuplicationGroup.Block("foo3", 13, 23)); + DefaultDuplication group1 = new DefaultDuplication() + .setOriginBlock(new Duplication.Block("foo", 1, 2)); + group1.duplicates().add(new Duplication.Block("foo", 1, 2)); + group1.duplicates().add(new Duplication.Block("foo2", 12, 22)); + group1.duplicates().add(new Duplication.Block("foo3", 13, 23)); - DuplicationGroup group2 = new DuplicationGroup(new DuplicationGroup.Block("2foo", 1, 2)) - .addDuplicate(new DuplicationGroup.Block("2foo", 1, 2)) - .addDuplicate(new DuplicationGroup.Block("2foo2", 12, 22)) - .addDuplicate(new DuplicationGroup.Block("2foo3", 13, 23)); + DefaultDuplication group2 = new DefaultDuplication() + .setOriginBlock(new Duplication.Block("2foo", 1, 2)); + group2.duplicates().add(new Duplication.Block("2foo", 1, 2)); + group2.duplicates().add(new Duplication.Block("2foo2", 12, 22)); + group2.duplicates().add(new Duplication.Block("2foo3", 13, 23)); - assertThat(cache.entries()).hasSize(0); + assertThat(cache.componentKeys()).hasSize(0); - cache.put("foo", new ArrayList(Arrays.asList(group1, group2))); + cache.put("foo", group1); + cache.put("foo", group2); - assertThat(cache.entries()).hasSize(1); + assertThat(cache.componentKeys()).hasSize(1); + assertThat(cache.byComponent("foo")).hasSize(2); - List entry = cache.byComponent("foo"); - assertThat(entry.get(0).originBlock().resourceKey()).isEqualTo("foo"); + Iterable entry = cache.byComponent("foo"); + assertThat(entry.iterator().next().originBlock().resourceKey()).isEqualTo("foo"); } diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/DuplicationPersisterTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/DuplicationPersisterTest.java index a62dd075bb5..c12eee78b4c 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/index/DuplicationPersisterTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/index/DuplicationPersisterTest.java @@ -22,7 +22,8 @@ package org.sonar.batch.index; import org.junit.Before; import org.junit.Test; import org.junit.rules.ExpectedException; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.database.model.Snapshot; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.MetricFinder; @@ -32,7 +33,6 @@ import org.sonar.batch.duplication.DuplicationCache; import org.sonar.core.persistence.AbstractDaoTestCase; import java.util.Arrays; -import java.util.List; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -71,13 +71,14 @@ public class DuplicationPersisterTest extends AbstractDaoTestCase { public void should_insert_duplications() { setupData("empty"); - DuplicationGroup.Block originBlock = new DuplicationGroup.Block("foo:org/foo/Bar.java", 1, 4); + Duplication.Block originBlock = new Duplication.Block("foo:org/foo/Bar.java", 1, 4); - DuplicationGroup group = new DuplicationGroup(originBlock) - .addDuplicate(new DuplicationGroup.Block("foo:org/foo/Foo.java", 5, 9)); + DefaultDuplication group = new DefaultDuplication().setOriginBlock(originBlock); + group.duplicates().add(new Duplication.Block("foo:org/foo/Foo.java", 5, 9)); - when(duplicationCache.entries()).thenReturn( - Arrays.>>asList(new Cache.Entry(new String[] {"foo:org/foo/Bar.java"}, Arrays.asList(group)))); + when(duplicationCache.componentKeys()).thenReturn(Arrays.asList("foo:org/foo/Bar.java")); + + when(duplicationCache.byComponent("foo:org/foo/Bar.java")).thenReturn(Arrays.asList(group)); duplicationPersister.persist(); diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/SourceDataFactoryTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/SourceDataFactoryTest.java index 3ced19c739b..d6e07f67f5f 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/index/SourceDataFactoryTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/index/SourceDataFactoryTest.java @@ -28,7 +28,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.sonar.api.batch.fs.internal.DefaultFileSystem; import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; +import org.sonar.api.batch.sensor.duplication.internal.DefaultDuplication; import org.sonar.api.batch.sensor.highlighting.TypeOfText; import org.sonar.api.measures.CoreMetrics; import org.sonar.api.measures.Measure; @@ -45,8 +46,10 @@ import org.sonar.server.source.db.FileSourceDb; import java.io.File; import java.util.Arrays; +import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -76,6 +79,7 @@ public class SourceDataFactoryTest { metadata = new InputFileMetadata(); FileUtils.write(inputFile.file(), "one\ntwo\nthree\n"); output = sut.createForSource(inputFile); + when(duplicationCache.byComponent(anyString())).thenReturn(Collections.emptyList()); } @Test @@ -170,13 +174,13 @@ public class SourceDataFactoryTest { @Test public void applyDuplications() throws Exception { - DuplicationGroup group1 = new DuplicationGroup(new DuplicationGroup.Block(inputFile.key(), 1, 1)) - .addDuplicate(new DuplicationGroup.Block(inputFile.key(), 3, 1)) - .addDuplicate(new DuplicationGroup.Block("anotherFile1", 12, 1)) - .addDuplicate(new DuplicationGroup.Block("anotherFile2", 13, 1)); - DuplicationGroup group2 = new DuplicationGroup(new DuplicationGroup.Block(inputFile.key(), 1, 2)) - .addDuplicate(new DuplicationGroup.Block("anotherFile1", 12, 2)) - .addDuplicate(new DuplicationGroup.Block("anotherFile2", 13, 2)); + DefaultDuplication group1 = new DefaultDuplication().setOriginBlock(new Duplication.Block(inputFile.key(), 1, 1)); + group1.duplicates().add(new Duplication.Block(inputFile.key(), 3, 1)); + group1.duplicates().add(new Duplication.Block("anotherFile1", 12, 1)); + group1.duplicates().add(new Duplication.Block("anotherFile2", 13, 1)); + DefaultDuplication group2 = new DefaultDuplication().setOriginBlock(new Duplication.Block(inputFile.key(), 1, 2)); + group2.duplicates().add(new Duplication.Block("anotherFile1", 12, 2)); + group2.duplicates().add(new Duplication.Block("anotherFile2", 13, 2)); when(duplicationCache.byComponent(inputFile.key())).thenReturn(Lists.newArrayList(group1, group2)); sut.applyDuplications(inputFile.key(), output); @@ -190,9 +194,9 @@ public class SourceDataFactoryTest { @Test public void applyDuplications_ignore_bad_lines() throws Exception { // duplication on 10 lines - DuplicationGroup group1 = new DuplicationGroup(new DuplicationGroup.Block(inputFile.key(), 1, 10)) - .addDuplicate(new DuplicationGroup.Block("anotherFile1", 12, 1)) - .addDuplicate(new DuplicationGroup.Block("anotherFile2", 13, 1)); + DefaultDuplication group1 = new DefaultDuplication().setOriginBlock(new Duplication.Block(inputFile.key(), 1, 10)); + group1.duplicates().add(new Duplication.Block("anotherFile1", 12, 1)); + group1.duplicates().add(new Duplication.Block("anotherFile2", 13, 1)); when(duplicationCache.byComponent(inputFile.key())).thenReturn(Lists.newArrayList(group1)); sut.applyDuplications(inputFile.key(), output); diff --git a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java index e5e6f6c1a1c..47ebaa81f8a 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java @@ -29,7 +29,7 @@ import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; -import org.sonar.api.batch.sensor.duplication.DuplicationGroup; +import org.sonar.api.batch.sensor.duplication.Duplication; import org.sonar.batch.mediumtest.BatchMediumTester; import org.sonar.batch.mediumtest.TaskResult; import org.sonar.xoo.XooPlugin; @@ -106,10 +106,10 @@ public class CpdMediumTest { InputFile inputFile1 = result.inputFile("src/sample1.xoo"); InputFile inputFile2 = result.inputFile("src/sample2.xoo"); // One clone group on each file - List duplicationGroupsFile1 = result.duplicationsFor(inputFile1); + List duplicationGroupsFile1 = result.duplicationsFor(inputFile1); assertThat(duplicationGroupsFile1).hasSize(1); - DuplicationGroup cloneGroupFile1 = duplicationGroupsFile1.get(0); + Duplication cloneGroupFile1 = duplicationGroupsFile1.get(0); assertThat(cloneGroupFile1.duplicates()).hasSize(1); assertThat(cloneGroupFile1.originBlock().startLine()).isEqualTo(1); assertThat(cloneGroupFile1.originBlock().length()).isEqualTo(17); @@ -117,10 +117,10 @@ public class CpdMediumTest { assertThat(cloneGroupFile1.duplicates()).hasSize(1); assertThat(cloneGroupFile1.duplicates().get(0).resourceKey()).isEqualTo(((DefaultInputFile) inputFile2).key()); - List duplicationGroupsFile2 = result.duplicationsFor(inputFile2); + List duplicationGroupsFile2 = result.duplicationsFor(inputFile2); assertThat(duplicationGroupsFile2).hasSize(1); - DuplicationGroup cloneGroupFile2 = duplicationGroupsFile2.get(0); + Duplication cloneGroupFile2 = duplicationGroupsFile2.get(0); assertThat(cloneGroupFile2.duplicates()).hasSize(1); assertThat(cloneGroupFile2.originBlock().startLine()).isEqualTo(1); assertThat(cloneGroupFile2.originBlock().length()).isEqualTo(17); @@ -153,10 +153,10 @@ public class CpdMediumTest { InputFile inputFile = result.inputFile("src/sample.xoo"); // One clone group - List duplicationGroups = result.duplicationsFor(inputFile); + List duplicationGroups = result.duplicationsFor(inputFile); assertThat(duplicationGroups).hasSize(1); - DuplicationGroup cloneGroup = duplicationGroups.get(0); + Duplication cloneGroup = duplicationGroups.get(0); assertThat(cloneGroup.duplicates()).hasSize(1); assertThat(cloneGroup.originBlock().startLine()).isEqualTo(1); assertThat(cloneGroup.originBlock().length()).isEqualTo(2); diff --git a/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorContextTest.java b/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorContextTest.java index 1812fde5cf4..4f7fa7507ac 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorContextTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorContextTest.java @@ -32,7 +32,6 @@ import org.sonar.api.batch.rule.internal.ActiveRulesBuilder; import org.sonar.api.batch.sensor.SensorStorage; import org.sonar.api.config.Settings; import org.sonar.api.measures.CoreMetrics; -import org.sonar.batch.duplication.BlockCache; import org.sonar.batch.duplication.DuplicationCache; import org.sonar.batch.index.ComponentDataCache; @@ -64,10 +63,9 @@ public class DefaultSensorContextTest { when(metricFinder.findByKey(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION_KEY)).thenReturn(CoreMetrics.FUNCTION_COMPLEXITY_DISTRIBUTION); settings = new Settings(); ComponentDataCache componentDataCache = mock(ComponentDataCache.class); - BlockCache blockCache = mock(BlockCache.class); sensorStorage = mock(SensorStorage.class); analysisMode = mock(AnalysisMode.class); - adaptor = new DefaultSensorContext(settings, fs, activeRules, analysisMode, componentDataCache, blockCache, mock(DuplicationCache.class), sensorStorage); + adaptor = new DefaultSensorContext(settings, fs, activeRules, analysisMode, componentDataCache, mock(DuplicationCache.class), sensorStorage); } @Test diff --git a/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java b/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java index 506bcd328e1..a4ed0b08571 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/sensor/DefaultSensorStorageTest.java @@ -51,7 +51,6 @@ import org.sonar.api.resources.File; import org.sonar.api.resources.Project; import org.sonar.api.resources.Resource; import org.sonar.api.rule.RuleKey; -import org.sonar.batch.duplication.BlockCache; import org.sonar.batch.duplication.DuplicationCache; import org.sonar.batch.index.ComponentDataCache; import org.sonar.batch.index.DefaultIndex; @@ -91,13 +90,12 @@ public class DefaultSensorStorageTest { settings = new Settings(); resourcePerspectives = mock(ResourcePerspectives.class); ComponentDataCache componentDataCache = mock(ComponentDataCache.class); - BlockCache blockCache = mock(BlockCache.class); project = new Project("myProject"); sonarIndex = mock(DefaultIndex.class); CoverageExclusions coverageExclusions = mock(CoverageExclusions.class); when(coverageExclusions.accept(any(Resource.class), any(Measure.class))).thenReturn(true); sensorStorage = new DefaultSensorStorage(metricFinder, project, - resourcePerspectives, settings, fs, activeRules, componentDataCache, blockCache, mock(DuplicationCache.class), sonarIndex, coverageExclusions); + resourcePerspectives, settings, fs, activeRules, componentDataCache, mock(DuplicationCache.class), sonarIndex, coverageExclusions); } @Test -- cgit v1.2.3