SONAR-7389 Support CPD exclusions in new API

14 files changed, 111 insertions, 36 deletions

diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdComponents.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdComponents.java
index 30f4863783d..a204f14d9fe 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdComponents.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdComponents.java
@@ -21,6 +21,10 @@ package org.sonar.batch.cpd;
 
 import com.google.common.collect.ImmutableList;
 import java.util.List;
+import org.sonar.batch.cpd.deprecated.CpdMappings;
+import org.sonar.batch.cpd.deprecated.DefaultCpdBlockIndexer;
+import org.sonar.batch.cpd.deprecated.DeprecatedCpdBlockIndexerSensor;
+import org.sonar.batch.cpd.deprecated.JavaCpdBlockIndexer;
 
 public final class CpdComponents {
 
@@ -29,7 +33,7 @@ public final class CpdComponents {
 
   public static List<Class<? extends Object>> all() {
     return ImmutableList.of(
-      CpdSensor.class,
+      DeprecatedCpdBlockIndexerSensor.class,
       CpdMappings.class,
       JavaCpdBlockIndexer.class,
       DefaultCpdBlockIndexer.class);
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/CpdBlockIndexer.java
index 9ebf80c88a7..fe58cb6558b 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/CpdBlockIndexer.java
@@ -17,13 +17,13 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import org.slf4j.Logger;
 import org.sonar.api.batch.BatchSide;
 
 @BatchSide
-public abstract class CpdIndexer {
+public abstract class CpdBlockIndexer {
 
   abstract boolean isLanguageSupported(String language);
 
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdMappings.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/CpdMappings.java
index 9a930d62fb5..ad2275e399c 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdMappings.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/CpdMappings.java
@@ -17,7 +17,7 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import org.sonar.api.batch.BatchSide;
 import org.sonar.api.batch.CpdMapping;
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/DefaultCpdBlockIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/DefaultCpdBlockIndexer.java
index 33b2f269595..db51246e1c8 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/DefaultCpdBlockIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/DefaultCpdBlockIndexer.java
@@ -17,7 +17,7 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
@@ -35,7 +35,7 @@ import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.duplications.block.Block;
 import org.sonar.duplications.internal.pmd.TokenizerBridge;
 
-public class DefaultCpdBlockIndexer extends CpdIndexer {
+public class DefaultCpdBlockIndexer extends CpdBlockIndexer {
 
   private static final Logger LOG = LoggerFactory.getLogger(DefaultCpdBlockIndexer.class);
 
@@ -83,7 +83,7 @@ public class DefaultCpdBlockIndexer extends CpdIndexer {
     TokenizerBridge bridge = new TokenizerBridge(mapping.getTokenizer(), fs.encoding().name(), getBlockSize(languageKey));
     for (InputFile inputFile : sourceFiles) {
       if (!index.isIndexed(inputFile)) {
-        LOG.debug("Populating index from {}", inputFile);
+        LOG.debug("Populating index from {}", inputFile.absolutePath());
         String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
         List<Block> blocks = bridge.chunk(resourceEffectiveKey, inputFile.file());
         index.insert(inputFile, blocks);
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdSensor.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/DeprecatedCpdBlockIndexerSensor.java
index 9ea1bac9254..ad72d673777 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/CpdSensor.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/DeprecatedCpdBlockIndexerSensor.java
@@ -17,12 +17,13 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.CpdMapping;
 import org.sonar.api.batch.Phase;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.sensor.Sensor;
@@ -30,34 +31,40 @@ import org.sonar.api.batch.sensor.SensorContext;
 import org.sonar.api.batch.sensor.SensorDescriptor;
 import org.sonar.api.config.Settings;
 
+/**
+ * Feed block index using deprecated {@link CpdMapping} extension point if not already
+ * fed by another Sensor using {@link SensorContext#newCpdTokens()}. Special case for Java
+ * that use a dedicated block indexer.
+ * Can be removed when {@link CpdMapping} extension is removed and Java specific part moved to Java plugin.
+ */
 @Phase(name = Phase.Name.POST)
-public class CpdSensor implements Sensor {
+public class DeprecatedCpdBlockIndexerSensor implements Sensor {
 
-  private static final Logger LOG = LoggerFactory.getLogger(CpdSensor.class);
+  private static final Logger LOG = LoggerFactory.getLogger(DeprecatedCpdBlockIndexerSensor.class);
 
-  private CpdIndexer sonarEngine;
-  private CpdIndexer sonarBridgeEngine;
+  private CpdBlockIndexer javaCpdBlockIndexer;
+  private CpdBlockIndexer defaultCpdBlockIndexer;
   private Settings settings;
   private FileSystem fs;
 
-  public CpdSensor(JavaCpdBlockIndexer sonarEngine, DefaultCpdBlockIndexer sonarBridgeEngine, Settings settings, FileSystem fs) {
-    this.sonarEngine = sonarEngine;
-    this.sonarBridgeEngine = sonarBridgeEngine;
+  public DeprecatedCpdBlockIndexerSensor(JavaCpdBlockIndexer javaCpdBlockIndexer, DefaultCpdBlockIndexer defaultCpdBlockIndexer, Settings settings, FileSystem fs) {
+    this.javaCpdBlockIndexer = javaCpdBlockIndexer;
+    this.defaultCpdBlockIndexer = defaultCpdBlockIndexer;
     this.settings = settings;
     this.fs = fs;
   }
 
   @Override
   public void describe(SensorDescriptor descriptor) {
-    descriptor.name("CPD Sensor");
+    descriptor.name("CPD Block Indexer");
   }
 
   @VisibleForTesting
-  CpdIndexer getEngine(String language) {
-    if (sonarEngine.isLanguageSupported(language)) {
-      return sonarEngine;
+  CpdBlockIndexer getBlockIndexer(String language) {
+    if (javaCpdBlockIndexer.isLanguageSupported(language)) {
+      return javaCpdBlockIndexer;
     }
-    return sonarBridgeEngine;
+    return defaultCpdBlockIndexer;
   }
 
   @VisibleForTesting
@@ -86,13 +93,13 @@ public class CpdSensor implements Sensor {
         continue;
       }
 
-      CpdIndexer engine = getEngine(language);
-      if (!engine.isLanguageSupported(language)) {
+      CpdBlockIndexer blockIndexer = getBlockIndexer(language);
+      if (!blockIndexer.isLanguageSupported(language)) {
         LOG.debug("Detection of duplicated code is not supported for {}", language);
         continue;
       }
-      LOG.info("{} is used for {}", engine, language);
-      engine.index(language);
+      LOG.info("{} is used for {}", blockIndexer, language);
+      blockIndexer.index(language);
     }
   }
 
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/JavaCpdBlockIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/JavaCpdBlockIndexer.java
index a2dde817c74..7e0772408b7 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/JavaCpdBlockIndexer.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/JavaCpdBlockIndexer.java
@@ -17,7 +17,7 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import com.google.common.collect.Lists;
 import java.io.FileInputStream;
@@ -43,7 +43,7 @@ import org.sonar.duplications.statement.Statement;
 import org.sonar.duplications.statement.StatementChunker;
 import org.sonar.duplications.token.TokenChunker;
 
-public class JavaCpdBlockIndexer extends CpdIndexer {
+public class JavaCpdBlockIndexer extends CpdBlockIndexer {
 
   private static final Logger LOG = LoggerFactory.getLogger(JavaCpdBlockIndexer.class);
 
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/package-info.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/package-info.java
new file mode 100644
index 00000000000..8a035753659
--- /dev/null
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/cpd/deprecated/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2016 SonarSource SA
+ * mailto:contact AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.batch.cpd.deprecated;
+
+import javax.annotation.ParametersAreNonnullByDefault;
+
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java
index 1080dab084a..e0a61eb9117 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorContext.java
@@ -109,7 +109,7 @@ public class DefaultSensorContext implements SensorContext {
     if (analysisMode.isIssues()) {
       return NO_OP_NEW_CPD_TOKENS;
     }
-    return new DefaultCpdTokens(sensorStorage);
+    return new DefaultCpdTokens(settings, sensorStorage);
   }
 
 }
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java b/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
index 62e7d982d34..c8de752737b 100644
--- a/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
+++ b/sonar-scanner-engine/src/main/java/org/sonar/batch/sensor/DefaultSensorStorage.java
@@ -53,7 +53,7 @@ import org.sonar.api.resources.Resource;
 import org.sonar.api.source.Symbol;
 import org.sonar.api.utils.KeyValueFormat;
 import org.sonar.api.utils.SonarException;
-import org.sonar.batch.cpd.DefaultCpdBlockIndexer;
+import org.sonar.batch.cpd.deprecated.DefaultCpdBlockIndexer;
 import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponent;
 import org.sonar.batch.index.BatchComponentCache;
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/DefaultCpdBlockIndexerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/deprecated/DefaultCpdBlockIndexerTest.java
index 9fcd03ac940..5dc28fde04c 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/DefaultCpdBlockIndexerTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/deprecated/DefaultCpdBlockIndexerTest.java
@@ -17,12 +17,13 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.sonar.api.config.Settings;
+import org.sonar.batch.cpd.deprecated.DefaultCpdBlockIndexer;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Matchers.anyString;
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/deprecated/DeprecatedCpdBlockIndexerSensorTest.java
index ce27f0776c4..4adc630247e 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/CpdSensorTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/deprecated/DeprecatedCpdBlockIndexerSensorTest.java
@@ -17,7 +17,7 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import java.io.IOException;
 import org.junit.Before;
@@ -28,17 +28,18 @@ import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.config.PropertyDefinitions;
 import org.sonar.api.config.Settings;
 import org.sonar.api.resources.Java;
+import org.sonar.batch.cpd.CpdComponents;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-public class CpdSensorTest {
+public class DeprecatedCpdBlockIndexerSensorTest {
 
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
 
   JavaCpdBlockIndexer sonarEngine;
   DefaultCpdBlockIndexer sonarBridgeEngine;
-  CpdSensor sensor;
+  DeprecatedCpdBlockIndexerSensor sensor;
   Settings settings;
 
   @Before
@@ -48,7 +49,7 @@ public class CpdSensorTest {
     settings = new Settings(new PropertyDefinitions(CpdComponents.class));
 
     DefaultFileSystem fs = new DefaultFileSystem(temp.newFolder().toPath());
-    sensor = new CpdSensor(sonarEngine, sonarBridgeEngine, settings, fs);
+    sensor = new DeprecatedCpdBlockIndexerSensor(sonarEngine, sonarBridgeEngine, settings, fs);
   }
 
   @Test
@@ -73,8 +74,8 @@ public class CpdSensorTest {
 
   @Test
   public void test_engine() {
-    assertThat(sensor.getEngine(Java.KEY)).isSameAs(sonarEngine);
-    assertThat(sensor.getEngine("PHP")).isSameAs(sonarBridgeEngine);
+    assertThat(sensor.getBlockIndexer(Java.KEY)).isSameAs(sonarEngine);
+    assertThat(sensor.getBlockIndexer("PHP")).isSameAs(sonarBridgeEngine);
   }
 
 }
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/JavaCpdBlockIndexerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/deprecated/JavaCpdBlockIndexerTest.java
index 851fb523815..b42843d91aa 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/JavaCpdBlockIndexerTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/batch/cpd/deprecated/JavaCpdBlockIndexerTest.java
@@ -17,7 +17,7 @@
  * along with this program; if not, write to the Free Software Foundation,
  * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
-package org.sonar.batch.cpd;
+package org.sonar.batch.cpd.deprecated;
 
 import org.apache.commons.io.FileUtils;
 import org.junit.Before;
@@ -33,6 +33,7 @@ import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.config.Settings;
+import org.sonar.batch.cpd.deprecated.JavaCpdBlockIndexer;
 import org.sonar.batch.cpd.index.SonarCpdBlockIndex;
 import org.sonar.batch.index.BatchComponentCache;
 import org.sonar.duplications.block.Block;
diff --git a/sonar-scanner-engine/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java
index 662ed7dbcfd..6613421a4da 100644
--- a/sonar-scanner-engine/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java
+++ b/sonar-scanner-engine/src/test/java/org/sonar/batch/mediumtest/cpd/CpdMediumTest.java
@@ -220,6 +220,43 @@ public class CpdMediumTest {
   }
 
   @Test
+  public void testExclusions() throws IOException {
+    File srcDir = new File(baseDir, "src");
+    srcDir.mkdir();
+
+    String duplicatedStuff = "Sample xoo\ncontent\n"
+      + "foo\nbar\ntoto\ntiti\n"
+      + "foo\nbar\ntoto\ntiti\n"
+      + "bar\ntoto\ntiti\n"
+      + "foo\nbar\ntoto\ntiti";
+
+    File xooFile1 = new File(srcDir, "sample1.xoo");
+    FileUtils.write(xooFile1, duplicatedStuff);
+
+    File xooFile2 = new File(srcDir, "sample2.xoo");
+    FileUtils.write(xooFile2, duplicatedStuff);
+
+    TaskResult result = tester.newTask()
+      .properties(builder
+        .put("sonar.sources", "src")
+        .put("sonar.cpd.xoo.minimumTokens", "10")
+        .put("sonar.cpd.exclusions", "src/sample1.xoo")
+        .build())
+      .start();
+
+    assertThat(result.inputFiles()).hasSize(2);
+
+    InputFile inputFile1 = result.inputFile("src/sample1.xoo");
+    InputFile inputFile2 = result.inputFile("src/sample2.xoo");
+
+    List<org.sonar.scanner.protocol.output.ScannerReport.Duplication> duplicationGroupsFile1 = result.duplicationsFor(inputFile1);
+    assertThat(duplicationGroupsFile1).isEmpty();
+
+    List<org.sonar.scanner.protocol.output.ScannerReport.Duplication> duplicationGroupsFile2 = result.duplicationsFor(inputFile2);
+    assertThat(duplicationGroupsFile2).isEmpty();
+  }
+
+  @Test
   public void enableCrossProjectDuplication() throws IOException {
     File srcDir = new File(baseDir, "src");
     srcDir.mkdir();
diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/batch/cpd/ManyStatements.java b/sonar-scanner-engine/src/test/resources/org/sonar/batch/cpd/deprecated/ManyStatements.java
index ed2297068e4..ed2297068e4 100644
--- a/sonar-scanner-engine/src/test/resources/org/sonar/batch/cpd/ManyStatements.java
+++ b/sonar-scanner-engine/src/test/resources/org/sonar/batch/cpd/deprecated/ManyStatements.java