SONAR-10138 Restore Java duplications

sonar-scanner-engine/build.gradle

@@ -47,7 +47,7 @@ dependencies {
 }
 
 license {
-  excludes(["**/Fake.java", "**/Fake.groovy", "org/sonar/scanner/cpd/deprecated/ManyStatements.java"])
+  excludes(["**/Fake.java", "**/Fake.groovy", "org/sonar/scanner/cpd/ManyStatements.java"])
 }
 
 artifactoryPublish.skip = false

sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensor.java

@@ -0,0 +1,113 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.cpd;
+
+import com.google.common.collect.Lists;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.Phase;
+import org.sonar.api.batch.fs.FilePredicates;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.DefaultInputFile;
+import org.sonar.api.batch.sensor.Sensor;
+import org.sonar.api.batch.sensor.SensorContext;
+import org.sonar.api.batch.sensor.SensorDescriptor;
+import org.sonar.duplications.block.Block;
+import org.sonar.duplications.block.BlockChunker;
+import org.sonar.duplications.java.JavaStatementBuilder;
+import org.sonar.duplications.java.JavaTokenProducer;
+import org.sonar.duplications.statement.Statement;
+import org.sonar.duplications.statement.StatementChunker;
+import org.sonar.duplications.token.TokenChunker;
+import org.sonar.scanner.cpd.index.SonarCpdBlockIndex;
+
+/**
+ * Special case for Java that use a dedicated block indexer.
+ */
+@Phase(name = Phase.Name.POST)
+public class JavaCpdBlockIndexerSensor implements Sensor {
+
+  private static final int BLOCK_SIZE = 10;
+  private static final Logger LOG = LoggerFactory.getLogger(JavaCpdBlockIndexerSensor.class);
+  private final SonarCpdBlockIndex index;
+
+  public JavaCpdBlockIndexerSensor(SonarCpdBlockIndex index) {
+    this.index = index;
+  }
+
+  @Override
+  public void describe(SensorDescriptor descriptor) {
+    descriptor.name("Java CPD Block Indexer")
+      .onlyOnLanguage("java")
+      .global();
+  }
+
+  @Override
+  public void execute(SensorContext context) {
+    String[] cpdExclusions = context.config().getStringArray(CoreProperties.CPD_EXCLUSIONS);
+    FilePredicates p = context.fileSystem().predicates();
+    List<InputFile> sourceFiles = Lists.newArrayList(context.fileSystem().inputFiles(p.and(
+      p.hasType(InputFile.Type.MAIN),
+      p.hasLanguage("java"),
+      p.doesNotMatchPathPatterns(cpdExclusions))));
+    if (sourceFiles.isEmpty()) {
+      return;
+    }
+    createIndex(sourceFiles);
+  }
+
+  private void createIndex(Iterable<InputFile> sourceFiles) {
+    TokenChunker tokenChunker = JavaTokenProducer.build();
+    StatementChunker statementChunker = JavaStatementBuilder.build();
+    BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
+
+    for (InputFile inputFile : sourceFiles) {
+      LOG.debug("Populating index from {}", inputFile);
+      String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
+
+      List<Statement> statements;
+
+      try (InputStream is = inputFile.inputStream();
+        Reader reader = new InputStreamReader(is, inputFile.charset())) {
+        statements = statementChunker.chunk(tokenChunker.chunk(reader));
+      } catch (FileNotFoundException e) {
+        throw new IllegalStateException("Cannot find file " + inputFile.file(), e);
+      } catch (IOException e) {
+        throw new IllegalStateException("Exception handling file: " + inputFile.file(), e);
+      }
+
+      List<Block> blocks;
+      try {
+        blocks = blockChunker.chunk(resourceEffectiveKey, statements);
+      } catch (Exception e) {
+        throw new IllegalStateException("Cannot process file " + inputFile.file(), e);
+      }
+      index.insert(inputFile, blocks);
+    }
+  }
+
+}

sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java

@@ -43,6 +43,7 @@ import org.sonar.scanner.bootstrap.GlobalAnalysisMode;
 import org.sonar.scanner.bootstrap.MetricProvider;
 import org.sonar.scanner.cpd.CpdExecutor;
 import org.sonar.scanner.cpd.CpdSettings;
+import org.sonar.scanner.cpd.JavaCpdBlockIndexerSensor;
 import org.sonar.scanner.cpd.index.SonarCpdBlockIndex;
 import org.sonar.scanner.deprecated.test.TestPlanBuilder;
 import org.sonar.scanner.deprecated.test.TestableBuilder;
@@ -205,6 +206,7 @@ public class ProjectScanContainer extends ComponentContainer {
       CpdExecutor.class,
       CpdSettings.class,
       SonarCpdBlockIndex.class,
+      JavaCpdBlockIndexerSensor.class,
 
       ScanTaskObservers.class);
 

sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensorTest.java

@@ -0,0 +1,93 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.scanner.cpd;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.sonar.api.CoreProperties;
+import org.sonar.api.batch.fs.InputFile;
+import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
+import org.sonar.api.batch.sensor.internal.SensorContextTester;
+import org.sonar.duplications.block.Block;
+import org.sonar.scanner.cpd.index.SonarCpdBlockIndex;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
+public class JavaCpdBlockIndexerSensorTest {
+
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+
+  private SensorContextTester context;
+
+  @Mock
+  private SonarCpdBlockIndex index;
+
+  @Captor
+  private ArgumentCaptor<List<Block>> blockCaptor;
+
+  private InputFile file;
+
+  @Before
+  public void prepare() throws IOException {
+    MockitoAnnotations.initMocks(this);
+    File baseDir = temp.newFolder();
+    context = SensorContextTester.create(baseDir);
+    file = new TestInputFileBuilder("foo", "src/ManyStatements.java")
+      .setModuleBaseDir(baseDir.toPath())
+      .setCharset(StandardCharsets.UTF_8)
+      .setLanguage("java").build();
+    context.fileSystem().add(file);
+    File ioFile = file.file();
+    FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile);
+  }
+
+  @Test
+  public void testExclusions() {
+    context.settings().setProperty(CoreProperties.CPD_EXCLUSIONS, "**");
+    new JavaCpdBlockIndexerSensor(index).execute(context);
+    verifyZeroInteractions(index);
+  }
+
+  @Test
+  public void testJavaIndexing() {
+    new JavaCpdBlockIndexerSensor(index).execute(context);
+
+    verify(index).insert(eq(file), blockCaptor.capture());
+    List<Block> blockList = blockCaptor.getValue();
+
+    assertThat(blockList).hasSize(26);
+  }
+
+}