From 20a3509beca56ee2c44212bded73ada3ef5aa8cd Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Wed, 27 Jun 2018 23:57:43 +0200 Subject: [PATCH] SONAR-10138 Restore Java duplications --- sonar-scanner-engine/build.gradle | 2 +- .../cpd/JavaCpdBlockIndexerSensor.java | 113 ++++++++++++++++++ .../scanner/scan/ProjectScanContainer.java | 2 + .../cpd/JavaCpdBlockIndexerSensorTest.java | 93 ++++++++++++++ .../cpd/{deprecated => }/ManyStatements.java | 0 5 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensor.java create mode 100644 sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensorTest.java rename sonar-scanner-engine/src/test/resources/org/sonar/scanner/cpd/{deprecated => }/ManyStatements.java (100%) diff --git a/sonar-scanner-engine/build.gradle b/sonar-scanner-engine/build.gradle index 6186e58573c..2442d85d34d 100644 --- a/sonar-scanner-engine/build.gradle +++ b/sonar-scanner-engine/build.gradle @@ -47,7 +47,7 @@ dependencies { } license { - excludes(["**/Fake.java", "**/Fake.groovy", "org/sonar/scanner/cpd/deprecated/ManyStatements.java"]) + excludes(["**/Fake.java", "**/Fake.groovy", "org/sonar/scanner/cpd/ManyStatements.java"]) } artifactoryPublish.skip = false diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensor.java new file mode 100644 index 00000000000..b4e0f3f3006 --- /dev/null +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensor.java @@ -0,0 +1,113 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cpd; + +import com.google.common.collect.Lists; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.CoreProperties; +import org.sonar.api.batch.Phase; +import org.sonar.api.batch.fs.FilePredicates; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputFile; +import org.sonar.api.batch.sensor.Sensor; +import org.sonar.api.batch.sensor.SensorContext; +import org.sonar.api.batch.sensor.SensorDescriptor; +import org.sonar.duplications.block.Block; +import org.sonar.duplications.block.BlockChunker; +import org.sonar.duplications.java.JavaStatementBuilder; +import org.sonar.duplications.java.JavaTokenProducer; +import org.sonar.duplications.statement.Statement; +import org.sonar.duplications.statement.StatementChunker; +import org.sonar.duplications.token.TokenChunker; +import org.sonar.scanner.cpd.index.SonarCpdBlockIndex; + +/** + * Special case for Java that use a dedicated block indexer. + */ +@Phase(name = Phase.Name.POST) +public class JavaCpdBlockIndexerSensor implements Sensor { + + private static final int BLOCK_SIZE = 10; + private static final Logger LOG = LoggerFactory.getLogger(JavaCpdBlockIndexerSensor.class); + private final SonarCpdBlockIndex index; + + public JavaCpdBlockIndexerSensor(SonarCpdBlockIndex index) { + this.index = index; + } + + @Override + public void describe(SensorDescriptor descriptor) { + descriptor.name("Java CPD Block Indexer") + .onlyOnLanguage("java") + .global(); + } + + @Override + public void execute(SensorContext context) { + String[] cpdExclusions = context.config().getStringArray(CoreProperties.CPD_EXCLUSIONS); + FilePredicates p = context.fileSystem().predicates(); + List sourceFiles = Lists.newArrayList(context.fileSystem().inputFiles(p.and( + p.hasType(InputFile.Type.MAIN), + p.hasLanguage("java"), + p.doesNotMatchPathPatterns(cpdExclusions)))); + if (sourceFiles.isEmpty()) { + return; + } + createIndex(sourceFiles); + } + + private void createIndex(Iterable sourceFiles) { + TokenChunker tokenChunker = JavaTokenProducer.build(); + StatementChunker statementChunker = JavaStatementBuilder.build(); + BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE); + + for (InputFile inputFile : sourceFiles) { + LOG.debug("Populating index from {}", inputFile); + String resourceEffectiveKey = ((DefaultInputFile) inputFile).key(); + + List statements; + + try (InputStream is = inputFile.inputStream(); + Reader reader = new InputStreamReader(is, inputFile.charset())) { + statements = statementChunker.chunk(tokenChunker.chunk(reader)); + } catch (FileNotFoundException e) { + throw new IllegalStateException("Cannot find file " + inputFile.file(), e); + } catch (IOException e) { + throw new IllegalStateException("Exception handling file: " + inputFile.file(), e); + } + + List blocks; + try { + blocks = blockChunker.chunk(resourceEffectiveKey, statements); + } catch (Exception e) { + throw new IllegalStateException("Cannot process file " + inputFile.file(), e); + } + index.insert(inputFile, blocks); + } + } + +} diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java index c079f64aff9..0bb31674dd4 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java @@ -43,6 +43,7 @@ import org.sonar.scanner.bootstrap.GlobalAnalysisMode; import org.sonar.scanner.bootstrap.MetricProvider; import org.sonar.scanner.cpd.CpdExecutor; import org.sonar.scanner.cpd.CpdSettings; +import org.sonar.scanner.cpd.JavaCpdBlockIndexerSensor; import org.sonar.scanner.cpd.index.SonarCpdBlockIndex; import org.sonar.scanner.deprecated.test.TestPlanBuilder; import org.sonar.scanner.deprecated.test.TestableBuilder; @@ -205,6 +206,7 @@ public class ProjectScanContainer extends ComponentContainer { CpdExecutor.class, CpdSettings.class, SonarCpdBlockIndex.class, + JavaCpdBlockIndexerSensor.class, ScanTaskObservers.class); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensorTest.java new file mode 100644 index 00000000000..de67f04859b --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/JavaCpdBlockIndexerSensorTest.java @@ -0,0 +1,93 @@ +/* + * SonarQube + * Copyright (C) 2009-2018 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner.cpd; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.sonar.api.CoreProperties; +import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.TestInputFileBuilder; +import org.sonar.api.batch.sensor.internal.SensorContextTester; +import org.sonar.duplications.block.Block; +import org.sonar.scanner.cpd.index.SonarCpdBlockIndex; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; + +public class JavaCpdBlockIndexerSensorTest { + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + private SensorContextTester context; + + @Mock + private SonarCpdBlockIndex index; + + @Captor + private ArgumentCaptor> blockCaptor; + + private InputFile file; + + @Before + public void prepare() throws IOException { + MockitoAnnotations.initMocks(this); + File baseDir = temp.newFolder(); + context = SensorContextTester.create(baseDir); + file = new TestInputFileBuilder("foo", "src/ManyStatements.java") + .setModuleBaseDir(baseDir.toPath()) + .setCharset(StandardCharsets.UTF_8) + .setLanguage("java").build(); + context.fileSystem().add(file); + File ioFile = file.file(); + FileUtils.copyURLToFile(this.getClass().getResource("ManyStatements.java"), ioFile); + } + + @Test + public void testExclusions() { + context.settings().setProperty(CoreProperties.CPD_EXCLUSIONS, "**"); + new JavaCpdBlockIndexerSensor(index).execute(context); + verifyZeroInteractions(index); + } + + @Test + public void testJavaIndexing() { + new JavaCpdBlockIndexerSensor(index).execute(context); + + verify(index).insert(eq(file), blockCaptor.capture()); + List blockList = blockCaptor.getValue(); + + assertThat(blockList).hasSize(26); + } + +} diff --git a/sonar-scanner-engine/src/test/resources/org/sonar/scanner/cpd/deprecated/ManyStatements.java b/sonar-scanner-engine/src/test/resources/org/sonar/scanner/cpd/ManyStatements.java similarity index 100% rename from sonar-scanner-engine/src/test/resources/org/sonar/scanner/cpd/deprecated/ManyStatements.java rename to sonar-scanner-engine/src/test/resources/org/sonar/scanner/cpd/ManyStatements.java -- 2.39.5