From ac7edbe5890ae7921593dda7520157dca652860d Mon Sep 17 00:00:00 2001 From: Julien HENRY Date: Tue, 1 Aug 2017 16:31:07 +0200 Subject: [PATCH] SONAR-9574 Same file indexed in different modules --- .../xoo/extensions/XooProjectBuilder.java | 4 +- .../batch/fs/internal/DefaultIndexedFile.java | 8 ++++ .../batch/fs/internal/DefaultInputFile.java | 8 ++++ .../scan/issue/filter/FilterableIssue.java | 6 ++- .../fs/internal/DefaultInputFileTest.java | 8 +++- .../cpd/internal/DefaultCpdTokensTest.java | 3 +- .../scanner/scan/filesystem/FileIndexer.java | 17 ++++---- .../scan/filesystem/InputComponentStore.java | 11 ++--- .../scanner/DefaultFileLinesContextTest.java | 4 +- .../mediumtest/fs/FileSystemMediumTest.java | 40 ++++++++++--------- .../fs/ProjectBuilderMediumTest.java | 2 +- .../scan/filesystem/InputFileBuilderTest.java | 5 ++- 12 files changed, 68 insertions(+), 48 deletions(-) diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/extensions/XooProjectBuilder.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/extensions/XooProjectBuilder.java index 28f1ec6a4eb..38e690c0671 100644 --- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/extensions/XooProjectBuilder.java +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/extensions/XooProjectBuilder.java @@ -19,12 +19,11 @@ */ package org.sonar.xoo.extensions; +import java.io.File; import org.sonar.api.batch.bootstrap.ProjectBuilder; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.config.Settings; -import java.io.File; - public class XooProjectBuilder extends ProjectBuilder { private Settings settings; @@ -39,6 +38,7 @@ public class XooProjectBuilder extends ProjectBuilder { return; } ProjectDefinition root = context.projectReactor().getRoot(); + root.resetSources(); ProjectDefinition module = ProjectDefinition.create() .setKey(root.getKey() + ":module1") diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultIndexedFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultIndexedFile.java index 0d07003dce5..2f456b489fc 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultIndexedFile.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultIndexedFile.java @@ -70,6 +70,14 @@ public class DefaultIndexedFile extends DefaultInputComponent implements Indexed return sensorStrategy.isGlobal() ? projectRelativePath : moduleRelativePath; } + public String getModuleRelativePath() { + return moduleRelativePath; + } + + public String getProjectRelativePath() { + return projectRelativePath; + } + @Override public String absolutePath() { return PathUtils.sanitize(path().toString()); diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java index 3fe157a43e2..effb5663f31 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/batch/fs/internal/DefaultInputFile.java @@ -125,6 +125,14 @@ public class DefaultInputFile extends DefaultInputComponent implements InputFile return indexedFile.relativePath(); } + public String getModuleRelativePath() { + return indexedFile.getModuleRelativePath(); + } + + public String getProjectRelativePath() { + return indexedFile.getProjectRelativePath(); + } + @Override public String absolutePath() { return indexedFile.absolutePath(); diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/scan/issue/filter/FilterableIssue.java b/sonar-plugin-api/src/main/java/org/sonar/api/scan/issue/filter/FilterableIssue.java index 76e1a74ec29..30d039436b7 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/scan/issue/filter/FilterableIssue.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/scan/issue/filter/FilterableIssue.java @@ -20,10 +20,8 @@ package org.sonar.api.scan.issue.filter; import java.util.Date; - import javax.annotation.CheckForNull; import javax.annotation.concurrent.ThreadSafe; - import org.sonar.api.rule.RuleKey; /** @@ -55,6 +53,10 @@ public interface FilterableIssue { @CheckForNull Double gap(); + /** + * @deprecated since 6.6 useless + */ + @Deprecated Date creationDate(); String projectKey(); diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java index 7d699d5bf39..5eb4b0a9d47 100644 --- a/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java +++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/fs/internal/DefaultInputFileTest.java @@ -74,8 +74,6 @@ public class DefaultInputFileTest { .setStatus(InputFile.Status.ADDED) .setCharset(StandardCharsets.ISO_8859_1); - assertThat(inputFile.relativePath()).isEqualTo(MODULE_RELATIVE_PATH); - assertThat(new File(inputFile.relativePath())).isRelative(); assertThat(inputFile.absolutePath()).endsWith("Foo.php"); assertThat(inputFile.filename()).isEqualTo("Foo.php"); assertThat(inputFile.uri()).hasPath(baseDir.resolve(PROJECT_RELATIVE_PATH).toUri().getPath()); @@ -86,8 +84,14 @@ public class DefaultInputFileTest { assertThat(inputFile.lines()).isEqualTo(42); assertThat(inputFile.charset()).isEqualTo(StandardCharsets.ISO_8859_1); + assertThat(inputFile.getModuleRelativePath()).isEqualTo(MODULE_RELATIVE_PATH); + assertThat(inputFile.getProjectRelativePath()).isEqualTo(PROJECT_RELATIVE_PATH); + + assertThat(inputFile.relativePath()).isEqualTo(MODULE_RELATIVE_PATH); + assertThat(new File(inputFile.relativePath())).isRelative(); sensorStrategy.setGlobal(true); assertThat(inputFile.relativePath()).isEqualTo(PROJECT_RELATIVE_PATH); + assertThat(new File(inputFile.relativePath())).isRelative(); } @Test diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/cpd/internal/DefaultCpdTokensTest.java b/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/cpd/internal/DefaultCpdTokensTest.java index 62a268f00f7..30126f0271d 100644 --- a/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/cpd/internal/DefaultCpdTokensTest.java +++ b/sonar-plugin-api/src/test/java/org/sonar/api/batch/sensor/cpd/internal/DefaultCpdTokensTest.java @@ -23,7 +23,6 @@ import org.junit.Test; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; import org.sonar.api.batch.sensor.internal.SensorStorage; -import org.sonar.api.config.Settings; import org.sonar.api.config.internal.MapSettings; import static org.assertj.core.api.Assertions.assertThat; @@ -146,7 +145,7 @@ public class DefaultCpdTokensTest { tokens.addToken(INPUT_FILE.newRange(1, 2, 1, 5), "foo"); fail("Expected exception"); } catch (Exception e) { - assertThat(e).hasMessage("Tokens of file [moduleKey=foo, relative=src/Foo.java, basedir=foo] should be provided in order.\n" + + assertThat(e).hasMessage("Tokens of file src/Foo.java should be provided in order.\n" + "Previous token: Range[from [line=1, lineOffset=6] to [line=1, lineOffset=10]]\n" + "Last token: Range[from [line=1, lineOffset=2] to [line=1, lineOffset=5]]"); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 4b78ce2f6c6..33cd3749c1b 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -31,9 +31,7 @@ import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -43,7 +41,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.ScannerSide; -import org.sonar.api.batch.fs.IndexedFile; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.InputFileFilter; @@ -184,8 +181,8 @@ public class FileIndexer { } String parentRelativePath = getParentRelativePath(inputFile); synchronized (this) { - indexFileAndParentDir(inputFile, parentRelativePath); progress.markAsIndexed(inputFile); + indexFileAndParentDir(inputFile, parentRelativePath); } LOG.debug("'{}' indexed {}with language '{}'", relativePath, type == Type.TEST ? "as test " : "", inputFile.language()); inputFileBuilder.checkMetadata(inputFile); @@ -271,16 +268,16 @@ public class FileIndexer { } private class Progress { - private final Set indexed = new HashSet<>(); + private AtomicInteger indexedCount = new AtomicInteger(0); private AtomicInteger excludedByPatternsCount = new AtomicInteger(0); - void markAsIndexed(IndexedFile inputFile) { - if (indexed.contains(inputFile.path())) { + void markAsIndexed(DefaultInputFile inputFile) { + if (componentStore.getFile(inputFile.getProjectRelativePath()) != null) { throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce " + "disjoint sets for main and test files"); } - indexed.add(inputFile.path()); - progressReport.message(indexed.size() + " " + pluralizeFiles(indexed.size()) + " indexed... (last one was " + inputFile.relativePath() + ")"); + int count = indexedCount.incrementAndGet(); + progressReport.message(count + " " + pluralizeFiles(count) + " indexed... (last one was " + inputFile.relativePath() + ")"); } void increaseExcludedByPatternsCount() { @@ -292,7 +289,7 @@ public class FileIndexer { } int count() { - return indexed.size(); + return indexedCount.get(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java index bec0e176f10..ee1b34a8d95 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java @@ -110,7 +110,7 @@ public class InputComponentStore { public InputComponentStore remove(InputFile inputFile) { DefaultInputFile file = (DefaultInputFile) inputFile; - inputFileCache.remove(file.moduleKey(), inputFile.relativePath()); + inputFileCache.remove(file.moduleKey(), file.getModuleRelativePath()); return this; } @@ -123,8 +123,8 @@ public class InputComponentStore { public InputComponentStore put(InputFile inputFile) { DefaultInputFile file = (DefaultInputFile) inputFile; addToLanguageCache(file); - inputFileCache.put(file.moduleKey(), inputFile.relativePath(), inputFile); - globalInputFileCache.put(getProjectRelativePath(file), inputFile); + inputFileCache.put(file.moduleKey(), file.getModuleRelativePath(), inputFile); + globalInputFileCache.put(file.getProjectRelativePath(), inputFile); inputComponents.put(inputFile.key(), inputFile); filesByNameCache.put(inputFile.filename(), inputFile); filesByExtensionCache.put(FileExtensionPredicate.getExtension(inputFile), inputFile); @@ -142,15 +142,12 @@ public class InputComponentStore { public InputComponentStore put(InputDir inputDir) { DefaultInputDir dir = (DefaultInputDir) inputDir; inputDirCache.put(dir.moduleKey(), inputDir.relativePath(), inputDir); + // FIXME an InputDir can be already indexed by another module globalInputDirCache.put(getProjectRelativePath(dir), inputDir); inputComponents.put(inputDir.key(), inputDir); return this; } - private String getProjectRelativePath(DefaultInputFile file) { - return PathResolver.relativePath(getProjectBaseDir(), file.path()); - } - private String getProjectRelativePath(DefaultInputDir dir) { return PathResolver.relativePath(getProjectBaseDir(), dir.path()); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/DefaultFileLinesContextTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/DefaultFileLinesContextTest.java index 76e59e815f4..f3a47b46ec8 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/DefaultFileLinesContextTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/DefaultFileLinesContextTest.java @@ -95,13 +95,13 @@ public class DefaultFileLinesContextTest { @Test public void validateLineGreaterThanZero() { - thrown.expectMessage("Line number should be positive for file [moduleKey=foo, relative=src/foo.php, basedir=foo]."); + thrown.expectMessage("Line number should be positive for file src/foo.php."); fileLineMeasures.setIntValue(HITS_METRIC_KEY, 0, 2); } @Test public void validateLineLowerThanLineCount() { - thrown.expectMessage("Line 4 is out of range for file [moduleKey=foo, relative=src/foo.php, basedir=foo]. File has 3 lines"); + thrown.expectMessage("Line 4 is out of range for file src/foo.php. File has 3 lines"); fileLineMeasures.setIntValue(HITS_METRIC_KEY, 4, 2); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java index 5f2add6955a..a48b94bf2a1 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java @@ -562,12 +562,32 @@ public class FileSystemMediumTest { FileUtils.write(xooFile, "Sample xoo\ncontent"); thrown.expect(MessageException.class); - thrown.expectMessage("can't be indexed twice. Please check that inclusion/exclusion patterns produce disjoint sets for main and test files"); + thrown.expectMessage("File src/sample.xoo can't be indexed twice. Please check that inclusion/exclusion patterns produce disjoint sets for main and test files"); tester.newTask() .properties(builder .put("sonar.sources", "src,src/sample.xoo") .build()) .execute(); + } + + // SONAR-9574 + @Test + public void failForDuplicateInputFileInDifferentModules() throws IOException { + File srcDir = new File(baseDir, "module1/src"); + srcDir.mkdir(); + + File xooFile = new File(srcDir, "sample.xoo"); + FileUtils.write(xooFile, "Sample xoo\ncontent"); + + thrown.expect(MessageException.class); + thrown.expectMessage("File module1/src/sample.xoo can't be indexed twice. Please check that inclusion/exclusion patterns produce disjoint sets for main and test files"); + tester.newTask() + .properties(builder + .put("sonar.sources", "module1/src") + .put("sonar.modules", "module1") + .put("module1.sonar.sources", "src") + .build()) + .execute(); } @@ -713,22 +733,4 @@ public class FileSystemMediumTest { assertThat(result.inputFiles()).hasSize(1); } - @Test - public void detectDuplicatedFilesInDifferentModules() throws IOException { - File srcDir = new File(baseDir, "module1/src"); - srcDir.mkdir(); - - File xooFile = new File(srcDir, "sample.xoo"); - FileUtils.write(xooFile, "Sample xoo\ncontent"); - - TaskResult result = tester.newTask() - .properties(builder - .put("sonar.sources", "module1/src") - .put("sonar.modules", "module1") - .put("module1.sonar.sources", "src") - .build()) - .execute(); - - assertThat(result.inputFiles()).hasSize(1); - } } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/ProjectBuilderMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/ProjectBuilderMediumTest.java index 35eab16b110..bf86ea126c1 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/ProjectBuilderMediumTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/ProjectBuilderMediumTest.java @@ -190,7 +190,7 @@ public class ProjectBuilderMediumTest { } private File prepareProject() throws IOException { - File baseDir = temp.getRoot(); + File baseDir = temp.newFolder(); File module1Dir = new File(baseDir, "module1"); module1Dir.mkdir(); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java index 6611f15302b..482b215cee0 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java @@ -21,6 +21,7 @@ package org.sonar.scanner.scan.filesystem; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import org.junit.Before; import org.junit.Rule; @@ -56,8 +57,10 @@ public class InputFileBuilderTest { .setBaseDir(baseDir.toFile()) .setWorkDir(workDir.toFile()) .setKey("root"), 0); + Path moduleBaseDir = baseDir.resolve("module1"); + Files.createDirectories(moduleBaseDir); DefaultInputModule module = new DefaultInputModule(ProjectDefinition.create() - .setBaseDir(baseDir.resolve("module1").toFile()) + .setBaseDir(moduleBaseDir.toFile()) .setWorkDir(workDir.toFile()) .setKey("module1"), 0); -- 2.39.5