diff options
author | Duarte Meneses <duarte.meneses@sonarsource.com> | 2017-01-24 18:02:09 +0100 |
---|---|---|
committer | Duarte Meneses <duarte.meneses@sonarsource.com> | 2017-01-27 16:26:30 +0100 |
commit | 3c0dd8758d56ce8583a3a74c17a181e4cfe5918c (patch) | |
tree | 47766f525c8fa28e4baf78dd5de48b6a5ff82b58 /sonar-scanner-engine | |
parent | 291382160309e0543067c59c193d126710370dee (diff) | |
download | sonarqube-3c0dd8758d56ce8583a3a74c17a181e4cfe5918c.tar.gz sonarqube-3c0dd8758d56ce8583a3a74c17a181e4cfe5918c.zip |
SONAR-8622 Lazily generate metadata for input files
Diffstat (limited to 'sonar-scanner-engine')
34 files changed, 397 insertions, 138 deletions
diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/ProjectAnalysisInfo.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/ProjectAnalysisInfo.java index 3441db14cef..98013b4881c 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/ProjectAnalysisInfo.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/ProjectAnalysisInfo.java @@ -28,8 +28,7 @@ import org.sonar.api.utils.SonarException; import org.sonar.api.utils.System2; /** - * Used by views !! - * + * @since 6.3 */ @ScannerSide public class ProjectAnalysisInfo implements Startable { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerExtensionDictionnary.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerExtensionDictionnary.java index 446edb161ef..fa9189c0d61 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerExtensionDictionnary.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/bootstrap/ScannerExtensionDictionnary.java @@ -63,8 +63,7 @@ public class ScannerExtensionDictionnary { private final PostJobOptimizer postJobOptimizer; public ScannerExtensionDictionnary(ComponentContainer componentContainer, DefaultSensorContext sensorContext, - SensorOptimizer sensorOptimizer, PostJobContext postJobContext, - PostJobOptimizer postJobOptimizer) { + SensorOptimizer sensorOptimizer, PostJobContext postJobContext, PostJobOptimizer postJobOptimizer) { this.componentContainer = componentContainer; this.sensorContext = sensorContext; this.sensorOptimizer = sensorOptimizer; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java index 451d0a92642..ac76cc22212 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/cpd/CpdExecutor.java @@ -110,7 +110,7 @@ public class CpdExecutor { void runCpdAnalysis(ExecutorService executorService, String componentKey, final Collection<Block> fileBlocks, long timeout) { DefaultInputComponent component = (DefaultInputComponent) componentStore.getByKey(componentKey); if (component == null) { - LOG.error("Resource not found in component cache: {}. Skipping CPD computation for it", componentKey); + LOG.error("Resource not found in component store: {}. Skipping CPD computation for it", componentKey); return; } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/deprecated/DeprecatedSensorContext.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/deprecated/DeprecatedSensorContext.java index cfc2d8cf89e..1c7b788edce 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/deprecated/DeprecatedSensorContext.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/deprecated/DeprecatedSensorContext.java @@ -55,12 +55,12 @@ public class DeprecatedSensorContext extends DefaultSensorContext implements Sen @Override public Resource getParent(Resource reference) { - return index.getParent(reference.getEffectiveKey()); + return index.getParent(getComponentKey(reference)); } @Override public Collection<Resource> getChildren(Resource reference) { - return index.getChildren(reference.getEffectiveKey()); + return index.getChildren(getComponentKey(reference)); } @Override @@ -68,6 +68,9 @@ public class DeprecatedSensorContext extends DefaultSensorContext implements Sen return index.getMeasure(module.key(), metric); } + /** + * Returns effective key of a resource, without branch. + */ private String getComponentKey(Resource r) { if (ResourceUtils.isProject(r) || /* For technical projects */ResourceUtils.isRootProject(r)) { return r.getKey(); @@ -93,7 +96,7 @@ public class DeprecatedSensorContext extends DefaultSensorContext implements Sen @Override public <G extends Serializable> Measure<G> getMeasure(Resource resource, Metric<G> metric) { - return index.getMeasure(resource.getEffectiveKey(), metric); + return index.getMeasure(getComponentKey(resource), metric); } @Override diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/index/DefaultIndex.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/index/DefaultIndex.java index 32019e1f0bd..9a4e49a4eb8 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/index/DefaultIndex.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/index/DefaultIndex.java @@ -62,7 +62,7 @@ public class DefaultIndex { this.metricFinder = metricFinder; } - public void setCurrentProject(DefaultSensorStorage sensorStorage) { + public void setCurrentStorage(DefaultSensorStorage sensorStorage) { // the following components depend on the current module, so they need to be reloaded. this.sensorStorage = sensorStorage; } @@ -156,6 +156,9 @@ public class DefaultIndex { return measure; } + /** + * @param key Effective key, without branch + */ @CheckForNull public Resource getParent(String key) { InputComponent component = componentStore.getByKey(key); @@ -170,6 +173,9 @@ public class DefaultIndex { return toResource(parent); } + /** + * @param key Effective key, without branch + */ public Collection<Resource> getChildren(String key) { InputComponent component = componentStore.getByKey(key); Collection<InputComponent> children = tree.getChildren(component); @@ -191,6 +197,10 @@ public class DefaultIndex { return r; } + /** + * Gets a component from the store as a resource. + * @param key Effective key, without branch + */ @CheckForNull public Resource getResource(String key) { InputComponent component = componentStore.getByKey(key); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/DeprecatedIssueAdapterForFilter.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/DeprecatedIssueAdapterForFilter.java index 00cf22d19b2..83ef4634dcb 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/DeprecatedIssueAdapterForFilter.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/DeprecatedIssueAdapterForFilter.java @@ -25,7 +25,6 @@ import java.util.Date; import java.util.List; import java.util.Map; -import org.sonar.api.batch.fs.InputModule; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.issue.Issue; import org.sonar.api.issue.IssueComment; @@ -43,9 +42,9 @@ class DeprecatedIssueAdapterForFilter implements Issue { private DefaultInputModule module; private ProjectAnalysisInfo projectAnalysisInfo; - DeprecatedIssueAdapterForFilter(InputModule module, ProjectAnalysisInfo projectAnalysisInfo, org.sonar.scanner.protocol.output.ScannerReport.Issue rawIssue, + DeprecatedIssueAdapterForFilter(DefaultInputModule module, ProjectAnalysisInfo projectAnalysisInfo, org.sonar.scanner.protocol.output.ScannerReport.Issue rawIssue, String componentKey) { - this.module = (DefaultInputModule) module; + this.module = module; this.projectAnalysisInfo = projectAnalysisInfo; this.rawIssue = rawIssue; this.componentKey = componentKey; diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssueFilters.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssueFilters.java index f485dec405c..7d961244f24 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssueFilters.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/issue/IssueFilters.java @@ -25,7 +25,7 @@ import org.sonar.api.scan.issue.filter.IssueFilterChain; import org.sonar.scanner.ProjectAnalysisInfo; import org.sonar.scanner.protocol.output.ScannerReport; import org.sonar.api.batch.ScannerSide; -import org.sonar.api.batch.fs.InputModule; +import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.api.issue.Issue; import org.sonar.api.scan.issue.filter.IssueFilter; @@ -33,25 +33,25 @@ import org.sonar.api.scan.issue.filter.IssueFilter; public class IssueFilters { private final IssueFilter[] filters; private final org.sonar.api.issue.batch.IssueFilter[] deprecatedFilters; - private final InputModule module; + private final DefaultInputModule module; private final ProjectAnalysisInfo projectAnalysisInfo; - public IssueFilters(InputModule module, ProjectAnalysisInfo projectAnalysisInfo, IssueFilter[] exclusionFilters, org.sonar.api.issue.batch.IssueFilter[] filters) { + public IssueFilters(DefaultInputModule module, ProjectAnalysisInfo projectAnalysisInfo, IssueFilter[] exclusionFilters, org.sonar.api.issue.batch.IssueFilter[] filters) { this.module = module; this.filters = exclusionFilters; this.deprecatedFilters = filters; this.projectAnalysisInfo = projectAnalysisInfo; } - public IssueFilters(InputModule module, ProjectAnalysisInfo projectAnalysisInfo, IssueFilter[] filters) { + public IssueFilters(DefaultInputModule module, ProjectAnalysisInfo projectAnalysisInfo, IssueFilter[] filters) { this(module, projectAnalysisInfo, filters, new org.sonar.api.issue.batch.IssueFilter[0]); } - public IssueFilters(InputModule module, ProjectAnalysisInfo projectAnalysisInfo, org.sonar.api.issue.batch.IssueFilter[] deprecatedFilters) { + public IssueFilters(DefaultInputModule module, ProjectAnalysisInfo projectAnalysisInfo, org.sonar.api.issue.batch.IssueFilter[] deprecatedFilters) { this(module, projectAnalysisInfo, new IssueFilter[0], deprecatedFilters); } - public IssueFilters(InputModule module, ProjectAnalysisInfo projectAnalysisInfo) { + public IssueFilters(DefaultInputModule module, ProjectAnalysisInfo projectAnalysisInfo) { this(module, projectAnalysisInfo, new IssueFilter[0], new org.sonar.api.issue.batch.IssueFilter[0]); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/phases/IssuesPhaseExecutor.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/phases/IssuesPhaseExecutor.java index 478b29bc5d6..c68bf066b49 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/phases/IssuesPhaseExecutor.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/phases/IssuesPhaseExecutor.java @@ -31,7 +31,6 @@ import org.sonar.scanner.rule.QProfileVerifier; import org.sonar.scanner.scan.filesystem.DefaultModuleFileSystem; import org.sonar.scanner.scan.filesystem.FileSystemLogger; import org.sonar.scanner.scan.report.IssuesReports; -import org.sonar.scanner.scm.ScmPublisher; public final class IssuesPhaseExecutor extends AbstractPhaseExecutor { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/CoveragePublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/CoveragePublisher.java index 2060505fca8..c59e26f21f6 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/CoveragePublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/CoveragePublisher.java @@ -25,7 +25,6 @@ import java.util.LinkedHashMap; import java.util.Map; import javax.annotation.Nonnull; import org.apache.commons.lang.StringUtils; -import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.sensor.measure.internal.DefaultMeasure; import org.sonar.api.measures.CoreMetrics; @@ -48,8 +47,7 @@ public class CoveragePublisher implements ReportPublisherStep { @Override public void publish(ScannerReportWriter writer) { - for (final InputFile file : componentCache.allFiles()) { - DefaultInputFile inputFile = (DefaultInputFile) file; + for (final DefaultInputFile inputFile : componentCache.allFilesToPublish()) { Map<Integer, LineCoverage.Builder> coveragePerLine = new LinkedHashMap<>(); int lineCount = inputFile.lines(); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MeasuresPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MeasuresPublisher.java index ed54d5261cb..ba283458e14 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MeasuresPublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/MeasuresPublisher.java @@ -68,12 +68,12 @@ import static org.sonar.api.measures.CoreMetrics.UNCOVERED_LINES_KEY; public class MeasuresPublisher implements ReportPublisherStep { - private final InputComponentStore componentCache; + private final InputComponentStore componentStore; private final MeasureCache measureCache; private final TestPlanBuilder testPlanBuilder; - public MeasuresPublisher(InputComponentStore componentCache, MeasureCache measureCache, TestPlanBuilder testPlanBuilder) { - this.componentCache = componentCache; + public MeasuresPublisher(InputComponentStore componentStore, MeasureCache measureCache, TestPlanBuilder testPlanBuilder) { + this.componentStore = componentStore; this.measureCache = measureCache; this.testPlanBuilder = testPlanBuilder; } @@ -82,7 +82,7 @@ public class MeasuresPublisher implements ReportPublisherStep { public void publish(ScannerReportWriter writer) { final ScannerReport.Measure.Builder builder = ScannerReport.Measure.newBuilder(); - for (final InputComponent c : componentCache.all()) { + for (final InputComponent c : componentStore.all()) { DefaultInputComponent component = (DefaultInputComponent) c; // Recompute all coverage measures from line data to take into account the possible merge of several reports updateCoverageFromLineData(component); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java index b270567f86f..17ef8bf7ffc 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/report/SourcePublisher.java @@ -22,7 +22,6 @@ package org.sonar.scanner.report; import org.apache.commons.io.ByteOrderMark; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BOMInputStream; -import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.scanner.protocol.output.ScannerReportWriter; import org.sonar.scanner.scan.filesystem.InputComponentStore; @@ -39,32 +38,36 @@ public class SourcePublisher implements ReportPublisherStep { private final InputComponentStore componentCache; - public SourcePublisher(InputComponentStore componentCache) { - this.componentCache = componentCache; + public SourcePublisher(InputComponentStore componentStore) { + this.componentCache = componentStore; } @Override public void publish(ScannerReportWriter writer) { - for (final InputFile file : componentCache.allFiles()) { - DefaultInputFile inputFile = (DefaultInputFile) file; + for (final DefaultInputFile inputFile : componentCache.allFilesToPublish()) { File iofile = writer.getSourceFile(inputFile.batchId()); - int line = 0; + try (FileOutputStream output = new FileOutputStream(iofile); BOMInputStream bomIn = new BOMInputStream(new FileInputStream(inputFile.file()), ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE); BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, inputFile.charset()))) { - String lineStr = reader.readLine(); - while (lineStr != null) { - IOUtils.write(lineStr, output, StandardCharsets.UTF_8); - line++; - if (line < inputFile.lines()) { - IOUtils.write("\n", output, StandardCharsets.UTF_8); - } - lineStr = reader.readLine(); - } + writeSource(reader, output, inputFile.lines()); } catch (IOException e) { throw new IllegalStateException("Unable to store file source in the report", e); } } } + + private static void writeSource(BufferedReader reader, FileOutputStream output, int lines) throws IOException { + int line = 0; + String lineStr = reader.readLine(); + while (lineStr != null) { + IOUtils.write(lineStr, output, StandardCharsets.UTF_8); + line++; + if (line < lines) { + IOUtils.write("\n", output, StandardCharsets.UTF_8); + } + lineStr = reader.readLine(); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleIndexer.java index 24c1557246a..31f75f3ef40 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleIndexer.java @@ -23,17 +23,24 @@ import org.picocontainer.Startable; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.fs.internal.DefaultInputModule; import org.sonar.scanner.scan.filesystem.BatchIdGenerator; +import org.sonar.scanner.scan.filesystem.InputComponentStore; +/** + * Indexes all modules into {@link DefaultComponentTree}, {@link DefaultInputModuleHierarchy) and {@link InputComponentStore}, using the + * project definitions provided by the {@link ImmutableProjectReactor}. + */ public class ModuleIndexer implements Startable { private final ImmutableProjectReactor projectReactor; private final DefaultComponentTree componentTree; private final DefaultInputModuleHierarchy moduleHierarchy; private final BatchIdGenerator batchIdGenerator; + private final InputComponentStore componentStore; public ModuleIndexer(ImmutableProjectReactor projectReactor, DefaultComponentTree componentTree, - BatchIdGenerator batchIdGenerator, DefaultInputModuleHierarchy moduleHierarchy) { + InputComponentStore componentStore, BatchIdGenerator batchIdGenerator, DefaultInputModuleHierarchy moduleHierarchy) { this.projectReactor = projectReactor; this.componentTree = componentTree; + this.componentStore = componentStore; this.moduleHierarchy = moduleHierarchy; this.batchIdGenerator = batchIdGenerator; } @@ -42,6 +49,7 @@ public class ModuleIndexer implements Startable { public void start() { DefaultInputModule root = new DefaultInputModule(projectReactor.getRoot(), batchIdGenerator.get()); moduleHierarchy.setRoot(root); + componentStore.put(root); createChildren(root); } @@ -50,6 +58,7 @@ public class ModuleIndexer implements Startable { DefaultInputModule child = new DefaultInputModule(def, batchIdGenerator.get()); moduleHierarchy.index(child, parent); componentTree.index(child, parent); + componentStore.put(child); createChildren(child); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleScanContainer.java index 9c087e1b1d7..b62ad60e1a8 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ModuleScanContainer.java @@ -167,7 +167,7 @@ public class ModuleScanContainer extends ComponentContainer { @Override protected void doAfterStart() { DefaultIndex index = getComponentByType(DefaultIndex.class); - index.setCurrentProject(getComponentByType(DefaultSensorStorage.class)); + index.setCurrentStorage(getComponentByType(DefaultSensorStorage.class)); getComponentByType(AbstractPhaseExecutor.class).execute(module); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java index 0144b2e7a4b..d9ddca72dbf 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/ProjectScanContainer.java @@ -139,7 +139,6 @@ public class ProjectScanContainer extends ComponentContainer { ProjectAnalysisInfo.class, DefaultIndex.class, Storages.class, - DefaultIssueCallback.class, new RulesProvider(), new ProjectRepositoriesProvider(), @@ -159,6 +158,7 @@ public class ProjectScanContainer extends ComponentContainer { new QualityProfileProvider(), // issues + DefaultIssueCallback.class, IssueCache.class, DefaultProjectIssues.class, IssueTransition.class, diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/BatchIdGenerator.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/BatchIdGenerator.java index 48d90078bae..6443c15dc50 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/BatchIdGenerator.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/BatchIdGenerator.java @@ -19,13 +19,21 @@ */ package org.sonar.scanner.scan.filesystem; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import org.sonar.api.batch.fs.InputComponent; + +/** + * Generates unique IDs for any {@link InputComponent}. + * The IDs must be unique among all types of components and for all modules in the project. + * The ID should never be 0, as it is sometimes used to indicate invalid components. + */ public class BatchIdGenerator implements Supplier<Integer> { - private int nextBatchId = 1; + private AtomicInteger nextBatchId = new AtomicInteger(1); @Override public Integer get() { - return nextBatchId++; + return nextBatchId.getAndIncrement(); } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java index 61ef1d38d27..c72c6ed09c0 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/FileIndexer.java @@ -29,11 +29,18 @@ import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.batch.ScannerSide; @@ -50,6 +57,8 @@ import org.sonar.api.utils.MessageException; import org.sonar.scanner.scan.DefaultComponentTree; import org.sonar.scanner.util.ProgressReport; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + /** * Index input files into {@link InputComponentStore}. */ @@ -65,6 +74,8 @@ public class FileIndexer { private final DefaultInputModule module; private final BatchIdGenerator batchIdGenerator; private final InputComponentStore componentStore; + private ExecutorService executorService; + private final List<Future<Void>> tasks; private ProgressReport progressReport; @@ -77,6 +88,7 @@ public class FileIndexer { this.inputFileBuilder = inputFileBuilder; this.filters = filters; this.exclusionFilters = exclusionFilters; + this.tasks = new ArrayList<>(); this.isAggregator = !def.getSubProjects().isEmpty(); } @@ -86,11 +98,14 @@ public class FileIndexer { } void index(DefaultModuleFileSystem fileSystem) { - fileSystem.add(module); if (isAggregator) { // No indexing for an aggregator module return; } + + int threads = Math.max(1, Runtime.getRuntime().availableProcessors() - 1); + this.executorService = Executors.newFixedThreadPool(threads, new ThreadFactoryBuilder().setNameFormat("FileIndexer-%d").build()); + progressReport = new ProgressReport("Report about progress of file indexation", TimeUnit.SECONDS.toMillis(10)); progressReport.start("Index files"); exclusionFilters.prepare(); @@ -100,20 +115,40 @@ public class FileIndexer { indexFiles(fileSystem, progress, fileSystem.sources(), InputFile.Type.MAIN); indexFiles(fileSystem, progress, fileSystem.tests(), InputFile.Type.TEST); - progressReport.stop(progress.count() + " files indexed"); + waitForTasksToComplete(); + + progressReport.stop(progress.count() + " " + pluralizeFiles(progress.count()) + " indexed"); if (exclusionFilters.hasPattern()) { - LOG.info("{} files ignored because of inclusion/exclusion patterns", progress.excludedByPatternsCount()); + LOG.info("{} {} ignored because of inclusion/exclusion patterns", progress.excludedByPatternsCount(), pluralizeFiles(progress.excludedByPatternsCount())); + } + } + + private void waitForTasksToComplete() { + executorService.shutdown(); + for (Future<Void> task : tasks) { + try { + task.get(); + } catch (ExecutionException e) { + // Unwrap ExecutionException + throw e.getCause() instanceof RuntimeException ? (RuntimeException) e.getCause() : new IllegalStateException(e.getCause()); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } } } + private static String pluralizeFiles(int count) { + return count == 1 ? "file" : "files"; + } + private void indexFiles(DefaultModuleFileSystem fileSystem, Progress progress, List<File> sources, InputFile.Type type) { try { for (File dirOrFile : sources) { if (dirOrFile.isDirectory()) { indexDirectory(fileSystem, progress, dirOrFile.toPath(), type); } else { - indexFile(fileSystem, progress, dirOrFile.toPath(), type); + tasks.add(executorService.submit(() -> indexFile(fileSystem, progress, dirOrFile.toPath(), type))); } } } catch (IOException e) { @@ -126,20 +161,24 @@ public class FileIndexer { new IndexFileVisitor(fileSystem, status, type)); } - private void indexFile(DefaultModuleFileSystem fileSystem, Progress progress, Path sourceFile, InputFile.Type type) throws IOException { + private Void indexFile(DefaultModuleFileSystem fileSystem, Progress progress, Path sourceFile, InputFile.Type type) throws IOException { // get case of real file without resolving link Path realFile = sourceFile.toRealPath(LinkOption.NOFOLLOW_LINKS); DefaultInputFile inputFile = inputFileBuilder.create(realFile, type, fileSystem.encoding()); if (inputFile != null) { if (exclusionFilters.accept(inputFile, type) && accept(inputFile)) { - fileSystem.add(inputFile); - indexParentDir(fileSystem, inputFile); - progress.markAsIndexed(inputFile); + synchronized (this) { + fileSystem.add(inputFile); + indexParentDir(fileSystem, inputFile); + progress.markAsIndexed(inputFile); + } LOG.debug("'{}' indexed {} with language '{}'", inputFile.relativePath(), type == Type.TEST ? "as test " : "", inputFile.language()); + inputFileBuilder.checkMetadata(inputFile); } else { progress.increaseExcludedByPatternsCount(); } } + return null; } private void indexParentDir(DefaultModuleFileSystem fileSystem, InputFile inputFile) { @@ -197,7 +236,7 @@ public class FileIndexer { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (!Files.isHidden(file)) { - indexFile(fileSystem, status, file, type); + tasks.add(executorService.submit(() -> indexFile(fileSystem, status, file, type))); } return FileVisitResult.CONTINUE; } @@ -220,23 +259,23 @@ public class FileIndexer { private class Progress { private final Set<Path> indexed = new HashSet<>(); - private int excludedByPatternsCount = 0; + private AtomicInteger excludedByPatternsCount = new AtomicInteger(0); - synchronized void markAsIndexed(IndexedFile inputFile) { + void markAsIndexed(IndexedFile inputFile) { if (indexed.contains(inputFile.path())) { throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce " + "disjoint sets for main and test files"); } indexed.add(inputFile.path()); - progressReport.message(indexed.size() + " files indexed... (last one was " + inputFile.relativePath() + ")"); + progressReport.message(indexed.size() + " " + pluralizeFiles(indexed.size()) + " indexed... (last one was " + inputFile.relativePath() + ")"); } void increaseExcludedByPatternsCount() { - excludedByPatternsCount++; + excludedByPatternsCount.incrementAndGet(); } public int excludedByPatternsCount() { - return excludedByPatternsCount; + return excludedByPatternsCount.get(); } int count() { diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java index ae6c52d38c5..54bf5b9e3e6 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputComponentStore.java @@ -59,6 +59,12 @@ public class InputComponentStore { return inputComponents.values(); } + public Iterable<DefaultInputFile> allFilesToPublish() { + return inputFileCache.values().stream() + .map(f -> (DefaultInputFile) f) + .filter(DefaultInputFile::publish)::iterator; + } + public Iterable<InputFile> allFiles() { return inputFileCache.values(); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java index c3826963bb6..4a41149f65b 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/InputFileBuilder.java @@ -30,9 +30,11 @@ import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultIndexedFile; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.config.Settings; import org.sonar.api.scan.filesystem.PathResolver; public class InputFileBuilder { + private static final String PRELOAD_FILE_METADATA_KEY = "sonar.preloadFileMetadata"; private static final Logger LOG = LoggerFactory.getLogger(InputFileBuilder.class); private final String moduleKey; private final Path moduleBaseDir; @@ -40,15 +42,17 @@ public class InputFileBuilder { private final LanguageDetection langDetection; private final BatchIdGenerator idGenerator; private final MetadataGenerator metadataGenerator; + private final boolean preloadMetadata; public InputFileBuilder(DefaultInputModule module, PathResolver pathResolver, LanguageDetection langDetection, MetadataGenerator metadataGenerator, - BatchIdGenerator idGenerator) { + BatchIdGenerator idGenerator, Settings settings) { this.moduleKey = module.key(); this.moduleBaseDir = module.definition().getBaseDir().toPath(); this.pathResolver = pathResolver; this.langDetection = langDetection; this.metadataGenerator = metadataGenerator; this.idGenerator = idGenerator; + this.preloadMetadata = settings.getBoolean(PRELOAD_FILE_METADATA_KEY); } @CheckForNull @@ -65,6 +69,13 @@ public class InputFileBuilder { return null; } indexedFile.setLanguage(language); + return new DefaultInputFile(indexedFile, f -> metadataGenerator.setMetadata(f, defaultEncoding)); } + + void checkMetadata(DefaultInputFile inputFile) { + if (preloadMetadata) { + inputFile.checkMetadata(); + } + } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGenerator.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGenerator.java index ace65041674..8d1f6d69c27 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGenerator.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/MetadataGenerator.java @@ -21,11 +21,12 @@ package org.sonar.scanner.scan.filesystem; import com.google.common.annotations.VisibleForTesting; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,11 +55,29 @@ class MetadataGenerator { } /** + * Sets all metadata in the file, including charset and status. + * It is an expensive computation, reading the entire file. + */ + public void setMetadata(final DefaultInputFile inputFile, Charset defaultEncoding) { + try { + Charset charset = detectCharset(inputFile.path(), defaultEncoding); + inputFile.setCharset(charset); + Metadata metadata = fileMetadata.readMetadata(inputFile.file(), charset); + inputFile.setMetadata(metadata); + inputFile.setStatus(statusDetection.status(inputModule.definition().getKeyWithBranch(), inputFile.relativePath(), metadata.hash())); + LOG.debug("'{}' generated metadata {} with charset '{}'", + inputFile.relativePath(), inputFile.type() == Type.TEST ? "as test " : "", charset); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + /** * @return charset detected from BOM in given file or given defaultCharset * @throws IllegalStateException if an I/O error occurs */ - private static Charset detectCharset(File file, Charset defaultCharset) { - try (FileInputStream inputStream = new FileInputStream(file)) { + private static Charset detectCharset(Path path, Charset defaultCharset) { + try (InputStream inputStream = Files.newInputStream(path)) { byte[] bom = new byte[4]; int n = inputStream.read(bom, 0, bom.length); if ((n >= 3) && (bom[0] == (byte) 0xEF) && (bom[1] == (byte) 0xBB) && (bom[2] == (byte) 0xBF)) { @@ -75,21 +94,7 @@ class MetadataGenerator { return defaultCharset; } } catch (IOException e) { - throw new IllegalStateException("Unable to read file " + file.getAbsolutePath(), e); - } - } - - public void setMetadata(final DefaultInputFile inputFile, Charset defaultEncoding) { - try { - Charset charset = detectCharset(inputFile.file(), defaultEncoding); - inputFile.setCharset(charset); - Metadata metadata = fileMetadata.readMetadata(inputFile.file(), charset); - inputFile.setMetadata(metadata); - inputFile.setStatus(statusDetection.status(inputModule.definition().getKeyWithBranch(), inputFile.relativePath(), metadata.hash())); - LOG.debug("'{}' generated metadata {} with and charset '{}'", - inputFile.relativePath(), inputFile.type() == Type.TEST ? "as test " : "", charset); - } catch (Exception e) { - throw new IllegalStateException(e); + throw new IllegalStateException("Unable to read file " + path.toAbsolutePath().toString(), e); } } } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java index dd518d9d152..d276b852cdd 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/filesystem/ModuleInputComponentStore.java @@ -62,16 +62,6 @@ public class ModuleInputComponentStore extends DefaultFileSystem.Cache { } @Override - protected void doAdd(InputModule inputModule) { - inputComponentStore.put(inputModule); - } - - @Override - public InputModule module() { - return inputComponentStore.getModule(moduleKey); - } - - @Override public Iterable<InputFile> getFilesByName(String filename) { return inputComponentStore.getFilesByName(filename); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/ConsoleReport.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/ConsoleReport.java index 94f7054a128..5dc8df42808 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/ConsoleReport.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/ConsoleReport.java @@ -100,7 +100,7 @@ public class ConsoleReport implements Reporter { if (settings.getBoolean(CONSOLE_REPORT_ENABLED_KEY)) { LOG.warn("Console report is deprecated. Use SonarLint CLI to have local reports of issues"); Report r = new Report(); - r.setNoFile(!inputPathCache.allFiles().iterator().hasNext()); + r.setNoFile(!inputPathCache.allFilesToPublish().iterator().hasNext()); for (TrackedIssue issue : issueCache.all()) { r.process(issue); } diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/IssuesReportBuilder.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/IssuesReportBuilder.java index 7d5e82a2f8f..a711166fe7e 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/IssuesReportBuilder.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/IssuesReportBuilder.java @@ -58,7 +58,7 @@ public class IssuesReportBuilder { public IssuesReport buildReport() { DefaultInputModule project = moduleHierarchy.root(); IssuesReport issuesReport = new IssuesReport(); - issuesReport.setNoFile(!inputComponentCache.allFiles().iterator().hasNext()); + issuesReport.setNoFile(!inputComponentCache.allFilesToPublish().iterator().hasNext()); issuesReport.setTitle(project.definition().getName()); issuesReport.setDate(projectAnalysisInfo.analysisDate()); diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/JSONReport.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/JSONReport.java index f1ce2013147..6551ddb2dc7 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/JSONReport.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scan/report/JSONReport.java @@ -40,7 +40,6 @@ import org.sonar.api.Property; import org.sonar.api.PropertyType; import org.sonar.api.batch.fs.FileSystem; import org.sonar.api.batch.fs.InputDir; -import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultInputDir; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; @@ -73,13 +72,13 @@ public class JSONReport implements Reporter { private final Server server; private final Rules rules; private final IssueCache issueCache; - private final InputComponentStore fileCache; + private final InputComponentStore componentStore; private final DefaultInputModule rootModule; private final UserRepositoryLoader userRepository; private final InputModuleHierarchy moduleHierarchy; public JSONReport(InputModuleHierarchy moduleHierarchy, Settings settings, FileSystem fileSystem, Server server, Rules rules, IssueCache issueCache, - DefaultInputModule rootModule, InputComponentStore fileCache, UserRepositoryLoader userRepository) { + DefaultInputModule rootModule, InputComponentStore componentStore, UserRepositoryLoader userRepository) { this.moduleHierarchy = moduleHierarchy; this.settings = settings; this.fileSystem = fileSystem; @@ -87,7 +86,7 @@ public class JSONReport implements Reporter { this.rules = rules; this.issueCache = issueCache; this.rootModule = rootModule; - this.fileCache = fileCache; + this.componentStore = componentStore; this.userRepository = userRepository; } @@ -166,17 +165,17 @@ public class JSONReport implements Reporter { json.name("components").beginArray(); // Dump modules writeJsonModuleComponents(json, rootModule); - for (InputFile inputFile : fileCache.allFiles()) { - String key = ((DefaultInputFile) inputFile).key(); + for (DefaultInputFile inputFile : componentStore.allFilesToPublish()) { + String key = inputFile.key(); json .beginObject() .prop("key", key) .prop("path", inputFile.relativePath()) - .prop("moduleKey", StringUtils.substringBeforeLast(key, ":")) + .prop("moduleKey", inputFile.moduleKey()) .prop("status", inputFile.status().name()) .endObject(); } - for (InputDir inputDir : fileCache.allDirs()) { + for (InputDir inputDir : componentStore.allDirs()) { String key = ((DefaultInputDir) inputDir).key(); json .beginObject() diff --git a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java index 8c8f8ff5e95..2159e09ad4f 100644 --- a/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java +++ b/sonar-scanner-engine/src/main/java/org/sonar/scanner/scm/ScmPublisher.java @@ -19,7 +19,6 @@ */ package org.sonar.scanner.scm; -import java.io.File; import java.util.LinkedList; import java.util.List; import org.apache.commons.lang.StringUtils; @@ -34,10 +33,10 @@ import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.scanner.protocol.output.ScannerReport; import org.sonar.scanner.protocol.output.ScannerReportWriter; +import org.sonar.scanner.report.ReportPublisher; import org.sonar.scanner.protocol.output.ScannerReport.Changesets.Builder; import org.sonar.scanner.repository.FileData; import org.sonar.scanner.repository.ProjectRepositories; -import org.sonar.scanner.scan.ImmutableProjectReactor; import org.sonar.scanner.scan.filesystem.DefaultModuleFileSystem; import org.sonar.scanner.scan.filesystem.ModuleInputComponentStore; @@ -51,19 +50,17 @@ public final class ScmPublisher { private final ScmConfiguration configuration; private final ProjectRepositories projectRepositories; private final ModuleInputComponentStore componentStore; - - private DefaultModuleFileSystem fs; - private ScannerReportWriter writer; + private final DefaultModuleFileSystem fs; + private final ScannerReportWriter writer; public ScmPublisher(DefaultInputModule inputModule, ScmConfiguration configuration, ProjectRepositories projectRepositories, - ModuleInputComponentStore componentStore, DefaultModuleFileSystem fs, ImmutableProjectReactor reactor) { + ModuleInputComponentStore componentStore, DefaultModuleFileSystem fs, ReportPublisher reportPublisher) { this.inputModule = inputModule; this.configuration = configuration; this.projectRepositories = projectRepositories; this.componentStore = componentStore; this.fs = fs; - File reportDir = new File(reactor.getRoot().getWorkDir(), "batch-report"); - writer = new ScannerReportWriter(reportDir); + this.writer = reportPublisher.getWriter(); } public void publish() { diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/ProjectAnalysisInfoTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/ProjectAnalysisInfoTest.java new file mode 100644 index 00000000000..8c0ff8b93ef --- /dev/null +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/ProjectAnalysisInfoTest.java @@ -0,0 +1,49 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.scanner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.Date; + +import org.junit.Test; +import org.sonar.api.CoreProperties; +import org.sonar.api.config.MapSettings; +import org.sonar.api.config.Settings; +import org.sonar.api.utils.System2; + +public class ProjectAnalysisInfoTest { + @Test + public void testSimpleDate() { + Settings settings = new MapSettings(); + settings.appendProperty(CoreProperties.PROJECT_DATE_PROPERTY, "2017-01-01"); + settings.appendProperty(CoreProperties.PROJECT_VERSION_PROPERTY, "version"); + System2 system = mock(System2.class); + ProjectAnalysisInfo info = new ProjectAnalysisInfo(settings, system); + info.start(); + LocalDate date = LocalDate.of(2017, 1, 1); + + assertThat(info.analysisDate()).isEqualTo(Date.from(date.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant())); + assertThat(info.analysisVersion()).isEqualTo("version"); + } +} diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java index 33a74f0662e..9a70906f106 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/cpd/CpdExecutorTest.java @@ -210,7 +210,7 @@ public class CpdExecutorTest { public void failOnMissingComponent() { executor.runCpdAnalysis(null, "unknown", Collections.emptyList(), 1); readDuplications(0); - assertThat(logTester.logs(LoggerLevel.ERROR)).contains("Resource not found in component cache: unknown. Skipping CPD computation for it"); + assertThat(logTester.logs(LoggerLevel.ERROR)).contains("Resource not found in component store: unknown. Skipping CPD computation for it"); } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/index/DefaultIndexTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/index/DefaultIndexTest.java index 2eadd1f5170..b796c4f322c 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/index/DefaultIndexTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/index/DefaultIndexTest.java @@ -93,7 +93,7 @@ public class DefaultIndexTest { rule = Rule.create("repoKey", "ruleKey", "Rule"); rule.setId(1); rulesProfile.activateRule(rule, null); - index.setCurrentProject(mock(DefaultSensorStorage.class)); + index.setCurrentStorage(mock(DefaultSensorStorage.class)); } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/LogOutputRecorder.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/LogOutputRecorder.java index e720681e671..1ef8e3835d1 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/LogOutputRecorder.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/LogOutputRecorder.java @@ -22,17 +22,19 @@ package org.sonar.scanner.mediumtest; import java.util.Collection; import java.util.LinkedList; import java.util.List; +import java.util.stream.Collectors; + import org.sonar.batch.bootstrapper.LogOutput; import com.google.common.collect.Multimap; import com.google.common.collect.HashMultimap; public class LogOutputRecorder implements LogOutput { - private Multimap<String, String> recordedByLevel = HashMultimap.create(); - private List<String> recorded = new LinkedList<>(); - private StringBuffer asString = new StringBuffer(); + private final Multimap<String, String> recordedByLevel = HashMultimap.create(); + private final List<String> recorded = new LinkedList<>(); + private final StringBuffer asString = new StringBuffer(); @Override - public void log(String formattedMessage, Level level) { + public synchronized void log(String formattedMessage, Level level) { recordedByLevel.put(level.toString(), formattedMessage); recorded.add(formattedMessage); asString.append(formattedMessage).append("\n"); @@ -42,6 +44,10 @@ public class LogOutputRecorder implements LogOutput { return recorded; } + public String getAllAsString() { + return recorded.stream().collect(Collectors.joining("\n")); + } + public Collection<String> get(String level) { return recordedByLevel.get(level); } diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java index 6c83a42f045..7cfe5091725 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/mediumtest/fs/FileSystemMediumTest.java @@ -28,12 +28,16 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.InputDir; import org.sonar.api.batch.fs.InputFile; +import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.utils.MessageException; import org.sonar.api.utils.System2; +import org.sonar.scanner.mediumtest.LogOutputRecorder; import org.sonar.scanner.mediumtest.ScannerMediumTester; import org.sonar.scanner.mediumtest.TaskResult; import org.sonar.xoo.XooPlugin; +import org.sonar.xoo.rule.XooRulesDefinition; import java.io.File; import java.io.IOException; @@ -48,13 +52,15 @@ public class FileSystemMediumTest { @Rule public ExpectedException thrown = ExpectedException.none(); + private LogOutputRecorder logs = new LogOutputRecorder(); + public ScannerMediumTester tester = ScannerMediumTester.builder() .registerPlugin("xoo", new XooPlugin()) .addDefaultQProfile("xoo", "Sonar Way") + .setLogOutput(logs) .build(); private File baseDir; - private ImmutableMap.Builder<String, String> builder; @Before @@ -75,6 +81,7 @@ public class FileSystemMediumTest { @After public void stop() { tester.stop(); + logs = new LogOutputRecorder(); } @Test @@ -103,9 +110,118 @@ public class FileSystemMediumTest { assertThat(result.getReportReader().readComponent(ref).getName()).isEmpty(); assertThat(result.inputFiles()).hasSize(1); assertThat(result.inputDirs()).hasSize(1); - assertThat(result.inputFile("src/sample.xoo").type()).isEqualTo(InputFile.Type.MAIN); - assertThat(result.inputFile("src/sample.xoo").relativePath()).isEqualTo("src/sample.xoo"); - assertThat(result.inputDir("src").relativePath()).isEqualTo("src"); + + DefaultInputFile file = (DefaultInputFile) result.inputFile("src/sample.xoo"); + InputDir dir = result.inputDir("src"); + assertThat(file.type()).isEqualTo(InputFile.Type.MAIN); + assertThat(file.relativePath()).isEqualTo("src/sample.xoo"); + assertThat(dir.relativePath()).isEqualTo("src"); + + // file and dirs were not published + assertThat(file.publish()).isFalse(); + assertThat(result.getReportComponent(dir.key())).isNull(); + assertThat(result.getReportComponent(file.key())).isNull(); + } + + @Test + public void onlyGenerateMetadataIfNeeded() throws IOException { + builder = ImmutableMap.<String, String>builder() + .put("sonar.task", "scan") + .put("sonar.verbose", "true") + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.projectVersion", "1.0-SNAPSHOT") + .put("sonar.projectDescription", "Description of Foo Project"); + + File srcDir = new File(baseDir, "src"); + srcDir.mkdir(); + + File xooFile = new File(srcDir, "sample.xoo"); + FileUtils.write(xooFile, "Sample xoo\ncontent"); + + File unknownFile = new File(srcDir, "sample.unknown"); + FileUtils.write(unknownFile, "Sample xoo\ncontent"); + + tester.newTask() + .properties(builder + .put("sonar.sources", "src") + .build()) + .start(); + + assertThat(logs.getAllAsString()).contains("2 files indexed"); + assertThat(logs.getAllAsString()).contains("'src/sample.xoo' generated metadata"); + assertThat(logs.getAllAsString()).doesNotContain("'src/sample.unknown' generated metadata"); + } + + @Test + public void preloadFileMetadata() throws IOException { + builder = ImmutableMap.<String, String>builder() + .put("sonar.task", "scan") + .put("sonar.verbose", "true") + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.projectVersion", "1.0-SNAPSHOT") + .put("sonar.preloadFileMetadata", "true") + .put("sonar.projectDescription", "Description of Foo Project"); + + File srcDir = new File(baseDir, "src"); + srcDir.mkdir(); + + File xooFile = new File(srcDir, "sample.xoo"); + FileUtils.write(xooFile, "Sample xoo\ncontent"); + + File unknownFile = new File(srcDir, "sample.unknown"); + FileUtils.write(unknownFile, "Sample xoo\ncontent"); + + tester.newTask() + .properties(builder + .put("sonar.sources", "src") + .build()) + .start(); + + assertThat(logs.getAllAsString()).contains("2 files indexed"); + assertThat(logs.getAllAsString()).contains("'src/sample.xoo' generated metadata"); + assertThat(logs.getAllAsString()).contains("'src/sample.unknown' generated metadata"); + } + + @Test + public void publishFilesWithIssues() throws IOException { + ScannerMediumTester tester2 = ScannerMediumTester.builder() + .registerPlugin("xoo", new XooPlugin()) + .addDefaultQProfile("xoo", "Sonar Way") + .addRules(new XooRulesDefinition()) + .addActiveRule("xoo", "OneIssueOnDirPerFile", null, "OneIssueOnDirPerFile", "MAJOR", null, "xoo") + .build(); + tester2.start(); + + builder = ImmutableMap.<String, String>builder() + .put("sonar.task", "scan") + .put("sonar.verbose", "true") + .put("sonar.projectBaseDir", baseDir.getAbsolutePath()) + .put("sonar.projectKey", "com.foo.project") + .put("sonar.projectVersion", "1.0-SNAPSHOT") + .put("sonar.projectDescription", "Description of Foo Project"); + + File srcDir = new File(baseDir, "src"); + srcDir.mkdir(); + + File xooFile = new File(srcDir, "sample.xoo"); + FileUtils.write(xooFile, "Sample xoo\ncontent"); + + TaskResult result = tester2.newTask() + .properties(builder + .put("sonar.sources", "src") + .build()) + .start(); + + DefaultInputFile file = (DefaultInputFile) result.inputFile("src/sample.xoo"); + InputDir dir = result.inputDir("src"); + + assertThat(file.publish()).isTrue(); + assertThat(result.getReportComponent(dir.key())).isNotNull(); + assertThat(result.getReportComponent(file.key())).isNotNull(); + + tester2.stop(); } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java index 00d7c123bf7..06f83958a4a 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/ModuleIndexerTest.java @@ -28,19 +28,22 @@ import org.junit.Test; import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.fs.InputModule; import org.sonar.scanner.scan.filesystem.BatchIdGenerator; +import org.sonar.scanner.scan.filesystem.InputComponentStore; public class ModuleIndexerTest { private ModuleIndexer indexer; private DefaultComponentTree tree; private DefaultInputModuleHierarchy moduleHierarchy; private ImmutableProjectReactor reactor; + private InputComponentStore componentStore; @Before public void setUp() { reactor = mock(ImmutableProjectReactor.class); + componentStore = new InputComponentStore(); tree = new DefaultComponentTree(); moduleHierarchy = new DefaultInputModuleHierarchy(); - indexer = new ModuleIndexer(reactor, tree, new BatchIdGenerator(), moduleHierarchy); + indexer = new ModuleIndexer(reactor, tree, componentStore, new BatchIdGenerator(), moduleHierarchy); } @Test diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java index 131eb7d6c6e..4fbb83c5804 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputComponentStoreTest.java @@ -19,18 +19,21 @@ */ package org.sonar.scanner.scan.filesystem; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.util.LinkedList; +import java.util.List; + import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.InputFile.Status; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.InputPath; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; -import org.sonar.scanner.scan.filesystem.InputComponentStore; -import java.nio.charset.StandardCharsets; - -import static org.assertj.core.api.Assertions.assertThat; public class InputComponentStoreTest { @Rule @@ -39,10 +42,14 @@ public class InputComponentStoreTest { @Test public void should_add_input_file() throws Exception { InputComponentStore cache = new InputComponentStore(); - DefaultInputFile fooFile = new TestInputFileBuilder("struts", "src/main/java/Foo.java").setModuleBaseDir(temp.newFolder().toPath()).build(); + DefaultInputFile fooFile = new TestInputFileBuilder("struts", "src/main/java/Foo.java") + .setModuleBaseDir(temp.newFolder().toPath()) + .setPublish(true) + .build(); cache.put(fooFile); cache.put(new TestInputFileBuilder("struts-core", "src/main/java/Bar.java") .setLanguage("bla") + .setPublish(false) .setType(Type.MAIN) .setStatus(Status.ADDED) .setLines(2) @@ -61,6 +68,10 @@ public class InputComponentStoreTest { assertThat(inputPath.relativePath()).startsWith("src/main/java/"); } + List<InputFile> toPublish = new LinkedList<>(); + cache.allFilesToPublish().forEach(toPublish::add); + assertThat(toPublish).containsOnly(fooFile); + cache.remove(fooFile); assertThat(cache.allFiles()).hasSize(1); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java index b1c514fd1a2..9856d8fabde 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/filesystem/InputFileBuilderTest.java @@ -34,6 +34,8 @@ import org.sonar.api.batch.bootstrap.ProjectDefinition; import org.sonar.api.batch.fs.InputFile.Type; import org.sonar.api.batch.fs.internal.DefaultInputFile; import org.sonar.api.batch.fs.internal.DefaultInputModule; +import org.sonar.api.config.MapSettings; +import org.sonar.api.config.Settings; import org.sonar.api.scan.filesystem.PathResolver; public class InputFileBuilderTest { @@ -54,14 +56,15 @@ public class InputFileBuilderTest { LanguageDetection langDetection = mock(LanguageDetection.class); MetadataGenerator metadataGenerator = mock(MetadataGenerator.class); BatchIdGenerator idGenerator = new BatchIdGenerator(); - builder = new InputFileBuilder(module, pathResolver, langDetection, metadataGenerator, idGenerator); + Settings settings = new MapSettings(); + builder = new InputFileBuilder(module, pathResolver, langDetection, metadataGenerator, idGenerator, settings); } @Test public void testBuild() { Path filePath = baseDir.resolve("src/File1.xoo"); DefaultInputFile inputFile = builder.create(filePath, Type.MAIN, StandardCharsets.UTF_8); - + assertThat(inputFile.moduleKey()).isEqualTo("module1"); assertThat(inputFile.absolutePath()).isEqualTo(filePath.toString()); assertThat(inputFile.key()).isEqualTo("module1:src/File1.xoo"); diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/ConsoleReportTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/ConsoleReportTest.java index a176c309830..b0ae6550d4e 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/ConsoleReportTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/ConsoleReportTest.java @@ -25,7 +25,6 @@ import javax.annotation.Nullable; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.TestInputFileBuilder; import org.sonar.api.config.Settings; import org.sonar.api.config.MapSettings; @@ -69,8 +68,8 @@ public class ConsoleReportTest { @Test public void testNoFile() { settings.setProperty(ConsoleReport.CONSOLE_REPORT_ENABLED_KEY, "true"); - when(inputPathCache.allFiles()).thenReturn(Collections.<InputFile>emptyList()); - when(issueCache.all()).thenReturn(Collections.<TrackedIssue>emptyList()); + when(inputPathCache.allFilesToPublish()).thenReturn(Collections.emptyList()); + when(issueCache.all()).thenReturn(Collections.emptyList()); report.execute(); assertDeprecated(); assertThat(getReportLog()).isEqualTo( @@ -82,7 +81,7 @@ public class ConsoleReportTest { @Test public void testNoNewIssue() { settings.setProperty(ConsoleReport.CONSOLE_REPORT_ENABLED_KEY, "true"); - when(inputPathCache.allFiles()).thenReturn(Arrays.<InputFile>asList(new TestInputFileBuilder("foo", "src/Foo.php").build())); + when(inputPathCache.allFilesToPublish()).thenReturn(Collections.singleton(new TestInputFileBuilder("foo", "src/Foo.php").build())); when(issueCache.all()).thenReturn(Arrays.asList(createIssue(false, null))); report.execute(); assertDeprecated(); @@ -95,7 +94,7 @@ public class ConsoleReportTest { @Test public void testOneNewIssue() { settings.setProperty(ConsoleReport.CONSOLE_REPORT_ENABLED_KEY, "true"); - when(inputPathCache.allFiles()).thenReturn(Arrays.<InputFile>asList(new TestInputFileBuilder("foo", "src/Foo.php").build())); + when(inputPathCache.allFilesToPublish()).thenReturn(Collections.singleton(new TestInputFileBuilder("foo", "src/Foo.php").build())); when(issueCache.all()).thenReturn(Arrays.asList(createIssue(true, Severity.BLOCKER))); report.execute(); assertDeprecated(); @@ -109,7 +108,7 @@ public class ConsoleReportTest { @Test public void testOneNewIssuePerSeverity() { settings.setProperty(ConsoleReport.CONSOLE_REPORT_ENABLED_KEY, "true"); - when(inputPathCache.allFiles()).thenReturn(Arrays.<InputFile>asList(new TestInputFileBuilder("foo", "src/Foo.php").build())); + when(inputPathCache.allFilesToPublish()).thenReturn(Collections.singleton(new TestInputFileBuilder("foo", "src/Foo.php").build())); when(issueCache.all()).thenReturn(Arrays.asList( createIssue(true, Severity.BLOCKER), createIssue(true, Severity.CRITICAL), diff --git a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/JSONReportTest.java b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/JSONReportTest.java index 01cce65f395..e464770bb66 100644 --- a/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/JSONReportTest.java +++ b/sonar-scanner-engine/src/test/java/org/sonar/scanner/scan/report/JSONReportTest.java @@ -19,7 +19,6 @@ */ package org.sonar.scanner.scan.report; -import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; import java.io.StringWriter; @@ -31,7 +30,6 @@ import org.apache.commons.io.IOUtils; import org.junit.Before; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.sonar.api.batch.fs.InputDir; import org.sonar.api.batch.fs.InputFile; import org.sonar.api.batch.fs.internal.DefaultFileSystem; import org.sonar.api.batch.fs.internal.DefaultInputDir; @@ -86,8 +84,8 @@ public class JSONReportTest { DefaultInputFile inputFile = new TestInputFileBuilder("struts", "src/main/java/org/apache/struts/Action.java").build(); inputFile.setStatus(InputFile.Status.CHANGED); InputComponentStore fileCache = mock(InputComponentStore.class); - when(fileCache.allFiles()).thenReturn(Arrays.<InputFile>asList(inputFile)); - when(fileCache.allDirs()).thenReturn(Arrays.<InputDir>asList(inputDir)); + when(fileCache.allFilesToPublish()).thenReturn(Collections.singleton(inputFile)); + when(fileCache.allDirs()).thenReturn(Collections.singleton(inputDir)); DefaultInputModule rootModule = new DefaultInputModule("struts"); DefaultInputModule moduleA = new DefaultInputModule("struts-core"); @@ -123,7 +121,7 @@ public class JSONReportTest { issue.setAssignee("simon"); issue.setCreationDate(SIMPLE_DATE_FORMAT.parse("2013-04-24")); issue.setNew(false); - when(issueCache.all()).thenReturn(Lists.newArrayList(issue)); + when(issueCache.all()).thenReturn(Collections.singleton(issue)); ScannerInput.User user = ScannerInput.User.newBuilder().setLogin("simon").setName("Simon").build(); when(userRepository.load("simon")).thenReturn(user); @@ -144,7 +142,7 @@ public class JSONReportTest { issue.setResolution(Issue.RESOLUTION_FIXED); issue.setCreationDate(SIMPLE_DATE_FORMAT.parse("2013-04-24")); issue.setNew(false); - when(issueCache.all()).thenReturn(Lists.newArrayList(issue)); + when(issueCache.all()).thenReturn(Collections.singleton(issue)); StringWriter writer = new StringWriter(); jsonReport.writeJson(writer); |