diff options
author | Julien HENRY <julien.henry@sonarsource.com> | 2013-10-02 16:00:16 +0200 |
---|---|---|
committer | Julien HENRY <julien.henry@sonarsource.com> | 2013-10-02 16:10:29 +0200 |
commit | d1ed91b890bfec5df267fff19faca781848be242 (patch) | |
tree | 12d724f59edc85c9bdc54d2c9fc5f4cde773b57e | |
parent | f6af69b24e93bb1262b9c02d8865e88cffa8d9a5 (diff) | |
download | sonarqube-d1ed91b890bfec5df267fff19faca781848be242.tar.gz sonarqube-d1ed91b890bfec5df267fff19faca781848be242.zip |
SONAR-2657 Expose changed files API in ModuleFileSystem
19 files changed, 575 insertions, 320 deletions
diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java index 4fcf1dc68b3..adea1718036 100644 --- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java +++ b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/CorePlugin.java @@ -19,11 +19,12 @@ */ package org.sonar.plugins.core; -import org.sonar.plugins.core.batch.PartialScanFilter; - -import org.sonar.plugins.core.utils.HashBuilder; import com.google.common.collect.ImmutableList; -import org.sonar.api.*; +import org.sonar.api.CoreProperties; +import org.sonar.api.Properties; +import org.sonar.api.Property; +import org.sonar.api.PropertyType; +import org.sonar.api.SonarPlugin; import org.sonar.api.checks.NoSonarFilter; import org.sonar.api.config.PropertyDefinition; import org.sonar.api.resources.Java; @@ -35,21 +36,91 @@ import org.sonar.plugins.core.charts.DistributionAreaChart; import org.sonar.plugins.core.charts.DistributionBarChart; import org.sonar.plugins.core.charts.XradarChart; import org.sonar.plugins.core.colorizers.JavaColorizerFormat; -import org.sonar.plugins.core.dashboards.*; -import org.sonar.plugins.core.issue.*; +import org.sonar.plugins.core.dashboards.GlobalDefaultDashboard; +import org.sonar.plugins.core.dashboards.ProjectDefaultDashboard; +import org.sonar.plugins.core.dashboards.ProjectHotspotDashboard; +import org.sonar.plugins.core.dashboards.ProjectIssuesDashboard; +import org.sonar.plugins.core.dashboards.ProjectTimeMachineDashboard; +import org.sonar.plugins.core.issue.CountFalsePositivesDecorator; +import org.sonar.plugins.core.issue.CountUnresolvedIssuesDecorator; +import org.sonar.plugins.core.issue.InitialOpenIssuesSensor; +import org.sonar.plugins.core.issue.InitialOpenIssuesStack; +import org.sonar.plugins.core.issue.IssueHandlers; +import org.sonar.plugins.core.issue.IssueTracking; +import org.sonar.plugins.core.issue.IssueTrackingDecorator; +import org.sonar.plugins.core.issue.IssuesDensityDecorator; +import org.sonar.plugins.core.issue.WeightedIssuesDecorator; import org.sonar.plugins.core.issue.ignore.IgnoreIssuesPlugin; -import org.sonar.plugins.core.issue.notification.*; +import org.sonar.plugins.core.issue.notification.ChangesOnMyIssueNotificationDispatcher; +import org.sonar.plugins.core.issue.notification.IssueChangesEmailTemplate; +import org.sonar.plugins.core.issue.notification.NewFalsePositiveNotificationDispatcher; +import org.sonar.plugins.core.issue.notification.NewIssuesEmailTemplate; +import org.sonar.plugins.core.issue.notification.NewIssuesNotificationDispatcher; +import org.sonar.plugins.core.issue.notification.SendIssueNotificationsPostJob; import org.sonar.plugins.core.measurefilters.MyFavouritesFilter; import org.sonar.plugins.core.measurefilters.ProjectFilter; import org.sonar.plugins.core.notifications.alerts.NewAlerts; import org.sonar.plugins.core.security.ApplyProjectRolesDecorator; -import org.sonar.plugins.core.sensors.*; +import org.sonar.plugins.core.sensors.BranchCoverageDecorator; +import org.sonar.plugins.core.sensors.CheckAlertThresholds; +import org.sonar.plugins.core.sensors.CommentDensityDecorator; +import org.sonar.plugins.core.sensors.CoverageDecorator; +import org.sonar.plugins.core.sensors.CoverageMeasurementFilter; +import org.sonar.plugins.core.sensors.DirectoriesDecorator; +import org.sonar.plugins.core.sensors.FileHashSensor; +import org.sonar.plugins.core.sensors.FilesDecorator; +import org.sonar.plugins.core.sensors.GenerateAlertEvents; +import org.sonar.plugins.core.sensors.ItBranchCoverageDecorator; +import org.sonar.plugins.core.sensors.ItCoverageDecorator; +import org.sonar.plugins.core.sensors.ItLineCoverageDecorator; +import org.sonar.plugins.core.sensors.LineCoverageDecorator; +import org.sonar.plugins.core.sensors.ManualMeasureDecorator; +import org.sonar.plugins.core.sensors.OverallBranchCoverageDecorator; +import org.sonar.plugins.core.sensors.OverallCoverageDecorator; +import org.sonar.plugins.core.sensors.OverallLineCoverageDecorator; +import org.sonar.plugins.core.sensors.ProfileEventsSensor; +import org.sonar.plugins.core.sensors.ProfileSensor; +import org.sonar.plugins.core.sensors.ProjectLinksSensor; +import org.sonar.plugins.core.sensors.UnitTestDecorator; +import org.sonar.plugins.core.sensors.VersionEventsSensor; import org.sonar.plugins.core.technicaldebt.TechnicalDebtDecorator; -import org.sonar.plugins.core.timemachine.*; +import org.sonar.plugins.core.timemachine.NewCoverageAggregator; +import org.sonar.plugins.core.timemachine.NewCoverageFileAnalyzer; +import org.sonar.plugins.core.timemachine.NewItCoverageFileAnalyzer; +import org.sonar.plugins.core.timemachine.NewOverallCoverageFileAnalyzer; +import org.sonar.plugins.core.timemachine.TendencyDecorator; +import org.sonar.plugins.core.timemachine.TimeMachineConfigurationPersister; +import org.sonar.plugins.core.timemachine.VariationDecorator; import org.sonar.plugins.core.web.Lcom4Viewer; import org.sonar.plugins.core.web.TestsViewer; -import org.sonar.plugins.core.widgets.*; -import org.sonar.plugins.core.widgets.issues.*; +import org.sonar.plugins.core.widgets.AlertsWidget; +import org.sonar.plugins.core.widgets.ComplexityWidget; +import org.sonar.plugins.core.widgets.CoverageWidget; +import org.sonar.plugins.core.widgets.CustomMeasuresWidget; +import org.sonar.plugins.core.widgets.DescriptionWidget; +import org.sonar.plugins.core.widgets.DocumentationCommentsWidget; +import org.sonar.plugins.core.widgets.DuplicationsWidget; +import org.sonar.plugins.core.widgets.EventsWidget; +import org.sonar.plugins.core.widgets.HotspotMetricWidget; +import org.sonar.plugins.core.widgets.HotspotMostViolatedResourcesWidget; +import org.sonar.plugins.core.widgets.HotspotMostViolatedRulesWidget; +import org.sonar.plugins.core.widgets.ItCoverageWidget; +import org.sonar.plugins.core.widgets.MeasureFilterListWidget; +import org.sonar.plugins.core.widgets.MeasureFilterTreemapWidget; +import org.sonar.plugins.core.widgets.RulesWidget; +import org.sonar.plugins.core.widgets.SizeWidget; +import org.sonar.plugins.core.widgets.TechnicalDebtPyramidWidget; +import org.sonar.plugins.core.widgets.TechnicalDebtWidget; +import org.sonar.plugins.core.widgets.TimeMachineWidget; +import org.sonar.plugins.core.widgets.TimelineWidget; +import org.sonar.plugins.core.widgets.TreemapWidget; +import org.sonar.plugins.core.widgets.WelcomeWidget; +import org.sonar.plugins.core.widgets.issues.ActionPlansWidget; +import org.sonar.plugins.core.widgets.issues.FalsePositiveIssuesWidget; +import org.sonar.plugins.core.widgets.issues.IssueFilterWidget; +import org.sonar.plugins.core.widgets.issues.MyUnresolvedIssuesWidget; +import org.sonar.plugins.core.widgets.issues.UnresolvedIssuesPerAssigneeWidget; +import org.sonar.plugins.core.widgets.issues.UnresolvedIssuesStatusesWidget; import java.util.Arrays; import java.util.List; @@ -209,127 +280,125 @@ public final class CorePlugin extends SonarPlugin { ImmutableList.Builder<Object> extensions = ImmutableList.builder(); extensions.add( - DefaultResourceTypes.class, - UserManagedMetrics.class, - Periods.class, + DefaultResourceTypes.class, + UserManagedMetrics.class, + Periods.class, - // pages - Lcom4Viewer.class, - TestsViewer.class, + // pages + Lcom4Viewer.class, + TestsViewer.class, - // measure filters - ProjectFilter.class, - MyFavouritesFilter.class, + // measure filters + ProjectFilter.class, + MyFavouritesFilter.class, - // widgets - AlertsWidget.class, - CoverageWidget.class, - ItCoverageWidget.class, - DescriptionWidget.class, - ComplexityWidget.class, - RulesWidget.class, - SizeWidget.class, - EventsWidget.class, - CustomMeasuresWidget.class, - TimelineWidget.class, - TimeMachineWidget.class, - HotspotMetricWidget.class, - TreemapWidget.class, - MeasureFilterListWidget.class, - MeasureFilterTreemapWidget.class, - WelcomeWidget.class, - DocumentationCommentsWidget.class, - DuplicationsWidget.class, - TechnicalDebtWidget.class, - TechnicalDebtPyramidWidget.class, + // widgets + AlertsWidget.class, + CoverageWidget.class, + ItCoverageWidget.class, + DescriptionWidget.class, + ComplexityWidget.class, + RulesWidget.class, + SizeWidget.class, + EventsWidget.class, + CustomMeasuresWidget.class, + TimelineWidget.class, + TimeMachineWidget.class, + HotspotMetricWidget.class, + TreemapWidget.class, + MeasureFilterListWidget.class, + MeasureFilterTreemapWidget.class, + WelcomeWidget.class, + DocumentationCommentsWidget.class, + DuplicationsWidget.class, + TechnicalDebtWidget.class, + TechnicalDebtPyramidWidget.class, - // dashboards - ProjectDefaultDashboard.class, - ProjectHotspotDashboard.class, - ProjectIssuesDashboard.class, - ProjectTimeMachineDashboard.class, - GlobalDefaultDashboard.class, + // dashboards + ProjectDefaultDashboard.class, + ProjectHotspotDashboard.class, + ProjectIssuesDashboard.class, + ProjectTimeMachineDashboard.class, + GlobalDefaultDashboard.class, - // chart - XradarChart.class, - DistributionBarChart.class, - DistributionAreaChart.class, + // chart + XradarChart.class, + DistributionBarChart.class, + DistributionAreaChart.class, - // colorizers - JavaColorizerFormat.class, + // colorizers + JavaColorizerFormat.class, - // issues - IssueTrackingDecorator.class, - IssueTracking.class, - IssueHandlers.class, - CountUnresolvedIssuesDecorator.class, - CountFalsePositivesDecorator.class, - WeightedIssuesDecorator.class, - IssuesDensityDecorator.class, - InitialOpenIssuesSensor.class, - InitialOpenIssuesStack.class, - HotspotMostViolatedResourcesWidget.class, - HotspotMostViolatedRulesWidget.class, - MyUnresolvedIssuesWidget.class, - FalsePositiveIssuesWidget.class, - ActionPlansWidget.class, - UnresolvedIssuesPerAssigneeWidget.class, - UnresolvedIssuesStatusesWidget.class, - IssueFilterWidget.class, - org.sonar.api.issue.NoSonarFilter.class, + // issues + IssueTrackingDecorator.class, + IssueTracking.class, + IssueHandlers.class, + CountUnresolvedIssuesDecorator.class, + CountFalsePositivesDecorator.class, + WeightedIssuesDecorator.class, + IssuesDensityDecorator.class, + InitialOpenIssuesSensor.class, + InitialOpenIssuesStack.class, + HotspotMostViolatedResourcesWidget.class, + HotspotMostViolatedRulesWidget.class, + MyUnresolvedIssuesWidget.class, + FalsePositiveIssuesWidget.class, + ActionPlansWidget.class, + UnresolvedIssuesPerAssigneeWidget.class, + UnresolvedIssuesStatusesWidget.class, + IssueFilterWidget.class, + org.sonar.api.issue.NoSonarFilter.class, - // issue notifications - SendIssueNotificationsPostJob.class, - NewIssuesEmailTemplate.class, - IssueChangesEmailTemplate.class, - ChangesOnMyIssueNotificationDispatcher.class, - ChangesOnMyIssueNotificationDispatcher.newMetadata(), - NewIssuesNotificationDispatcher.class, - NewIssuesNotificationDispatcher.newMetadata(), - NewFalsePositiveNotificationDispatcher.class, - NewFalsePositiveNotificationDispatcher.newMetadata(), + // issue notifications + SendIssueNotificationsPostJob.class, + NewIssuesEmailTemplate.class, + IssueChangesEmailTemplate.class, + ChangesOnMyIssueNotificationDispatcher.class, + ChangesOnMyIssueNotificationDispatcher.newMetadata(), + NewIssuesNotificationDispatcher.class, + NewIssuesNotificationDispatcher.newMetadata(), + NewFalsePositiveNotificationDispatcher.class, + NewFalsePositiveNotificationDispatcher.newMetadata(), - // batch - ProfileSensor.class, - ProfileEventsSensor.class, - ProjectLinksSensor.class, - UnitTestDecorator.class, - VersionEventsSensor.class, - CheckAlertThresholds.class, - GenerateAlertEvents.class, - LineCoverageDecorator.class, - CoverageDecorator.class, - BranchCoverageDecorator.class, - ItLineCoverageDecorator.class, - ItCoverageDecorator.class, - ItBranchCoverageDecorator.class, - OverallLineCoverageDecorator.class, - OverallCoverageDecorator.class, - OverallBranchCoverageDecorator.class, - CoverageMeasurementFilter.class, - ApplyProjectRolesDecorator.class, - CommentDensityDecorator.class, - NoSonarFilter.class, - DirectoriesDecorator.class, - FilesDecorator.class, - IndexProjectPostJob.class, - ManualMeasureDecorator.class, - HashBuilder.class, - FileHashSensor.class, - PartialScanFilter.class, + // batch + ProfileSensor.class, + ProfileEventsSensor.class, + ProjectLinksSensor.class, + UnitTestDecorator.class, + VersionEventsSensor.class, + CheckAlertThresholds.class, + GenerateAlertEvents.class, + LineCoverageDecorator.class, + CoverageDecorator.class, + BranchCoverageDecorator.class, + ItLineCoverageDecorator.class, + ItCoverageDecorator.class, + ItBranchCoverageDecorator.class, + OverallLineCoverageDecorator.class, + OverallCoverageDecorator.class, + OverallBranchCoverageDecorator.class, + CoverageMeasurementFilter.class, + ApplyProjectRolesDecorator.class, + CommentDensityDecorator.class, + NoSonarFilter.class, + DirectoriesDecorator.class, + FilesDecorator.class, + IndexProjectPostJob.class, + ManualMeasureDecorator.class, + FileHashSensor.class, - // time machine - TendencyDecorator.class, - VariationDecorator.class, - TimeMachineConfigurationPersister.class, - NewCoverageFileAnalyzer.class, - NewItCoverageFileAnalyzer.class, - NewOverallCoverageFileAnalyzer.class, - NewCoverageAggregator.class, + // time machine + TendencyDecorator.class, + VariationDecorator.class, + TimeMachineConfigurationPersister.class, + NewCoverageFileAnalyzer.class, + NewItCoverageFileAnalyzer.class, + NewOverallCoverageFileAnalyzer.class, + NewCoverageAggregator.class, - // Notify alerts on my favourite projects - NewAlerts.class, - NewAlerts.newMetadata()); + // Notify alerts on my favourite projects + NewAlerts.class, + NewAlerts.newMetadata()); extensions.addAll(ExclusionProperties.definitions()); extensions.addAll(IgnoreIssuesPlugin.getExtensions()); @@ -341,7 +410,6 @@ public final class CorePlugin extends SonarPlugin { return extensions.build(); } - static List<PropertyDefinition> propertyDefinitions() { return Arrays.asList( PropertyDefinition.builder(CoreProperties.CORE_VIOLATION_LOCALE_PROPERTY) diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/sensors/FileHashSensor.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/sensors/FileHashSensor.java index 8e3f8d0600e..93c943a3490 100644 --- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/sensors/FileHashSensor.java +++ b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/sensors/FileHashSensor.java @@ -27,30 +27,30 @@ import org.sonar.api.scan.filesystem.FileType; import org.sonar.api.scan.filesystem.ModuleFileSystem; import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.batch.index.ComponentDataCache; +import org.sonar.batch.scan.filesystem.FileHashCache; import org.sonar.core.source.SnapshotDataType; -import org.sonar.plugins.core.utils.HashBuilder; import java.io.File; import java.util.List; /** - * This sensor will compute md5 checksum of each file of the current module and store it in DB + * This sensor will retrieve hash of each file of the current module and store it in DB * in order to compare it during next analysis and see if the file was modified. - * This is used by the partial analysis mode. - * @see org.sonar.plugins.core.batch.PartialScanFilter + * This is used by the incremental preview mode. + * @see org.sonar.plugins.core.batch.IncrementalPreviewFilter * @since 4.0 */ public final class FileHashSensor implements Sensor { private ModuleFileSystem moduleFileSystem; private PathResolver pathResolver; - private HashBuilder hashBuilder; private ComponentDataCache componentDataCache; + private FileHashCache fileHashCache; - public FileHashSensor(ModuleFileSystem moduleFileSystem, PathResolver pathResolver, HashBuilder hashBuilder, ComponentDataCache componentDataCache) { + public FileHashSensor(FileHashCache fileHashCache, ModuleFileSystem moduleFileSystem, PathResolver pathResolver, ComponentDataCache componentDataCache) { + this.fileHashCache = fileHashCache; this.moduleFileSystem = moduleFileSystem; this.pathResolver = pathResolver; - this.hashBuilder = hashBuilder; this.componentDataCache = componentDataCache; } @@ -69,8 +69,8 @@ public final class FileHashSensor implements Sensor { private void analyse(StringBuilder fileHashMap, Project project, FileType fileType) { List<File> files = moduleFileSystem.files(FileQuery.on(fileType).onLanguage(project.getLanguageKey())); for (File file : files) { - String md5 = hashBuilder.computeHash(file); - fileHashMap.append(pathResolver.relativePath(moduleFileSystem.baseDir(), file)).append("=").append(md5).append("\n"); + String hash = fileHashCache.getCurrentHash(file); + fileHashMap.append(pathResolver.relativePath(moduleFileSystem.baseDir(), file)).append("=").append(hash).append("\n"); } } diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/utils/package-info.java b/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/utils/package-info.java deleted file mode 100644 index 6d0f75f915b..00000000000 --- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/utils/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2013 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.plugins.core.utils; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/sensors/FileHashSensorTest.java b/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/sensors/FileHashSensorTest.java index 0f3d59db57b..a7bbfa49697 100644 --- a/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/sensors/FileHashSensorTest.java +++ b/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/sensors/FileHashSensorTest.java @@ -33,7 +33,7 @@ import org.sonar.api.scan.filesystem.FileQuery; import org.sonar.api.scan.filesystem.ModuleFileSystem; import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.batch.index.ComponentDataCache; -import org.sonar.plugins.core.utils.HashBuilder; +import org.sonar.batch.scan.filesystem.FileHashCache; import java.io.File; import java.util.Arrays; @@ -61,11 +61,14 @@ public class FileHashSensorTest { private Project project; + private FileHashCache fileHashCache; + @Before public void prepare() { fileSystem = mock(ModuleFileSystem.class); componentDataCache = mock(ComponentDataCache.class); - sensor = new FileHashSensor(fileSystem, new PathResolver(), new HashBuilder(), componentDataCache); + fileHashCache = mock(FileHashCache.class); + sensor = new FileHashSensor(fileHashCache, fileSystem, new PathResolver(), componentDataCache); PropertiesConfiguration conf = new PropertiesConfiguration(); conf.setProperty("sonar.language", "java"); project = new Project("java_project").setConfiguration(conf).setLanguage(Java.INSTANCE); @@ -82,15 +85,16 @@ public class FileHashSensorTest { File baseDir = temp.newFolder(); File file1 = new File(baseDir, "src/com/foo/Bar.java"); FileUtils.write(file1, "Bar"); + when(fileHashCache.getCurrentHash(file1)).thenReturn("barhash"); File file2 = new File(baseDir, "src/com/foo/Foo.java"); FileUtils.write(file2, "Foo"); + when(fileHashCache.getCurrentHash(file2)).thenReturn("foohash"); when(fileSystem.baseDir()).thenReturn(baseDir); when(fileSystem.files(any(FileQuery.class))).thenReturn(Arrays.asList(file1, file2)).thenReturn(Collections.<File> emptyList()); sensor.analyse(project, mock(SensorContext.class)); verify(componentDataCache).setStringData("java_project", "hash", - "src/com/foo/Bar.java=ddc35f88fa71b6ef142ae61f35364653\n" - + "src/com/foo/Foo.java=1356c67d7ad1638d816bfb822dd2c25d\n"); + "src/com/foo/Bar.java=barhash\n" + + "src/com/foo/Foo.java=foohash\n"); } - } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java index b97118be253..37ef6fcedb4 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java @@ -51,6 +51,7 @@ import org.sonar.batch.phases.PhaseExecutor; import org.sonar.batch.phases.PhasesTimeProfiler; import org.sonar.batch.scan.filesystem.DeprecatedFileSystemAdapter; import org.sonar.batch.scan.filesystem.ExclusionFilters; +import org.sonar.batch.scan.filesystem.FileHashCache; import org.sonar.batch.scan.filesystem.FileSystemLogger; import org.sonar.batch.scan.filesystem.LanguageFilters; import org.sonar.batch.scan.filesystem.ModuleFileSystemProvider; @@ -104,6 +105,7 @@ public class ModuleScanContainer extends ComponentContainer { new ModuleFileSystemProvider(), DeprecatedFileSystemAdapter.class, FileSystemLogger.class, + FileHashCache.class, // the Snapshot component will be removed when asynchronous measures are improved (required for AsynchronousMeasureSensor) getComponentByType(ResourcePersister.class).getSnapshot(module), diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java index ec79143de77..0615528efa0 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java @@ -33,11 +33,33 @@ import org.sonar.batch.DefaultFileLinesContextFactory; import org.sonar.batch.DefaultResourceCreationLock; import org.sonar.batch.ProjectConfigurator; import org.sonar.batch.ProjectTree; -import org.sonar.batch.bootstrap.*; -import org.sonar.batch.index.*; -import org.sonar.batch.issue.*; +import org.sonar.batch.bootstrap.BootstrapSettings; +import org.sonar.batch.bootstrap.ExtensionInstaller; +import org.sonar.batch.bootstrap.ExtensionMatcher; +import org.sonar.batch.bootstrap.ExtensionUtils; +import org.sonar.batch.bootstrap.MetricProvider; +import org.sonar.batch.index.Caches; +import org.sonar.batch.index.ComponentDataCache; +import org.sonar.batch.index.ComponentDataPersister; +import org.sonar.batch.index.DefaultIndex; +import org.sonar.batch.index.DefaultPersistenceManager; +import org.sonar.batch.index.DefaultResourcePersister; +import org.sonar.batch.index.DependencyPersister; +import org.sonar.batch.index.EventPersister; +import org.sonar.batch.index.LinkPersister; +import org.sonar.batch.index.MeasurePersister; +import org.sonar.batch.index.MemoryOptimizer; +import org.sonar.batch.index.ResourceCache; +import org.sonar.batch.index.SnapshotCache; +import org.sonar.batch.index.SourcePersister; +import org.sonar.batch.issue.DefaultProjectIssues; +import org.sonar.batch.issue.DeprecatedViolations; +import org.sonar.batch.issue.IssueCache; +import org.sonar.batch.issue.IssuePersister; +import org.sonar.batch.issue.ScanIssueStorage; import org.sonar.batch.phases.GraphPersister; import org.sonar.batch.profiling.PhasesSumUpTimeProfiler; +import org.sonar.batch.scan.filesystem.HashBuilder; import org.sonar.batch.scan.maven.FakeMavenPluginExecutor; import org.sonar.batch.scan.maven.MavenPluginExecutor; import org.sonar.batch.source.HighlightableBuilder; @@ -50,7 +72,11 @@ import org.sonar.core.issue.workflow.IssueWorkflow; import org.sonar.core.notification.DefaultNotificationManager; import org.sonar.core.technicaldebt.TechnicalDebtModel; import org.sonar.core.technicaldebt.WorkUnitConverter; -import org.sonar.core.technicaldebt.functions.*; +import org.sonar.core.technicaldebt.functions.ConstantFunction; +import org.sonar.core.technicaldebt.functions.Functions; +import org.sonar.core.technicaldebt.functions.LinearFunction; +import org.sonar.core.technicaldebt.functions.LinearWithOffsetFunction; +import org.sonar.core.technicaldebt.functions.LinearWithThresholdFunction; import org.sonar.core.test.TestPlanBuilder; import org.sonar.core.test.TestPlanPerspectiveLoader; import org.sonar.core.test.TestableBuilder; @@ -116,6 +142,7 @@ public class ProjectScanContainer extends ComponentContainer { ResourceCache.class, ComponentDataCache.class, ComponentDataPersister.class, + HashBuilder.class, // issues IssueUpdater.class, diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/utils/HashBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ChangedFileFilter.java index 87b78a11c77..af0968d4137 100644 --- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/utils/HashBuilder.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ChangedFileFilter.java @@ -17,30 +17,32 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.plugins.core.utils; +package org.sonar.batch.scan.filesystem; -import org.apache.commons.io.IOUtils; -import org.sonar.api.BatchExtension; -import org.sonar.api.utils.SonarException; +import org.sonar.api.scan.filesystem.FileSystemFilter; import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; /** + * When enabled this filter will only allow modified files to be analyzed. * @since 4.0 */ -public final class HashBuilder implements BatchExtension { +class ChangedFileFilter implements FileSystemFilter { - public String computeHash(File file) { - FileInputStream fis = null; - try { - fis = new FileInputStream(file); - return org.apache.commons.codec.digest.DigestUtils.md5Hex(fis); - } catch (IOException e) { - throw new SonarException("Unable to compute file hash", e); - } finally { - IOUtils.closeQuietly(fis); + private FileHashCache fileHashCache; + + public ChangedFileFilter(FileHashCache fileHashCache) { + this.fileHashCache = fileHashCache; + } + + @Override + public boolean accept(File file, Context context) { + String previousHash = fileHashCache.getPreviousHash(file); + if (previousHash == null) { + return true; } + String currentHash = fileHashCache.getCurrentHash(file); + return !currentHash.equals(previousHash); } + } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java index 22ab1f9d892..1d0c76a7bee 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java @@ -34,6 +34,7 @@ import org.sonar.api.scan.filesystem.FileSystemFilter; import org.sonar.api.scan.filesystem.FileType; import org.sonar.api.scan.filesystem.ModuleFileSystem; import org.sonar.api.scan.filesystem.PathResolver; +import org.sonar.api.utils.SonarException; import java.io.File; import java.io.FileFilter; @@ -58,8 +59,10 @@ public class DefaultModuleFileSystem implements ModuleFileSystem { private PathResolver pathResolver = new PathResolver(); private List<FileSystemFilter> fsFilters = Lists.newArrayList(); private LanguageFilters languageFilters; + private FileHashCache fileHashCache; - DefaultModuleFileSystem() { + DefaultModuleFileSystem(FileHashCache fileHashCache) { + this.fileHashCache = fileHashCache; } public File baseDir() { @@ -110,7 +113,26 @@ public class DefaultModuleFileSystem implements ModuleFileSystem { } public List<File> files(FileQuery query) { + boolean changedFilesOnly = false; + if (settings.getBoolean(CoreProperties.INCREMENTAL_PREVIEW)) { + if (!settings.getBoolean(CoreProperties.DRY_RUN)) { + throw new SonarException("Incremental preview is only supported with preview mode"); + } + changedFilesOnly = true; + } + return files(query, changedFilesOnly); + } + + @Override + public List<File> changedFiles(FileQuery query) { + return files(query, true); + } + + private List<File> files(FileQuery query, boolean changedFilesOnly) { List<FileSystemFilter> filters = Lists.newArrayList(fsFilters); + if (changedFilesOnly) { + filters.add(new ChangedFileFilter(fileHashCache)); + } for (FileFilter fileFilter : query.filters()) { filters.add(new FileFilterWrapper(fileFilter)); } @@ -142,7 +164,7 @@ public class DefaultModuleFileSystem implements ModuleFileSystem { } private void applyFilters(List<File> result, FileFilterContext context, - Collection<FileSystemFilter> filters, Collection<File> dirs) { + Collection<FileSystemFilter> filters, Collection<File> dirs) { for (File dir : dirs) { if (dir.exists()) { context.setRelativeDir(dir); diff --git a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/batch/PartialScanFilter.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileHashCache.java index 903a5ca36e4..34452fa2305 100644 --- a/plugins/sonar-core-plugin/src/main/java/org/sonar/plugins/core/batch/PartialScanFilter.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileHashCache.java @@ -17,17 +17,16 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.plugins.core.batch; +package org.sonar.batch.scan.filesystem; import com.google.common.collect.Maps; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.picocontainer.Startable; -import org.sonar.api.CoreProperties; +import org.sonar.api.BatchComponent; import org.sonar.api.batch.bootstrap.ProjectDefinition; -import org.sonar.api.config.Settings; import org.sonar.api.database.model.Snapshot; -import org.sonar.api.scan.filesystem.FileSystemFilter; +import org.sonar.api.scan.filesystem.ModuleFileSystem; import org.sonar.api.scan.filesystem.PathResolver; import org.sonar.api.utils.SonarException; import org.sonar.batch.components.PastSnapshot; @@ -35,7 +34,8 @@ import org.sonar.batch.components.PastSnapshotFinder; import org.sonar.core.source.SnapshotDataType; import org.sonar.core.source.jdbc.SnapshotDataDao; import org.sonar.core.source.jdbc.SnapshotDataDto; -import org.sonar.plugins.core.utils.HashBuilder; + +import javax.annotation.CheckForNull; import java.io.File; import java.io.IOException; @@ -45,27 +45,24 @@ import java.util.Collection; import java.util.List; import java.util.Map; -/** - * When enabled this filter will only allow modified files to be analyzed. - * @since 4.0 - */ -public class PartialScanFilter implements FileSystemFilter, Startable { +public class FileHashCache implements BatchComponent, Startable { + + private Map<String, String> currentHashCache = Maps.newHashMap(); + private Map<String, String> previousHashCache = Maps.newHashMap(); private PathResolver pathResolver; private HashBuilder hashBuilder; - private Settings settings; private SnapshotDataDao snapshotDataDao; private PastSnapshotFinder pastSnapshotFinder; private Snapshot snapshot; private ProjectDefinition module; + private ModuleFileSystem fs; - private Map<String, String> fileHashMap = Maps.newHashMap(); - - public PartialScanFilter(Settings settings, ProjectDefinition module, PathResolver pathResolver, HashBuilder hashBuilder, + public FileHashCache(ModuleFileSystem fs, ProjectDefinition module, PathResolver pathResolver, HashBuilder hashBuilder, Snapshot snapshot, SnapshotDataDao snapshotDataDao, PastSnapshotFinder pastSnapshotFinder) { - this.settings = settings; + this.fs = fs; this.module = module; this.pathResolver = pathResolver; this.hashBuilder = hashBuilder; @@ -76,51 +73,44 @@ public class PartialScanFilter implements FileSystemFilter, Startable { @Override public void start() { - // Extract previous checksum of all files of this module and store - // them in a map - if (settings.getBoolean(CoreProperties.PARTIAL_ANALYSIS)) { - if (!settings.getBoolean(CoreProperties.DRY_RUN)) { - throw new SonarException("Partial analysis is only supported with dry run mode"); - } - PastSnapshot pastSnapshot = pastSnapshotFinder.findPreviousAnalysis(snapshot); - if (pastSnapshot.isRelatedToSnapshot()) { - Collection<SnapshotDataDto> selectSnapshotData = snapshotDataDao.selectSnapshotData(pastSnapshot.getProjectSnapshot().getId().longValue(), - Arrays.asList(SnapshotDataType.FILE_HASH.getValue())); - if (!selectSnapshotData.isEmpty()) { - SnapshotDataDto snapshotDataDto = selectSnapshotData.iterator().next(); - String data = snapshotDataDto.getData(); - try { - List<String> lines = IOUtils.readLines(new StringReader(data)); - for (String line : lines) { - String[] keyValue = StringUtils.split(line, "="); - if (keyValue.length == 2) { - fileHashMap.put(keyValue[0], keyValue[1]); - } + // Extract previous checksum of all files of this module and store them in a map + PastSnapshot pastSnapshot = pastSnapshotFinder.findPreviousAnalysis(snapshot); + if (pastSnapshot.isRelatedToSnapshot()) { + Collection<SnapshotDataDto> selectSnapshotData = snapshotDataDao.selectSnapshotData(pastSnapshot.getProjectSnapshot().getId().longValue(), + Arrays.asList(SnapshotDataType.FILE_HASH.getValue())); + if (!selectSnapshotData.isEmpty()) { + SnapshotDataDto snapshotDataDto = selectSnapshotData.iterator().next(); + String data = snapshotDataDto.getData(); + try { + List<String> lines = IOUtils.readLines(new StringReader(data)); + for (String line : lines) { + String[] keyValue = StringUtils.split(line, "="); + if (keyValue.length == 2) { + previousHashCache.put(keyValue[0], keyValue[1]); } - } catch (IOException e) { - throw new SonarException("Unable to read previous file hashes", e); } + } catch (IOException e) { + throw new SonarException("Unable to read previous file hashes", e); } } } } - @Override - public void stop() { + public String getCurrentHash(File file) { + String relativePath = pathResolver.relativePath(module.getBaseDir(), file); + if (!currentHashCache.containsKey(relativePath)) { + currentHashCache.put(relativePath, hashBuilder.computeHashNormalizeLineEnds(file, fs.sourceCharset())); + } + return currentHashCache.get(relativePath); } - @Override - public boolean accept(File file, Context context) { - if (!settings.getBoolean(CoreProperties.PARTIAL_ANALYSIS)) { - return true; - } + @CheckForNull + public String getPreviousHash(File file) { String relativePath = pathResolver.relativePath(module.getBaseDir(), file); - String previousHash = fileHashMap.get(relativePath); - if (previousHash == null) { - return true; - } - String currentHash = hashBuilder.computeHash(file); - return !currentHash.equals(previousHash); + return previousHashCache.get(relativePath); } + @Override + public void stop() { + } } diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/HashBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/HashBuilder.java new file mode 100644 index 00000000000..2db7946e67c --- /dev/null +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/HashBuilder.java @@ -0,0 +1,91 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.batch.scan.filesystem; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.IOUtils; +import org.sonar.api.BatchExtension; +import org.sonar.api.batch.InstantiationStrategy; +import org.sonar.api.utils.SonarException; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.Charset; +import java.security.MessageDigest; + +/** + * @since 4.0 + */ +@InstantiationStrategy(InstantiationStrategy.PER_BATCH) +public final class HashBuilder implements BatchExtension { + + /** + * Compute hash of a file ignoring line ends differences. + * Maximum performance is needed. + */ + public String computeHashNormalizeLineEnds(File file, Charset charset) { + Reader reader = null; + try { + MessageDigest md5Digest = DigestUtils.getMd5Digest(); + md5Digest.reset(); + reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset)); + int i = reader.read(); + boolean afterCR = true; + while (i != -1) { + char c = (char) i; + if (afterCR) { + afterCR = false; + if (c == '\n') { + // Ignore + i = reader.read(); + continue; + } + } + if (c == '\r') { + afterCR = true; + c = '\n'; + } + md5Digest.update(charToBytesUTF(c)); + i = reader.read(); + } + return Hex.encodeHexString(md5Digest.digest()); + } catch (IOException e) { + throw new SonarException("Unable to compute file hash", e); + } finally { + IOUtils.closeQuietly(reader); + } + } + + public static byte[] charToBytesUTF(char c) { + char[] buffer = new char[] {c}; + byte[] b = new byte[buffer.length << 1]; + for (int i = 0; i < buffer.length; i++) { + int bpos = i << 1; + b[bpos] = (byte) ((buffer[i] & 0xFF00) >> 8); + b[bpos + 1] = (byte) (buffer[i] & 0x00FF); + } + return b; + } +} diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java index 1cb0bc76fe5..3efae8a0ec0 100644 --- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java +++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java @@ -40,9 +40,10 @@ public class ModuleFileSystemProvider extends ProviderAdapter { private DefaultModuleFileSystem singleton; public DefaultModuleFileSystem provide(ProjectDefinition module, PathResolver pathResolver, TempDirectories tempDirectories, - LanguageFilters languageFilters, Settings settings, FileSystemFilter[] pluginFileFilters) { + LanguageFilters languageFilters, Settings settings, FileSystemFilter[] pluginFileFilters, + FileHashCache fileHashCache) { if (singleton == null) { - DefaultModuleFileSystem fs = new DefaultModuleFileSystem(); + DefaultModuleFileSystem fs = new DefaultModuleFileSystem(fileHashCache); fs.setLanguageFilters(languageFilters); fs.setBaseDir(module.getBaseDir()); fs.setBuildDir(module.getBuildDir()); @@ -97,7 +98,6 @@ public class ModuleFileSystemProvider extends ProviderAdapter { } } - private void initBinaryDirs(ProjectDefinition module, PathResolver pathResolver, DefaultModuleFileSystem fs) { for (String path : module.getBinaries()) { File dir = pathResolver.relativeFile(module.getBaseDir(), path); diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java index c9e7ee4d478..30ca6a3fa6f 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystemTest.java @@ -22,6 +22,7 @@ package org.sonar.batch.scan.filesystem; import org.apache.commons.io.filefilter.FileFilterUtils; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.sonar.api.CoreProperties; import org.sonar.api.config.Settings; @@ -29,6 +30,7 @@ import org.sonar.api.resources.AbstractLanguage; import org.sonar.api.resources.Languages; import org.sonar.api.scan.filesystem.FileQuery; import org.sonar.api.scan.filesystem.FileSystemFilter; +import org.sonar.api.utils.SonarException; import java.io.File; import java.io.IOException; @@ -38,11 +40,15 @@ import java.util.List; import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class DefaultModuleFileSystemTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); + @Rule + public ExpectedException thrown = ExpectedException.none(); + @Test public void test_new_file_system() throws IOException { File basedir = temp.newFolder("base"); @@ -50,7 +56,7 @@ public class DefaultModuleFileSystemTest { LanguageFilters languageFilters = mock(LanguageFilters.class); FileSystemFilter fileFilter = mock(FileSystemFilter.class); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setWorkingDir(workingDir) .addBinaryDir(new File(basedir, "target/classes")) @@ -73,7 +79,7 @@ public class DefaultModuleFileSystemTest { @Test public void default_source_encoding() { File basedir = temp.newFolder("base"); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setSettings(new Settings()); @@ -86,7 +92,7 @@ public class DefaultModuleFileSystemTest { File basedir = temp.newFolder("base"); Settings settings = new Settings(); settings.setProperty(CoreProperties.ENCODING_PROPERTY, "UTF-8"); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setSettings(settings); @@ -97,10 +103,11 @@ public class DefaultModuleFileSystemTest { @Test public void should_exclude_dirs_starting_with_dot() throws IOException { File basedir = new File(resourcesDir(), "exclude_dir_starting_with_dot"); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setWorkingDir(temp.newFolder()) - .addSourceDir(new File(basedir, "src")); + .addSourceDir(new File(basedir, "src")) + .setSettings(new Settings()); List<File> files = fileSystem.files(FileQuery.onSource()); assertThat(files).hasSize(1); @@ -110,12 +117,13 @@ public class DefaultModuleFileSystemTest { @Test public void should_load_source_files_by_language() throws IOException { File basedir = new File(resourcesDir(), "main_and_test_files"); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setWorkingDir(temp.newFolder()) .addSourceDir(new File(basedir, "src/main/java")) .addTestDir(new File(basedir, "src/test/java")) - .setLanguageFilters(new LanguageFilters(new Languages(new Java(), new Php()))); + .setLanguageFilters(new LanguageFilters(new Languages(new Java(), new Php()))) + .setSettings(new Settings()); List<File> files = fileSystem.files(FileQuery.onSource().onLanguage("java")); assertThat(files).hasSize(2); @@ -129,11 +137,12 @@ public class DefaultModuleFileSystemTest { @Test public void should_load_test_files() throws IOException { File basedir = new File(resourcesDir(), "main_and_test_files"); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setWorkingDir(temp.newFolder()) .addSourceDir(new File(basedir, "src/main/java")) - .addTestDir(new File(basedir, "src/test/java")); + .addTestDir(new File(basedir, "src/test/java")) + .setSettings(new Settings()); assertThat(fileSystem.testDirs()).hasSize(1); List<File> testFiles = fileSystem.files(FileQuery.onTest()); @@ -147,12 +156,13 @@ public class DefaultModuleFileSystemTest { @Test public void should_load_test_files_by_language() throws IOException { File basedir = new File(resourcesDir(), "main_and_test_files"); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setWorkingDir(temp.newFolder()) .addSourceDir(new File(basedir, "src/main/java")) .addTestDir(new File(basedir, "src/test/java")) - .setLanguageFilters(new LanguageFilters(new Languages(new Java(), new Php()))); + .setLanguageFilters(new LanguageFilters(new Languages(new Java(), new Php()))) + .setSettings(new Settings()); List<File> testFiles = fileSystem.files(FileQuery.onTest().onLanguage("java")); assertThat(testFiles).hasSize(2); @@ -174,11 +184,12 @@ public class DefaultModuleFileSystemTest { @Test public void should_apply_file_filters() throws IOException { File basedir = new File(resourcesDir(), "main_and_test_files"); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setWorkingDir(temp.newFolder()) .addSourceDir(new File(basedir, "src/main/java")) - .addFilters(new FileFilterWrapper(FileFilterUtils.nameFileFilter("Foo.java"))); + .addFilters(new FileFilterWrapper(FileFilterUtils.nameFileFilter("Foo.java"))) + .setSettings(new Settings()); List<File> files = fileSystem.files(FileQuery.onSource()); assertThat(files).hasSize(1); @@ -191,7 +202,7 @@ public class DefaultModuleFileSystemTest { } public String[] getFileSuffixes() { - return new String[]{"php"}; + return new String[] {"php"}; } } @@ -201,14 +212,14 @@ public class DefaultModuleFileSystemTest { } public String[] getFileSuffixes() { - return new String[]{"java", "jav"}; + return new String[] {"java", "jav"}; } } @Test public void test_reset_dirs() throws IOException { File basedir = temp.newFolder(); - DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem() + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) .setBaseDir(basedir) .setWorkingDir(basedir) .addSourceDir(new File(basedir, "src/main/java")) @@ -229,4 +240,63 @@ public class DefaultModuleFileSystemTest { assertThat(fileSystem.binaryDirs()).hasSize(1); assertThat(fileSystem.binaryDirs().get(0).getCanonicalPath()).isEqualTo(existingDir.getCanonicalPath()); } + + @Test + public void should_throw_if_incremental_mode_and_not_in_dryrun() throws Exception { + File basedir = temp.newFolder(); + Settings settings = new Settings(); + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(mock(FileHashCache.class)) + .setBaseDir(basedir) + .setWorkingDir(temp.newFolder()) + .addSourceDir(new File(basedir, "src/main/java")) + .setSettings(settings); + + settings.setProperty(CoreProperties.INCREMENTAL_PREVIEW, true); + + thrown.expect(SonarException.class); + thrown.expectMessage("Incremental preview is only supported with preview mode"); + fileSystem.files(FileQuery.onSource()); + } + + @Test + public void should_filter_changed_files() throws Exception { + File basedir = new File(resourcesDir(), "main_and_test_files"); + Settings settings = new Settings(); + File mainDir = new File(basedir, "src/main/java"); + File testDir = new File(basedir, "src/test/java"); + File foo = new File(mainDir, "Foo.java"); + File hello = new File(mainDir, "Hello.java"); + File fooTest = new File(testDir, "FooTest.java"); + File helloTest = new File(testDir, "HelloTest.java"); + + FileHashCache fileHashCache = mock(FileHashCache.class); + when(fileHashCache.getPreviousHash(foo)).thenReturn("oldfoohash"); + when(fileHashCache.getCurrentHash(foo)).thenReturn("foohash"); + when(fileHashCache.getPreviousHash(hello)).thenReturn("oldhellohash"); + when(fileHashCache.getCurrentHash(hello)).thenReturn("oldhellohash"); + when(fileHashCache.getPreviousHash(fooTest)).thenReturn("oldfooTesthash"); + when(fileHashCache.getCurrentHash(fooTest)).thenReturn("fooTesthash"); + when(fileHashCache.getPreviousHash(helloTest)).thenReturn("oldhelloTesthash"); + when(fileHashCache.getCurrentHash(helloTest)).thenReturn("oldhelloTesthash"); + + DefaultModuleFileSystem fileSystem = new DefaultModuleFileSystem(fileHashCache) + .setBaseDir(basedir) + .setWorkingDir(temp.newFolder()) + .addSourceDir(mainDir) + .addTestDir(testDir) + .setSettings(settings); + + assertThat(fileSystem.files(FileQuery.onSource())).containsExactly(foo, hello); + assertThat(fileSystem.files(FileQuery.onTest())).containsExactly(fooTest, helloTest); + + assertThat(fileSystem.changedFiles(FileQuery.onSource())).containsExactly(foo); + assertThat(fileSystem.changedFiles(FileQuery.onTest())).containsExactly(fooTest); + + settings.setProperty(CoreProperties.INCREMENTAL_PREVIEW, true); + settings.setProperty(CoreProperties.DRY_RUN, true); + + assertThat(fileSystem.files(FileQuery.onSource())).containsExactly(foo); + assertThat(fileSystem.files(FileQuery.onTest())).containsExactly(fooTest); + } + } diff --git a/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/batch/PartialScanFilterTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileHashCacheTest.java index 7cd784988d9..1342044a151 100644 --- a/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/batch/PartialScanFilterTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileHashCacheTest.java @@ -17,27 +17,24 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.plugins.core.batch; +package org.sonar.batch.scan.filesystem; +import com.google.common.base.Charsets; import org.apache.commons.io.FileUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; -import org.sonar.api.CoreProperties; import org.sonar.api.batch.bootstrap.ProjectDefinition; -import org.sonar.api.config.Settings; import org.sonar.api.database.model.Snapshot; -import org.sonar.api.scan.filesystem.FileSystemFilter; +import org.sonar.api.scan.filesystem.ModuleFileSystem; import org.sonar.api.scan.filesystem.PathResolver; -import org.sonar.api.utils.SonarException; import org.sonar.batch.components.PastSnapshot; import org.sonar.batch.components.PastSnapshotFinder; import org.sonar.core.source.SnapshotDataType; import org.sonar.core.source.jdbc.SnapshotDataDao; import org.sonar.core.source.jdbc.SnapshotDataDto; -import org.sonar.plugins.core.utils.HashBuilder; import java.io.File; import java.util.Arrays; @@ -47,7 +44,7 @@ import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class PartialScanFilterTest { +public class FileHashCacheTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); @@ -55,9 +52,7 @@ public class PartialScanFilterTest { @Rule public ExpectedException thrown = ExpectedException.none(); - private PartialScanFilter filter; - - private Settings settings; + private FileHashCache cache; private PastSnapshotFinder pastSnapshotFinder; @@ -67,61 +62,39 @@ public class PartialScanFilterTest { private SnapshotDataDao snapshotDataDao; + private ModuleFileSystem moduleFileSystem; + @Before public void prepare() throws Exception { - settings = new Settings(); pastSnapshotFinder = mock(PastSnapshotFinder.class); snapshot = mock(Snapshot.class); baseDir = temp.newFolder(); snapshotDataDao = mock(SnapshotDataDao.class); - filter = new PartialScanFilter(settings, ProjectDefinition.create().setBaseDir(baseDir), new PathResolver(), new HashBuilder(), snapshot, + moduleFileSystem = mock(ModuleFileSystem.class); + cache = new FileHashCache(moduleFileSystem, ProjectDefinition.create().setBaseDir(baseDir), new PathResolver(), new HashBuilder(), snapshot, snapshotDataDao, pastSnapshotFinder); } @Test - public void should_not_run_by_default() throws Exception { - filter.start(); - assertThat(filter.accept(temp.newFile(), mock(FileSystemFilter.Context.class))).isTrue(); - } - - @Test - public void should_throw_if_partial_mode_and_not_in_dryrun() throws Exception { - settings.setProperty(CoreProperties.PARTIAL_ANALYSIS, true); - - thrown.expect(SonarException.class); - thrown.expectMessage("Partial analysis is only supported with dry run mode"); - filter.start(); - } - - @Test - public void should_include_if_no_previous_snapshot() throws Exception { - settings.setProperty(CoreProperties.PARTIAL_ANALYSIS, true); - settings.setProperty(CoreProperties.DRY_RUN, true); - + public void should_return_null_if_no_previous_snapshot() throws Exception { when(pastSnapshotFinder.findPreviousAnalysis(snapshot)).thenReturn(new PastSnapshot("foo")); - filter.start(); - assertThat(filter.accept(new File(baseDir, "src/main/java/foo/Bar.java"), mock(FileSystemFilter.Context.class))).isTrue(); + cache.start(); + assertThat(cache.getPreviousHash(new File(baseDir, "src/main/java/foo/Bar.java"))).isNull(); } @Test - public void should_include_if_no_previous_snapshot_data() throws Exception { - settings.setProperty(CoreProperties.PARTIAL_ANALYSIS, true); - settings.setProperty(CoreProperties.DRY_RUN, true); - + public void should_return_null_if_no_previous_snapshot_data() throws Exception { Snapshot previousSnapshot = mock(Snapshot.class); PastSnapshot pastSnapshot = new PastSnapshot("foo", new Date(), previousSnapshot); when(pastSnapshotFinder.findPreviousAnalysis(snapshot)).thenReturn(pastSnapshot); - filter.start(); - assertThat(filter.accept(new File(baseDir, "src/main/java/foo/Bar.java"), mock(FileSystemFilter.Context.class))).isTrue(); + cache.start(); + assertThat(cache.getPreviousHash(new File(baseDir, "src/main/java/foo/Bar.java"))).isNull(); } @Test - public void should_include_if_different_hash() throws Exception { - settings.setProperty(CoreProperties.PARTIAL_ANALYSIS, true); - settings.setProperty(CoreProperties.DRY_RUN, true); - + public void should_return_previous_hash() throws Exception { Snapshot previousSnapshot = mock(Snapshot.class); when(previousSnapshot.getId()).thenReturn(123); PastSnapshot pastSnapshot = new PastSnapshot("foo", new Date(), previousSnapshot); @@ -132,30 +105,23 @@ public class PartialScanFilterTest { when(snapshotDataDao.selectSnapshotData(123, Arrays.asList(SnapshotDataType.FILE_HASH.getValue()))) .thenReturn(Arrays.asList(snapshotDataDto)); - filter.start(); File file = new File(baseDir, "src/main/java/foo/Bar.java"); FileUtils.write(file, "foo"); - assertThat(filter.accept(file, mock(FileSystemFilter.Context.class))).isTrue(); + cache.start(); + assertThat(cache.getPreviousHash(file)).isEqualTo("abcd1234"); } @Test - public void should_exclude_if_same_hash() throws Exception { - settings.setProperty(CoreProperties.PARTIAL_ANALYSIS, true); - settings.setProperty(CoreProperties.DRY_RUN, true); - - Snapshot previousSnapshot = mock(Snapshot.class); - when(previousSnapshot.getId()).thenReturn(123); - PastSnapshot pastSnapshot = new PastSnapshot("foo", new Date(), previousSnapshot); - when(pastSnapshotFinder.findPreviousAnalysis(snapshot)).thenReturn(pastSnapshot); + public void should_compute_and_cache_current_hash() throws Exception { + when(moduleFileSystem.sourceCharset()).thenReturn(Charsets.UTF_8); - SnapshotDataDto snapshotDataDto = new SnapshotDataDto(); - snapshotDataDto.setData("src/main/java/foo/Bar.java=acbd18db4cc2f85cedef654fccc4a4d8\n"); - when(snapshotDataDao.selectSnapshotData(123, Arrays.asList(SnapshotDataType.FILE_HASH.getValue()))) - .thenReturn(Arrays.asList(snapshotDataDto)); - - filter.start(); File file = new File(baseDir, "src/main/java/foo/Bar.java"); FileUtils.write(file, "foo"); - assertThat(filter.accept(file, mock(FileSystemFilter.Context.class))).isFalse(); + String hash = "9a8742076ef9ffa5591f633704c2286b"; + assertThat(cache.getCurrentHash(file)).isEqualTo(hash); + + // Modify file + FileUtils.write(file, "bar"); + assertThat(cache.getCurrentHash(file)).isEqualTo(hash); } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileSystemLoggerTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileSystemLoggerTest.java index 065dd3490e6..19f2ae4c62d 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileSystemLoggerTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/FileSystemLoggerTest.java @@ -38,7 +38,7 @@ public class FileSystemLoggerTest { @Test public void log() { - DefaultModuleFileSystem fs = new DefaultModuleFileSystem(); + DefaultModuleFileSystem fs = new DefaultModuleFileSystem(mock(FileHashCache.class)); File src = temp.newFolder("src"); File test = temp.newFolder("test"); File base = temp.newFolder("base"); diff --git a/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/utils/HashBuilderTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/HashBuilderTest.java index a0819b57450..f15876f8d54 100644 --- a/plugins/sonar-core-plugin/src/test/java/org/sonar/plugins/core/utils/HashBuilderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/HashBuilderTest.java @@ -17,8 +17,9 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.plugins.core.utils; +package org.sonar.batch.scan.filesystem; +import com.google.common.base.Charsets; import org.apache.commons.io.FileUtils; import org.junit.Rule; import org.junit.Test; @@ -33,17 +34,42 @@ public class HashBuilderTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); + private HashBuilder hashBuilder = new HashBuilder(); @Test public void should_compute_hash() throws Exception { File tempFile = temp.newFile(); - FileUtils.write(tempFile, "foobar"); - assertThat(new HashBuilder().computeHash(tempFile)).isEqualTo("3858f62230ac3c915f300c664312c63f"); + FileUtils.write(tempFile, "foo\r\nbar", Charsets.UTF_8, true); + + assertThat(hashBuilder.computeHashNormalizeLineEnds(tempFile, Charsets.UTF_8)).isEqualTo("daef8a22a3f12580beadf086a9e11519"); + } + + @Test + public void should_normalize_line_ends() throws Exception { + File file1 = temp.newFile(); + FileUtils.write(file1, "foobar\nfofo", Charsets.UTF_8); + String hash1 = hashBuilder.computeHashNormalizeLineEnds(file1, Charsets.UTF_8); + + File file2 = temp.newFile(); + FileUtils.write(file2, "foobar\r\nfofo", Charsets.UTF_8); + String hash2 = hashBuilder.computeHashNormalizeLineEnds(file2, Charsets.UTF_8); + + File file3 = temp.newFile(); + FileUtils.write(file3, "foobar\rfofo", Charsets.UTF_8); + String hash3 = hashBuilder.computeHashNormalizeLineEnds(file3, Charsets.UTF_8); + + File file4 = temp.newFile(); + FileUtils.write(file4, "foobar\nfofo\n", Charsets.UTF_8); + String hash4 = hashBuilder.computeHashNormalizeLineEnds(file4, Charsets.UTF_8); + + assertThat(hash1).isEqualTo(hash2); + assertThat(hash1).isEqualTo(hash3); + assertThat(hash1).isNotEqualTo(hash4); } @Test(expected = SonarException.class) public void should_throw_on_not_existing_file() throws Exception { File tempFolder = temp.newFolder(); - new HashBuilder().computeHash(new File(tempFolder, "unknowFile.txt")); + hashBuilder.computeHashNormalizeLineEnds(new File(tempFolder, "unknowFile.txt"), Charsets.UTF_8); } } diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProviderTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProviderTest.java index cc387e9f6f4..600fa6a4aac 100644 --- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProviderTest.java +++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProviderTest.java @@ -54,7 +54,7 @@ public class ModuleFileSystemProviderTest { .setBaseDir(baseDir) .setWorkDir(workDir); ModuleFileSystem fs = provider.provide(module, new PathResolver(), new TempDirectories(), mock(LanguageFilters.class), - new Settings(), new FileSystemFilter[0]); + new Settings(), new FileSystemFilter[0], mock(FileHashCache.class)); assertThat(fs).isNotNull(); assertThat(fs.baseDir().getCanonicalPath()).isEqualTo(baseDir.getCanonicalPath()); @@ -70,7 +70,7 @@ public class ModuleFileSystemProviderTest { ModuleFileSystemProvider provider = new ModuleFileSystemProvider(); ModuleFileSystem fs = provider.provide(newSimpleModule(), new PathResolver(), new TempDirectories(), mock(LanguageFilters.class), - new Settings(), new FileSystemFilter[0]); + new Settings(), new FileSystemFilter[0], mock(FileHashCache.class)); assertThat(fs.sourceCharset()).isEqualTo(Charset.defaultCharset()); } @@ -83,7 +83,7 @@ public class ModuleFileSystemProviderTest { settings.setProperty(CoreProperties.ENCODING_PROPERTY, Charsets.ISO_8859_1.name()); ModuleFileSystem fs = provider.provide(module, new PathResolver(), new TempDirectories(), mock(LanguageFilters.class), - settings, new FileSystemFilter[0]); + settings, new FileSystemFilter[0], mock(FileHashCache.class)); assertThat(fs.sourceCharset()).isEqualTo(Charsets.ISO_8859_1); } @@ -109,7 +109,7 @@ public class ModuleFileSystemProviderTest { .addBinaryDir("target/classes"); ModuleFileSystem fs = provider.provide(project, new PathResolver(), new TempDirectories(), mock(LanguageFilters.class), - new Settings(), new FileSystemFilter[0]); + new Settings(), new FileSystemFilter[0], mock(FileHashCache.class)); assertThat(fs.baseDir().getCanonicalPath()).isEqualTo(baseDir.getCanonicalPath()); assertThat(fs.buildDir().getCanonicalPath()).isEqualTo(buildDir.getCanonicalPath()); diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java index f654f1b0d57..673c18fc7d2 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/CoreProperties.java @@ -473,5 +473,5 @@ public interface CoreProperties { /** * @since 4.0 */ - String PARTIAL_ANALYSIS = "sonar.partialAnalysis"; + String INCREMENTAL_PREVIEW = "sonar.incrementalPreview"; } diff --git a/sonar-plugin-api/src/main/java/org/sonar/api/scan/filesystem/ModuleFileSystem.java b/sonar-plugin-api/src/main/java/org/sonar/api/scan/filesystem/ModuleFileSystem.java index f85b89e868a..31882d0ac28 100644 --- a/sonar-plugin-api/src/main/java/org/sonar/api/scan/filesystem/ModuleFileSystem.java +++ b/sonar-plugin-api/src/main/java/org/sonar/api/scan/filesystem/ModuleFileSystem.java @@ -71,6 +71,11 @@ public interface ModuleFileSystem extends BatchComponent { */ List<File> files(FileQuery query); + /** + * Search for changed files. Never return null. + * @since 4.0 + */ + List<File> changedFiles(FileQuery query); /** * Charset of source and test files. If it's not defined, then return the platform default charset. diff --git a/sonar-plugin-api/src/test/java/org/sonar/api/scan/filesystem/SimpleModuleFileSystem.java b/sonar-plugin-api/src/test/java/org/sonar/api/scan/filesystem/SimpleModuleFileSystem.java index 8e0d8e5bcf1..48bb0a3fcb0 100644 --- a/sonar-plugin-api/src/test/java/org/sonar/api/scan/filesystem/SimpleModuleFileSystem.java +++ b/sonar-plugin-api/src/test/java/org/sonar/api/scan/filesystem/SimpleModuleFileSystem.java @@ -81,6 +81,11 @@ public class SimpleModuleFileSystem implements ModuleFileSystem { return Collections.emptyList(); } + @Override + public List<File> changedFiles(FileQuery query) { + return Collections.emptyList(); + } + public Charset sourceCharset() { return Charset.forName(CharEncoding.UTF_8); } |