SONAR-2657 Expose changed files API in ModuleFileSystem

7 files changed, 315 insertions, 9 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java
index b97118be253..37ef6fcedb4 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ModuleScanContainer.java
@@ -51,6 +51,7 @@ import org.sonar.batch.phases.PhaseExecutor;
 import org.sonar.batch.phases.PhasesTimeProfiler;
 import org.sonar.batch.scan.filesystem.DeprecatedFileSystemAdapter;
 import org.sonar.batch.scan.filesystem.ExclusionFilters;
+import org.sonar.batch.scan.filesystem.FileHashCache;
 import org.sonar.batch.scan.filesystem.FileSystemLogger;
 import org.sonar.batch.scan.filesystem.LanguageFilters;
 import org.sonar.batch.scan.filesystem.ModuleFileSystemProvider;
@@ -104,6 +105,7 @@ public class ModuleScanContainer extends ComponentContainer {
       new ModuleFileSystemProvider(),
       DeprecatedFileSystemAdapter.class,
       FileSystemLogger.class,
+      FileHashCache.class,
 
       // the Snapshot component will be removed when asynchronous measures are improved (required for AsynchronousMeasureSensor)
       getComponentByType(ResourcePersister.class).getSnapshot(module),
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
index ec79143de77..0615528efa0 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/ProjectScanContainer.java
@@ -33,11 +33,33 @@ import org.sonar.batch.DefaultFileLinesContextFactory;
 import org.sonar.batch.DefaultResourceCreationLock;
 import org.sonar.batch.ProjectConfigurator;
 import org.sonar.batch.ProjectTree;
-import org.sonar.batch.bootstrap.*;
-import org.sonar.batch.index.*;
-import org.sonar.batch.issue.*;
+import org.sonar.batch.bootstrap.BootstrapSettings;
+import org.sonar.batch.bootstrap.ExtensionInstaller;
+import org.sonar.batch.bootstrap.ExtensionMatcher;
+import org.sonar.batch.bootstrap.ExtensionUtils;
+import org.sonar.batch.bootstrap.MetricProvider;
+import org.sonar.batch.index.Caches;
+import org.sonar.batch.index.ComponentDataCache;
+import org.sonar.batch.index.ComponentDataPersister;
+import org.sonar.batch.index.DefaultIndex;
+import org.sonar.batch.index.DefaultPersistenceManager;
+import org.sonar.batch.index.DefaultResourcePersister;
+import org.sonar.batch.index.DependencyPersister;
+import org.sonar.batch.index.EventPersister;
+import org.sonar.batch.index.LinkPersister;
+import org.sonar.batch.index.MeasurePersister;
+import org.sonar.batch.index.MemoryOptimizer;
+import org.sonar.batch.index.ResourceCache;
+import org.sonar.batch.index.SnapshotCache;
+import org.sonar.batch.index.SourcePersister;
+import org.sonar.batch.issue.DefaultProjectIssues;
+import org.sonar.batch.issue.DeprecatedViolations;
+import org.sonar.batch.issue.IssueCache;
+import org.sonar.batch.issue.IssuePersister;
+import org.sonar.batch.issue.ScanIssueStorage;
 import org.sonar.batch.phases.GraphPersister;
 import org.sonar.batch.profiling.PhasesSumUpTimeProfiler;
+import org.sonar.batch.scan.filesystem.HashBuilder;
 import org.sonar.batch.scan.maven.FakeMavenPluginExecutor;
 import org.sonar.batch.scan.maven.MavenPluginExecutor;
 import org.sonar.batch.source.HighlightableBuilder;
@@ -50,7 +72,11 @@ import org.sonar.core.issue.workflow.IssueWorkflow;
 import org.sonar.core.notification.DefaultNotificationManager;
 import org.sonar.core.technicaldebt.TechnicalDebtModel;
 import org.sonar.core.technicaldebt.WorkUnitConverter;
-import org.sonar.core.technicaldebt.functions.*;
+import org.sonar.core.technicaldebt.functions.ConstantFunction;
+import org.sonar.core.technicaldebt.functions.Functions;
+import org.sonar.core.technicaldebt.functions.LinearFunction;
+import org.sonar.core.technicaldebt.functions.LinearWithOffsetFunction;
+import org.sonar.core.technicaldebt.functions.LinearWithThresholdFunction;
 import org.sonar.core.test.TestPlanBuilder;
 import org.sonar.core.test.TestPlanPerspectiveLoader;
 import org.sonar.core.test.TestableBuilder;
@@ -116,6 +142,7 @@ public class ProjectScanContainer extends ComponentContainer {
       ResourceCache.class,
       ComponentDataCache.class,
       ComponentDataPersister.class,
+      HashBuilder.class,
 
       // issues
       IssueUpdater.class,
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ChangedFileFilter.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ChangedFileFilter.java
new file mode 100644
index 00000000000..af0968d4137
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ChangedFileFilter.java
@@ -0,0 +1,48 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import org.sonar.api.scan.filesystem.FileSystemFilter;
+
+import java.io.File;
+
+/**
+ * When enabled this filter will only allow modified files to be analyzed.
+ * @since 4.0
+ */
+class ChangedFileFilter implements FileSystemFilter {
+
+  private FileHashCache fileHashCache;
+
+  public ChangedFileFilter(FileHashCache fileHashCache) {
+    this.fileHashCache = fileHashCache;
+  }
+
+  @Override
+  public boolean accept(File file, Context context) {
+    String previousHash = fileHashCache.getPreviousHash(file);
+    if (previousHash == null) {
+      return true;
+    }
+    String currentHash = fileHashCache.getCurrentHash(file);
+    return !currentHash.equals(previousHash);
+  }
+
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
index 22ab1f9d892..1d0c76a7bee 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/DefaultModuleFileSystem.java
@@ -34,6 +34,7 @@ import org.sonar.api.scan.filesystem.FileSystemFilter;
 import org.sonar.api.scan.filesystem.FileType;
 import org.sonar.api.scan.filesystem.ModuleFileSystem;
 import org.sonar.api.scan.filesystem.PathResolver;
+import org.sonar.api.utils.SonarException;
 
 import java.io.File;
 import java.io.FileFilter;
@@ -58,8 +59,10 @@ public class DefaultModuleFileSystem implements ModuleFileSystem {
   private PathResolver pathResolver = new PathResolver();
   private List<FileSystemFilter> fsFilters = Lists.newArrayList();
   private LanguageFilters languageFilters;
+  private FileHashCache fileHashCache;
 
-  DefaultModuleFileSystem() {
+  DefaultModuleFileSystem(FileHashCache fileHashCache) {
+    this.fileHashCache = fileHashCache;
   }
 
   public File baseDir() {
@@ -110,7 +113,26 @@ public class DefaultModuleFileSystem implements ModuleFileSystem {
   }
 
   public List<File> files(FileQuery query) {
+    boolean changedFilesOnly = false;
+    if (settings.getBoolean(CoreProperties.INCREMENTAL_PREVIEW)) {
+      if (!settings.getBoolean(CoreProperties.DRY_RUN)) {
+        throw new SonarException("Incremental preview is only supported with preview mode");
+      }
+      changedFilesOnly = true;
+    }
+    return files(query, changedFilesOnly);
+  }
+
+  @Override
+  public List<File> changedFiles(FileQuery query) {
+    return files(query, true);
+  }
+
+  private List<File> files(FileQuery query, boolean changedFilesOnly) {
     List<FileSystemFilter> filters = Lists.newArrayList(fsFilters);
+    if (changedFilesOnly) {
+      filters.add(new ChangedFileFilter(fileHashCache));
+    }
     for (FileFilter fileFilter : query.filters()) {
       filters.add(new FileFilterWrapper(fileFilter));
     }
@@ -142,7 +164,7 @@ public class DefaultModuleFileSystem implements ModuleFileSystem {
   }
 
   private void applyFilters(List<File> result, FileFilterContext context,
-                            Collection<FileSystemFilter> filters, Collection<File> dirs) {
+    Collection<FileSystemFilter> filters, Collection<File> dirs) {
     for (File dir : dirs) {
       if (dir.exists()) {
         context.setRelativeDir(dir);
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileHashCache.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileHashCache.java
new file mode 100644
index 00000000000..34452fa2305
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/FileHashCache.java
@@ -0,0 +1,116 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.picocontainer.Startable;
+import org.sonar.api.BatchComponent;
+import org.sonar.api.batch.bootstrap.ProjectDefinition;
+import org.sonar.api.database.model.Snapshot;
+import org.sonar.api.scan.filesystem.ModuleFileSystem;
+import org.sonar.api.scan.filesystem.PathResolver;
+import org.sonar.api.utils.SonarException;
+import org.sonar.batch.components.PastSnapshot;
+import org.sonar.batch.components.PastSnapshotFinder;
+import org.sonar.core.source.SnapshotDataType;
+import org.sonar.core.source.jdbc.SnapshotDataDao;
+import org.sonar.core.source.jdbc.SnapshotDataDto;
+
+import javax.annotation.CheckForNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public class FileHashCache implements BatchComponent, Startable {
+
+  private Map<String, String> currentHashCache = Maps.newHashMap();
+  private Map<String, String> previousHashCache = Maps.newHashMap();
+
+  private PathResolver pathResolver;
+  private HashBuilder hashBuilder;
+  private SnapshotDataDao snapshotDataDao;
+  private PastSnapshotFinder pastSnapshotFinder;
+  private Snapshot snapshot;
+  private ProjectDefinition module;
+  private ModuleFileSystem fs;
+
+  public FileHashCache(ModuleFileSystem fs, ProjectDefinition module, PathResolver pathResolver, HashBuilder hashBuilder,
+    Snapshot snapshot,
+    SnapshotDataDao snapshotDataDao,
+    PastSnapshotFinder pastSnapshotFinder) {
+    this.fs = fs;
+    this.module = module;
+    this.pathResolver = pathResolver;
+    this.hashBuilder = hashBuilder;
+    this.snapshot = snapshot;
+    this.snapshotDataDao = snapshotDataDao;
+    this.pastSnapshotFinder = pastSnapshotFinder;
+  }
+
+  @Override
+  public void start() {
+    // Extract previous checksum of all files of this module and store them in a map
+    PastSnapshot pastSnapshot = pastSnapshotFinder.findPreviousAnalysis(snapshot);
+    if (pastSnapshot.isRelatedToSnapshot()) {
+      Collection<SnapshotDataDto> selectSnapshotData = snapshotDataDao.selectSnapshotData(pastSnapshot.getProjectSnapshot().getId().longValue(),
+        Arrays.asList(SnapshotDataType.FILE_HASH.getValue()));
+      if (!selectSnapshotData.isEmpty()) {
+        SnapshotDataDto snapshotDataDto = selectSnapshotData.iterator().next();
+        String data = snapshotDataDto.getData();
+        try {
+          List<String> lines = IOUtils.readLines(new StringReader(data));
+          for (String line : lines) {
+            String[] keyValue = StringUtils.split(line, "=");
+            if (keyValue.length == 2) {
+              previousHashCache.put(keyValue[0], keyValue[1]);
+            }
+          }
+        } catch (IOException e) {
+          throw new SonarException("Unable to read previous file hashes", e);
+        }
+      }
+    }
+  }
+
+  public String getCurrentHash(File file) {
+    String relativePath = pathResolver.relativePath(module.getBaseDir(), file);
+    if (!currentHashCache.containsKey(relativePath)) {
+      currentHashCache.put(relativePath, hashBuilder.computeHashNormalizeLineEnds(file, fs.sourceCharset()));
+    }
+    return currentHashCache.get(relativePath);
+  }
+
+  @CheckForNull
+  public String getPreviousHash(File file) {
+    String relativePath = pathResolver.relativePath(module.getBaseDir(), file);
+    return previousHashCache.get(relativePath);
+  }
+
+  @Override
+  public void stop() {
+  }
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/HashBuilder.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/HashBuilder.java
new file mode 100644
index 00000000000..2db7946e67c
--- /dev/null
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/HashBuilder.java
@@ -0,0 +1,91 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2013 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.batch.scan.filesystem;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.IOUtils;
+import org.sonar.api.BatchExtension;
+import org.sonar.api.batch.InstantiationStrategy;
+import org.sonar.api.utils.SonarException;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+
+/**
+ * @since 4.0
+ */
+@InstantiationStrategy(InstantiationStrategy.PER_BATCH)
+public final class HashBuilder implements BatchExtension {
+
+  /**
+   * Compute hash of a file ignoring line ends differences.
+   * Maximum performance is needed.
+   */
+  public String computeHashNormalizeLineEnds(File file, Charset charset) {
+    Reader reader = null;
+    try {
+      MessageDigest md5Digest = DigestUtils.getMd5Digest();
+      md5Digest.reset();
+      reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
+      int i = reader.read();
+      boolean afterCR = true;
+      while (i != -1) {
+        char c = (char) i;
+        if (afterCR) {
+          afterCR = false;
+          if (c == '\n') {
+            // Ignore
+            i = reader.read();
+            continue;
+          }
+        }
+        if (c == '\r') {
+          afterCR = true;
+          c = '\n';
+        }
+        md5Digest.update(charToBytesUTF(c));
+        i = reader.read();
+      }
+      return Hex.encodeHexString(md5Digest.digest());
+    } catch (IOException e) {
+      throw new SonarException("Unable to compute file hash", e);
+    } finally {
+      IOUtils.closeQuietly(reader);
+    }
+  }
+
+  public static byte[] charToBytesUTF(char c) {
+    char[] buffer = new char[] {c};
+    byte[] b = new byte[buffer.length << 1];
+    for (int i = 0; i < buffer.length; i++) {
+      int bpos = i << 1;
+      b[bpos] = (byte) ((buffer[i] & 0xFF00) >> 8);
+      b[bpos + 1] = (byte) (buffer[i] & 0x00FF);
+    }
+    return b;
+  }
+}
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java
index 1cb0bc76fe5..3efae8a0ec0 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ModuleFileSystemProvider.java
@@ -40,9 +40,10 @@ public class ModuleFileSystemProvider extends ProviderAdapter {
   private DefaultModuleFileSystem singleton;
 
   public DefaultModuleFileSystem provide(ProjectDefinition module, PathResolver pathResolver, TempDirectories tempDirectories,
-                                         LanguageFilters languageFilters, Settings settings, FileSystemFilter[] pluginFileFilters) {
+    LanguageFilters languageFilters, Settings settings, FileSystemFilter[] pluginFileFilters,
+    FileHashCache fileHashCache) {
     if (singleton == null) {
-      DefaultModuleFileSystem fs = new DefaultModuleFileSystem();
+      DefaultModuleFileSystem fs = new DefaultModuleFileSystem(fileHashCache);
       fs.setLanguageFilters(languageFilters);
       fs.setBaseDir(module.getBaseDir());
       fs.setBuildDir(module.getBuildDir());
@@ -97,7 +98,6 @@ public class ModuleFileSystemProvider extends ProviderAdapter {
     }
   }
 
-
   private void initBinaryDirs(ProjectDefinition module, PathResolver pathResolver, DefaultModuleFileSystem fs) {
     for (String path : module.getBinaries()) {
       File dir = pathResolver.relativeFile(module.getBaseDir(), path);