SONAR-5869 Remove SnapshotSourceDto and all related code. Remove colorizer stuff on server side.

9 files changed, 24 insertions, 221 deletions

diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java
index b1235f09964..a3133af5c25 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultIndex.java
@@ -22,6 +22,7 @@ package org.sonar.batch.index;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.ObjectUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
@@ -60,6 +61,7 @@ import org.sonar.core.component.ScanGraph;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -475,8 +477,19 @@ public class DefaultIndex extends SonarIndex {
   }
 
   @Override
-  public String getSource(Resource resource) {
-    return persistence.getSource(resource);
+  public String getSource(Resource reference) {
+    Resource resource = getResource(reference);
+    if (resource instanceof File) {
+      File file = (File) resource;
+      Project module = (Project) file.getParent().getParent();
+      ProjectDefinition def = projectTree.getProjectDefinition(module);
+      try {
+        return FileUtils.readFileToString(new java.io.File(def.getBaseDir(), file.getPath()));
+      } catch (IOException e) {
+        throw new IllegalStateException("Unable to read file content " + reference, e);
+      }
+    }
+    return null;
   }
 
   /**
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java
index e13b34235ae..79a09a7d6f4 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/DefaultPersistenceManager.java
@@ -67,11 +67,6 @@ public final class DefaultPersistenceManager implements PersistenceManager {
   }
 
   @Override
-  public String getSource(Resource resource) {
-    return sourcePersister.getSource(resource);
-  }
-
-  @Override
   public void saveDependency(Project project, Dependency dependency, Dependency parentDependency) {
     if (ResourceUtils.isPersistable(dependency.getFrom()) && ResourceUtils.isPersistable(dependency.getTo())) {
       dependencyPersister.saveDependency(project, dependency, parentDependency);
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java b/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java
index bcf8c6cf565..2e5de75acb1 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/PersistenceManager.java
@@ -37,8 +37,6 @@ public interface PersistenceManager {
 
   Snapshot saveResource(Project project, Resource resource, @Nullable Resource parent);
 
-  String getSource(Resource resource);
-
   void saveDependency(Project project, Dependency dependency, Dependency parentDependency);
 
   void saveLink(Project project, ProjectLink link);
diff --git a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
index edd0c01da3f..f10ea9d4a55 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/index/SourcePersister.java
@@ -32,10 +32,8 @@ import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.sensor.duplication.DuplicationGroup;
 import org.sonar.api.batch.sensor.duplication.DuplicationGroup.Block;
 import org.sonar.api.batch.sensor.symbol.Symbol;
-import org.sonar.api.database.model.Snapshot;
 import org.sonar.api.measures.CoreMetrics;
 import org.sonar.api.measures.Measure;
-import org.sonar.api.resources.Resource;
 import org.sonar.api.utils.KeyValueFormat;
 import org.sonar.api.utils.System2;
 import org.sonar.api.utils.text.CsvWriter;
@@ -52,8 +50,6 @@ import org.sonar.core.persistence.MyBatis;
 import org.sonar.core.source.SnapshotDataTypes;
 import org.sonar.core.source.db.FileSourceDto;
 import org.sonar.core.source.db.FileSourceMapper;
-import org.sonar.core.source.db.SnapshotSourceDao;
-import org.sonar.core.source.db.SnapshotSourceDto;
 
 import javax.annotation.CheckForNull;
 import javax.annotation.Nullable;
@@ -74,7 +70,6 @@ public class SourcePersister implements ScanPersister {
 
   private static final String BOM = "\uFEFF";
   private final ResourcePersister resourcePersister;
-  private final SnapshotSourceDao sourceDao;
   private final InputPathCache inputPathCache;
   private final MyBatis mybatis;
   private final MeasureCache measureCache;
@@ -85,11 +80,10 @@ public class SourcePersister implements ScanPersister {
   private CodeColorizers codeColorizers;
   private DuplicationCache duplicationCache;
 
-  public SourcePersister(ResourcePersister resourcePersister, SnapshotSourceDao sourceDao, InputPathCache inputPathCache,
+  public SourcePersister(ResourcePersister resourcePersister, InputPathCache inputPathCache,
     MyBatis mybatis, MeasureCache measureCache, ComponentDataCache componentDataCache, ProjectTree projectTree, System2 system2,
     ResourceCache resourceCache, CodeColorizers codeColorizers, DuplicationCache duplicationCache) {
     this.resourcePersister = resourcePersister;
-    this.sourceDao = sourceDao;
     this.inputPathCache = inputPathCache;
     this.mybatis = mybatis;
     this.measureCache = measureCache;
@@ -101,24 +95,6 @@ public class SourcePersister implements ScanPersister {
     this.duplicationCache = duplicationCache;
   }
 
-  public void saveSource(Resource resource, String source, Date updatedAt) {
-    Snapshot snapshot = resourcePersister.getSnapshotOrFail(resource);
-    SnapshotSourceDto dto = new SnapshotSourceDto();
-    dto.setSnapshotId(snapshot.getId().longValue());
-    dto.setData(source);
-    dto.setUpdatedAt(updatedAt);
-    sourceDao.insert(dto);
-  }
-
-  @CheckForNull
-  public String getSource(Resource resource) {
-    Snapshot snapshot = resourcePersister.getSnapshot(resource);
-    if (snapshot != null && snapshot.getId() != null) {
-      return sourceDao.selectSnapshotSource(snapshot.getId());
-    }
-    return null;
-  }
-
   @Override
   public void persist() {
     // Don't use batch insert for file_sources since keeping all data in memory can produce OOM for big files
diff --git a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
index a148c03d190..6e2d5769a29 100644
--- a/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
+++ b/sonar-batch/src/main/java/org/sonar/batch/scan/filesystem/ComponentIndexer.java
@@ -19,28 +19,18 @@
  */
 package org.sonar.batch.scan.filesystem;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.CharMatcher;
-import com.google.common.io.Files;
 import org.sonar.api.BatchComponent;
 import org.sonar.api.batch.SonarIndex;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
-import org.sonar.api.batch.fs.InputFile.Status;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
 import org.sonar.api.resources.File;
 import org.sonar.api.resources.Languages;
 import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Resource;
-import org.sonar.api.utils.SonarException;
 import org.sonar.batch.index.ResourceKeyMigration;
-import org.sonar.batch.index.SnapshotCache;
-import org.sonar.batch.index.SourcePersister;
-import org.sonar.batch.protocol.input.ProjectReferentials;
 import org.sonar.batch.util.DeprecatedKeyUtils;
 
-import java.util.Date;
-
 /**
  * Index all files/directories of the module in SQ database and importing source code.
  *
@@ -52,19 +42,12 @@ public class ComponentIndexer implements BatchComponent {
   private final SonarIndex sonarIndex;
   private final ResourceKeyMigration migration;
   private final Project module;
-  private final SourcePersister sourcePersister;
-  private final ProjectReferentials projectReferentials;
-  private final Date projectAnalysisDate;
 
-  public ComponentIndexer(Project module, Languages languages, SonarIndex sonarIndex, ResourceKeyMigration migration, SourcePersister sourcePersister,
-    ProjectReferentials projectReferentials, SnapshotCache snapshotCache) {
+  public ComponentIndexer(Project module, Languages languages, SonarIndex sonarIndex, ResourceKeyMigration migration) {
     this.module = module;
     this.languages = languages;
     this.sonarIndex = sonarIndex;
     this.migration = migration;
-    this.sourcePersister = sourcePersister;
-    this.projectReferentials = projectReferentials;
-    this.projectAnalysisDate = snapshotCache.get(module.getEffectiveKey()).getBuildDate();
   }
 
   public void execute(FileSystem fs) {
@@ -84,28 +67,7 @@ public class ComponentIndexer implements BatchComponent {
         sonarFile.setDeprecatedKey(pathFromSourceDir);
       }
       sonarIndex.index(sonarFile);
-
-      importSources(fs, inputFile, sonarFile);
     }
   }
 
-  @VisibleForTesting
-  void importSources(FileSystem fs, InputFile inputFile, Resource sonarFile) {
-    try {
-      // TODO this part deserves optimization.
-      // We should try to remove BOM and count lines in a single pass
-      String source = Files.toString(inputFile.file(), fs.encoding());
-      // SONAR-3860 Remove BOM character from source
-      source = CharMatcher.anyOf("\uFEFF").removeFrom(source);
-      if (inputFile.status() == Status.SAME) {
-        sourcePersister.saveSource(sonarFile, source, projectReferentials.lastAnalysisDate());
-      } else {
-        sourcePersister.saveSource(sonarFile, source, this.projectAnalysisDate);
-      }
-
-    } catch (Exception e) {
-      throw new SonarException("Unable to read and import the source file : '" + inputFile.absolutePath() + "' with the charset : '"
-        + fs.encoding() + "'.", e);
-    }
-  }
 }
diff --git a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
index 52a6a9b2414..2d771d166f6 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/index/SourcePersisterTest.java
@@ -49,7 +49,6 @@ import org.sonar.core.persistence.AbstractDaoTestCase;
 import org.sonar.core.source.SnapshotDataTypes;
 import org.sonar.core.source.db.FileSourceDao;
 import org.sonar.core.source.db.FileSourceDto;
-import org.sonar.core.source.db.SnapshotSourceDao;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -94,7 +93,7 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
     when(measureCache.byMetric(anyString(), anyString())).thenReturn(Collections.<org.sonar.api.measures.Measure>emptyList());
     componentDataCache = mock(ComponentDataCache.class);
     duplicationCache = mock(DuplicationCache.class);
-    sourcePersister = new SourcePersister(resourcePersister, new SnapshotSourceDao(getMyBatis()), inputPathCache,
+    sourcePersister = new SourcePersister(resourcePersister, inputPathCache,
       getMyBatis(), measureCache, componentDataCache, projectTree, system2,
       resourceCache, mock(CodeColorizers.class), duplicationCache);
     Project project = new Project(PROJECT_KEY);
@@ -104,13 +103,6 @@ public class SourcePersisterTest extends AbstractDaoTestCase {
   }
 
   @Test
-  public void shouldSaveSource() {
-    setupData("shouldSaveSource");
-    sourcePersister.saveSource(new File("org/foo/Bar.java"), "this is the file content", DateUtils.parseDateTime("2014-10-31T16:44:02+0100"));
-    checkTables("shouldSaveSource", "snapshot_sources");
-  }
-
-  @Test
   public void testPersistDontTouchUnchanged() throws Exception {
     setupData("file_sources");
     when(system2.newDate()).thenReturn(DateUtils.parseDateTime("2014-10-29T16:44:02+0100"));
diff --git a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ComponentIndexerTest.java b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ComponentIndexerTest.java
index 127d344250e..b5e858e2873 100644
--- a/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ComponentIndexerTest.java
+++ b/sonar-batch/src/test/java/org/sonar/batch/scan/filesystem/ComponentIndexerTest.java
@@ -19,39 +19,28 @@
  */
 package org.sonar.batch.scan.filesystem;
 
-import com.google.common.base.Charsets;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.CharEncoding;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.mockito.ArgumentMatcher;
-import org.mockito.exceptions.verification.junit.ArgumentsAreDifferent;
 import org.sonar.api.batch.SonarIndex;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.internal.DefaultFileSystem;
 import org.sonar.api.batch.fs.internal.DefaultInputFile;
 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
-import org.sonar.api.database.model.Snapshot;
 import org.sonar.api.resources.AbstractLanguage;
 import org.sonar.api.resources.Java;
 import org.sonar.api.resources.Languages;
 import org.sonar.api.resources.Project;
 import org.sonar.api.resources.Qualifiers;
-import org.sonar.api.resources.Resource;
 import org.sonar.batch.index.ResourceKeyMigration;
-import org.sonar.batch.index.SnapshotCache;
-import org.sonar.batch.index.SourcePersister;
-import org.sonar.batch.protocol.input.ProjectReferentials;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.Date;
 
 import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
@@ -59,21 +48,16 @@ public class ComponentIndexerTest {
 
   @Rule
   public TemporaryFolder temp = new TemporaryFolder();
-  File baseDir;
-  DefaultFileSystem fs = new DefaultFileSystem();
-  SonarIndex sonarIndex;
-  AbstractLanguage cobolLanguage;
-  Project project;
-
-  String aClaess;
-  String explicacao;
-  private SourcePersister sourcePersister;
+  private File baseDir;
+  private DefaultFileSystem fs = new DefaultFileSystem();
+  private SonarIndex sonarIndex;
+  private AbstractLanguage cobolLanguage;
+  private Project project;
 
   @Before
   public void prepare() throws IOException {
     baseDir = temp.newFolder();
     sonarIndex = mock(SonarIndex.class);
-    sourcePersister = mock(SourcePersister.class);
     project = new Project("myProject");
     cobolLanguage = new AbstractLanguage("cobol") {
       @Override
@@ -81,9 +65,6 @@ public class ComponentIndexerTest {
         return new String[] {"cbl"};
       }
     };
-
-    aClaess = new String(new byte[] {65, 67, 108, 97, -61, -88, 115, 115, 40, 41}, CharEncoding.UTF_8);
-    explicacao = new String(new byte[] {101, 120, 112, 108, 105, 99, 97, -61, -89, -61, -93, 111, 40, 41}, CharEncoding.UTF_8);
   }
 
   @Test
@@ -109,11 +90,7 @@ public class ComponentIndexerTest {
   }
 
   private ComponentIndexer createIndexer(Languages languages) {
-    SnapshotCache snapshotCache = new SnapshotCache();
-    snapshotCache.put("myProject", mock(Snapshot.class));
-    ComponentIndexer indexer = new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class), sourcePersister, new ProjectReferentials(),
-      snapshotCache);
-    return indexer;
+    return new ComponentIndexer(project, languages, sonarIndex, mock(ResourceKeyMigration.class));
   }
 
   @Test
@@ -131,90 +108,6 @@ public class ComponentIndexerTest {
     verify(sonarIndex).index(org.sonar.api.resources.File.create("/src/test/foo/bar/FooTest.cbl", "foo/bar/FooTest.cbl", cobolLanguage, true));
   }
 
-  @Test
-  public void shouldImportSource() throws IOException {
-    fs.add(newInputFile("src/main/java/foo/bar/Foo.java", "sample code", "foo/bar/Foo.java", "java", false));
-    Languages languages = new Languages(Java.INSTANCE);
-    ComponentIndexer indexer = createIndexer(languages);
-    indexer.execute(fs);
-
-    Resource sonarFile = org.sonar.api.resources.File.create("src/main/java/foo/bar/Foo.java", "foo/bar/Foo.java", Java.INSTANCE, false);
-    verify(sonarIndex).index(sonarFile);
-    verify(sourcePersister).saveSource(sonarFile, "sample code", null);
-  }
-
-  @Test
-  public void should_use_mac_roman_charset_for_reading_source_files() throws Exception {
-    String encoding = "MacRoman";
-    String testFile = "MacRomanEncoding.java";
-    fileEncodingTest(encoding, testFile);
-  }
-
-  @Test
-  public void should_use_CP1252_charset_for_reading_source_files() throws Exception {
-    String encoding = "CP1252";
-    String testFile = "CP1252Encoding.java";
-    fileEncodingTest(encoding, testFile);
-  }
-
-  @Test(expected = ArgumentsAreDifferent.class)
-  public void should_fail_with_wrong_charset_for_reading_source_files() throws Exception {
-    String encoding = CharEncoding.UTF_8;
-    String testFile = "CP1252Encoding.java";
-    fileEncodingTest(encoding, testFile);
-  }
-
-  @Test
-  public void remove_byte_order_mark_character() throws Exception {
-    File javaFile1 = new File(baseDir, "src/main/java/foo/bar/Foo.java");
-    FileUtils.write(javaFile1, "\uFEFFpublic class Test", Charsets.UTF_8);
-    fs.add(new DeprecatedDefaultInputFile("foo", "src/main/java/foo/bar/Foo.java")
-      .setPathRelativeToSourceDir("foo/bar/Foo.java")
-      .setFile(javaFile1)
-      .setLanguage("java"));
-    Languages languages = new Languages(Java.INSTANCE);
-    ComponentIndexer indexer = createIndexer(languages);
-    indexer.execute(fs);
-
-    Resource sonarFile = org.sonar.api.resources.File.create("src/main/java/foo/bar/Foo.java", "foo/bar/Foo.java", Java.INSTANCE, false);
-
-    verify(sourcePersister).saveSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
-      @Override
-      public boolean matches(Object arg0) {
-        String source = (String) arg0;
-        return !source.contains("\uFEFF");
-      }
-    }), (Date) eq(null));
-  }
-
-  private void fileEncodingTest(String encoding, String testFile) throws Exception {
-    fs.setEncoding(Charset.forName(encoding));
-
-    File javaFile1 = new File(baseDir, "src/main/java/foo/bar/Foo.java");
-    FileUtils.copyFile(getFile(testFile), javaFile1);
-    fs.add(new DeprecatedDefaultInputFile("foo", "src/main/java/foo/bar/Foo.java")
-      .setPathRelativeToSourceDir("foo/bar/Foo.java")
-      .setFile(javaFile1)
-      .setLanguage("java"));
-    Languages languages = new Languages(Java.INSTANCE);
-    ComponentIndexer indexer = createIndexer(languages);
-    indexer.execute(fs);
-
-    Resource sonarFile = org.sonar.api.resources.File.create("/src/main/java/foo/bar/Foo.java", "foo/bar/Foo.java", Java.INSTANCE, false);
-
-    verify(sourcePersister).saveSource(eq(sonarFile), argThat(new ArgumentMatcher<String>() {
-      @Override
-      public boolean matches(Object arg0) {
-        String source = (String) arg0;
-        return source.contains(aClaess) && source.contains(explicacao);
-      }
-    }), (Date) eq(null));
-  }
-
-  private File getFile(String testFile) {
-    return new File("test-resources/org/sonar/batch/phases/ComponentIndexerTest/encoding/" + testFile);
-  }
-
   private DefaultInputFile newInputFile(String path, String content, String sourceRelativePath, String languageKey, boolean unitTest) throws IOException {
     File file = new File(baseDir, path);
     FileUtils.write(file, content);
diff --git a/sonar-batch/test-resources/org/sonar/batch/phases/ComponentIndexerTest/encoding/CP1252Encoding.java b/sonar-batch/test-resources/org/sonar/batch/phases/ComponentIndexerTest/encoding/CP1252Encoding.java
deleted file mode 100644
index 5f80ef59765..00000000000
--- a/sonar-batch/test-resources/org/sonar/batch/phases/ComponentIndexerTest/encoding/CP1252Encoding.java
+++ /dev/null
@@ -1,13 +0,0 @@
-public class Car {
-
-	public AClaèss() {
-	}
-
-	public int explicação() {
-	  return 1;
-	}
-
-	public String getS() {
-		return "";
-	}
-}
diff --git a/sonar-batch/test-resources/org/sonar/batch/phases/ComponentIndexerTest/encoding/MacRomanEncoding.java b/sonar-batch/test-resources/org/sonar/batch/phases/ComponentIndexerTest/encoding/MacRomanEncoding.java
deleted file mode 100644
index 30e52000eaf..00000000000
--- a/sonar-batch/test-resources/org/sonar/batch/phases/ComponentIndexerTest/encoding/MacRomanEncoding.java
+++ /dev/null
@@ -1,13 +0,0 @@
-public class Car {
-
-  public ACla�ss() {
-  }
-
-  public int explica�‹o() {
-    return 1;
-  }
-
-  public String getS() {
-    return "";
-  }
-}