SONAR-5801 support empty files

+ fix quality flaws

21 files changed, 198 insertions, 134 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java
index f98f46b2e4b..5f88f181b3c 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java
@@ -37,12 +37,12 @@ public abstract class BaseIndexer implements ServerComponent, Startable {
   protected final EsClient esClient;
   private volatile long lastUpdatedAt = 0L;
 
-  protected BaseIndexer(EsClient client, long threadKeepAliveMilliseconds, String indexName, String typeName) {
+  protected BaseIndexer(EsClient client, long threadKeepAliveSeconds, String indexName, String typeName) {
     this.indexName = indexName;
     this.typeName = typeName;
     this.esClient = client;
     this.executor = new ThreadPoolExecutor(0, 1,
-      threadKeepAliveMilliseconds, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
+      threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
   }
 
   public void index() {
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IssueAuthorizationIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/IssueAuthorizationIndexer.java
index 872f3ffe625..a6adcae37bd 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/IssueAuthorizationIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/IssueAuthorizationIndexer.java
@@ -45,7 +45,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer {
   private final DbClient dbClient;
 
   public IssueAuthorizationIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 0L, IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE_AUTHORIZATION);
+    super(esClient, 0L, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION);
     this.dbClient = dbClient;
   }
 
@@ -53,7 +53,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer {
   protected long doIndex(long lastUpdatedAt) {
     // warning - do not enable large mode, else disabling of replicas
     // will impact the type "issue" which is much bigger than issueAuthorization
-    final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX_ISSUES);
+    final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX);
 
     DbSession dbSession = dbClient.openSession(false);
     Connection dbConnection = dbSession.getConnection();
@@ -70,7 +70,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer {
 
   @VisibleForTesting
   void index(Collection<IssueAuthorizationDao.Dto> authorizations) {
-    final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX_ISSUES);
+    final BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX);
     doIndex(bulk, authorizations);
   }
 
@@ -86,10 +86,10 @@ public class IssueAuthorizationIndexer extends BaseIndexer {
   }
 
   public void deleteProject(String uuid, boolean refresh) {
-    esClient.prepareDelete(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE_AUTHORIZATION, uuid).get();
-    if (refresh) {
-      esClient.prepareRefresh(IssueIndexDefinition.INDEX_ISSUES).get();
-    }
+    esClient
+      .prepareDelete(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, uuid)
+      .setRefresh(refresh)
+      .get();
   }
 
   private ActionRequest newUpdateRequest(IssueAuthorizationDao.Dto dto) {
@@ -97,7 +97,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer {
     if (dto.hasNoGroupsNorUsers()) {
       // project still exists but there are no permissions
       // TODO do we really need to delete the document ? Pushing empty groups/users should be enough
-      request = new DeleteRequest(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE_AUTHORIZATION, dto.getProjectUuid())
+      request = new DeleteRequest(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, dto.getProjectUuid())
         .routing(dto.getProjectUuid());
     } else {
       Map<String, Object> doc = ImmutableMap.of(
@@ -105,7 +105,7 @@ public class IssueAuthorizationIndexer extends BaseIndexer {
         IssueIndexDefinition.FIELD_AUTHORIZATION_GROUPS, dto.getGroups(),
         IssueIndexDefinition.FIELD_AUTHORIZATION_USERS, dto.getUsers(),
         IssueIndexDefinition.FIELD_AUTHORIZATION_UPDATED_AT, new Date(dto.getUpdatedAt()));
-      request = new UpdateRequest(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE_AUTHORIZATION, dto.getProjectUuid())
+      request = new UpdateRequest(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_AUTHORIZATION, dto.getProjectUuid())
         .routing(dto.getProjectUuid())
         .doc(doc)
         .upsert(doc);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java
index dd57e2d1562..6abcf219f90 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexDefinition.java
@@ -30,9 +30,9 @@ import org.sonar.server.search.BaseNormalizer;
  */
 public class IssueIndexDefinition implements IndexDefinition {
 
-  public static final String INDEX_ISSUES = "issues";
+  public static final String INDEX = "issues";
 
-  public static final String TYPE_ISSUE_AUTHORIZATION = "issueAuthorization";
+  public static final String TYPE_AUTHORIZATION = "issueAuthorization";
   public static final String TYPE_ISSUE = "issue";
 
   public static final String FIELD_AUTHORIZATION_PROJECT_UUID = "project";
@@ -75,7 +75,7 @@ public class IssueIndexDefinition implements IndexDefinition {
 
   @Override
   public void define(IndexDefinitionContext context) {
-    NewIndex index = context.create(INDEX_ISSUES);
+    NewIndex index = context.create(INDEX);
 
     // shards
     boolean clusterMode = settings.getBoolean(ProcessConstants.CLUSTER_ACTIVATE);
@@ -86,7 +86,7 @@ public class IssueIndexDefinition implements IndexDefinition {
     }
 
     // type "issueAuthorization"
-    NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_ISSUE_AUTHORIZATION);
+    NewIndex.NewIndexType authorizationMapping = index.createType(TYPE_AUTHORIZATION);
     authorizationMapping.setAttribute("_id", ImmutableMap.of("path", FIELD_AUTHORIZATION_PROJECT_UUID));
     authorizationMapping.createDateTimeField(FIELD_AUTHORIZATION_UPDATED_AT);
     authorizationMapping.stringFieldBuilder(FIELD_AUTHORIZATION_PROJECT_UUID).build();
@@ -96,7 +96,7 @@ public class IssueIndexDefinition implements IndexDefinition {
     // type "issue"
     NewIndex.NewIndexType issueMapping = index.createType(TYPE_ISSUE);
     issueMapping.setAttribute("_id", ImmutableMap.of("path", FIELD_ISSUE_KEY));
-    issueMapping.setAttribute("_parent", ImmutableMap.of("type", TYPE_ISSUE_AUTHORIZATION));
+    issueMapping.setAttribute("_parent", ImmutableMap.of("type", TYPE_AUTHORIZATION));
     issueMapping.setAttribute("_routing", ImmutableMap.of("required", true, "path", FIELD_ISSUE_PROJECT_UUID));
     issueMapping.stringFieldBuilder(FIELD_ISSUE_ACTION_PLAN).build();
     // TODO do we really sort by assignee ?
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java
index 218db7b2594..97034d4c9b2 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/IssueIndexer.java
@@ -27,7 +27,6 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.sonar.core.persistence.DbSession;
 import org.sonar.server.db.DbClient;
 import org.sonar.server.issue.index.IssueDoc;
-import org.sonar.server.issue.index.IssueNormalizer;
 
 import java.sql.Connection;
 import java.util.Iterator;
@@ -37,7 +36,7 @@ public class IssueIndexer extends BaseIndexer {
   private final DbClient dbClient;
 
   public IssueIndexer(DbClient dbClient, EsClient esClient) {
-    super(esClient, 300000L, IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE);
+    super(esClient, 300, IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE);
     this.dbClient = dbClient;
   }
 
@@ -79,14 +78,14 @@ public class IssueIndexer extends BaseIndexer {
       QueryBuilders.matchAllQuery(),
       FilterBuilders.boolFilter().must(FilterBuilders.termsFilter(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, uuid))
     );
-    esClient.prepareDeleteByQuery(IssueIndexDefinition.INDEX_ISSUES).setQuery(query).get();
+    esClient.prepareDeleteByQuery(IssueIndexDefinition.INDEX).setQuery(query).get();
     if (refresh) {
-      esClient.prepareRefresh(IssueIndexDefinition.INDEX_ISSUES).get();
+      esClient.prepareRefresh(IssueIndexDefinition.INDEX).get();
     }
   }
 
   BulkIndexer createBulkIndexer(boolean large) {
-    BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX_ISSUES);
+    BulkIndexer bulk = new BulkIndexer(esClient, IssueIndexDefinition.INDEX);
     bulk.setLarge(large);
     return bulk;
   }
@@ -97,7 +96,7 @@ public class IssueIndexer extends BaseIndexer {
     // parent doc is issueAuthorization
     issue.setField("_parent", projectUuid);
 
-    return new UpdateRequest(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE, issue.key())
+    return new UpdateRequest(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE, issue.key())
       .routing(projectUuid)
       .parent(projectUuid)
       .doc(issue.getFields())
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java
index de3cc8307d9..bf44b764a28 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/IssueResultSetIterator.java
@@ -34,8 +34,8 @@ import java.sql.Timestamp;
 import java.util.Date;
 
 /**
- * Scroll over table ISSUES and directly read the maps required to
- * post index requests
+ * Scrolls over table ISSUES and reads documents to populate
+ * the issues index
  */
 class IssueResultSetIterator extends ResultSetIterator<IssueDoc> {
 
diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
index 5a5079ee9d5..ffd43ddc8fa 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
@@ -293,7 +293,7 @@ public class IssueIndex extends BaseIndex<Issue, IssueDto, String> {
     for (String group : groups) {
       groupsAndUser.add(FilterBuilders.termFilter(IssueIndexDefinition.FIELD_AUTHORIZATION_GROUPS, group));
     }
-    return FilterBuilders.hasParentFilter(IssueIndexDefinition.TYPE_ISSUE_AUTHORIZATION,
+    return FilterBuilders.hasParentFilter(IssueIndexDefinition.TYPE_AUTHORIZATION,
       QueryBuilders.filteredQuery(
         QueryBuilders.matchAllQuery(),
         FilterBuilders.boolFilter()
diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java
index 0e7d2ab59fd..543431f72b1 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexSynchronizer.java
@@ -72,7 +72,7 @@ public class IndexSynchronizer {
       synchronize(session, db.activityDao(), index.get(ActivityIndex.class));
 
       LOG.info("Indexing of sourceLine records");
-      sourceLineIndexer.indexSourceLines(true);
+      sourceLineIndexer.index();
 
       session.commit();
       LOG.info("Synchronization done in {}ms...", System.currentTimeMillis() - start);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java b/server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java
index ea842928e5b..a03d1ba5b23 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/IndexSourceLinesStep.java
@@ -27,7 +27,7 @@ import org.sonar.server.source.index.SourceLineIndexer;
 
 public class IndexSourceLinesStep implements ComputationStep {
 
-  private SourceLineIndexer indexer;
+  private final SourceLineIndexer indexer;
 
   public IndexSourceLinesStep(SourceLineIndexer indexer) {
     this.indexer = indexer;
@@ -35,7 +35,7 @@ public class IndexSourceLinesStep implements ComputationStep {
 
   @Override
   public void execute(DbSession session, AnalysisReportDto analysisReportDto, ComponentDto project) {
-    indexer.indexSourceLines(false);
+    indexer.index();
   }
 
   @Override
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
index bbaffed232a..7269ceb8777 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
@@ -20,7 +20,7 @@
 package org.sonar.server.source.index;
 
 import com.google.common.base.Preconditions;
-import org.elasticsearch.common.collect.Lists;
+import com.google.common.collect.Lists;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.sort.SortOrder;
@@ -38,18 +38,6 @@ public class SourceLineIndex implements ServerComponent {
   }
 
   /**
-   * Unindex all lines in file with UUID <code>fileUuid</code> above line <code>lastLine</code>
-   */
-  public void deleteLinesFromFileAbove(String fileUuid, int lastLine) {
-    esClient.prepareDeleteByQuery(SourceLineIndexDefinition.INDEX_SOURCE_LINES)
-      .setTypes(SourceLineIndexDefinition.TYPE_SOURCE_LINE)
-      .setQuery(QueryBuilders.boolQuery()
-        .must(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid))
-        .must(QueryBuilders.rangeQuery(SourceLineIndexDefinition.FIELD_LINE).gt(lastLine))
-      ).get();
-  }
-
-  /**
    * Get lines of code for file with UUID <code>fileUuid</code> with line numbers
    * between <code>from</code> and <code>to</code> (both inclusive). Line numbers
    * start at 1.
@@ -62,8 +50,8 @@ public class SourceLineIndex implements ServerComponent {
     Preconditions.checkArgument(to >= from, "'to' must be larger than or equal to 'from'");
     List<SourceLineDoc> lines = Lists.newArrayList();
 
-    for (SearchHit hit: esClient.prepareSearch(SourceLineIndexDefinition.INDEX_SOURCE_LINES)
-      .setTypes(SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+    for (SearchHit hit: esClient.prepareSearch(SourceLineIndexDefinition.INDEX)
+      .setTypes(SourceLineIndexDefinition.TYPE)
       .setSize(1 + to - from)
       .setQuery(QueryBuilders.boolQuery()
         .must(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid))
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
index fc2eb002ed6..c3950f178f5 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexDefinition.java
@@ -37,9 +37,9 @@ public class SourceLineIndexDefinition implements IndexDefinition {
   public static final String FIELD_HIGHLIGHTING = "highlighting";
   public static final String FIELD_SOURCE = "source";
 
-  public static final String INDEX_SOURCE_LINES = "sourcelines";
+  public static final String INDEX = "sourcelines";
 
-  public static final String TYPE_SOURCE_LINE = "sourceline";
+  public static final String TYPE = "sourceline";
 
 
   private final Settings settings;
@@ -50,7 +50,7 @@ public class SourceLineIndexDefinition implements IndexDefinition {
 
   @Override
   public void define(IndexDefinitionContext context) {
-    NewIndex index = context.create(INDEX_SOURCE_LINES);
+    NewIndex index = context.create(INDEX);
 
     // shards
     boolean clusterMode = settings.getBoolean(ProcessConstants.CLUSTER_ACTIVATE);
@@ -61,7 +61,7 @@ public class SourceLineIndexDefinition implements IndexDefinition {
     }
 
     // type "sourceline"
-    NewIndex.NewIndexType sourceLineMapping = index.createType(TYPE_SOURCE_LINE);
+    NewIndex.NewIndexType sourceLineMapping = index.createType(TYPE);
     sourceLineMapping.stringFieldBuilder(FIELD_PROJECT_UUID).build();
     sourceLineMapping.stringFieldBuilder(FIELD_FILE_UUID).build();
     sourceLineMapping.createIntegerField(FIELD_LINE);
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
index 6116ea942d7..4ed8d7abe80 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
@@ -19,83 +19,83 @@
  */
 package org.sonar.server.source.index;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.elasticsearch.action.update.UpdateRequest;
-import org.sonar.api.ServerComponent;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.sonar.core.persistence.DbSession;
 import org.sonar.server.db.DbClient;
+import org.sonar.server.es.BaseIndexer;
 import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.EsClient;
 
 import java.sql.Connection;
-import java.util.Collection;
 import java.util.Iterator;
 
-public class SourceLineIndexer implements ServerComponent {
+public class SourceLineIndexer extends BaseIndexer {
 
   private final DbClient dbClient;
-  private final EsClient esClient;
-  private final SourceLineIndex index;
-  private long lastUpdatedAt = 0L;
 
-  public SourceLineIndexer(DbClient dbClient, EsClient esClient, SourceLineIndex index) {
+  public SourceLineIndexer(DbClient dbClient, EsClient esClient) {
+    super(esClient, 0L, SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE);
     this.dbClient = dbClient;
-    this.esClient = esClient;
-    this.index = index;
   }
 
-  public void indexSourceLines(boolean large) {
-    final BulkIndexer bulk = new BulkIndexer(esClient, SourceLineIndexDefinition.INDEX_SOURCE_LINES);
-    bulk.setLarge(large);
+  @Override
+  protected long doIndex(long lastUpdatedAt) {
+    final BulkIndexer bulk = new BulkIndexer(esClient, SourceLineIndexDefinition.INDEX);
+    bulk.setLarge(lastUpdatedAt == 0L);
 
     DbSession dbSession = dbClient.openSession(false);
     Connection dbConnection = dbSession.getConnection();
     try {
-      SourceLineResultSetIterator rowIt = SourceLineResultSetIterator.create(dbClient, dbConnection, getLastUpdatedAt());
-      indexSourceLines(bulk, rowIt);
+      SourceLineResultSetIterator rowIt = SourceLineResultSetIterator.create(dbClient, dbConnection, lastUpdatedAt);
+      long maxUpdatedAt = doIndex(bulk, rowIt);
       rowIt.close();
+      return maxUpdatedAt;
 
     } finally {
       dbSession.close();
     }
   }
 
-  public void indexSourceLines(BulkIndexer bulk, Iterator<Collection<SourceLineDoc>> sourceLines) {
+  @VisibleForTesting
+  long index(Iterator<SourceLineResultSetIterator.SourceFile> sourceFiles) {
+    final BulkIndexer bulk = new BulkIndexer(esClient, SourceLineIndexDefinition.INDEX);
+    return doIndex(bulk, sourceFiles);
+  }
+
+  private long doIndex(BulkIndexer bulk, Iterator<SourceLineResultSetIterator.SourceFile> files) {
+    long maxUpdatedAt = 0L;
     bulk.start();
-    while (sourceLines.hasNext()) {
-      Collection<SourceLineDoc> lineDocs = sourceLines.next();
-      String fileUuid = null;
-      int lastLine = 0;
-      for (SourceLineDoc sourceLine: lineDocs) {
-        lastLine ++;
-        fileUuid = sourceLine.fileUuid();
-        bulk.add(newUpsertRequest(sourceLine));
-        long dtoUpdatedAt = sourceLine.updateDate().getTime();
-        if (lastUpdatedAt < dtoUpdatedAt) {
-          lastUpdatedAt = dtoUpdatedAt;
-        }
+    while (files.hasNext()) {
+      SourceLineResultSetIterator.SourceFile file = files.next();
+      for (SourceLineDoc line : file.getLines()) {
+        bulk.add(newUpsertRequest(line));
       }
-      index.deleteLinesFromFileAbove(fileUuid, lastLine);
+      deleteLinesFromFileAbove(file.getFileUuid(), file.getLines().size());
+      maxUpdatedAt = Math.max(maxUpdatedAt, file.getUpdatedAt());
     }
     bulk.stop();
-  }
-
-  private long getLastUpdatedAt() {
-    long result;
-    if (lastUpdatedAt <= 0L) {
-      // request ES to get the max(updatedAt)
-      result = esClient.getLastUpdatedAt(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE);
-    } else {
-      // use cache. Will not work with Tomcat cluster.
-      result = lastUpdatedAt;
-    }
-    return result;
+    return maxUpdatedAt;
   }
 
   private UpdateRequest newUpsertRequest(SourceLineDoc lineDoc) {
     String projectUuid = lineDoc.projectUuid();
-    return new UpdateRequest(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE, lineDoc.key())
+    return new UpdateRequest(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE, lineDoc.key())
       .routing(projectUuid)
       .doc(lineDoc.getFields())
       .upsert(lineDoc.getFields());
   }
+
+  /**
+   * Unindex all lines in file with UUID <code>fileUuid</code> above line <code>lastLine</code>
+   */
+  private void deleteLinesFromFileAbove(String fileUuid, int lastLine) {
+    esClient.prepareDeleteByQuery(SourceLineIndexDefinition.INDEX)
+      .setTypes(SourceLineIndexDefinition.TYPE)
+      .setQuery(QueryBuilders.boolQuery()
+        .must(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid))
+        .must(QueryBuilders.rangeQuery(SourceLineIndexDefinition.FIELD_LINE).gt(lastLine))
+      ).get();
+  }
 }
diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java
index 0bafa2fc512..504ae9f4618 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineResultSetIterator.java
@@ -37,15 +37,41 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.Collection;
 import java.util.Date;
 import java.util.List;
 
 /**
- * Scroll over table ISSUES and directly read the maps required to
- * post index requests
+ * Scroll over table FILE_SOURCES and directly parse CSV field required to
+ * populate the index sourcelines
  */
-class SourceLineResultSetIterator extends ResultSetIterator<Collection<SourceLineDoc>> {
+class SourceLineResultSetIterator extends ResultSetIterator<SourceLineResultSetIterator.SourceFile> {
+
+  static class SourceFile {
+    private final String fileUuid;
+    private final long updatedAt;
+    private final List<SourceLineDoc> lines = Lists.newArrayList();
+
+    SourceFile(String fileUuid, long updatedAt) {
+      this.fileUuid = fileUuid;
+      this.updatedAt = updatedAt;
+    }
+
+    String getFileUuid() {
+      return fileUuid;
+    }
+
+    long getUpdatedAt() {
+      return updatedAt;
+    }
+
+    List<SourceLineDoc> getLines() {
+      return lines;
+    }
+
+    void addLine(SourceLineDoc line) {
+      this.lines.add(line);
+    }
+  }
 
   private static final String[] FIELDS = {
     // column 1
@@ -77,13 +103,15 @@ class SourceLineResultSetIterator extends ResultSetIterator<Collection<SourceLin
   }
 
   @Override
-  protected Collection<SourceLineDoc> read(ResultSet rs) throws SQLException {
+  protected SourceFile read(ResultSet rs) throws SQLException {
     String projectUuid = rs.getString(1);
     String fileUuid = rs.getString(2);
-    Date updatedAt = new Date(SqlUtil.getLong(rs, 3));
+    long updatedAt = SqlUtil.getLong(rs, 3);
+    Date updatedDate = new Date(updatedAt);
+    SourceFile result = new SourceFile(fileUuid, updatedAt);
+
     String csv = rs.getString(4);
 
-    List<SourceLineDoc> lines = Lists.newArrayList();
     if (StringUtils.isNotEmpty(csv)) {
       int line = 1;
       CSVParser csvParser = null;
@@ -96,7 +124,7 @@ class SourceLineResultSetIterator extends ResultSetIterator<Collection<SourceLin
           doc.setProjectUuid(projectUuid);
           doc.setFileUuid(fileUuid);
           doc.setLine(line);
-          doc.setUpdateDate(updatedAt);
+          doc.setUpdateDate(updatedDate);
           doc.setScmRevision(csvRecord.get(0));
           doc.setScmAuthor(csvRecord.get(1));
           doc.setScmDate(DateUtils.parseDateTimeQuietly(csvRecord.get(2)));
@@ -106,7 +134,7 @@ class SourceLineResultSetIterator extends ResultSetIterator<Collection<SourceLin
           doc.setHighlighting(csvRecord.get(6));
           doc.setSource(csvRecord.get(csvRecord.size() - 1));
 
-          lines.add(doc);
+          result.addLine(doc);
 
           line++;
         }
@@ -120,6 +148,6 @@ class SourceLineResultSetIterator extends ResultSetIterator<Collection<SourceLin
       }
     }
 
-    return lines;
+    return result;
   }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java
index 382cc01fe73..de764280847 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/component/ComponentCleanerServiceMediumTest.java
@@ -135,6 +135,6 @@ public class ComponentCleanerServiceMediumTest {
   }
 
   private long countIssueAuthorizationDocs() {
-    return tester.get(EsClient.class).prepareCount(IssueIndexDefinition.INDEX_ISSUES).setTypes(IssueIndexDefinition.TYPE_ISSUE_AUTHORIZATION).get().getCount();
+    return tester.get(EsClient.class).prepareCount(IssueIndexDefinition.INDEX).setTypes(IssueIndexDefinition.TYPE_AUTHORIZATION).get().getCount();
   }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
index 9b0e4db0506..fba0b280625 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
@@ -115,7 +115,7 @@ public class EsTester extends ExternalResource {
     }
   }
 
-  public void truncateAllIndices() {
+  public void truncateIndices() {
     client.prepareDeleteByQuery(client.prepareState().get()
       .getState().getMetaData().concreteAllIndices())
       .setQuery(QueryBuilders.matchAllQuery())
diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexerTest.java
index bb91349d7be..e439d95f54c 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/IssueIndexerTest.java
@@ -36,7 +36,7 @@ public class IssueIndexerTest {
   public void index_nothing() throws Exception {
     IssueIndexer indexer = new IssueIndexer(null, esTester.client());
     indexer.index(indexer.createBulkIndexer(false), Iterators.<IssueDoc>emptyIterator());
-    assertThat(esTester.countDocuments(IssueIndexDefinition.INDEX_ISSUES, IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0L);
+    assertThat(esTester.countDocuments(IssueIndexDefinition.INDEX, IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0L);
   }
 
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java
index c27acbf14e0..444ef8d60a3 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/permission/InternalPermissionServiceMediumTest.java
@@ -176,6 +176,6 @@ public class InternalPermissionServiceMediumTest {
   }
 
   private long countIssueAuthorizationDocs() {
-    return tester.get(EsClient.class).prepareCount(IssueIndexDefinition.INDEX_ISSUES).setTypes(IssueIndexDefinition.TYPE_ISSUE_AUTHORIZATION).get().getCount();
+    return tester.get(EsClient.class).prepareCount(IssueIndexDefinition.INDEX).setTypes(IssueIndexDefinition.TYPE_AUTHORIZATION).get().getCount();
   }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexDefinitionTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexDefinitionTest.java
new file mode 100644
index 00000000000..15778924882
--- /dev/null
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexDefinitionTest.java
@@ -0,0 +1,42 @@
+package org.sonar.server.source.index;
+
+import org.junit.Test;
+import org.sonar.api.config.Settings;
+import org.sonar.process.ProcessConstants;
+import org.sonar.server.es.IndexDefinition;
+import org.sonar.server.es.NewIndex;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class SourceLineIndexDefinitionTest {
+
+  IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
+
+  @Test
+  public void define() throws Exception {
+    IndexDefinition def = new SourceLineIndexDefinition(new Settings());
+    def.define(context);
+
+    assertThat(context.getIndices()).hasSize(1);
+    NewIndex index = context.getIndices().get("sourcelines");
+    assertThat(index).isNotNull();
+    assertThat(index.getTypes().keySet()).containsOnly("sourceline");
+
+    // no cluster by default
+    assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("1");
+    assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0");
+  }
+
+  @Test
+  public void enable_cluster() throws Exception {
+    Settings settings = new Settings();
+    settings.setProperty(ProcessConstants.CLUSTER_ACTIVATE, true);
+    IndexDefinition def = new SourceLineIndexDefinition(settings);
+    def.define(context);
+
+    NewIndex issuesIndex = context.getIndices().get("sourcelines");
+    assertThat(issuesIndex.getSettings().get("index.number_of_shards")).isEqualTo("4");
+    assertThat(issuesIndex.getSettings().get("index.number_of_replicas")).isEqualTo("1");
+  }
+
+}
diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java
index ea3feff8b4b..ced283fc0bb 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexTest.java
@@ -41,7 +41,7 @@ public class SourceLineIndexTest {
 
   @Test
   public void should_retrieve_line_range() throws Exception {
-    es.putDocuments(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE,
+    es.putDocuments(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE,
       this.getClass(),
       "file1_line1.json",
       "file1_line2.json",
diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
index 6d38d142899..b4c16ec448c 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
@@ -19,9 +19,8 @@
  */
 package org.sonar.server.source.index;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
+import com.google.common.collect.Iterators;
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -31,48 +30,50 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.config.Settings;
 import org.sonar.api.utils.DateUtils;
+import org.sonar.core.persistence.TestDatabase;
 import org.sonar.server.db.DbClient;
-import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.EsTester;
 import org.sonar.server.search.BaseNormalizer;
 import org.sonar.test.TestUtils;
 
 import java.io.FileInputStream;
-import java.util.Collection;
 import java.util.Date;
-import java.util.List;
 import java.util.Map;
 
 import static org.fest.assertions.Assertions.assertThat;
-import static org.mockito.Mockito.mock;
 
 public class SourceLineIndexerTest {
 
   @Rule
   public EsTester es = new EsTester().addDefinitions(new SourceLineIndexDefinition(new Settings()));
 
-  private SourceLineIndex index;
+  @Rule
+  public TestDatabase db = new TestDatabase();
 
   private SourceLineIndexer indexer;
 
   @Before
   public void setUp() {
-    index = new SourceLineIndex(es.client());
-    indexer = new SourceLineIndexer(mock(DbClient.class), es.client(), index);
+    indexer = new SourceLineIndexer(new DbClient(db.database(), db.myBatis()), es.client());
+  }
+
+  @Test
+  public void index_source_lines_from_db() throws Exception {
+    db.prepareDbUnit(getClass(), "db.xml");
+    indexer.index();
+    assertThat(countDocuments()).isEqualTo(2);
   }
 
   @Test
-  public void should_index_source_lines() throws Exception {
-    es.client().prepareIndex(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+  public void update_already_indexed_lines() throws Exception {
+    es.client().prepareIndex(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE)
       .setSource(IOUtils.toString(new FileInputStream(TestUtils.getResource(this.getClass(), "line2.json"))))
       .get();
-    es.client().prepareIndex(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+    es.client().prepareIndex(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE)
       .setSource(IOUtils.toString(new FileInputStream(TestUtils.getResource(this.getClass(), "line2_other_file.json"))))
       .setRefresh(true)
       .get();
 
-    BulkIndexer bulk = new BulkIndexer(es.client(), SourceLineIndexDefinition.INDEX_SOURCE_LINES);
-
     SourceLineDoc line1 = new SourceLineDoc(ImmutableMap.<String, Object>builder()
       .put(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "abcd")
       .put(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh")
@@ -83,16 +84,14 @@ public class SourceLineIndexerTest {
       .put(SourceLineIndexDefinition.FIELD_SOURCE, "package org.sonar.server.source;")
       .put(BaseNormalizer.UPDATED_AT_FIELD, new Date())
       .build());
-    Collection<SourceLineDoc> sourceLines = ImmutableList.of(line1);
-
-    List<Collection<SourceLineDoc>> sourceLineContainer = Lists.newArrayList();
-    sourceLineContainer.add(sourceLines);
-    indexer.indexSourceLines(bulk, sourceLineContainer.iterator());
+    SourceLineResultSetIterator.SourceFile file = new SourceLineResultSetIterator.SourceFile("efgh", System.currentTimeMillis());
+    file.addLine(line1);
+    indexer.index(Iterators.singletonIterator(file));
 
-    assertThat(es.countDocuments(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE)).isEqualTo(2L);
+    assertThat(countDocuments()).isEqualTo(2L);
 
-    SearchResponse fileSearch = es.client().prepareSearch(SourceLineIndexDefinition.INDEX_SOURCE_LINES)
-      .setTypes(SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+    SearchResponse fileSearch = es.client().prepareSearch(SourceLineIndexDefinition.INDEX)
+      .setTypes(SourceLineIndexDefinition.TYPE)
       .setQuery(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh"))
       .get();
     assertThat(fileSearch.getHits().getTotalHits()).isEqualTo(1L);
@@ -106,6 +105,10 @@ public class SourceLineIndexerTest {
       MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_DATE, "2014-01-01T11:34:56.000Z"),
       MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "polop"),
       MapAssert.entry(SourceLineIndexDefinition.FIELD_SOURCE, "package org.sonar.server.source;")
-    );
+      );
+  }
+
+  private long countDocuments() {
+    return es.countDocuments(SourceLineIndexDefinition.INDEX, SourceLineIndexDefinition.TYPE);
   }
 }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java
index 57ffb0f6e4d..8639ec88f8b 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineResultSetIteratorTest.java
@@ -19,7 +19,6 @@
  */
 package org.sonar.server.source.index;
 
-import com.google.common.collect.Lists;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ClassRule;
@@ -29,8 +28,6 @@ import org.sonar.server.db.DbClient;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
-import java.util.Collection;
-import java.util.List;
 
 import static org.fest.assertions.Assertions.assertThat;
 import static org.fest.assertions.Fail.fail;
@@ -68,9 +65,9 @@ public class SourceLineResultSetIteratorTest {
 
     SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, connection, 0L);
     assertThat(iterator.hasNext()).isTrue();
-    List<SourceLineDoc> sourceLines = Lists.newArrayList(iterator.next());
-    assertThat(sourceLines).hasSize(4);
-    SourceLineDoc firstLine = sourceLines.get(0);
+    SourceLineResultSetIterator.SourceFile file = iterator.next();
+    assertThat(file.getLines()).hasSize(4);
+    SourceLineDoc firstLine = file.getLines().get(0);
     assertThat(firstLine.projectUuid()).isEqualTo("uuid-MyProject");
     assertThat(firstLine.fileUuid()).isEqualTo("uuid-MyFile.xoo");
     assertThat(firstLine.line()).isEqualTo(1);
@@ -97,8 +94,9 @@ public class SourceLineResultSetIteratorTest {
 
     SourceLineResultSetIterator iterator = SourceLineResultSetIterator.create(dbClient, db.openConnection(), 0L);
     assertThat(iterator.hasNext()).isTrue();
-    Collection<SourceLineDoc> lines = iterator.next();
-    assertThat(lines).isEmpty();
+    SourceLineResultSetIterator.SourceFile file = iterator.next();
+    assertThat(file.getFileUuid()).isEqualTo("uuid-MyFile.xoo");
+    assertThat(file.getLines()).isEmpty();
   }
 
   @Test
diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/db.xml b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/db.xml
new file mode 100644
index 00000000000..eaaeae9f4de
--- /dev/null
+++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/db.xml
@@ -0,0 +1,6 @@
+<dataset>
+
+  <file_sources id="1" project_uuid="uuid-MyProject" file_uuid="uuid-MyFile.xoo" created_at="1416238020000" updated_at="1416239042000"
+                data="aef12a,alice,2014-04-25T12:34:56+0100,,,,polop,class Foo {&#10;aef12a,alice,2014-04-25T12:34:56+0100,,,,polop,}" data_hash="THE_HASH" />
+
+</dataset>