SONAR-5801 Delete unneeded lines after indexation of source

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
index 35cb739359af3c2e8929db364110428b19f505bb..9e8a9566d69bdb97a0b84275f1927cac72f81541 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/EsClient.java
@@ -36,6 +36,7 @@ import org.elasticsearch.action.count.CountRequestBuilder;
 import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
 import org.elasticsearch.action.get.GetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.client.Client;
@@ -46,23 +47,7 @@ import org.picocontainer.Startable;
 import org.sonar.core.profiling.Profiling;
 import org.sonar.server.search.ClusterHealth;
 import org.sonar.server.search.SearchClient;
-import org.sonar.server.search.request.ProxyBulkRequestBuilder;
-import org.sonar.server.search.request.ProxyClusterHealthRequestBuilder;
-import org.sonar.server.search.request.ProxyClusterStateRequestBuilder;
-import org.sonar.server.search.request.ProxyClusterStatsRequestBuilder;
-import org.sonar.server.search.request.ProxyCountRequestBuilder;
-import org.sonar.server.search.request.ProxyCreateIndexRequestBuilder;
-import org.sonar.server.search.request.ProxyDeleteByQueryRequestBuilder;
-import org.sonar.server.search.request.ProxyFlushRequestBuilder;
-import org.sonar.server.search.request.ProxyGetRequestBuilder;
-import org.sonar.server.search.request.ProxyIndicesExistsRequestBuilder;
-import org.sonar.server.search.request.ProxyIndicesStatsRequestBuilder;
-import org.sonar.server.search.request.ProxyMultiGetRequestBuilder;
-import org.sonar.server.search.request.ProxyNodesStatsRequestBuilder;
-import org.sonar.server.search.request.ProxyPutMappingRequestBuilder;
-import org.sonar.server.search.request.ProxyRefreshRequestBuilder;
-import org.sonar.server.search.request.ProxySearchRequestBuilder;
-import org.sonar.server.search.request.ProxySearchScrollRequestBuilder;
+import org.sonar.server.search.request.*;
 
 /**
  * Facade to connect to Elasticsearch node. Handles correctly errors (logging + exceptions
@@ -169,6 +154,10 @@ public class EsClient implements Startable {
     return new ProxyDeleteByQueryRequestBuilder(client, profiling).setIndices(indices);
   }
 
+  public IndexRequestBuilder prepareIndex(String index, String type) {
+    return new ProxyIndexRequestBuilder(client, profiling).setIndex(index).setType(type);
+  }
+
   public long getLastUpdatedAt(String indexName, String typeName) {
     SearchRequestBuilder request = prepareSearch(indexName)
       .setTypes(typeName)

diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java
index 2dc9b64f54d1589f5689d37e2dd4464bdd525783..b7a594aa4f8540bde242556593a38841d9ad0a4f 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java
@@ -159,6 +159,7 @@ import org.sonar.server.rule.index.RuleNormalizer;
 import org.sonar.server.rule.ws.*;
 import org.sonar.server.search.*;
 import org.sonar.server.source.*;
+import org.sonar.server.source.index.SourceLineIndex;
 import org.sonar.server.source.index.SourceLineIndexDefinition;
 import org.sonar.server.source.index.SourceLineIndexer;
 import org.sonar.server.source.ws.*;
@@ -548,6 +549,7 @@ class ServerComponents {
     pico.addSingleton(RawAction.class);
     pico.addSingleton(ScmAction.class);
     pico.addSingleton(SourceLineIndexDefinition.class);
+    pico.addSingleton(SourceLineIndex.class);
     pico.addSingleton(SourceLineIndexer.class);
     pico.addSingleton(IndexSourceLinesStep.class);
 

diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/request/ProxyIndexRequestBuilder.java b/server/sonar-server/src/main/java/org/sonar/server/search/request/ProxyIndexRequestBuilder.java
new file mode 100644 (file)
index 0000000..26751f0
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/search/request/ProxyIndexRequestBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.search.request;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.unit.TimeValue;
+import org.sonar.core.profiling.Profiling;
+import org.sonar.core.profiling.StopWatch;
+
+public class ProxyIndexRequestBuilder extends IndexRequestBuilder {
+
+  private final Profiling profiling;
+
+  public ProxyIndexRequestBuilder(Client client, Profiling profiling) {
+    super(client);
+    this.profiling = profiling;
+  }
+
+  @Override
+  public IndexResponse get() {
+    StopWatch fullProfile = profiling.start("index", Profiling.Level.FULL);
+    try {
+      return super.execute().actionGet();
+    } catch (Exception e) {
+      throw new IllegalStateException(String.format("Fail to execute %s", toString()), e);
+    } finally {
+      if (profiling.isProfilingEnabled(Profiling.Level.FULL)) {
+        fullProfile.stop("%s", toString());
+      }
+    }
+  }
+
+  @Override
+  public IndexResponse get(TimeValue timeout) {
+    throw new IllegalStateException("Not yet implemented");
+  }
+
+  @Override
+  public IndexResponse get(String timeout) {
+    throw new IllegalStateException("Not yet implemented");
+  }
+
+  @Override
+  public ListenableActionFuture<IndexResponse> execute() {
+    throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous");
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder message = new StringBuilder().append("ES index request");
+    message.append(String.format(" for key '%s'", request.id()));
+    message.append(String.format(" on index '%s'", request.index()));
+    message.append(String.format(" on type '%s'", request.type()));
+    return message.toString();
+  }
+}

diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
new file mode 100644 (file)
index 0000000..36b39b9
--- /dev/null
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndex.java
@@ -0,0 +1,42 @@
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.source.index;
+
+import org.elasticsearch.index.query.QueryBuilders;
+import org.sonar.api.ServerComponent;
+import org.sonar.server.es.EsClient;
+
+public class SourceLineIndex implements ServerComponent {
+
+  private final EsClient esClient;
+
+  public SourceLineIndex(EsClient esClient) {
+    this.esClient = esClient;
+  }
+
+  public void deleteLinesFromFileAbove(String fileUuid, int lastLine) {
+    esClient.prepareDeleteByQuery(SourceLineIndexDefinition.INDEX_SOURCE_LINES)
+      .setTypes(SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+      .setQuery(QueryBuilders.boolQuery()
+        .must(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, fileUuid))
+        .must(QueryBuilders.rangeQuery(SourceLineIndexDefinition.FIELD_LINE).gt(lastLine))
+      ).get();
+  }
+}

diff --git a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
index 8dfa2cedf2abf1062393f1ead0f43e219706127f..2895663d2cc3bc8ec81223be372818744f7b04e3 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/source/index/SourceLineIndexer.java
@@ -37,11 +37,13 @@ public class SourceLineIndexer implements ServerComponent {
 
   private final DbClient dbClient;
   private final EsClient esClient;
+  private final SourceLineIndex index;
   private long lastUpdatedAt = 0L;
 
-  public SourceLineIndexer(DbClient dbClient, EsClient esClient) {
+  public SourceLineIndexer(DbClient dbClient, EsClient esClient, SourceLineIndex index) {
     this.dbClient = dbClient;
     this.esClient = esClient;
+    this.index = index;
   }
 
   public void indexSourceLines(boolean large) {
@@ -65,14 +67,18 @@ public class SourceLineIndexer implements ServerComponent {
     bulk.start();
     while (sourceLines.hasNext()) {
       Collection<SourceLineDoc> lineDocs = sourceLines.next();
+      String fileUuid = null;
+      int lastLine = 0;
       for (SourceLineDoc sourceLine: lineDocs) {
+        lastLine ++;
+        fileUuid = sourceLine.fileUuid();
         bulk.add(newUpsertRequest(sourceLine));
         long dtoUpdatedAt = sourceLine.updateDate().getTime();
         if (lastUpdatedAt < dtoUpdatedAt) {
           lastUpdatedAt = dtoUpdatedAt;
         }
       }
-
+      index.deleteLinesFromFileAbove(fileUuid, lastLine);
     }
     bulk.stop();
   }

diff --git a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
index 3cc23526c6f322c46c90fd84333987e5903cd5f6..43ebbfcb8bdf4825eaf8357730882d8f9fe8a8c9 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/source/index/SourceLineIndexerTest.java
@@ -22,6 +22,10 @@ package org.sonar.server.source.index;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.fest.assertions.MapAssert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -30,14 +34,15 @@ import org.sonar.server.db.DbClient;
 import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.EsTester;
 import org.sonar.server.search.BaseNormalizer;
-import org.sonar.server.source.index.SourceLineDoc;
-import org.sonar.server.source.index.SourceLineIndexDefinition;
-import org.sonar.server.source.index.SourceLineIndexer;
+import org.sonar.test.TestUtils;
 
+import java.io.FileInputStream;
 import java.util.Collection;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 
+import static org.fest.assertions.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 
 public class SourceLineIndexerTest {
@@ -45,21 +50,32 @@ public class SourceLineIndexerTest {
   @Rule
   public EsTester es = new EsTester().addDefinitions(new SourceLineIndexDefinition(new Settings()));
 
+  private SourceLineIndex index;
+
   private SourceLineIndexer indexer;
 
   @Before
   public void setUp() {
-    indexer = new SourceLineIndexer(mock(DbClient.class), es.client());
+    index = new SourceLineIndex(es.client());
+    indexer = new SourceLineIndexer(mock(DbClient.class), es.client(), index);
   }
 
   @Test
-  public void should_index_source_lines() {
+  public void should_index_source_lines() throws Exception {
+    es.client().prepareIndex(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+      .setSource(IOUtils.toString(new FileInputStream(TestUtils.getResource(this.getClass(), "line2.json"))))
+      .get();
+    es.client().prepareIndex(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+      .setSource(IOUtils.toString(new FileInputStream(TestUtils.getResource(this.getClass(), "line2_other_file.json"))))
+      .setRefresh(true)
+      .get();
+
     BulkIndexer bulk = new BulkIndexer(es.client(), SourceLineIndexDefinition.INDEX_SOURCE_LINES);
 
     SourceLineDoc line1 = new SourceLineDoc(ImmutableMap.<String, Object>builder()
       .put(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "abcd")
       .put(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh")
-      .put(SourceLineIndexDefinition.FIELD_LINE, 42)
+      .put(SourceLineIndexDefinition.FIELD_LINE, 1)
       .put(SourceLineIndexDefinition.FIELD_SCM_REVISION, "cafebabe")
       .put(SourceLineIndexDefinition.FIELD_SCM_DATE, "2014-01-01T12:34:56.7+0100")
       .put(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "polop")
@@ -71,5 +87,24 @@ public class SourceLineIndexerTest {
     List<Collection<SourceLineDoc>> sourceLineContainer = Lists.newArrayList();
     sourceLineContainer.add(sourceLines);
     indexer.indexSourceLines(bulk, sourceLineContainer.iterator());
+
+    assertThat(es.countDocuments(SourceLineIndexDefinition.INDEX_SOURCE_LINES, SourceLineIndexDefinition.TYPE_SOURCE_LINE)).isEqualTo(2L);
+
+    SearchResponse fileSearch = es.client().prepareSearch(SourceLineIndexDefinition.INDEX_SOURCE_LINES)
+      .setTypes(SourceLineIndexDefinition.TYPE_SOURCE_LINE)
+      .setQuery(QueryBuilders.termQuery(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh"))
+      .get();
+    assertThat(fileSearch.getHits().getTotalHits()).isEqualTo(1L);
+    Map<String, Object> fields = fileSearch.getHits().getHits()[0].sourceAsMap();
+    assertThat(fields).hasSize(8);
+    assertThat(fields).includes(
+      MapAssert.entry(SourceLineIndexDefinition.FIELD_PROJECT_UUID, "abcd"),
+      MapAssert.entry(SourceLineIndexDefinition.FIELD_FILE_UUID, "efgh"),
+      MapAssert.entry(SourceLineIndexDefinition.FIELD_LINE, 1),
+      MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_REVISION, "cafebabe"),
+      MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_DATE, "2014-01-01T12:34:56.7+0100"),
+      MapAssert.entry(SourceLineIndexDefinition.FIELD_SCM_AUTHOR, "polop"),
+      MapAssert.entry(SourceLineIndexDefinition.FIELD_SOURCE, "package org.sonar.server.source;")
+    );
   }
 }

diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2.json b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2.json
new file mode 100644 (file)
index 0000000..422dc02
--- /dev/null
+++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2.json
@@ -0,0 +1,10 @@
+{
+  "projectUuid": "abcd",
+  "fileUuid": "efgh",
+  "line": 2,
+  "scmAuthor": "polop",
+  "scmDate": "2014-01-01T12:34:56.7+01:00",
+  "scmRevision": "cafebabe",
+  "source": "// Empty",
+  "updatedAt": "2014-01-01T23:45:01.8+01:00"
+}
\ No newline at end of file

diff --git a/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2_other_file.json b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2_other_file.json
new file mode 100644 (file)
index 0000000..495d268
--- /dev/null
+++ b/server/sonar-server/src/test/resources/org/sonar/server/source/index/SourceLineIndexerTest/line2_other_file.json
@@ -0,0 +1,10 @@
+{
+  "projectUuid": "abcd",
+  "fileUuid": "fdsq",
+  "line": 2,
+  "scmAuthor": "polop",
+  "scmDate": "2014-01-01T12:34:56.7+01:00",
+  "scmRevision": "cafebabe",
+  "source": "// Empty",
+  "updatedAt": "2014-01-01T23:45:01.8+01:00"
+}
\ No newline at end of file