SONAR-7588 stop using delete_by_query

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
index c78f30802c12606a6d7a047b5934f7ca0fd60157..6c404a5f1b2f0730bdd7ad244b79df848c8aeaf6 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java
@@ -21,6 +21,10 @@ package org.sonar.server.es;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
@@ -41,11 +45,6 @@ import org.sonar.api.utils.log.Logger;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.core.util.ProgressLogger;
 
-import java.util.Map;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
 import static java.lang.String.format;
 
 /**
@@ -174,6 +173,8 @@ public class BulkIndexer implements Startable {
   /**
    * Delete all the documents matching the given search request. This method is blocking.
    * Index is refreshed, so docs are not searchable as soon as method is executed.
+   *
+   * Note that the parameter indexName could be removed if progress logs are not needed.
    */
   public static void delete(EsClient client, String indexName, SearchRequestBuilder searchRequest) {
     BulkIndexer bulk = new BulkIndexer(client, indexName);

diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java
index a8c2c2931c2005ec9c6fcaf17022da96ffe0c186..d6b55b2d57332f9365775add8baba0a990b8fbb6 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java
@@ -25,16 +25,18 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import org.apache.commons.dbutils.DbUtils;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.sonar.api.server.ServerSide;
 import org.sonar.api.utils.log.Loggers;
 import org.sonar.db.DbSession;
 import org.sonar.db.MyBatis;
 import org.sonar.db.version.DatabaseVersion;
+import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.EsClient;
 import org.sonar.server.issue.index.IssueIndexDefinition;
 import org.sonar.server.view.index.ViewIndexDefinition;
 
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+
 @ServerSide
 public class BackendCleanup {
 
@@ -82,18 +84,11 @@ public class BackendCleanup {
   public void clearIndexes() {
     Loggers.get(getClass()).info("Truncate Elasticsearch indices");
     try {
-      esClient.prepareClearCache()
-        .get();
-      esClient.prepareDeleteByQuery(esClient.prepareState().get()
-        .getState().getMetaData().concreteAllIndices())
-        .setQuery(QueryBuilders.matchAllQuery())
-        .get();
-      esClient.prepareRefresh(esClient.prepareState().get()
-        .getState().getMetaData().concreteAllIndices())
-        .get();
-      esClient.prepareFlush(esClient.prepareState().get()
-        .getState().getMetaData().concreteAllIndices())
-        .get();
+      esClient.prepareClearCache().get();
+
+      for (String index : esClient.prepareState().get().getState().getMetaData().concreteAllIndices()) {
+        clearIndex(index);
+      }
     } catch (Exception e) {
       throw new IllegalStateException("Unable to clear indexes", e);
     }
@@ -168,10 +163,10 @@ public class BackendCleanup {
    * Completely remove a index with all types
    */
   public void clearIndex(String indexName) {
-    esClient.prepareDeleteByQuery(esClient.prepareState().get()
-      .getState().getMetaData().concreteIndices(IndicesOptions.strictExpand(), indexName))
-      .setQuery(QueryBuilders.matchAllQuery())
-      .get();
+    String[] indicesToClear = esClient.prepareState().get().getState().getMetaData().concreteIndices(IndicesOptions.strictExpand(), indexName);
+    for (String index : indicesToClear) {
+      BulkIndexer.delete(esClient, index, esClient.prepareSearch(index).setQuery(matchAllQuery()));
+    }
   }
 
 }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java
index 50bcb813a54ab5fdca1478300fd2cdac1fce873a..e8c4f6e8ed8d4f2f13fded267c787cb4d10ce7c1 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndex.java
@@ -25,14 +25,16 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.sonar.api.ce.ComputeEngineSide;
 import org.sonar.api.server.ServerSide;
+import org.sonar.server.es.BulkIndexer;
 import org.sonar.server.es.EsClient;
 
 import static com.google.common.collect.Lists.newArrayList;
+import static org.elasticsearch.index.query.FilterBuilders.termsFilter;
+import static org.elasticsearch.index.query.QueryBuilders.filteredQuery;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
 @ServerSide
 @ComputeEngineSide
@@ -53,7 +55,7 @@ public class ViewIndex {
       .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES))
       .setFetchSource(ViewIndexDefinition.FIELD_UUID, null)
       .setSize(100)
-      .setQuery(QueryBuilders.matchAllQuery());
+      .setQuery(matchAllQuery());
 
     SearchResponse response = esSearch.get();
     List<String> result = newArrayList();
@@ -74,12 +76,10 @@ public class ViewIndex {
   }
 
   public void delete(Collection<String> viewUuids) {
-    esClient
-      .prepareDeleteByQuery(ViewIndexDefinition.INDEX)
+    SearchRequestBuilder searchRequest = esClient.prepareSearch(ViewIndexDefinition.INDEX)
       .setTypes(ViewIndexDefinition.TYPE_VIEW)
-      .setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(),
-        FilterBuilders.termsFilter(ViewIndexDefinition.FIELD_UUID, viewUuids)
-        ))
-      .get();
+      .setQuery(filteredQuery(matchAllQuery(), termsFilter(ViewIndexDefinition.FIELD_UUID, viewUuids)
+        ));
+    BulkIndexer.delete(esClient, ViewIndexDefinition.INDEX, searchRequest);
   }
 }

diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
index 7c089e869ec15e561b3ff884baa9f52740006a74..cb09028917d964e930c4874af8afb1db8c6918df 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/es/EsTester.java
@@ -55,6 +55,7 @@ import org.sonar.test.TestUtils;
 
 import static com.google.common.collect.Lists.newArrayList;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
 public class EsTester extends ExternalResource {
 
@@ -124,17 +125,10 @@ public class EsTester extends ExternalResource {
   }
 
   public void truncateIndices() {
-    client.prepareDeleteByQuery(client.prepareState().get()
-      .getState().getMetaData().concreteAllIndices())
-      .setQuery(QueryBuilders.matchAllQuery())
-      .get();
-    client.prepareRefresh(client.prepareState().get()
-      .getState().getMetaData().concreteAllIndices())
-      .setForce(true)
-      .get();
-    client.prepareFlush(client.prepareState().get()
-      .getState().getMetaData().concreteAllIndices())
-      .get();
+    String[] indices = client.prepareState().get().getState().getMetaData().concreteAllIndices();
+    for (String index : indices) {
+      BulkIndexer.delete(client, index, client().prepareSearch(index).setQuery(matchAllQuery()));
+    }
   }
 
   public void putDocuments(String index, String type, Class<?> testClass, String... jsonPaths) throws Exception {