Fix TestIndexer resiliency

2 files changed, 76 insertions, 84 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java
index ebd02c87a23..f09036bd748 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java
@@ -19,10 +19,8 @@
  */
 package org.sonar.server.test.index;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import org.elasticsearch.action.search.SearchRequestBuilder;
@@ -71,7 +69,7 @@ public class TestIndexer implements ProjectIndexer {
     try (DbSession dbSession = dbClient.openSession(false);
          TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, null)) {
 
-      BulkIndexer bulkIndexer = new BulkIndexer(esClient, TestIndexDefinition.INDEX_TYPE_TEST, Size.LARGE);
+      BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.LARGE);
       bulkIndexer.start();
       addTestsToBulkIndexer(rowIt, bulkIndexer);
       bulkIndexer.stop();
@@ -80,7 +78,7 @@ public class TestIndexer implements ProjectIndexer {
 
   @Override
   public void indexOnAnalysis(String projectUuid) {
-    BulkIndexer bulkIndexer = new BulkIndexer(esClient, TestIndexDefinition.INDEX_TYPE_TEST, Size.REGULAR);
+    BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.REGULAR);
     bulkIndexer.start();
     addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid);
     try (DbSession dbSession = dbClient.openSession(false);
@@ -115,17 +113,6 @@ public class TestIndexer implements ProjectIndexer {
     }
   }
 
-  @VisibleForTesting
-  protected IndexingResult doIndex(Iterator<FileSourcesUpdaterHelper.Row> dbRows, Size bulkSize, IndexingListener listener) {
-    BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_TEST, bulkSize, listener);
-    bulk.start();
-    while (dbRows.hasNext()) {
-      FileSourcesUpdaterHelper.Row row = dbRows.next();
-      row.getUpdateRequests().forEach(bulk::add);
-    }
-    return bulk.stop();
-  }
-
   public void deleteByFile(String fileUuid) {
     SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX_TYPE_TEST)
       .setQuery(QueryBuilders.termQuery(FIELD_FILE_UUID, fileUuid));
@@ -134,19 +121,17 @@ public class TestIndexer implements ProjectIndexer {
 
   @Override
   public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+    // The items are to be deleted
     if (items.isEmpty()) {
       return new IndexingResult();
     }
 
     IndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items);
-    BulkIndexer bulkIndexer = new BulkIndexer(esClient, TestIndexDefinition.INDEX_TYPE_TEST, Size.REGULAR, listener);
+    BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.REGULAR, listener);
     bulkIndexer.start();
     items.forEach(i -> {
       String projectUuid = i.getDocId();
       addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid);
-      try (TestResultSetIterator rowIt = TestResultSetIterator.create(dbClient, dbSession, projectUuid)) {
-        addTestsToBulkIndexer(rowIt, bulkIndexer);
-      }
     });
 
     return bulkIndexer.stop();
diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java
index 4feb494fae3..60e297dbcae 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java
@@ -19,44 +19,35 @@
  */
 package org.sonar.server.test.index;
 
-import com.google.common.collect.Iterators;
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Date;
+import java.sql.SQLException;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.junit.Rule;
 import org.junit.Test;
 import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.utils.System2;
 import org.sonar.db.DbTester;
-import org.sonar.db.protobuf.DbFileSources;
-import org.sonar.server.es.BulkIndexer;
+import org.sonar.db.es.EsQueueDto;
 import org.sonar.server.es.EsTester;
-import org.sonar.server.es.IndexingListener;
-import org.sonar.server.source.index.FileSourcesUpdaterHelper;
+import org.sonar.server.es.IndexingResult;
+import org.sonar.server.es.ProjectIndexer;
 import org.sonar.server.test.db.TestTesting;
 
 import static java.lang.String.format;
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.entry;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
 import static org.sonar.server.es.DefaultIndexSettings.REFRESH_IMMEDIATE;
-import static org.sonar.server.test.index.TestIndexDefinition.FIELD_DURATION_IN_MS;
 import static org.sonar.server.test.index.TestIndexDefinition.FIELD_FILE_UUID;
-import static org.sonar.server.test.index.TestIndexDefinition.FIELD_MESSAGE;
 import static org.sonar.server.test.index.TestIndexDefinition.FIELD_NAME;
-import static org.sonar.server.test.index.TestIndexDefinition.FIELD_PROJECT_UUID;
-import static org.sonar.server.test.index.TestIndexDefinition.FIELD_STACKTRACE;
-import static org.sonar.server.test.index.TestIndexDefinition.FIELD_STATUS;
-import static org.sonar.server.test.index.TestIndexDefinition.FIELD_TEST_UUID;
 import static org.sonar.server.test.index.TestIndexDefinition.INDEX_TYPE_TEST;
 
 public class TestIndexerTest {
@@ -110,43 +101,70 @@ public class TestIndexerTest {
   }
 
   /**
-   * File F1 in project P1 has one test -> to be updated
-   * File F2 in project P1 has one test -> untouched
+   * Indexing recovery is handled by Compute Engine, without using
+   * the table es_queue
    */
+  @Test
+  public void indexOnAnalysis_does_not_fail_on_errors_and_does_not_enable_recovery_mode() throws IOException, SQLException {
+    db.prepareDbUnit(getClass(), "db.xml");
+
+    es.lockWrites(INDEX_TYPE_TEST);
+    TestTesting.updateDataColumn(db.getSession(), "FILE_UUID", TestTesting.newRandomTests(3));
+
+    underTest.indexOnAnalysis("PROJECT_UUID");
+    es.unlockWrites(INDEX_TYPE_TEST);
+
+    assertThat(countDocuments()).isEqualTo(0);
+    assertThat(db.countRowsOfTable("es_queue")).isEqualTo(0);
+  }
 
   @Test
-  public void update_already_indexed_test() throws Exception {
-    indexTest("P1", "F1", "T1", "U111");
-    indexTest("P1", "F2", "T1", "U121");
+  public void prepareForRecovery_must_be_empty_unless_cause_is_PROJECT_DELETION() {
+    db.prepareDbUnit(getClass(), "db.xml");
+    assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_CREATION))
+      .isEmpty();
+    assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_KEY_UPDATE))
+      .isEmpty();
+    assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_TAGS_UPDATE))
+      .isEmpty();
+    assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PERMISSION_CHANGE))
+      .isEmpty();
+
+    // Only deletion is resilient with recovery
+    assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_DELETION))
+      .isNotEmpty();
+  }
 
-    FileSourcesUpdaterHelper.Row dbRow = TestResultSetIterator.toRow("P1", "F1", new Date(), Arrays.asList(
-      DbFileSources.Test.newBuilder()
-        .setUuid("U111")
-        .setName("NAME_1")
-        .setStatus(DbFileSources.Test.TestStatus.FAILURE)
-        .setMsg("NEW_MESSAGE_1")
-        .setStacktrace("NEW_STACKTRACE_1")
-        .setExecutionTimeMs(123_456L)
-        .addCoveredFile(DbFileSources.Test.CoveredFile.newBuilder().setFileUuid("MAIN_UUID_1").addCoveredLine(42))
-        .build()));
-    underTest.doIndex(Iterators.singletonIterator(dbRow), BulkIndexer.Size.REGULAR, IndexingListener.NOOP);
-
-    assertThat(countDocuments()).isEqualTo(2L);
-
-    SearchResponse fileSearch = prepareSearch()
-      .setQuery(QueryBuilders.termQuery(FIELD_FILE_UUID, "F1"))
-      .get();
-    assertThat(fileSearch.getHits().getTotalHits()).isEqualTo(1L);
-    Map<String, Object> fields = fileSearch.getHits().getHits()[0].sourceAsMap();
-    assertThat(fields).contains(
-      entry(FIELD_PROJECT_UUID, "P1"),
-      entry(FIELD_FILE_UUID, "F1"),
-      entry(FIELD_TEST_UUID, "U111"),
-      entry(FIELD_NAME, "NAME_1"),
-      entry(FIELD_STATUS, "FAILURE"),
-      entry(FIELD_MESSAGE, "NEW_MESSAGE_1"),
-      entry(FIELD_STACKTRACE, "NEW_STACKTRACE_1"),
-      entry(FIELD_DURATION_IN_MS, 123_456));
+  @Test
+  public void errors_during_project_deletion_are_recovered() throws IOException, SQLException, InterruptedException {
+    // Create a project with 3 tests
+    db.prepareDbUnit(getClass(), "db.xml");
+    TestTesting.updateDataColumn(db.getSession(), "FILE_UUID", TestTesting.newRandomTests(3));
+    underTest.indexOnAnalysis("PROJECT_UUID"); //index(db.getSession(), items);
+    assertThat(countDocuments()).isEqualTo(3);
+
+    // Now delete the files
+    es.lockWrites(INDEX_TYPE_TEST);
+    Collection<EsQueueDto> items = underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_DELETION);
+    db.commit();
+
+    underTest.deleteByFile("FILE_UUID");
+    es.unlockWrites(INDEX_TYPE_TEST);
+    // Still 3 tests
+    assertThat(countDocuments()).isEqualTo(3);
+
+    // Recover must delete the 3 tests
+    IndexingResult result = recover();
+    assertThat(result.getTotal()).isEqualTo(3);
+
+    assertThat(countDocuments()).isEqualTo(0);
+  }
+
+  @Test
+  public void indexing_with_empty_esqueue_dto_does_nothing() {
+    assertThat(underTest.index(db.getSession(), emptyList()))
+      .extracting(IndexingResult::getTotal, IndexingResult::getFailures, IndexingResult::getSuccess)
+      .containsExactly(0L, 0L, 0L);
   }
 
   @Test
@@ -164,22 +182,6 @@ public class TestIndexerTest {
     assertThat(document.get(FIELD_FILE_UUID)).isEqualTo("F2");
   }
 
-//  @Test
-//  public void delete_project_by_uuid() throws Exception {
-//    indexTest("P1", "F1", "T1", "U111");
-//    indexTest("P1", "F1", "T2", "U112");
-//    indexTest("P1", "F2", "T1", "U121");
-//    indexTest("P2", "F3", "T1", "U231");
-//
-//    underTest.deleteProject("P1");
-//
-//    List<SearchHit> hits = getDocuments();
-//    assertThat(hits).hasSize(1);
-//    Map<String, Object> document = hits.get(0).getSource();
-//    assertThat(hits).hasSize(1);
-//    assertThat(document.get(FIELD_PROJECT_UUID)).isEqualTo("P2");
-//  }
-
   private void indexTest(String projectUuid, String fileUuid, String testName, String uuid) throws IOException {
     es.client().prepareIndex(INDEX_TYPE_TEST)
       .setId(uuid)
@@ -200,4 +202,9 @@ public class TestIndexerTest {
   private long countDocuments() {
     return es.countDocuments(INDEX_TYPE_TEST);
   }
+
+  private IndexingResult recover() {
+    Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), System.currentTimeMillis() + 10_000L, 10);
+    return underTest.index(db.getSession(), items);
+  }
 }