*/
package org.sonar.server.view.index;
-import java.util.Collection;
import java.util.List;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.sort.SortOrder;
import org.sonar.api.ce.ComputeEngineSide;
import org.sonar.api.server.ServerSide;
-import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.EsClient;
import static com.google.common.collect.Lists.newArrayList;
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
@ServerSide
@ComputeEngineSide
}
return result;
}
-
- public void delete(Collection<String> viewUuids) {
- SearchRequestBuilder searchRequest = esClient.prepareSearch(ViewIndexDefinition.INDEX_TYPE_VIEW)
- .setQuery(boolQuery().must(matchAllQuery()).filter(termsQuery(ViewIndexDefinition.FIELD_UUID, viewUuids)));
- BulkIndexer.delete(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, searchRequest);
- }
}
package org.sonar.server.view.index;
import com.google.common.collect.ImmutableSet;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.elasticsearch.action.index.IndexRequest;
+import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.component.UuidWithProjectUuidDto;
+import org.sonar.db.es.EsQueueDto;
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.BulkIndexer.Size;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.StartupIndexer;
+import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.IndexingResult;
+import org.sonar.server.es.OneToOneResilientIndexingListener;
+import org.sonar.server.es.ResilientIndexer;
import static com.google.common.collect.Maps.newHashMap;
+import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW;
-public class ViewIndexer implements StartupIndexer {
+public class ViewIndexer implements ResilientIndexer {
private final DbClient dbClient;
private final EsClient esClient;
* The views lookup cache will be cleared
*/
public void index(String rootViewUuid) {
- DbSession dbSession = dbClient.openSession(false);
- try {
+ try (DbSession dbSession = dbClient.openSession(false)) {
Map<String, String> viewAndProjectViewUuidMap = newHashMap();
for (ComponentDto viewOrSubView : dbClient.componentDao().selectEnabledDescendantModules(dbSession, rootViewUuid)) {
viewAndProjectViewUuidMap.put(viewOrSubView.uuid(), viewOrSubView.projectUuid());
}
index(dbSession, viewAndProjectViewUuidMap, true, Size.REGULAR);
- } finally {
- dbSession.close();
}
}
}
}
+ /**
+ * This is based on the fact that a WebService is only calling {@link ViewIndexer#delete(DbSession, Collection)}
+ * So the resiliency is only taking in account a deletion of view component
+ * A safety check is done by not deleting any component that still exist in database.
+ *
+ * This should not occur but prevent any misuse on this resiliency
+ */
+ @Override
+ public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
+ if (items.isEmpty()) {
+ return new IndexingResult();
+ }
+
+ Set<String> views = items
+ .stream()
+ .map(EsQueueDto::getDocId)
+ .collect(toHashSet(items.size()));
+
+ BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items));
+ bulkIndexer.start();
+
+ // Safety check to remove all views that may not have been deleted
+ views.removeAll(dbClient.componentDao().selectExistingUuids(dbSession, views));
+ views.forEach(v -> bulkIndexer.addDeletion(INDEX_TYPE_VIEW, v));
+ return bulkIndexer.stop();
+ }
+
+ public void delete(DbSession dbSession, Collection<String> viewUuids) {
+ List<EsQueueDto> items = viewUuids.stream()
+ .map(l -> EsQueueDto.create(INDEX_TYPE_VIEW.format(), l))
+ .collect(MoreCollectors.toArrayList());
+
+ dbClient.esQueueDao().insert(dbSession, items);
+ dbSession.commit();
+ index(dbSession, items);
+ }
+
+ private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) {
+ return new BulkIndexer(esClient, INDEX_TYPE_VIEW, bulkSize, listener);
+ }
}
assertThat(result).isEmpty();
}
-
- @Test
- public void delete_views() throws Exception {
- ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(asList("P1"));
- ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(asList("P2", "P3", "P4"));
- ViewDoc view3 = new ViewDoc().setUuid("UUID3").setProjects(asList("P2", "P3", "P4"));
- esTester.putDocuments(INDEX_TYPE_VIEW, view1);
- esTester.putDocuments(INDEX_TYPE_VIEW, view2);
- esTester.putDocuments(INDEX_TYPE_VIEW, view3);
-
- index.delete(asList(view1.uuid(), view2.uuid()));
-
- assertThat(esTester.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID)).containsOnly(view3.uuid());
- }
-
}
import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
+import java.util.function.BooleanSupplier;
import org.elasticsearch.action.search.SearchResponse;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.System2;
import org.sonar.db.DbClient;
import org.sonar.db.rule.RuleDto;
import org.sonar.db.rule.RuleTesting;
import org.sonar.server.es.EsTester;
+import org.sonar.server.es.RecoveryIndexer;
import org.sonar.server.es.SearchOptions;
import org.sonar.server.issue.IssueQuery;
import org.sonar.server.issue.index.IssueIndex;
import org.sonar.server.tester.UserSessionRule;
import static com.google.common.collect.Lists.newArrayList;
+import static java.util.Arrays.asList;
import static java.util.Collections.emptySet;
import static org.assertj.core.api.Assertions.assertThat;
import static org.sonar.db.component.ComponentTesting.newProjectCopy;
+import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW;
public class ViewIndexerTest {
private System2 system2 = System2.INSTANCE;
+ @Rule
+ public TestRule safeguardTimeout = new DisableOnDebug(Timeout.seconds(60));
+
@Rule
public DbTester dbTester = DbTester.create(system2);
assertThat(issueIndex.search(IssueQuery.builder().viewUuids(newArrayList(viewUuid)).build(), new SearchOptions()).getHits()).hasSize(2);
}
+ @Test
+ public void delete_should_delete_the_view() {
+ ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(asList("P1"));
+ ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(asList("P2", "P3", "P4"));
+ ViewDoc view3 = new ViewDoc().setUuid("UUID3").setProjects(asList("P2", "P3", "P4"));
+ esTester.putDocuments(INDEX_TYPE_VIEW, view1);
+ esTester.putDocuments(INDEX_TYPE_VIEW, view2);
+ esTester.putDocuments(INDEX_TYPE_VIEW, view3);
+
+ assertThat(esTester.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
+ .containsOnly(view1.uuid(), view2.uuid(), view3.uuid());
+
+ underTest.delete(dbSession, asList(view1.uuid(), view2.uuid()));
+
+ assertThat(esTester.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
+ .containsOnly(view3.uuid());
+ }
+
+ @Test
+ public void delete_should_be_resilient() throws InterruptedException {
+ ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(asList("P1"));
+ ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(asList("P2", "P3", "P4"));
+ ViewDoc view3 = new ViewDoc().setUuid("UUID3").setProjects(asList("P2", "P3", "P4"));
+ esTester.putDocuments(INDEX_TYPE_VIEW, view1);
+ esTester.putDocuments(INDEX_TYPE_VIEW, view2);
+ esTester.putDocuments(INDEX_TYPE_VIEW, view3);
+
+ assertThat(esTester.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
+ .containsOnly(view1.uuid(), view2.uuid(), view3.uuid());
+
+ // Lock writes
+ esTester.lockWrites(INDEX_TYPE_VIEW);
+ underTest.delete(dbSession, asList(view1.uuid(), view2.uuid()));
+
+ assertThat(esTester.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
+ .containsOnly(view1.uuid(), view2.uuid(), view3.uuid());
+
+ // Unlock writes
+ esTester.unlockWrites(INDEX_TYPE_VIEW);
+
+ doRecover(() -> esTester.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID).size() == 3);
+
+ assertThat(esTester.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
+ .containsOnly(view3.uuid());
+ }
+
private ComponentDto addProjectWithIssue(RuleDto rule, OrganizationDto org) {
ComponentDto project = ComponentTesting.newPublicProjectDto(org);
ComponentDto file = ComponentTesting.newFileDto(project, null);
return project;
}
+ private void doRecover(BooleanSupplier condition) throws InterruptedException {
+ MapSettings settings = new MapSettings()
+ .setProperty("sonar.search.recovery.initialDelayInMs", "0")
+ .setProperty("sonar.search.recovery.minAgeInMs", "1")
+ .setProperty("sonar.search.recovery.delayInMs", "1");
+
+ RecoveryIndexer recoveryIndexer = new RecoveryIndexer(System2.INSTANCE, settings.asConfig(), dbClient, underTest);
+ recoveryIndexer.start();
+
+ // Wait for recovery
+ while (condition.getAsBoolean()) {
+ Thread.sleep(1_000);
+ }
+
+ recoveryIndexer.stop();
+ }
}
projects / sonarqube.git / commitdiff