SONAR-9514 SONAR-9516 SONAR-9517 ES resilience from POST WS

8 files changed, 426 insertions, 13 deletions

diff --git a/tests/src/test/java/org/sonarqube/tests/Category1Suite.java b/tests/src/test/java/org/sonarqube/tests/Category1Suite.java
index b48b1283b8a..f1e0fb726fc 100644
--- a/tests/src/test/java/org/sonarqube/tests/Category1Suite.java
+++ b/tests/src/test/java/org/sonarqube/tests/Category1Suite.java
@@ -38,7 +38,7 @@ import org.sonarqube.tests.measure.SincePreviousVersionHistoryTest;
 import org.sonarqube.tests.measure.SinceXDaysHistoryTest;
 import org.sonarqube.tests.measure.TimeMachineTest;
 import org.sonarqube.tests.projectAdministration.BackgroundTasksTest;
-import org.sonarqube.tests.projectAdministration.BulkDeletionTest;
+import org.sonarqube.tests.projectAdministration.ProjectBulkDeletionPageTest;
 import org.sonarqube.tests.projectAdministration.ProjectAdministrationTest;
 import org.sonarqube.tests.projectAdministration.ProjectLinksPageTest;
 import org.sonarqube.tests.projectSearch.ProjectsPageTest;
@@ -65,7 +65,7 @@ import static util.ItUtils.xooPlugin;
   UsersPageTest.class,
   ProjectVisibilityTest.class,
   // project administration
-  BulkDeletionTest.class,
+  ProjectBulkDeletionPageTest.class,
   ProjectAdministrationTest.class,
   ProjectLinksPageTest.class,
   BackgroundTasksTest.class,
diff --git a/tests/src/test/java/org/sonarqube/tests/Category6Suite.java b/tests/src/test/java/org/sonarqube/tests/Category6Suite.java
index eae3e358b8a..cf02b593415 100644
--- a/tests/src/test/java/org/sonarqube/tests/Category6Suite.java
+++ b/tests/src/test/java/org/sonarqube/tests/Category6Suite.java
@@ -20,6 +20,8 @@
 package org.sonarqube.tests;
 
 import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.util.NetworkUtils;
+import java.net.InetAddress;
 import org.junit.ClassRule;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
@@ -31,6 +33,8 @@ import org.sonarqube.tests.organization.OrganizationMembershipUiTest;
 import org.sonarqube.tests.organization.OrganizationTest;
 import org.sonarqube.tests.organization.PersonalOrganizationTest;
 import org.sonarqube.tests.organization.RootUserOnOrganizationTest;
+import org.sonarqube.tests.projectAdministration.ProjectDeletionTest;
+import org.sonarqube.tests.projectAdministration.ProjectProvisioningTest;
 import org.sonarqube.tests.projectSearch.LeakProjectsPageTest;
 import org.sonarqube.tests.projectSearch.SearchProjectsTest;
 import org.sonarqube.tests.qualityProfile.BuiltInQualityProfilesTest;
@@ -65,12 +69,22 @@ import static util.ItUtils.xooPlugin;
   IssueTagsTest.class,
   LeakProjectsPageTest.class,
   SearchProjectsTest.class,
-  RulesWsTest.class
+  RulesWsTest.class,
+  ProjectDeletionTest.class,
+  ProjectProvisioningTest.class
 })
 public class Category6Suite {
 
+  public static final int SEARCH_HTTP_PORT = NetworkUtils.getNextAvailablePort(InetAddress.getLoopbackAddress());
+
   @ClassRule
   public static final Orchestrator ORCHESTRATOR = Orchestrator.builderEnv()
+
+    // for ES resiliency tests
+    .setServerProperty("sonar.search.httpPort", "" + SEARCH_HTTP_PORT)
+    .setServerProperty("sonar.search.recovery.delayInMs", "1000")
+    .setServerProperty("sonar.search.recovery.minAgeInMs", "3000")
+
     .addPlugin(xooPlugin())
     .addPlugin(pluginArtifact("base-auth-plugin"))
     .addPlugin(pluginArtifact("fake-billing-plugin"))
diff --git a/tests/src/test/java/org/sonarqube/tests/Elasticsearch.java b/tests/src/test/java/org/sonarqube/tests/Elasticsearch.java
new file mode 100644
index 00000000000..60df943b9b1
--- /dev/null
+++ b/tests/src/test/java/org/sonarqube/tests/Elasticsearch.java
@@ -0,0 +1,51 @@
+package org.sonarqube.tests;
+
+import java.net.InetAddress;
+import okhttp3.MediaType;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.RequestBody;
+import okhttp3.Response;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Helper to directly access Elasticsearch. It requires the HTTP port
+ * to be open.
+ */
+public class Elasticsearch {
+
+  private final int httpPort;
+
+  Elasticsearch(int httpPort) {
+    this.httpPort = httpPort;
+  }
+
+  /**
+   * Forbid indexing requests on the specified index. Index becomes read-only.
+   */
+  public void lockWrites(String index) throws Exception {
+    putIndexSetting(httpPort, index, "blocks.write", "true");
+  }
+
+  /**
+   * Enable indexing requests on the specified index.
+   * @see #lockWrites(String)
+   */
+  public void unlockWrites(String index) throws Exception {
+    putIndexSetting(httpPort, index, "blocks.write", "false");
+  }
+
+  private void putIndexSetting(int searchHttpPort, String index, String key, String value) throws Exception {
+    Request.Builder request = new Request.Builder()
+      .url("http://" + InetAddress.getLoopbackAddress().getHostAddress() + ":" + searchHttpPort + "/" + index + "/_settings")
+      .put(RequestBody.create(MediaType.parse("application/json"), "{" +
+        "    \"index\" : {" +
+        "        \"" + key + "\" : \"" + value + "\"" +
+        "    }" +
+        "}"));
+    OkHttpClient okClient = new OkHttpClient.Builder().build();
+    Response response = okClient.newCall(request.build()).execute();
+    assertThat(response.isSuccessful()).isTrue();
+  }
+}
diff --git a/tests/src/test/java/org/sonarqube/tests/Tester.java b/tests/src/test/java/org/sonarqube/tests/Tester.java
index f6a8f55162a..7ab41b836d5 100644
--- a/tests/src/test/java/org/sonarqube/tests/Tester.java
+++ b/tests/src/test/java/org/sonarqube/tests/Tester.java
@@ -22,23 +22,24 @@ package org.sonarqube.tests;
 import com.sonar.orchestrator.Orchestrator;
 import javax.annotation.Nullable;
 import org.junit.rules.ExternalResource;
-import org.sonarqube.ws.client.WsClient;
 import org.sonarqube.pageobjects.Navigation;
+import org.sonarqube.ws.client.WsClient;
 import util.selenium.Selenese;
 
+import static java.util.Objects.requireNonNull;
 import static util.ItUtils.newUserWsClient;
 
 /**
  * This JUnit rule wraps an {@link Orchestrator} instance and provides :
  * <ul>
- *   <li>enabling the organization feature by default</li>
- *   <li>clean-up of organizations between tests</li>
- *   <li>clean-up of users between tests</li>
- *   <li>clean-up of session when opening a browser (cookies, local storage)</li>
- *   <li>quick access to {@link WsClient} instances</li>
- *   <li>helpers to generate organizations and users</li>
+ * <li>enabling the organization feature by default</li>
+ * <li>clean-up of organizations between tests</li>
+ * <li>clean-up of users between tests</li>
+ * <li>clean-up of session when opening a browser (cookies, local storage)</li>
+ * <li>quick access to {@link WsClient} instances</li>
+ * <li>helpers to generate organizations and users</li>
  * </ul>
- *
+ * <p>
  * Recommendation is to define a {@code @Rule} instance. If not possible, then
  * {@code @ClassRule} must be used through a {@link org.junit.rules.RuleChain}
  * around {@link Orchestrator}.
@@ -49,6 +50,7 @@ public class Tester extends ExternalResource implements Session {
 
   // configuration before startup
   private boolean disableOrganizations = false;
+  private Elasticsearch elasticsearch = null;
 
   // initialized in #before()
   private boolean beforeCalled = false;
@@ -64,6 +66,18 @@ public class Tester extends ExternalResource implements Session {
     return this;
   }
 
+  /**
+   * Enables Elasticsearch debugging, see {@link #elasticsearch()}.
+   *
+   * The property "sonar.search.httpPort" must be defined before
+   * starting SonarQube server.
+   */
+  public Tester setElasticsearchHttpPort(int port) {
+    verifyNotStarted();
+    elasticsearch = new Elasticsearch(port);
+    return this;
+  }
+
   @Override
   protected void before() {
     verifyNotStarted();
@@ -98,6 +112,10 @@ public class Tester extends ExternalResource implements Session {
     return new SessionImpl(orchestrator, login, password);
   }
 
+  public Elasticsearch elasticsearch() {
+    return requireNonNull(elasticsearch, "Elasticsearch HTTP port is not defined. See #setElasticsearchHttpPort()");
+  }
+
   /**
    * Open a new browser session. Cookies are deleted.
    */
diff --git a/tests/src/test/java/org/sonarqube/tests/projectAdministration/BulkDeletionTest.java b/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectBulkDeletionPageTest.java
index e31020a3700..c0cc2a71e9e 100644
--- a/tests/src/test/java/org/sonarqube/tests/projectAdministration/BulkDeletionTest.java
+++ b/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectBulkDeletionPageTest.java
@@ -32,7 +32,7 @@ import util.user.UserRule;
 import static util.ItUtils.projectDir;
 import static util.selenium.Selenese.runSelenese;
 
-public class BulkDeletionTest {
+public class ProjectBulkDeletionPageTest {
 
   private static final String ADMIN_USER_LOGIN = "admin-user";
 
@@ -63,7 +63,7 @@ public class BulkDeletionTest {
     executeBuild("cameleon-2", "Foo-Application");
     executeBuild("cameleon-3", "Bar-Sonar-Plugin");
 
-    runSelenese(orchestrator, "/projectAdministration/BulkDeletionTest/bulk-delete-filter-projects.html");
+    runSelenese(orchestrator, "/projectAdministration/ProjectBulkDeletionPageTest/bulk-delete-filter-projects.html");
   }
 
   private void executeBuild(String projectKey, String projectName) {
diff --git a/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectDeletionTest.java b/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectDeletionTest.java
new file mode 100644
index 00000000000..d9ea32b0bb9
--- /dev/null
+++ b/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectDeletionTest.java
@@ -0,0 +1,197 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.tests.projectAdministration;
+
+import com.sonar.orchestrator.Orchestrator;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonarqube.tests.Category6Suite;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.Organizations;
+import org.sonarqube.ws.WsComponents;
+import org.sonarqube.ws.WsProjects;
+import org.sonarqube.ws.WsProjects.CreateWsResponse.Project;
+import org.sonarqube.ws.client.GetRequest;
+import org.sonarqube.ws.client.WsResponse;
+import org.sonarqube.ws.client.component.SearchProjectsRequest;
+import org.sonarqube.ws.client.project.BulkDeleteRequest;
+import org.sonarqube.ws.client.project.CreateRequest;
+import org.sonarqube.ws.client.project.DeleteRequest;
+import org.sonarqube.ws.client.project.SearchWsRequest;
+import util.ItUtils;
+
+import static java.util.Collections.singletonList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ProjectDeletionTest {
+
+  @ClassRule
+  public static final Orchestrator orchestrator = Category6Suite.ORCHESTRATOR;
+
+  @Rule
+  public TestRule safeguard = new DisableOnDebug(Timeout.seconds(300));
+  @Rule
+  public Tester tester = new Tester(orchestrator)
+    .setElasticsearchHttpPort(Category6Suite.SEARCH_HTTP_PORT);
+
+  @Test
+  public void deletion_removes_project_from_search_engines() {
+    Organizations.Organization organization = tester.organizations().generate();
+    Project project1 = createProject(organization, "one", "Foo");
+    Project project2 = createProject(organization, "two", "Bar");
+    assertThatProjectIsSearchable(organization, "Foo");
+    assertThatProjectIsSearchable(organization, "Bar");
+
+    deleteProject(project1);
+    assertThatProjectIsNotSearchable(organization, project1.getName());
+    assertThatProjectIsSearchable(organization, project2.getName());
+
+    deleteProject(project2);
+    assertThatProjectIsNotSearchable(organization, project1.getName());
+    assertThatProjectIsNotSearchable(organization, project2.getName());
+  }
+
+  @Test
+  public void indexing_errors_are_recovered_asynchronously_when_deleting_project() throws Exception {
+    Organizations.Organization organization = tester.organizations().generate();
+    Project project = createProject(organization, "one", "Foo");
+
+    tester.elasticsearch().lockWrites("components");
+    tester.elasticsearch().lockWrites("projectmeasures");
+    deleteProject(project);
+    // WS reloads from database the results returned by Elasticsearch. That's
+    // why the project does not appear in search engine.
+    // However this test is still useful to verify that WS do not
+    // fail during this Elasticsearch inconsistency.
+    assertThatProjectIsNotSearchable(organization, project.getName());
+
+    tester.elasticsearch().unlockWrites("components");
+    tester.elasticsearch().unlockWrites("projectmeasures");
+    // TODO verify that recovery daemon successfully updated indices
+  }
+
+  @Test
+  public void bulk_deletion_removes_projects_from_search_engines() {
+    Organizations.Organization organization = tester.organizations().generate();
+    Project project1 = createProject(organization, "one", "Foo");
+    Project project2 = createProject(organization, "two", "Bar");
+    Project project3 = createProject(organization, "three", "Baz");
+
+    bulkDeleteProjects(organization, project1, project3);
+    assertThatProjectIsNotSearchable(organization, project1.getName());
+    assertThatProjectIsSearchable(organization, project2.getName());
+    assertThatProjectIsNotSearchable(organization, project3.getName());
+  }
+
+  @Test
+  public void indexing_errors_are_recovered_asynchronously_when_bulk_deleting_projects() throws Exception {
+    Organizations.Organization organization = tester.organizations().generate();
+    Project project1 = createProject(organization, "one", "Foo");
+    Project project2 = createProject(organization, "two", "Bar");
+    Project project3 = createProject(organization, "three", "Baz");
+
+    tester.elasticsearch().lockWrites("components");
+    tester.elasticsearch().lockWrites("projectmeasures");
+    bulkDeleteProjects(organization, project1, project3);
+
+    // WS reloads from database the results returned by Elasticsearch. That's
+    // why the project does not appear in search engine.
+    // However this test is still useful to verify that WS do not
+    // fail during this Elasticsearch inconsistency.
+    assertThatProjectIsNotSearchable(organization, project1.getName());
+    assertThatProjectIsSearchable(organization, project2.getName());
+    assertThatProjectIsNotSearchable(organization, project3.getName());
+
+    tester.elasticsearch().unlockWrites("components");
+    tester.elasticsearch().unlockWrites("projectmeasures");
+    // TODO verify that recovery daemon successfully updated indices
+  }
+
+  private void deleteProject(Project project) {
+    tester.wsClient().projects().delete(DeleteRequest.builder().setKey(project.getKey()).build());
+  }
+
+  private void bulkDeleteProjects(Organizations.Organization organization, Project... projects) {
+    BulkDeleteRequest request = BulkDeleteRequest.builder()
+      .setOrganization(organization.getKey())
+      .setProjectKeys(Arrays.stream(projects).map(Project::getKey).collect(Collectors.toList()))
+      .build();
+    tester.wsClient().projects().bulkDelete(request);
+  }
+
+  private Project createProject(Organizations.Organization organization, String key, String name) {
+    CreateRequest createRequest = CreateRequest.builder().setKey(key).setName(name).setOrganization(organization.getKey()).build();
+    return tester.wsClient().projects().create(createRequest).getProject();
+  }
+
+  private void assertThatProjectIsSearchable(Organizations.Organization organization, String name) {
+    assertThat(isInProjectsSearch(organization, name)).isTrue();
+    assertThat(isInComponentSearchProjects(name)).isTrue();
+    assertThat(isInComponentSuggestions(name)).isTrue();
+  }
+
+  private void assertThatProjectIsNotSearchable(Organizations.Organization organization, String name) {
+    assertThat(isInProjectsSearch(organization, name)).isFalse();
+    assertThat(isInComponentSearchProjects(name)).isFalse();
+    assertThat(isInComponentSuggestions(name)).isFalse();
+  }
+
+  /**
+   * Projects administration page - uses database
+   */
+  private boolean isInProjectsSearch(Organizations.Organization organization, String name) {
+    WsProjects.SearchWsResponse response = tester.wsClient().projects().search(
+      SearchWsRequest.builder().setOrganization(organization.getKey()).setQuery(name).setQualifiers(singletonList("TRK")).build());
+    return response.getComponentsCount() > 0;
+  }
+
+  /**
+   * Projects page - api/components/search_projects - uses ES + DB
+   */
+  private boolean isInComponentSearchProjects(String name) {
+    WsComponents.SearchProjectsWsResponse response = tester.wsClient().components().searchProjects(
+      SearchProjectsRequest.builder().setFilter("query=\"" + name + "\"").build());
+    return response.getComponentsCount() > 0;
+  }
+
+  /**
+   * Top-right search engine - api/components/suggestions - uses ES + DB
+   */
+  private boolean isInComponentSuggestions(String name) {
+    GetRequest request = new GetRequest("api/components/suggestions").setParam("s", name);
+    WsResponse response = tester.wsClient().wsConnector().call(request);
+    Map<String, Object> json = ItUtils.jsonToMap(response.content());
+    Collection<Map<String, Object>> results = (Collection<Map<String, Object>>) json.get("results");
+    Collection items = results.stream()
+      .filter(map -> "TRK".equals(map.get("q")))
+      .map(map -> (Collection) map.get("items"))
+      .findFirst()
+      .orElseThrow(() -> new IllegalStateException("missing field results/[q=TRK]"));
+    return !items.isEmpty();
+  }
+}
diff --git a/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectProvisioningTest.java b/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectProvisioningTest.java
new file mode 100644
index 00000000000..1ff7a8f24d6
--- /dev/null
+++ b/tests/src/test/java/org/sonarqube/tests/projectAdministration/ProjectProvisioningTest.java
@@ -0,0 +1,133 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarqube.tests.projectAdministration;
+
+import com.sonar.orchestrator.Orchestrator;
+import java.util.Collection;
+import java.util.Map;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonarqube.tests.Category6Suite;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.Organizations;
+import org.sonarqube.ws.WsComponents;
+import org.sonarqube.ws.WsProjects;
+import org.sonarqube.ws.WsProjects.CreateWsResponse.Project;
+import org.sonarqube.ws.client.GetRequest;
+import org.sonarqube.ws.client.WsResponse;
+import org.sonarqube.ws.client.component.SearchProjectsRequest;
+import org.sonarqube.ws.client.project.CreateRequest;
+import org.sonarqube.ws.client.project.SearchWsRequest;
+import util.ItUtils;
+
+import static java.util.Collections.singletonList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ProjectProvisioningTest {
+
+  @ClassRule
+  public static final Orchestrator orchestrator = Category6Suite.ORCHESTRATOR;
+
+  @Rule
+  public TestRule safeguard = new DisableOnDebug(Timeout.seconds(300));
+  @Rule
+  public Tester tester = new Tester(orchestrator)
+    .setElasticsearchHttpPort(Category6Suite.SEARCH_HTTP_PORT);
+
+  @Test
+  public void provisioned_project_is_available_in_search_engines() {
+    Organizations.Organization organization = tester.organizations().generate();
+
+    createProject(organization, "one", "Foo");
+
+    assertThat(isInProjectsSearch(organization, "Foo")).isTrue();
+    assertThat(isInComponentSearchProjects("Foo")).isTrue();
+    assertThat(isInComponentSuggestions("Foo")).isTrue();
+  }
+
+  @Test
+  public void indexing_errors_are_recovered_asynchronously_when_provisioning_project() throws Exception {
+    tester.elasticsearch().lockWrites("components");
+    tester.elasticsearch().lockWrites("projectmeasures");
+
+    Organizations.Organization organization = tester.organizations().generate();
+    Project project = createProject(organization, "one", "Foo");
+
+    // no ES requests but only DB
+    assertThat(isInProjectsSearch(organization, project.getName())).isTrue();
+
+    // these WS use ES so they are temporarily inconsistent
+    assertThat(isInComponentSearchProjects(project.getName())).isFalse();
+    assertThat(isInComponentSuggestions(project.getName())).isFalse();
+
+    tester.elasticsearch().unlockWrites("components");
+    tester.elasticsearch().unlockWrites("projectmeasures");
+
+    boolean found = false;
+    while (!found) {
+      // recovery daemon runs every second (see Category6Suite)
+      Thread.sleep(1_000L);
+      found = isInComponentSearchProjects(project.getName()) && isInComponentSuggestions(project.getName());
+    }
+  }
+
+  private Project createProject(Organizations.Organization organization, String key, String name) {
+    CreateRequest createRequest = CreateRequest.builder().setKey(key).setName(name).setOrganization(organization.getKey()).build();
+    return tester.wsClient().projects().create(createRequest).getProject();
+  }
+
+  /**
+   * Projects administration page - uses database
+   */
+  private boolean isInProjectsSearch(Organizations.Organization organization, String name) {
+    WsProjects.SearchWsResponse response = tester.wsClient().projects().search(
+      SearchWsRequest.builder().setOrganization(organization.getKey()).setQuery(name).setQualifiers(singletonList("TRK")).build());
+    return response.getComponentsCount() > 0;
+  }
+
+  /**
+   * Projects page - api/components/search_projects - uses ES + DB
+   */
+  private boolean isInComponentSearchProjects(String name) {
+    WsComponents.SearchProjectsWsResponse response = tester.wsClient().components().searchProjects(
+      SearchProjectsRequest.builder().setFilter("query=\"" + name + "\"").build());
+    return response.getComponentsCount() > 0;
+  }
+
+  /**
+   * Top-right search engine - api/components/suggestions - uses ES + DB
+   */
+  private boolean isInComponentSuggestions(String name) {
+    GetRequest request = new GetRequest("api/components/suggestions").setParam("s", name);
+    WsResponse response = tester.wsClient().wsConnector().call(request);
+    Map<String, Object> json = ItUtils.jsonToMap(response.content());
+    Collection<Map<String, Object>> results = (Collection<Map<String, Object>>) json.get("results");
+    Collection items = results.stream()
+      .filter(map -> "TRK".equals(map.get("q")))
+      .map(map -> (Collection) map.get("items"))
+      .findFirst()
+      .orElseThrow(() -> new IllegalStateException("missing field results/[q=TRK]"));
+    return !items.isEmpty();
+  }
+}
diff --git a/tests/src/test/resources/projectAdministration/BulkDeletionTest/bulk-delete-filter-projects.html b/tests/src/test/resources/projectAdministration/ProjectBulkDeletionPageTest/bulk-delete-filter-projects.html
index b6256e49b35..b6256e49b35 100644
--- a/tests/src/test/resources/projectAdministration/BulkDeletionTest/bulk-delete-filter-projects.html
+++ b/tests/src/test/resources/projectAdministration/ProjectBulkDeletionPageTest/bulk-delete-filter-projects.html