SONAR-9762 Implement resiliency on startup

server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java

@@ -36,7 +36,7 @@ public interface IndexingListener {
     @Override
     public void onFinish(IndexingResult result) {
       if (result.getFailures() > 0) {
-        throw new IllegalStateException("Indexation failures");
+        throw new IllegalStateException("Unrecoverable indexation failures");
       }
     }
   };

server/sonar-server/src/test/java/org/sonar/server/es/FailOnErrorIndexingListenerTest.java

@@ -0,0 +1,52 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class FailOnErrorIndexingListenerTest {
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  @Test
+  public void onFinish_must_throw_ISE_when_an_error_is_present() {
+    IndexingResult indexingResult = new IndexingResult();
+
+    indexingResult.incrementRequests();
+
+    expectedException.expect(IllegalStateException.class);
+    expectedException.expectMessage("Unrecoverable indexation failures");
+
+    IndexingListener.FAIL_ON_ERROR.onFinish(indexingResult);
+  }
+
+  @Test
+  public void onFinish_must_not_throw_any_exception_if_no_failure() {
+    IndexingResult indexingResult = new IndexingResult();
+
+    indexingResult.incrementRequests();
+    indexingResult.incrementSuccess();
+
+    IndexingListener.FAIL_ON_ERROR.onFinish(indexingResult);
+  }
+}

tests/plugins/pom.xml

@@ -60,6 +60,7 @@
     <module>sonar-subcategories-plugin</module>
     <module>ui-extensions-plugin</module>
     <module>posttask-plugin</module>
+    <module>wait-at-platform-level4-plugin</module>
     <module>ws-plugin</module>
     <module>backdating-plugin-v1</module>
     <module>backdating-plugin-v2</module>

tests/plugins/wait-at-platform-level4-plugin/pom.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.sonarsource.sonarqube.tests</groupId>
+    <artifactId>plugins</artifactId>
+    <version>6.6-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>wait-at-platform-level4-plugin</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <packaging>sonar-plugin</packaging>
+  <name>Plugins :: Wait at platform level4 initialization phase</name>
+  <description>Test for failing Elasticsearch on platform4</description>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.sonarsource.sonarqube</groupId>
+      <artifactId>sonar-plugin-api</artifactId>
+      <version>${apiVersion}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>17.0</version>
+      <exclusions>
+        <exclusion>
+          <!-- should be declared with scope provided -->
+          <groupId>com.google.code.findbugs</groupId>
+          <artifactId>jsr305</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.sonarsource.sonar-packaging-maven-plugin</groupId>
+        <artifactId>sonar-packaging-maven-plugin</artifactId>
+        <version>1.15</version>
+        <configuration>
+          <pluginClass>WaitAtPlaformLevel4Plugin</pluginClass>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+</project>

tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlaformLevel4Plugin.java

@@ -0,0 +1,32 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+import java.util.ArrayList;
+import java.util.List;
+import org.sonar.api.SonarPlugin;
+
+public final class WaitAtPlaformLevel4Plugin extends SonarPlugin {
+
+  public List getExtensions() {
+    List extensions = new ArrayList();
+    extensions.add(WaitAtPlatformLevel4.class);
+    return extensions;
+  }
+
+}

tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlatformLevel4.java

@@ -0,0 +1,65 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+import java.io.File;
+import java.util.Optional;
+import org.sonar.api.Startable;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.server.ServerSide;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+
+@ServerSide
+public class WaitAtPlatformLevel4 implements Startable {
+
+  private static final Logger LOGGER = Loggers.get(WaitAtPlatformLevel4.class);
+
+  private final Configuration configuration;
+
+  public WaitAtPlatformLevel4(Configuration configuration) {
+    this.configuration = configuration;
+  }
+
+  @Override
+  public void start() {
+    Optional<String> path = configuration.get("sonar.web.pause.path");
+    path.ifPresent(WaitAtPlatformLevel4::waitForFileToBeDeleted);
+  }
+
+  @Override
+  public void stop() {
+    // Nothing to do
+  }
+
+  private static void waitForFileToBeDeleted(String path) {
+    LOGGER.info("PlatformLevel4 initialization phase is paused. Waiting for file to be deleted: " + path);
+    File file = new File(path);
+    try {
+      while (file.exists()) {
+        Thread.sleep(500L);
+      }
+      LOGGER.info("PlatformLevel4 initilization is resumed");
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      LOGGER.info("PlatformLevel4 pause has been interrupted");
+      throw new IllegalStateException("Platform4 pause has been interrupted");
+    }
+  }
+}

tests/resilience/exception_on_listener.btm

@@ -0,0 +1,8 @@
+RULE throw an exception on IndexingListener#onFinish
+CLASS org.sonar.server.es.IndexingListener
+METHOD onFinish
+COMPILE
+AT ENTRY
+IF true
+DO THROW new IllegalStateException("Indexation failures from byteman")
+ENDRULE

tests/src/test/java/org/sonarqube/tests/Category5Suite.java

@@ -35,6 +35,7 @@ import org.sonarqube.tests.serverSystem.SystemStateTest;
 import org.sonarqube.tests.settings.ElasticsearchSettingsTest;
 import org.sonarqube.tests.settings.LicensesPageTest;
 import org.sonarqube.tests.settings.SettingsTestRestartingOrchestrator;
+import org.sonarqube.tests.startup.StartupIndexation;
 import org.sonarqube.tests.telemetry.TelemetryOptOutTest;
 import org.sonarqube.tests.telemetry.TelemetryUploadTest;
 import org.sonarqube.tests.updateCenter.UpdateCenterTest;
@@ -75,7 +76,7 @@ import org.sonarqube.tests.user.UserEsResilienceTest;
 
   // elasticsearch
   ElasticsearchSettingsTest.class,
-
+  StartupIndexation.class,
   SystemPasscodeTest.class
 })
 public class Category5Suite {

tests/src/test/java/org/sonarqube/tests/LogsTailer.java

@@ -25,6 +25,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -149,6 +150,13 @@ public class LogsTailer implements AutoCloseable {
       foundSignal.await();
     }
 
+    /**
+     * Blocks until the expected log appears in watched files with timeout
+     */
+    public void waitForLog(long timeout, TimeUnit timeUnit) throws InterruptedException {
+      foundSignal.await(timeout, timeUnit);
+    }
+
     public Optional<String> getLog() {
       return Optional.ofNullable(log);
     }

tests/src/test/java/org/sonarqube/tests/Tester.java

@@ -21,6 +21,7 @@ package org.sonarqube.tests;
 
 import com.sonar.orchestrator.Orchestrator;
 import javax.annotation.Nullable;
+import org.apache.commons.lang.StringUtils;
 import org.junit.rules.ExternalResource;
 import org.sonarqube.pageobjects.Navigation;
 import org.sonarqube.ws.client.WsClient;
@@ -58,6 +59,10 @@ public class Tester extends ExternalResource implements Session {
 
   public Tester(Orchestrator orchestrator) {
     this.orchestrator = orchestrator;
+    String elasticsearchHttpPort = orchestrator.getDistribution().getServerProperty("sonar.search.httpPort");
+    if (StringUtils.isNotBlank(elasticsearchHttpPort)) {
+      this.elasticsearch = new Elasticsearch(Integer.parseInt(elasticsearchHttpPort));
+    }
   }
 
   public Tester disableOrganizations() {
@@ -79,7 +84,7 @@ public class Tester extends ExternalResource implements Session {
   }
 
   @Override
-  protected void before() {
+  public void before() {
     verifyNotStarted();
     rootSession = new SessionImpl(orchestrator, "admin", "admin");
 
@@ -91,7 +96,7 @@ public class Tester extends ExternalResource implements Session {
   }
 
   @Override
-  protected void after() {
+  public void after() {
     if (!disableOrganizations) {
       organizations().deleteNonGuardedOrganizations();
     }

tests/src/test/java/org/sonarqube/tests/startup/StartupIndexation.java

@@ -0,0 +1,138 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package org.sonarqube.tests.startup;
+
+import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.util.NetworkUtils;
+import java.io.File;
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.io.FileUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonarqube.tests.LogsTailer;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.WsUsers;
+import org.sonarqube.ws.client.user.SearchRequest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static util.ItUtils.pluginArtifact;
+
+public class StartupIndexation {
+  @Rule
+  public TemporaryFolder temp = new TemporaryFolder();
+  @Rule
+  public TestRule safeguard = new DisableOnDebug(Timeout.seconds(600));
+
+  @Test
+  public void elasticsearch_error_at_startup_must_shutdown_node() throws Exception {
+    try (SonarQube sonarQube = new SonarQube();
+         LogsTailer.Watch failedInitialization = sonarQube.logsTailer.watch("Background initialization failed. Stopping SonarQube");
+         LogsTailer.Watch stopWatcher = sonarQube.logsTailer.watch("SonarQube is stopped")) {
+      sonarQube.lockAllElasticsearchWrites();
+      sonarQube.resume();
+      stopWatcher.waitForLog(10, TimeUnit.SECONDS);
+      assertThat(stopWatcher.getLog()).isPresent();
+      assertThat(failedInitialization.getLog()).isPresent();
+    }
+
+    // Restarting is recreating the indexes
+    try (SonarQube sonarQube = new SonarQube();
+         LogsTailer.Watch sonarQubeIsUpWatcher = sonarQube.logsTailer.watch("SonarQube is up")) {
+      sonarQube.resume();
+      sonarQubeIsUpWatcher.waitForLog(10, TimeUnit.SECONDS);
+      SearchRequest searchRequest = SearchRequest.builder().setQuery("admin").build();
+      WsUsers.SearchWsResponse searchWsResponse = sonarQube.tester.wsClient().users().search(searchRequest);
+      assertThat(searchWsResponse.getUsersCount()).isEqualTo(1);
+      assertThat(searchWsResponse.getUsers(0).getName()).isEqualTo("Administrator");
+    }
+  }
+
+  private class SonarQube implements AutoCloseable {
+    private final Orchestrator orchestrator;
+    private final Tester tester;
+    private final File pauseFile;
+    private final LogsTailer logsTailer;
+    private final int esHttpPort = NetworkUtils.getNextAvailablePort(InetAddress.getLoopbackAddress());
+
+    SonarQube() throws Exception {
+      pauseFile = temp.newFile();
+      FileUtils.touch(pauseFile);
+
+      orchestrator = Orchestrator.builderEnv()
+        .setServerProperty("sonar.web.pause.path", pauseFile.getAbsolutePath())
+        .addPlugin(pluginArtifact("wait-at-platform-level4-plugin"))
+        .setStartupLogWatcher(l -> l.contains("PlatformLevel4 initialization phase is paused"))
+        .setServerProperty("sonar.search.httpPort", "" + esHttpPort)
+        .build();
+
+      tester = new Tester(orchestrator);
+      orchestrator.start();
+      tester.before();
+
+      logsTailer = LogsTailer.builder()
+        .addFile(orchestrator.getServer().getWebLogs())
+        .addFile(orchestrator.getServer().getCeLogs())
+        .addFile(orchestrator.getServer().getAppLogs())
+        .build();
+    }
+
+    LogsTailer logs() {
+      return logsTailer;
+    }
+
+    void resume() throws Exception {
+      FileUtils.forceDelete(pauseFile);
+    }
+
+    void lockElasticsearchWritesOn(String index) throws Exception {
+      tester.elasticsearch().lockWrites(index);
+    }
+
+    void lockAllElasticsearchWrites() throws Exception {
+      for (String index : Arrays.asList("metadatas", "components", "tests", "projectmeasures", "rules", "issues", "users", "views")) {
+        lockElasticsearchWritesOn(index);
+      }
+    }
+
+    @Override
+    public void close() throws Exception {
+      if (tester != null) {
+        try {
+          tester.after();
+        } catch (Exception e) {
+          e.printStackTrace(System.err);
+        }
+      }
+      if (orchestrator != null) {
+        orchestrator.stop();
+      }
+      if (logsTailer != null) {
+        logsTailer.close();
+      }
+    }
+  }
+}