diff options
11 files changed, 365 insertions, 4 deletions
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java index d4748a5652a..aa5cd895b2b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java @@ -36,7 +36,7 @@ public interface IndexingListener { @Override public void onFinish(IndexingResult result) { if (result.getFailures() > 0) { - throw new IllegalStateException("Indexation failures"); + throw new IllegalStateException("Unrecoverable indexation failures"); } } }; diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/FailOnErrorIndexingListenerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/FailOnErrorIndexingListenerTest.java new file mode 100644 index 00000000000..7753063c63c --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/es/FailOnErrorIndexingListenerTest.java @@ -0,0 +1,52 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class FailOnErrorIndexingListenerTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void onFinish_must_throw_ISE_when_an_error_is_present() { + IndexingResult indexingResult = new IndexingResult(); + + indexingResult.incrementRequests(); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Unrecoverable indexation failures"); + + IndexingListener.FAIL_ON_ERROR.onFinish(indexingResult); + } + + @Test + public void onFinish_must_not_throw_any_exception_if_no_failure() { + IndexingResult indexingResult = new IndexingResult(); + + indexingResult.incrementRequests(); + indexingResult.incrementSuccess(); + + IndexingListener.FAIL_ON_ERROR.onFinish(indexingResult); + } +} diff --git a/tests/plugins/pom.xml b/tests/plugins/pom.xml index 70015985e6b..7ce44f0d4df 100644 --- a/tests/plugins/pom.xml +++ b/tests/plugins/pom.xml @@ -60,6 +60,7 @@ <module>sonar-subcategories-plugin</module> <module>ui-extensions-plugin</module> <module>posttask-plugin</module> + <module>wait-at-platform-level4-plugin</module> <module>ws-plugin</module> <module>backdating-plugin-v1</module> <module>backdating-plugin-v2</module> diff --git a/tests/plugins/wait-at-platform-level4-plugin/pom.xml b/tests/plugins/wait-at-platform-level4-plugin/pom.xml new file mode 100644 index 00000000000..07d4196c888 --- /dev/null +++ b/tests/plugins/wait-at-platform-level4-plugin/pom.xml @@ -0,0 +1,51 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.sonarsource.sonarqube.tests</groupId> + <artifactId>plugins</artifactId> + <version>6.6-SNAPSHOT</version> + </parent> + + <artifactId>wait-at-platform-level4-plugin</artifactId> + <version>1.0-SNAPSHOT</version> + <packaging>sonar-plugin</packaging> + <name>Plugins :: Wait at platform level4 initialization phase</name> + <description>Test for failing Elasticsearch on platform4</description> + + <dependencies> + <dependency> + <groupId>org.sonarsource.sonarqube</groupId> + <artifactId>sonar-plugin-api</artifactId> + <version>${apiVersion}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>17.0</version> + <exclusions> + <exclusion> + <!-- should be declared with scope provided --> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.sonarsource.sonar-packaging-maven-plugin</groupId> + <artifactId>sonar-packaging-maven-plugin</artifactId> + <version>1.15</version> + <configuration> + <pluginClass>WaitAtPlaformLevel4Plugin</pluginClass> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlaformLevel4Plugin.java b/tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlaformLevel4Plugin.java new file mode 100644 index 00000000000..c4713f9f49d --- /dev/null +++ b/tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlaformLevel4Plugin.java @@ -0,0 +1,32 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +import java.util.ArrayList; +import java.util.List; +import org.sonar.api.SonarPlugin; + +public final class WaitAtPlaformLevel4Plugin extends SonarPlugin { + + public List getExtensions() { + List extensions = new ArrayList(); + extensions.add(WaitAtPlatformLevel4.class); + return extensions; + } + +} diff --git a/tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlatformLevel4.java b/tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlatformLevel4.java new file mode 100644 index 00000000000..8bb87583865 --- /dev/null +++ b/tests/plugins/wait-at-platform-level4-plugin/src/main/java/WaitAtPlatformLevel4.java @@ -0,0 +1,65 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +import java.io.File; +import java.util.Optional; +import org.sonar.api.Startable; +import org.sonar.api.config.Configuration; +import org.sonar.api.server.ServerSide; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; + +@ServerSide +public class WaitAtPlatformLevel4 implements Startable { + + private static final Logger LOGGER = Loggers.get(WaitAtPlatformLevel4.class); + + private final Configuration configuration; + + public WaitAtPlatformLevel4(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public void start() { + Optional<String> path = configuration.get("sonar.web.pause.path"); + path.ifPresent(WaitAtPlatformLevel4::waitForFileToBeDeleted); + } + + @Override + public void stop() { + // Nothing to do + } + + private static void waitForFileToBeDeleted(String path) { + LOGGER.info("PlatformLevel4 initialization phase is paused. Waiting for file to be deleted: " + path); + File file = new File(path); + try { + while (file.exists()) { + Thread.sleep(500L); + } + LOGGER.info("PlatformLevel4 initilization is resumed"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.info("PlatformLevel4 pause has been interrupted"); + throw new IllegalStateException("Platform4 pause has been interrupted"); + } + } +} diff --git a/tests/resilience/exception_on_listener.btm b/tests/resilience/exception_on_listener.btm new file mode 100644 index 00000000000..d0416ff0d11 --- /dev/null +++ b/tests/resilience/exception_on_listener.btm @@ -0,0 +1,8 @@ +RULE throw an exception on IndexingListener#onFinish +CLASS org.sonar.server.es.IndexingListener +METHOD onFinish +COMPILE +AT ENTRY +IF true +DO THROW new IllegalStateException("Indexation failures from byteman") +ENDRULE diff --git a/tests/src/test/java/org/sonarqube/tests/Category5Suite.java b/tests/src/test/java/org/sonarqube/tests/Category5Suite.java index 951185485a8..4aa5c0c62a5 100644 --- a/tests/src/test/java/org/sonarqube/tests/Category5Suite.java +++ b/tests/src/test/java/org/sonarqube/tests/Category5Suite.java @@ -35,6 +35,7 @@ import org.sonarqube.tests.serverSystem.SystemStateTest; import org.sonarqube.tests.settings.ElasticsearchSettingsTest; import org.sonarqube.tests.settings.LicensesPageTest; import org.sonarqube.tests.settings.SettingsTestRestartingOrchestrator; +import org.sonarqube.tests.startup.StartupIndexation; import org.sonarqube.tests.telemetry.TelemetryOptOutTest; import org.sonarqube.tests.telemetry.TelemetryUploadTest; import org.sonarqube.tests.updateCenter.UpdateCenterTest; @@ -75,7 +76,7 @@ import org.sonarqube.tests.user.UserEsResilienceTest; // elasticsearch ElasticsearchSettingsTest.class, - + StartupIndexation.class, SystemPasscodeTest.class }) public class Category5Suite { diff --git a/tests/src/test/java/org/sonarqube/tests/LogsTailer.java b/tests/src/test/java/org/sonarqube/tests/LogsTailer.java index e367ba3d7cc..eb093b1c345 100644 --- a/tests/src/test/java/org/sonarqube/tests/LogsTailer.java +++ b/tests/src/test/java/org/sonarqube/tests/LogsTailer.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -149,6 +150,13 @@ public class LogsTailer implements AutoCloseable { foundSignal.await(); } + /** + * Blocks until the expected log appears in watched files with timeout + */ + public void waitForLog(long timeout, TimeUnit timeUnit) throws InterruptedException { + foundSignal.await(timeout, timeUnit); + } + public Optional<String> getLog() { return Optional.ofNullable(log); } diff --git a/tests/src/test/java/org/sonarqube/tests/Tester.java b/tests/src/test/java/org/sonarqube/tests/Tester.java index e0d5d4468d1..43ea1fe380b 100644 --- a/tests/src/test/java/org/sonarqube/tests/Tester.java +++ b/tests/src/test/java/org/sonarqube/tests/Tester.java @@ -21,6 +21,7 @@ package org.sonarqube.tests; import com.sonar.orchestrator.Orchestrator; import javax.annotation.Nullable; +import org.apache.commons.lang.StringUtils; import org.junit.rules.ExternalResource; import org.sonarqube.pageobjects.Navigation; import org.sonarqube.ws.client.WsClient; @@ -58,6 +59,10 @@ public class Tester extends ExternalResource implements Session { public Tester(Orchestrator orchestrator) { this.orchestrator = orchestrator; + String elasticsearchHttpPort = orchestrator.getDistribution().getServerProperty("sonar.search.httpPort"); + if (StringUtils.isNotBlank(elasticsearchHttpPort)) { + this.elasticsearch = new Elasticsearch(Integer.parseInt(elasticsearchHttpPort)); + } } public Tester disableOrganizations() { @@ -79,7 +84,7 @@ public class Tester extends ExternalResource implements Session { } @Override - protected void before() { + public void before() { verifyNotStarted(); rootSession = new SessionImpl(orchestrator, "admin", "admin"); @@ -91,7 +96,7 @@ public class Tester extends ExternalResource implements Session { } @Override - protected void after() { + public void after() { if (!disableOrganizations) { organizations().deleteNonGuardedOrganizations(); } diff --git a/tests/src/test/java/org/sonarqube/tests/startup/StartupIndexation.java b/tests/src/test/java/org/sonarqube/tests/startup/StartupIndexation.java new file mode 100644 index 00000000000..f4c6e029191 --- /dev/null +++ b/tests/src/test/java/org/sonarqube/tests/startup/StartupIndexation.java @@ -0,0 +1,138 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonarqube.tests.startup; + +import com.sonar.orchestrator.Orchestrator; +import com.sonar.orchestrator.util.NetworkUtils; +import java.io.File; +import java.net.InetAddress; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; +import org.apache.commons.io.FileUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; +import org.sonarqube.tests.LogsTailer; +import org.sonarqube.tests.Tester; +import org.sonarqube.ws.WsUsers; +import org.sonarqube.ws.client.user.SearchRequest; + +import static org.assertj.core.api.Assertions.assertThat; +import static util.ItUtils.pluginArtifact; + +public class StartupIndexation { + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + @Rule + public TestRule safeguard = new DisableOnDebug(Timeout.seconds(600)); + + @Test + public void elasticsearch_error_at_startup_must_shutdown_node() throws Exception { + try (SonarQube sonarQube = new SonarQube(); + LogsTailer.Watch failedInitialization = sonarQube.logsTailer.watch("Background initialization failed. Stopping SonarQube"); + LogsTailer.Watch stopWatcher = sonarQube.logsTailer.watch("SonarQube is stopped")) { + sonarQube.lockAllElasticsearchWrites(); + sonarQube.resume(); + stopWatcher.waitForLog(10, TimeUnit.SECONDS); + assertThat(stopWatcher.getLog()).isPresent(); + assertThat(failedInitialization.getLog()).isPresent(); + } + + // Restarting is recreating the indexes + try (SonarQube sonarQube = new SonarQube(); + LogsTailer.Watch sonarQubeIsUpWatcher = sonarQube.logsTailer.watch("SonarQube is up")) { + sonarQube.resume(); + sonarQubeIsUpWatcher.waitForLog(10, TimeUnit.SECONDS); + SearchRequest searchRequest = SearchRequest.builder().setQuery("admin").build(); + WsUsers.SearchWsResponse searchWsResponse = sonarQube.tester.wsClient().users().search(searchRequest); + assertThat(searchWsResponse.getUsersCount()).isEqualTo(1); + assertThat(searchWsResponse.getUsers(0).getName()).isEqualTo("Administrator"); + } + } + + private class SonarQube implements AutoCloseable { + private final Orchestrator orchestrator; + private final Tester tester; + private final File pauseFile; + private final LogsTailer logsTailer; + private final int esHttpPort = NetworkUtils.getNextAvailablePort(InetAddress.getLoopbackAddress()); + + SonarQube() throws Exception { + pauseFile = temp.newFile(); + FileUtils.touch(pauseFile); + + orchestrator = Orchestrator.builderEnv() + .setServerProperty("sonar.web.pause.path", pauseFile.getAbsolutePath()) + .addPlugin(pluginArtifact("wait-at-platform-level4-plugin")) + .setStartupLogWatcher(l -> l.contains("PlatformLevel4 initialization phase is paused")) + .setServerProperty("sonar.search.httpPort", "" + esHttpPort) + .build(); + + tester = new Tester(orchestrator); + orchestrator.start(); + tester.before(); + + logsTailer = LogsTailer.builder() + .addFile(orchestrator.getServer().getWebLogs()) + .addFile(orchestrator.getServer().getCeLogs()) + .addFile(orchestrator.getServer().getAppLogs()) + .build(); + } + + LogsTailer logs() { + return logsTailer; + } + + void resume() throws Exception { + FileUtils.forceDelete(pauseFile); + } + + void lockElasticsearchWritesOn(String index) throws Exception { + tester.elasticsearch().lockWrites(index); + } + + void lockAllElasticsearchWrites() throws Exception { + for (String index : Arrays.asList("metadatas", "components", "tests", "projectmeasures", "rules", "issues", "users", "views")) { + lockElasticsearchWritesOn(index); + } + } + + @Override + public void close() throws Exception { + if (tester != null) { + try { + tester.after(); + } catch (Exception e) { + e.printStackTrace(System.err); + } + } + if (orchestrator != null) { + orchestrator.stop(); + } + if (logsTailer != null) { + logsTailer.close(); + } + } + } +} |