From: Simon Brandhof Date: Sat, 24 May 2014 14:22:38 +0000 (+0200) Subject: SONAR-5007 move org.sonar.server.es to org.sonar.server.search X-Git-Tag: 4.4-RC1~833 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=28d8c4a2241b3590bd23f1e6662cb397285d70dc;p=sonarqube.git SONAR-5007 move org.sonar.server.es to org.sonar.server.search --- diff --git a/sonar-server/src/main/java/org/sonar/server/es/ESNode.java b/sonar-server/src/main/java/org/sonar/server/es/ESNode.java deleted file mode 100644 index 380a90f5387..00000000000 --- a/sonar-server/src/main/java/org/sonar/server/es/ESNode.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package org.sonar.server.es; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.picocontainer.Startable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.api.config.Settings; -import org.sonar.api.platform.ServerFileSystem; - -import java.io.File; -import java.io.IOException; - -/** - * ElasticSearch Node used to connect to index. - */ -public class ESNode implements Startable { - - private static final Logger LOG = LoggerFactory.getLogger(ESNode.class); - - private static final String HTTP_ENABLED = "http.enabled"; - static final String DATA_DIR = "data/es"; - - private static final String DEFAULT_HEALTH_TIMEOUT = "30s"; - - private final ServerFileSystem fileSystem; - private final Settings settings; - private final String healthTimeout; - - // available only after startup - private Node node; - - public ESNode(ServerFileSystem fileSystem, Settings settings) { - this(fileSystem, settings, DEFAULT_HEALTH_TIMEOUT); - } - - @VisibleForTesting - ESNode(ServerFileSystem fileSystem, Settings settings, String healthTimeout) { - this.fileSystem = fileSystem; - this.settings = settings; - this.healthTimeout = healthTimeout; - } - - @Override - public void start() { - LOG.info("Starting Elasticsearch..."); - - initLogging(); - ImmutableSettings.Builder esSettings = ImmutableSettings.builder() - .loadFromUrl(getClass().getResource("config/elasticsearch.json")); - initDirs(esSettings); - initRestConsole(esSettings); - initNetwork(esSettings); - - node = NodeBuilder.nodeBuilder() - .settings(esSettings) - .node(); - node.start(); - - addIndexTemplates(); - - if ( - node.client().admin().cluster().prepareHealth() - .setWaitForYellowStatus() - .setTimeout(healthTimeout) - .execute().actionGet() - .getStatus() == ClusterHealthStatus.RED) { - throw new IllegalStateException( - String.format("Elasticsearch index is corrupt, please delete directory '%s/%s' and relaunch the SonarQube server.", fileSystem.getHomeDir().getAbsolutePath(), DATA_DIR)); - } - - LOG.info("Elasticsearch started"); - } - - private void addIndexTemplates() { - try { - node.client().admin().indices().preparePutTemplate("default") - .setSource(IOUtils.toString(getClass().getResource("config/templates/default.json"))) - .execute().actionGet(); - } catch (IOException ioe) { - throw new IllegalStateException("Unable to load index templates", ioe); - } - } - - private void initLogging() { - ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory()); - } - - private void initNetwork(ImmutableSettings.Builder esSettings) { - esSettings.put("network.bind_host", "127.0.0.1"); - } - - private void initRestConsole(ImmutableSettings.Builder esSettings) { - int httpPort = settings.getInt("sonar.es.http.port"); - if (httpPort > 0) { - LOG.warn("Elasticsearch HTTP console enabled on port {}. Only for debugging purpose.", httpPort); - esSettings.put(HTTP_ENABLED, true); - esSettings.put("http.host", "127.0.0.1"); - esSettings.put("http.port", httpPort); - } else { - esSettings.put(HTTP_ENABLED, false); - } - } - - private void initDirs(ImmutableSettings.Builder esSettings) { - File esDir = new File(fileSystem.getHomeDir(), DATA_DIR); - try { - FileUtils.forceMkdir(esDir); - esSettings.put("path.home", esDir.getAbsolutePath()); - LOG.debug("Elasticsearch data stored in {}", esDir.getAbsolutePath()); - } catch (Exception e) { - throw new IllegalStateException("Fail to create directory " + esDir.getAbsolutePath(), e); - } - } - - @Override - public void stop() { - if (node != null) { - node.close(); - node = null; - } - } - - public Client client() { - if (node == null) { - throw new IllegalStateException("Elasticsearch is not started"); - } - return node.client(); - } -} diff --git a/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index 6bdb234aa20..b665c00243d 100644 --- a/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -102,7 +102,7 @@ import org.sonar.server.debt.DebtModelXMLExporter; import org.sonar.server.debt.DebtRulesXMLImporter; import org.sonar.server.duplication.ws.DuplicationsWriter; import org.sonar.server.duplication.ws.DuplicationsWs; -import org.sonar.server.es.ESNode; +import org.sonar.server.search.ESNode; import org.sonar.server.issue.ActionService; import org.sonar.server.issue.AssignAction; import org.sonar.server.issue.CommentAction; diff --git a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index 9db06d73859..00e52e9da38 100644 --- a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -48,7 +48,7 @@ import org.sonar.core.cluster.WorkQueue; import org.sonar.core.qualityprofile.db.ActiveRuleDto; import org.sonar.core.qualityprofile.db.ActiveRuleKey; import org.sonar.core.qualityprofile.db.QualityProfileKey; -import org.sonar.server.es.ESNode; +import org.sonar.server.search.ESNode; import org.sonar.server.qualityprofile.ActiveRule; import org.sonar.server.search.BaseIndex; import org.sonar.server.search.IndexDefinition; diff --git a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index bf7a8d044f0..ae83823767e 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -37,7 +37,7 @@ import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.core.cluster.WorkQueue; import org.sonar.core.rule.RuleDto; -import org.sonar.server.es.ESNode; +import org.sonar.server.search.ESNode; import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; import org.sonar.server.rule.Rule; import org.sonar.server.search.BaseIndex; diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index cc4cc006b65..670374438a1 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -36,7 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.core.cluster.WorkQueue; import org.sonar.core.persistence.Dto; -import org.sonar.server.es.ESNode; import java.io.IOException; import java.io.Serializable; diff --git a/sonar-server/src/main/java/org/sonar/server/search/ESNode.java b/sonar-server/src/main/java/org/sonar/server/search/ESNode.java new file mode 100644 index 00000000000..5ab1e56201d --- /dev/null +++ b/sonar-server/src/main/java/org/sonar/server/search/ESNode.java @@ -0,0 +1,158 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.search; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.picocontainer.Startable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.config.Settings; +import org.sonar.api.platform.ServerFileSystem; + +import java.io.File; +import java.io.IOException; + +/** + * ElasticSearch Node used to connect to index. + */ +public class ESNode implements Startable { + + private static final Logger LOG = LoggerFactory.getLogger(ESNode.class); + + private static final String HTTP_ENABLED = "http.enabled"; + static final String DATA_DIR = "data/es"; + + private static final String DEFAULT_HEALTH_TIMEOUT = "30s"; + + private final ServerFileSystem fileSystem; + private final Settings settings; + private final String healthTimeout; + + // available only after startup + private Node node; + + public ESNode(ServerFileSystem fileSystem, Settings settings) { + this(fileSystem, settings, DEFAULT_HEALTH_TIMEOUT); + } + + @VisibleForTesting + ESNode(ServerFileSystem fileSystem, Settings settings, String healthTimeout) { + this.fileSystem = fileSystem; + this.settings = settings; + this.healthTimeout = healthTimeout; + } + + @Override + public void start() { + LOG.info("Start Elasticsearch..."); + + initLogging(); + ImmutableSettings.Builder esSettings = ImmutableSettings.builder() + .loadFromUrl(getClass().getResource("config/elasticsearch.json")); + initDirs(esSettings); + initRestConsole(esSettings); + initNetwork(esSettings); + + node = NodeBuilder.nodeBuilder() + .settings(esSettings) + .node(); + node.start(); + + addIndexTemplates(); + + if ( + node.client().admin().cluster().prepareHealth() + .setWaitForYellowStatus() + .setTimeout(healthTimeout) + .execute().actionGet() + .getStatus() == ClusterHealthStatus.RED) { + throw new IllegalStateException( + String.format("Elasticsearch index is corrupt, please delete directory '%s/%s' and relaunch the SonarQube server.", fileSystem.getHomeDir().getAbsolutePath(), DATA_DIR)); + } + + LOG.info("Elasticsearch started"); + } + + private void addIndexTemplates() { + try { + node.client().admin().indices().preparePutTemplate("default") + .setSource(IOUtils.toString(getClass().getResource("config/templates/default.json"))) + .execute().actionGet(); + } catch (IOException ioe) { + throw new IllegalStateException("Unable to load index templates", ioe); + } + } + + private void initLogging() { + ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory()); + } + + private void initNetwork(ImmutableSettings.Builder esSettings) { + esSettings.put("network.bind_host", "127.0.0.1"); + } + + private void initRestConsole(ImmutableSettings.Builder esSettings) { + int httpPort = settings.getInt("sonar.es.http.port"); + if (httpPort > 0) { + LOG.warn("Elasticsearch HTTP console enabled on port {}. Only for debugging purpose.", httpPort); + esSettings.put(HTTP_ENABLED, true); + esSettings.put("http.host", "127.0.0.1"); + esSettings.put("http.port", httpPort); + } else { + esSettings.put(HTTP_ENABLED, false); + } + } + + private void initDirs(ImmutableSettings.Builder esSettings) { + File esDir = new File(fileSystem.getHomeDir(), DATA_DIR); + try { + FileUtils.forceMkdir(esDir); + esSettings.put("path.home", esDir.getAbsolutePath()); + LOG.debug("Elasticsearch data stored in {}", esDir.getAbsolutePath()); + } catch (Exception e) { + throw new IllegalStateException("Fail to create directory " + esDir.getAbsolutePath(), e); + } + } + + @Override + public void stop() { + if (node != null) { + node.close(); + node = null; + } + } + + public Client client() { + if (node == null) { + throw new IllegalStateException("Elasticsearch is not started"); + } + return node.client(); + } +} diff --git a/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json b/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json deleted file mode 100644 index 64913338ee0..00000000000 --- a/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "cluster": { - "name": "sonarqube" - }, - "node": { - "name": "sonarqube", - "local": true, - "data": true - }, - "discovery": { - "zen": { - "multicast": { - "enabled": false - } - } - }, - "index": { - "number_of_shards": 1, - "number_of_replicas": 0, - "mapper": { - "dynamic": false - }, - "analysis": { - "analyzer": { - "sortable": { - "type": "custom", - "tokenizer": "keyword", - "filter": "lowercase" - }, - "rule_name": { - "type": "custom", - "tokenizer": "standard", - "filter": ["lowercase", "rule_name_ngram"] - } - }, - "filter": { - "rule_name_ngram": { - "type": "nGram", - "min_gram": 3, - "max_gram": 5, - "token_chars": [ "letter", "digit" ] - } - } - } - } -} diff --git a/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json b/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json deleted file mode 100644 index 0e7a25d7c67..00000000000 --- a/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "template": "*", - "mappings": { - "_default_": { - "dynamic": "strict" - } - } -} diff --git a/sonar-server/src/main/resources/org/sonar/server/search/config/elasticsearch.json b/sonar-server/src/main/resources/org/sonar/server/search/config/elasticsearch.json new file mode 100644 index 00000000000..64913338ee0 --- /dev/null +++ b/sonar-server/src/main/resources/org/sonar/server/search/config/elasticsearch.json @@ -0,0 +1,46 @@ +{ + "cluster": { + "name": "sonarqube" + }, + "node": { + "name": "sonarqube", + "local": true, + "data": true + }, + "discovery": { + "zen": { + "multicast": { + "enabled": false + } + } + }, + "index": { + "number_of_shards": 1, + "number_of_replicas": 0, + "mapper": { + "dynamic": false + }, + "analysis": { + "analyzer": { + "sortable": { + "type": "custom", + "tokenizer": "keyword", + "filter": "lowercase" + }, + "rule_name": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "rule_name_ngram"] + } + }, + "filter": { + "rule_name_ngram": { + "type": "nGram", + "min_gram": 3, + "max_gram": 5, + "token_chars": [ "letter", "digit" ] + } + } + } + } +} diff --git a/sonar-server/src/main/resources/org/sonar/server/search/config/templates/default.json b/sonar-server/src/main/resources/org/sonar/server/search/config/templates/default.json new file mode 100644 index 00000000000..0e7a25d7c67 --- /dev/null +++ b/sonar-server/src/main/resources/org/sonar/server/search/config/templates/default.json @@ -0,0 +1,8 @@ +{ + "template": "*", + "mappings": { + "_default_": { + "dynamic": "strict" + } + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/es/ESNodeTest.java b/sonar-server/src/test/java/org/sonar/server/es/ESNodeTest.java deleted file mode 100644 index 19b66f6e3ef..00000000000 --- a/sonar-server/src/test/java/org/sonar/server/es/ESNodeTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.io.Resources; -import org.apache.commons.io.FileUtils; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.client.AdminClient; -import org.elasticsearch.client.ClusterAdminClient; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.StrictDynamicMappingException; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.sonar.api.config.Settings; -import org.sonar.api.platform.ServerFileSystem; -import org.sonar.api.utils.ZipUtils; - -import java.io.File; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; - -import static org.fest.assertions.Assertions.assertThat; -import static org.fest.assertions.Fail.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class ESNodeTest { - - ServerFileSystem fs; - File homedir; - File dataDir; - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Before - public void createMocks() throws IOException { - homedir = temp.newFolder(); - fs = mock(ServerFileSystem.class); - when(fs.getHomeDir()).thenReturn(homedir); - dataDir = new File(homedir, ESNode.DATA_DIR); - } - - @After - public void cleanUp() { - FileUtils.deleteQuietly(homedir); - } - - @Test - public void start_and_stop_es_node() throws Exception { - assertThat(dataDir).doesNotExist(); - - ESNode node = new ESNode(fs, new Settings()); - node.start(); - - ClusterAdminClient cluster = node.client().admin().cluster(); - ClusterState state = cluster.state(cluster.prepareState().request()).actionGet().getState(); - assertThat(state.getNodes().size()).isEqualTo(1); - assertThat(state.getNodes().getMasterNode().isDataNode()).isTrue(); - assertThat(dataDir).exists().isDirectory(); - - // REST console is disabled by default - assertThat(state.getMetaData().settings().get("http.port")).isNull(); - - node.stop(); - - // data dir is persistent - assertThat(dataDir).exists().isDirectory(); - } - - @Test(expected = StrictDynamicMappingException.class) - public void should_use_default_settings_for_index() throws Exception { - ESNode node = new ESNode(fs, new Settings()); - node.start(); - - node.client().admin().indices().prepareCreate("polop") - .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") - .execute().actionGet(); - node.client().admin().cluster().prepareHealth("polop").setWaitForYellowStatus().execute().actionGet(); - - // default "sortable" analyzer is defined for all indices - assertThat(node.client().admin().indices().prepareAnalyze("polop", "This Is A Wonderful Text").setAnalyzer("sortable").execute().actionGet() - .getTokens().get(0).getTerm()).isEqualTo("this is a wonderful text"); - - // strict mapping is enforced - try { - node.client().prepareIndex("polop", "type1", "666").setSource( - XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject() - ).execute().actionGet(); - } finally { - node.stop(); - } - } - - @Test - public void should_restore_status_on_startup() throws Exception { - File zip = new File(Resources.getResource(getClass(), "ESNodeTest/data-es-clean.zip").toURI()); - ZipUtils.unzip(zip, dataDir); - - ESNode node = new ESNode(fs, new Settings()); - node.start(); - - AdminClient admin = node.client().admin(); - assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue(); - assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isIn(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW); - - node.stop(); - } - - @Test(expected = IllegalStateException.class) - public void should_fail_on_corrupt_index() throws Exception { - File zip = new File(Resources.getResource(getClass(), "ESNodeTest/data-es-corrupt.zip").toURI()); - ZipUtils.unzip(zip, dataDir); - - ESNode node = new ESNode(fs, new Settings(), "5s"); - try { - node.start(); - } finally { - node.stop(); - } - } - - @Test - public void should_fail_to_get_client_if_not_started() { - ESNode node = new ESNode(fs, new Settings()); - try { - node.client(); - fail(); - } catch (IllegalStateException e) { - assertThat(e).hasMessage("Elasticsearch is not started"); - } - } - - @Test - public void should_enable_rest_console() throws Exception { - Settings settings = new Settings(); - int httpPort = NetworkUtils.freePort(); - settings.setProperty("sonar.es.http.port", httpPort); - ESNode node = new ESNode(fs, settings); - node.start(); - - URL url = URI.create("http://localhost:" + httpPort).toURL(); - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.connect(); - assertThat(connection.getResponseCode()).isEqualTo(200); - - node.stop(); - connection = (HttpURLConnection) url.openConnection(); - try { - connection.connect(); - fail(); - } catch (Exception e) { - // ok, console is down - } - } -} diff --git a/sonar-server/src/test/java/org/sonar/server/es/NetworkUtils.java b/sonar-server/src/test/java/org/sonar/server/es/NetworkUtils.java deleted file mode 100644 index 4b38d4b96d9..00000000000 --- a/sonar-server/src/test/java/org/sonar/server/es/NetworkUtils.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import java.io.IOException; -import java.net.ServerSocket; - -class NetworkUtils { - - static int freePort() { - for (int index = 0; index < 5; index++) { - try { - ServerSocket socket = new ServerSocket(0); - int unusedPort = socket.getLocalPort(); - socket.close(); - if (isValidPort(unusedPort)) { - return unusedPort; - } - - } catch (IOException e) { - throw new IllegalStateException("Can not find an open network port", e); - } - } - throw new IllegalStateException("Can not find an open network port"); - } - - - private static boolean isValidPort(int port) { - return port > 1023; - } -} diff --git a/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java b/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java new file mode 100644 index 00000000000..1c98d7020ed --- /dev/null +++ b/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java @@ -0,0 +1,179 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search; + +import com.google.common.io.Resources; +import org.apache.commons.io.FileUtils; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.ClusterAdminClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.StrictDynamicMappingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.sonar.api.config.Settings; +import org.sonar.api.platform.ServerFileSystem; +import org.sonar.api.utils.ZipUtils; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; + +import static org.fest.assertions.Assertions.assertThat; +import static org.fest.assertions.Fail.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ESNodeTest { + + ServerFileSystem fs; + File homedir; + File dataDir; + + @Rule + public TemporaryFolder temp = new TemporaryFolder(); + + @Before + public void createMocks() throws IOException { + homedir = temp.newFolder(); + fs = mock(ServerFileSystem.class); + when(fs.getHomeDir()).thenReturn(homedir); + dataDir = new File(homedir, ESNode.DATA_DIR); + } + + @After + public void cleanUp() { + FileUtils.deleteQuietly(homedir); + } + + @Test + public void start_and_stop_es_node() throws Exception { + assertThat(dataDir).doesNotExist(); + + ESNode node = new ESNode(fs, new Settings()); + node.start(); + + ClusterAdminClient cluster = node.client().admin().cluster(); + ClusterState state = cluster.state(cluster.prepareState().request()).actionGet().getState(); + assertThat(state.getNodes().size()).isEqualTo(1); + assertThat(state.getNodes().getMasterNode().isDataNode()).isTrue(); + assertThat(dataDir).exists().isDirectory(); + + // REST console is disabled by default + assertThat(state.getMetaData().settings().get("http.port")).isNull(); + + node.stop(); + + // data dir is persistent + assertThat(dataDir).exists().isDirectory(); + } + + @Test(expected = StrictDynamicMappingException.class) + public void should_use_default_settings_for_index() throws Exception { + ESNode node = new ESNode(fs, new Settings()); + node.start(); + + node.client().admin().indices().prepareCreate("polop") + .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") + .execute().actionGet(); + node.client().admin().cluster().prepareHealth("polop").setWaitForYellowStatus().execute().actionGet(); + + // default "sortable" analyzer is defined for all indices + assertThat(node.client().admin().indices().prepareAnalyze("polop", "This Is A Wonderful Text").setAnalyzer("sortable").execute().actionGet() + .getTokens().get(0).getTerm()).isEqualTo("this is a wonderful text"); + + // strict mapping is enforced + try { + node.client().prepareIndex("polop", "type1", "666").setSource( + XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject() + ).execute().actionGet(); + } finally { + node.stop(); + } + } + + @Test + public void should_restore_status_on_startup() throws Exception { + File zip = new File(Resources.getResource(getClass(), "ESNodeTest/data-es-clean.zip").toURI()); + ZipUtils.unzip(zip, dataDir); + + ESNode node = new ESNode(fs, new Settings()); + node.start(); + + AdminClient admin = node.client().admin(); + assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue(); + assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isIn(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW); + + node.stop(); + } + + @Test(expected = IllegalStateException.class) + public void should_fail_on_corrupt_index() throws Exception { + File zip = new File(Resources.getResource(getClass(), "ESNodeTest/data-es-corrupt.zip").toURI()); + ZipUtils.unzip(zip, dataDir); + + ESNode node = new ESNode(fs, new Settings(), "5s"); + try { + node.start(); + } finally { + node.stop(); + } + } + + @Test + public void should_fail_to_get_client_if_not_started() { + ESNode node = new ESNode(fs, new Settings()); + try { + node.client(); + fail(); + } catch (IllegalStateException e) { + assertThat(e).hasMessage("Elasticsearch is not started"); + } + } + + @Test + public void should_enable_rest_console() throws Exception { + Settings settings = new Settings(); + int httpPort = NetworkUtils.freePort(); + settings.setProperty("sonar.es.http.port", httpPort); + ESNode node = new ESNode(fs, settings); + node.start(); + + URL url = URI.create("http://localhost:" + httpPort).toURL(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.connect(); + assertThat(connection.getResponseCode()).isEqualTo(200); + + node.stop(); + connection = (HttpURLConnection) url.openConnection(); + try { + connection.connect(); + fail(); + } catch (Exception e) { + // ok, console is down + } + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/search/NetworkUtils.java b/sonar-server/src/test/java/org/sonar/server/search/NetworkUtils.java new file mode 100644 index 00000000000..8a76d42d29a --- /dev/null +++ b/sonar-server/src/test/java/org/sonar/server/search/NetworkUtils.java @@ -0,0 +1,48 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search; + +import java.io.IOException; +import java.net.ServerSocket; + +class NetworkUtils { + + static int freePort() { + for (int index = 0; index < 5; index++) { + try { + ServerSocket socket = new ServerSocket(0); + int unusedPort = socket.getLocalPort(); + socket.close(); + if (isValidPort(unusedPort)) { + return unusedPort; + } + + } catch (IOException e) { + throw new IllegalStateException("Can not find an open network port", e); + } + } + throw new IllegalStateException("Can not find an open network port"); + } + + + private static boolean isValidPort(int port) { + return port > 1023; + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/tester/DataStoreCleanup.java b/sonar-server/src/test/java/org/sonar/server/tester/DataStoreCleanup.java index c04c70da395..889a12f58c3 100644 --- a/sonar-server/src/test/java/org/sonar/server/tester/DataStoreCleanup.java +++ b/sonar-server/src/test/java/org/sonar/server/tester/DataStoreCleanup.java @@ -27,7 +27,7 @@ import org.sonar.api.ServerComponent; import org.sonar.core.persistence.DatabaseVersion; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.MyBatis; -import org.sonar.server.es.ESNode; +import org.sonar.server.search.ESNode; import java.sql.Connection; diff --git a/sonar-server/src/test/resources/org/sonar/server/es/ESNodeTest/data-es-clean.zip b/sonar-server/src/test/resources/org/sonar/server/es/ESNodeTest/data-es-clean.zip deleted file mode 100644 index b6d286012c9..00000000000 Binary files a/sonar-server/src/test/resources/org/sonar/server/es/ESNodeTest/data-es-clean.zip and /dev/null differ diff --git a/sonar-server/src/test/resources/org/sonar/server/es/ESNodeTest/data-es-corrupt.zip b/sonar-server/src/test/resources/org/sonar/server/es/ESNodeTest/data-es-corrupt.zip deleted file mode 100644 index 9f59a91cd37..00000000000 Binary files a/sonar-server/src/test/resources/org/sonar/server/es/ESNodeTest/data-es-corrupt.zip and /dev/null differ diff --git a/sonar-server/src/test/resources/org/sonar/server/search/ESNodeTest/data-es-clean.zip b/sonar-server/src/test/resources/org/sonar/server/search/ESNodeTest/data-es-clean.zip new file mode 100644 index 00000000000..b6d286012c9 Binary files /dev/null and b/sonar-server/src/test/resources/org/sonar/server/search/ESNodeTest/data-es-clean.zip differ diff --git a/sonar-server/src/test/resources/org/sonar/server/search/ESNodeTest/data-es-corrupt.zip b/sonar-server/src/test/resources/org/sonar/server/search/ESNodeTest/data-es-corrupt.zip new file mode 100644 index 00000000000..9f59a91cd37 Binary files /dev/null and b/sonar-server/src/test/resources/org/sonar/server/search/ESNodeTest/data-es-corrupt.zip differ