From: Stephane Gamard Date: Mon, 11 Aug 2014 08:10:24 +0000 (+0200) Subject: Extracted ES from sonar-server & updated ServerTester for mediumTests X-Git-Tag: 4.5-RC1~189 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=f1c87c9f0411ae6067408c9d0625c2e7f1f4d79a;p=sonarqube.git Extracted ES from sonar-server & updated ServerTester for mediumTests --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/ESNode.java b/server/sonar-server/src/main/java/org/sonar/server/search/ESNode.java deleted file mode 100644 index f8b3a2d8ea8..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/search/ESNode.java +++ /dev/null @@ -1,430 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package org.sonar.server.search; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.ActionRequestBuilder; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.action.ListenableActionFuture; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodeResponse; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; -import org.elasticsearch.common.network.NetworkUtils; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; -import org.picocontainer.Startable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonar.api.config.Settings; -import org.sonar.core.profiling.Profiling; -import org.sonar.core.profiling.StopWatch; -import org.sonar.server.search.es.ListUpdate; - -import javax.annotation.CheckForNull; -import java.io.File; -import java.net.InetAddress; - -/** - * ElasticSearch Node used to connect to index. - */ -public class ESNode implements Startable { - - private static final Logger LOG = LoggerFactory.getLogger(ESNode.class); - private static final String HTTP_ENABLED = "http.enabled"; - private static final String DEFAULT_HEALTH_TIMEOUT = "30s"; - - private final Settings settings; - private final String healthTimeout; - - // available only after startup - private Client client; - private Node node; - protected final Profiling profiling; - - public ESNode(Settings settings) { - this(settings, DEFAULT_HEALTH_TIMEOUT); - } - - @VisibleForTesting - ESNode(Settings settings, String healthTimeout) { - this.settings = settings; - this.healthTimeout = healthTimeout; - this.profiling = new Profiling(settings); - } - - @Override - public void start() { - initLogging(); - - String typeValue = settings.getString(IndexProperties.TYPE); - IndexProperties.ES_TYPE type = - typeValue != null ? - IndexProperties.ES_TYPE.valueOf(typeValue) : - IndexProperties.ES_TYPE.DATA; - - ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder() - .put("index.merge.policy.max_merge_at_once", "200") - .put("index.merge.policy.segments_per_tier", "200") - - .put("indices.store.throttle.type", "merge") - .put("indices.store.throttle.max_bytes_per_sec", "200mb") - - // Set cluster coordinates - .put("cluster.name", StringUtils.defaultIfBlank(settings.getString(IndexProperties.CLUSTER_NAME), "sonarqube")) - .put("node.rack_id", StringUtils.defaultIfEmpty(settings.getString(IndexProperties.NODE_NAME), "unknown")) - - .put("path.home", esHomeDir().getAbsolutePath()); - - - initAnalysis(esSettings); - - if (IndexProperties.ES_TYPE.TRANSPORT.equals(type)) { - initRemoteClient(esSettings); - } else { - initLocalClient(type, esSettings); - } - - if (getClusterHealthStatus() == ClusterHealthStatus.RED) { - throw new IllegalStateException(String.format("Elasticsearch index is corrupt, please delete directory '%s' " + - "and relaunch the SonarQube server.", esDataDir())); - } - - addIndexTemplates(); - - LOG.info("Elasticsearch started"); - } - - public NodeHealth getNodeHealth() { - NodeHealth health = new NodeHealth(); - ClusterStatsResponse clusterStatsResponse = client().admin().cluster().prepareClusterStats().get(); - - // Cluster health - health.setClusterAvailable(clusterStatsResponse.getStatus() != ClusterHealthStatus.RED); - - ClusterStatsNodes nodesStats = clusterStatsResponse.getNodesStats(); - - // JVM Heap Usage - health.setJvmHeapMax(nodesStats.getJvm().getHeapMax().bytes()); - health.setJvmHeapUsed(nodesStats.getJvm().getHeapUsed().bytes()); - - // OS Memory Usage ? - - // Disk Usage - health.setFsTotal(nodesStats.getFs().getTotal().bytes()); - health.setFsAvailable(nodesStats.getFs().getAvailable().bytes()); - - // Ping ? - - // Threads - health.setJvmThreads(nodesStats.getJvm().getThreads()); - - // CPU - health.setProcessCpuPercent(nodesStats.getProcess().getCpuPercent()); - - // Open Files - health.setOpenFiles(nodesStats.getProcess().getAvgOpenFileDescriptors()); - - // Uptime - health.setJvmUptimeMillis(nodesStats.getJvm().getMaxUpTime().getMillis()); - - return health; - } - - private ClusterHealthStatus getClusterHealthStatus() { - return client.admin().cluster().prepareHealth() - .setWaitForYellowStatus() - .setTimeout(healthTimeout) - .get() - .getStatus(); - } - - private void initRemoteClient(ImmutableSettings.Builder esSettings) { - int port = settings.getInt(IndexProperties.NODE_PORT); - client = new TransportClient(esSettings) - .addTransportAddress(new InetSocketTransportAddress("localhost", - port)); - LOG.info("Elasticsearch port: " + port); - } - - @CheckForNull - private NodeInfo getLocalNodeInfoByHostName(String hostname) { - for (ClusterStatsNodeResponse nodeResp : client().admin().cluster().prepareClusterStats().get().getNodes()) { - if (hostname.equals(nodeResp.nodeInfo().getHostname())) { - return nodeResp.nodeInfo(); - } - } - return null; - } - - @CheckForNull - private NodeInfo getLocalNodeInfoByNodeName(String nodeName) { - return client().admin().cluster().prepareClusterStats().get().getNodesMap().get(nodeName).nodeInfo(); - } - - @CheckForNull - private NodeInfo getLocalNodeInfo() { - if (settings.hasKey(IndexProperties.NODE_NAME)) { - return getLocalNodeInfoByNodeName(settings.getString(IndexProperties.NODE_NAME)); - } else { - try { - String LocalhostName = InetAddress.getLocalHost().getHostName(); - return getLocalNodeInfoByHostName(LocalhostName); - } catch (Exception exception) { - throw new IllegalStateException("Could not get localhost hostname", exception); - } - } - } - - private void initLocalClient(IndexProperties.ES_TYPE type, ImmutableSettings.Builder esSettings) { - if (IndexProperties.ES_TYPE.MEMORY.equals(type)) { - initMemoryES(esSettings); - } - - initNetwork(esSettings); - - node = NodeBuilder.nodeBuilder() - .settings(esSettings) - .node(); - node.start(); - - client = node.client(); - } - - private void initMemoryES(ImmutableSettings.Builder builder) { - builder - .put("node.name", "node-mem-" + System.currentTimeMillis()) - .put("node.data", true) - .put("cluster.name", "cluster-mem-" + NetworkUtils.getLocalAddress().getHostName()) - .put("index.store.type", "memory") - .put("index.store.fs.memory.enabled", "true") - .put("gateway.type", "none") - .put("index.number_of_shards", "1") - .put("index.number_of_replicas", "0") - .put("node.local", true) - - // Initialize our own ListUpdate - .put("script.default_lang", "native") - .put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName()) - - - .put("index.search.slowlog.threshold.query.warn", "10ms") - // Cannot use anything else but warn - .put("index.search.slowlog.threshold.query.info", "10ms") - .put("index.search.slowlog.threshold.query.debug", "10ms") - .put("index.search.slowlog.threshold.query.trace", "10ms") - - .put("index.search.slowlog.threshold.fetch.warn", "10ms") - // Cannot use anything else but warn - .put("index.search.slowlog.threshold.fetch.info", "10ms") - .put("index.search.slowlog.threshold.fetch.debug", "10ms") - .put("index.search.slowlog.threshold.fetch.trace", "10ms") - - .put("index.indexing.slowlog.threshold.index.warn", "10ms") - // Cannot use anything else but warn - .put("index.indexing.slowlog.threshold.index.info", "10ms") - .put("index.indexing.slowlog.threshold.index.debug", "10ms") - .put("index.indexing.slowlog.threshold.index.trace", "10ms") - - .put("path.data", esDataDir().getAbsolutePath()) - - .put("path.logs", esLogDir().getAbsolutePath()); - - int httpPort = settings.getInt(IndexProperties.HTTP_PORT); - if (httpPort > 0) { - LOG.warn("Elasticsearch HTTP console enabled on port {}. Only for debugging purpose.", httpPort); - builder.put(HTTP_ENABLED, true); - builder.put("http.host", "127.0.0.1"); - builder.put("http.port", httpPort); - } else { - builder.put(HTTP_ENABLED, false); - } - } - - private void addIndexTemplates() { - client.admin().indices() - .preparePutTemplate("default") - .setTemplate("*") - .addMapping("_default_", "{\"dynamic\": \"strict\"}") - .get(); - } - - private void initLogging() { - ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory()); - } - - private void initAnalysis(ImmutableSettings.Builder esSettings) { - esSettings - .put("index.mapper.dynamic", false) - - // Sortable text analyzer - .put("index.analysis.analyzer.sortable.type", "custom") - .put("index.analysis.analyzer.sortable.tokenizer", "keyword") - .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") - - // Edge NGram index-analyzer - .put("index.analysis.analyzer.index_grams.type", "custom") - .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") - - // Edge NGram search-analyzer - .put("index.analysis.analyzer.search_grams.type", "custom") - .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") - .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") - - // Word index-analyzer - .put("index.analysis.analyzer.index_words.type", "custom") - .put("index.analysis.analyzer.index_words.tokenizer", "standard") - .putArray("index.analysis.analyzer.index_words.filter", - "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") - - // Word search-analyzer - .put("index.analysis.analyzer.search_words.type", "custom") - .put("index.analysis.analyzer.search_words.tokenizer", "standard") - .putArray("index.analysis.analyzer.search_words.filter", - "standard", "lowercase", "stop", "asciifolding", "porter_stem") - - // Edge NGram filter - .put("index.analysis.filter.gram_filter.type", "edgeNGram") - .put("index.analysis.filter.gram_filter.min_gram", 2) - .put("index.analysis.filter.gram_filter.max_gram", 15) - .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") - - // Word filter - .put("index.analysis.filter.word_filter.type", "word_delimiter") - .put("index.analysis.filter.word_filter.generate_word_parts", true) - .put("index.analysis.filter.word_filter.catenate_words", true) - .put("index.analysis.filter.word_filter.catenate_numbers", true) - .put("index.analysis.filter.word_filter.catenate_all", true) - .put("index.analysis.filter.word_filter.split_on_case_change", true) - .put("index.analysis.filter.word_filter.preserve_original", true) - .put("index.analysis.filter.word_filter.split_on_numerics", true) - .put("index.analysis.filter.word_filter.stem_english_possessive", true) - - // Path Analyzer - .put("index.analysis.analyzer.path_analyzer.type", "custom") - .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); - - } - - private void initNetwork(ImmutableSettings.Builder esSettings) { - esSettings.put("network.bind_host", "127.0.0.1"); - } - - private File esHomeDir() { - if (!settings.hasKey("sonar.path.home")) { - throw new IllegalStateException("property 'sonar.path.home' is required"); - } - return new File(settings.getString("sonar.path.home")); - } - - private File esDataDir() { - if (settings.hasKey("sonar.path.data")) { - return new File(settings.getString("sonar.path.data"), "es"); - } else { - return new File(settings.getString("sonar.path.home"), "data/es"); - } - } - - private File esLogDir() { - if (settings.hasKey("sonar.path.log")) { - return new File(settings.getString("sonar.path.log")); - } else { - return new File(settings.getString("sonar.path.home"), "log"); - } - } - - @Override - public void stop() { - if (client != null) { - client.close(); - client = null; - } - if (node != null) { - LoggerFactory.getLogger(this.getClass()).info("Shutting down In Memory ES"); - node.close(); - while (!node.isClosed()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - // ignore - } - } - node = null; - } - } - - public Client client() { - if (client == null) { - throw new IllegalStateException("Elasticsearch is not started"); - } - return client; - } - - public K execute(ActionRequestBuilder request) { - StopWatch basicProfile = profiling.start("search", Profiling.Level.BASIC); - StopWatch fullProfile = profiling.start("search", Profiling.Level.FULL); - ListenableActionFuture acc = request.execute(); - try { - - K response = (K) acc.get(); - - if (profiling.isProfilingEnabled(Profiling.Level.BASIC)) { - if (ToXContent.class.isAssignableFrom(request.getClass())) { - XContentBuilder debugResponse = XContentFactory.jsonBuilder(); - debugResponse.startObject(); - ((ToXContent) request).toXContent(debugResponse, ToXContent.EMPTY_PARAMS); - debugResponse.endObject(); - fullProfile.stop("ES Request: %s", debugResponse.string()); - } else { - fullProfile.stop("ES Request: %s", request.toString().replaceAll("\n", "")); - } - } - - if (profiling.isProfilingEnabled(Profiling.Level.FULL)) { - if (ToXContent.class.isAssignableFrom(response.getClass())) { - XContentBuilder debugResponse = XContentFactory.jsonBuilder(); - debugResponse.startObject(); - ((ToXContent) response).toXContent(debugResponse, ToXContent.EMPTY_PARAMS); - debugResponse.endObject(); - fullProfile.stop("ES Response: %s", debugResponse.string()); - } else { - fullProfile.stop("ES Response: %s", response.toString()); - } - } - - return response; - } catch (Exception e) { - throw new IllegalStateException("ES error: ", e); - } - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java deleted file mode 100644 index 59963c4fd0f..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/search/ESNodeTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.search; - -import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.StrictDynamicMappingException; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.sonar.api.config.Settings; - -import java.io.File; -import java.io.IOException; - -import static org.fest.assertions.Assertions.assertThat; -import static org.fest.assertions.Fail.fail; - -public class ESNodeTest { - - File dataDir; - Settings settings; - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Before - public void createMocks() throws IOException { - dataDir = temp.newFolder(); - settings = new Settings(); - settings.setProperty("sonar.path.home", dataDir.getAbsolutePath()); - settings.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name()); - } - - @Test(expected = StrictDynamicMappingException.class) - public void should_use_default_settings_for_index() throws Exception { - ESNode node = new ESNode(settings); - node.start(); - - node.client().admin().indices().prepareCreate("strict") - .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") - .execute().actionGet(); - node.client().admin().cluster().prepareHealth("strict").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); - - // strict mapping is enforced - try { - node.client().prepareIndex("strict", "type1", "666").setSource( - XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject() - ).get(); - } finally { - node.stop(); - } - } - - @Test - public void check_path_analyzer() throws Exception { - ESNode node = new ESNode(settings); - node.start(); - - node.client().admin().indices().prepareCreate("path") - .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") - .execute().actionGet(); - node.client().admin().cluster().prepareHealth("path").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); - - // default "path_analyzer" analyzer is defined for all indices - AnalyzeResponse response = node.client().admin().indices() - .prepareAnalyze("path", "/temp/65236/test path/MyFile.java").setAnalyzer("path_analyzer").get(); - // default "path_analyzer" analyzer is defined for all indices - assertThat(response.getTokens()).hasSize(4); - assertThat(response.getTokens().get(0).getTerm()).isEqualTo("/temp"); - assertThat(response.getTokens().get(1).getTerm()).isEqualTo("/temp/65236"); - assertThat(response.getTokens().get(2).getTerm()).isEqualTo("/temp/65236/test path"); - assertThat(response.getTokens().get(3).getTerm()).isEqualTo("/temp/65236/test path/MyFile.java"); - - node.stop(); - } - - @Test - public void check_sortable_analyzer() throws Exception { - ESNode node = new ESNode(settings); - node.start(); - - node.client().admin().indices().prepareCreate("sort") - .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") - .execute().actionGet(); - node.client().admin().cluster().prepareHealth("sort").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); - - // default "sortable" analyzer is defined for all indices - assertThat(node.client().admin().indices() - .prepareAnalyze("sort", "This Is A Wonderful Text").setAnalyzer("sortable").get() - .getTokens().get(0).getTerm()).isEqualTo("this is a "); - - node.stop(); - } - - @Test - public void check_gram_analyzer() throws Exception { - ESNode node = new ESNode(settings); - node.start(); - - node.client().admin().indices().prepareCreate("gram") - .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") - .execute().actionGet(); - node.client().admin().cluster().prepareHealth("gram").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); - - // default "string_gram" analyzer is defined for all indices - AnalyzeResponse response = node.client().admin().indices() - .prepareAnalyze("gram", "he.llo w@rl#d").setAnalyzer("index_grams").get(); - assertThat(response.getTokens()).hasSize(10); - assertThat(response.getTokens().get(0).getTerm()).isEqualTo("he"); - assertThat(response.getTokens().get(7).getTerm()).isEqualTo("w@rl"); - - node.stop(); - } - - @Test - public void should_fail_to_get_client_if_not_started() { - ESNode node = new ESNode(settings); - try { - node.client(); - fail(); - } catch (IllegalStateException e) { - assertThat(e).hasMessage("Elasticsearch is not started"); - } - } -}