From: Jean-Baptiste Lievremont Date: Mon, 13 Jan 2014 17:20:37 +0000 (+0100) Subject: SONAR-4923 Externalize default settings for ES node / indices into JSON files X-Git-Tag: 4.2~711 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=b2a2ccef47421f5266249c13d26bafdfd2908d24;p=sonarqube.git SONAR-4923 Externalize default settings for ES node / indices into JSON files --- diff --git a/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java b/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java index 39c40d419ad..de703cc8128 100644 --- a/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java @@ -38,8 +38,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.Requests; import org.elasticsearch.common.io.BytesStream; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; @@ -66,14 +64,6 @@ public class SearchIndex implements Startable { private static final String PROFILE_DOMAIN = "es"; private static final Logger LOG = LoggerFactory.getLogger(SearchIndex.class); - private static final Settings INDEX_DEFAULT_SETTINGS = ImmutableSettings.builder() - .put("number_of_shards", 1) - .put("number_of_replicas", 0) - .put("mapper.dynamic", false) - .build(); - - private static final String INDEX_DEFAULT_MAPPING = "{ \"_default_\": { \"dynamic\": \"strict\" } }"; - private SearchNode searchNode; private Client client; private Profiling profiling; @@ -172,8 +162,6 @@ public class SearchIndex implements Startable { try { if (! indices.exists(indices.prepareExists(index).request()).get().isExists()) { indices.prepareCreate(index) - .setSettings(INDEX_DEFAULT_SETTINGS) - .addMapping("_default_", INDEX_DEFAULT_MAPPING) .execute().actionGet(); } } catch (Exception e) { diff --git a/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java b/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java index 78338697e04..c91d73f2cec 100644 --- a/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java +++ b/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java @@ -22,6 +22,7 @@ package org.sonar.server.es; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.Client; import org.elasticsearch.common.logging.ESLoggerFactory; @@ -36,6 +37,7 @@ import org.sonar.api.config.Settings; import org.sonar.api.platform.ServerFileSystem; import java.io.File; +import java.io.IOException; /** * Manages the ElasticSearch Node instance used to connect to the index. @@ -74,18 +76,18 @@ public class SearchNode implements Startable { LOG.info("Starting Elasticsearch..."); initLogging(); - ImmutableSettings.Builder esSettings = ImmutableSettings.builder().put("node.name", INSTANCE_NAME); + ImmutableSettings.Builder esSettings = ImmutableSettings.builder() + .loadFromUrl(getClass().getResource("config/elasticsearch.json")); initDirs(esSettings); initRestConsole(esSettings); node = NodeBuilder.nodeBuilder() - .clusterName(INSTANCE_NAME) - .local(true) - .data(true) .settings(esSettings) .node(); node.start(); + addIndexTemplates(); + if ( node.client().admin().cluster().prepareHealth() .setWaitForYellowStatus() @@ -99,6 +101,16 @@ public class SearchNode implements Startable { LOG.info("Elasticsearch started"); } + private void addIndexTemplates() { + try { + node.client().admin().indices().preparePutTemplate("default") + .setSource(IOUtils.toString(getClass().getResource("config/templates/default.json"))) + .execute().actionGet(); + } catch(IOException ioe) { + throw new IllegalStateException("Unable to load index templates", ioe); + } + } + private void initLogging() { ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory()); } diff --git a/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json b/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json new file mode 100644 index 00000000000..37c8d5ac636 --- /dev/null +++ b/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json @@ -0,0 +1,33 @@ +{ + "cluster": { + "name": "sonarqube", + "local": true, + "data": true + }, + "node": { + "name": "sonarqube" + }, + "discovery": { + "zen": { + "multicast": { + "enabled": false + } + } + }, + "index": { + "number_of_shards": 1, + "number_of_replicas": 0, + "mapper": { + "dynamic": false + }, + "analysis": { + "analyzer": { + "sortable": { + "type": "custom", + "tokenizer": "keyword", + "filter": "lowercase" + } + } + } + } +} diff --git a/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json b/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json new file mode 100644 index 00000000000..0e7a25d7c67 --- /dev/null +++ b/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json @@ -0,0 +1,8 @@ +{ + "template": "*", + "mappings": { + "_default_": { + "dynamic": "strict" + } + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java b/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java index b23f4a8ae6d..5d627c33d1d 100644 --- a/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java +++ b/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java @@ -23,7 +23,6 @@ package org.sonar.server.es; import com.github.tlrx.elasticsearch.test.EsSetup; import org.elasticsearch.common.io.BytesStream; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.StrictDynamicMappingException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -45,7 +44,7 @@ public class SearchIndexTest { private SearchIndex searchIndex; @Before - public void setUp() { + public void setUp() throws Exception { esSetup = new EsSetup(); esSetup.execute(EsSetup.deleteAll()); @@ -147,11 +146,4 @@ public class SearchIndexTest { assertThat(docIds).hasSize(numberOfDocuments); } - @Test(expected = StrictDynamicMappingException.class) - public void should_forbid_dynamic_mapping() throws Exception { - searchIndex.addMappingFromClasspath("index", "type1", "/org/sonar/server/es/SearchIndexTest/correct_mapping1.json"); - searchIndex.putSynchronous("index", "type1", "666", - XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject() - ); - } } diff --git a/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java b/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java index 7a2276f2238..4b30d11814a 100644 --- a/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java +++ b/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java @@ -19,13 +19,13 @@ */ package org.sonar.server.es; -import org.sonar.server.es.SearchNode; - import org.apache.commons.io.FileUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.client.AdminClient; import org.elasticsearch.client.ClusterAdminClient; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.mapper.StrictDynamicMappingException; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -91,6 +91,30 @@ public class SearchNodeTest { assertThat(dataDir).exists().isDirectory(); } + @Test(expected = StrictDynamicMappingException.class) + public void should_use_default_settings_for_index() throws Exception { + SearchNode node = new SearchNode(fs, new Settings().setProperty("sonar.es.http.port", 9200)); + node.start(); + + node.client().admin().indices().prepareCreate("polop") + .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") + .execute().actionGet(); + node.client().admin().cluster().prepareHealth("polop").setWaitForYellowStatus().execute().actionGet(); + + // default "sortable" analyzer is defined for all indices + assertThat(node.client().admin().indices().prepareAnalyze("polop", "This Is A Wonderful Text").setAnalyzer("sortable").execute().actionGet() + .getTokens().get(0).getTerm()).isEqualTo("this is a wonderful text"); + + // strict mapping is enforced + try { + node.client().prepareIndex("polop", "type1", "666").setSource( + XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject() + ).execute().actionGet(); + } finally { + node.stop(); + } + } + @Test public void should_restore_status_on_startup() throws Exception { ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-clean.zip"), dataDir); @@ -99,8 +123,8 @@ public class SearchNodeTest { node.start(); AdminClient admin = node.client().admin(); - assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue();; - assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isEqualTo(ClusterHealthStatus.YELLOW); + assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue(); + assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isIn(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW); node.stop(); }