SONAR-4923 Externalize default settings for ES node / indices into JSON files

diff --git a/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java b/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java
index 39c40d419ad64f57aff4a3f12f8c57ce452da7bb..de703cc81286a89f593d1a92f1cf2b7c9f2e46e4 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java
+++ b/sonar-server/src/main/java/org/sonar/server/es/SearchIndex.java
@@ -38,8 +38,6 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.IndicesAdminClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.io.BytesStream;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -66,14 +64,6 @@ public class SearchIndex implements Startable {
   private static final String PROFILE_DOMAIN = "es";
   private static final Logger LOG = LoggerFactory.getLogger(SearchIndex.class);
 
-  private static final Settings INDEX_DEFAULT_SETTINGS = ImmutableSettings.builder()
-    .put("number_of_shards", 1)
-    .put("number_of_replicas", 0)
-    .put("mapper.dynamic", false)
-    .build();
-
-  private static final String INDEX_DEFAULT_MAPPING = "{ \"_default_\": { \"dynamic\": \"strict\" } }";
-
   private SearchNode searchNode;
   private Client client;
   private Profiling profiling;
@@ -172,8 +162,6 @@ public class SearchIndex implements Startable {
     try {
       if (! indices.exists(indices.prepareExists(index).request()).get().isExists()) {
         indices.prepareCreate(index)
-          .setSettings(INDEX_DEFAULT_SETTINGS)
-          .addMapping("_default_", INDEX_DEFAULT_MAPPING)
           .execute().actionGet();
       }
     } catch (Exception e) {

diff --git a/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java b/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java
index 78338697e047a5f2c502394bcecf496cd0806b69..c91d73f2cecf676b07d7981b5b5501ee1eddb137 100644 (file)
--- a/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java
+++ b/sonar-server/src/main/java/org/sonar/server/es/SearchNode.java
@@ -22,6 +22,7 @@ package org.sonar.server.es;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.logging.ESLoggerFactory;
@@ -36,6 +37,7 @@ import org.sonar.api.config.Settings;
 import org.sonar.api.platform.ServerFileSystem;
 
 import java.io.File;
+import java.io.IOException;
 
 /**
  * Manages the ElasticSearch Node instance used to connect to the index.
@@ -74,18 +76,18 @@ public class SearchNode implements Startable {
     LOG.info("Starting Elasticsearch...");
 
     initLogging();
-    ImmutableSettings.Builder esSettings = ImmutableSettings.builder().put("node.name", INSTANCE_NAME);
+    ImmutableSettings.Builder esSettings = ImmutableSettings.builder()
+      .loadFromUrl(getClass().getResource("config/elasticsearch.json"));
     initDirs(esSettings);
     initRestConsole(esSettings);
 
     node = NodeBuilder.nodeBuilder()
-      .clusterName(INSTANCE_NAME)
-      .local(true)
-      .data(true)
       .settings(esSettings)
       .node();
     node.start();
 
+    addIndexTemplates();
+
     if (
       node.client().admin().cluster().prepareHealth()
       .setWaitForYellowStatus()
@@ -99,6 +101,16 @@ public class SearchNode implements Startable {
     LOG.info("Elasticsearch started");
   }
 
+  private void addIndexTemplates() {
+    try {
+      node.client().admin().indices().preparePutTemplate("default")
+        .setSource(IOUtils.toString(getClass().getResource("config/templates/default.json")))
+        .execute().actionGet();
+    } catch(IOException ioe) {
+      throw new IllegalStateException("Unable to load index templates", ioe);
+    }
+  }
+
   private void initLogging() {
     ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory());
   }

diff --git a/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json b/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json
new file mode 100644 (file)
index 0000000..37c8d5a
--- /dev/null
+++ b/sonar-server/src/main/resources/org/sonar/server/es/config/elasticsearch.json
@@ -0,0 +1,33 @@
+{
+  "cluster": {
+    "name": "sonarqube",
+    "local": true,
+    "data": true
+  },
+  "node": {
+    "name": "sonarqube"
+  },
+  "discovery": {
+    "zen": {
+      "multicast": {
+        "enabled": false
+      }
+    }
+  },
+  "index": {
+    "number_of_shards": 1,
+    "number_of_replicas": 0,
+    "mapper": {
+      "dynamic": false
+    },
+    "analysis": {
+      "analyzer": {
+        "sortable": {
+          "type": "custom",
+          "tokenizer": "keyword",
+          "filter": "lowercase"
+        }
+      }
+    }
+  }
+}

diff --git a/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json b/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json
new file mode 100644 (file)
index 0000000..0e7a25d
--- /dev/null
+++ b/sonar-server/src/main/resources/org/sonar/server/es/config/templates/default.json
@@ -0,0 +1,8 @@
+{
+  "template": "*",
+  "mappings": {
+    "_default_": {
+      "dynamic": "strict"
+    }
+  }
+}

diff --git a/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java b/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java
index b23f4a8ae6db6bb1b2cc59f589dee6362e0d5026..5d627c33d1dccb13e33f0e028d1ac583e5584010 100644 (file)
--- a/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/es/SearchIndexTest.java
@@ -23,7 +23,6 @@ package org.sonar.server.es;
 import com.github.tlrx.elasticsearch.test.EsSetup;
 import org.elasticsearch.common.io.BytesStream;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.mapper.StrictDynamicMappingException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -45,7 +44,7 @@ public class SearchIndexTest {
   private SearchIndex searchIndex;
 
   @Before
-  public void setUp() {
+  public void setUp() throws Exception {
     esSetup = new EsSetup();
     esSetup.execute(EsSetup.deleteAll());
 
@@ -147,11 +146,4 @@ public class SearchIndexTest {
     assertThat(docIds).hasSize(numberOfDocuments);
   }
 
-  @Test(expected = StrictDynamicMappingException.class)
-  public void should_forbid_dynamic_mapping() throws Exception {
-    searchIndex.addMappingFromClasspath("index", "type1", "/org/sonar/server/es/SearchIndexTest/correct_mapping1.json");
-    searchIndex.putSynchronous("index", "type1", "666",
-      XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject()
-    );
-  }
 }

diff --git a/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java b/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java
index 7a2276f22388a0e6c2cf46c57a31257a0b7ea112..4b30d11814a645ae8ea397ca53dbbdace4ed73bf 100644 (file)
--- a/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java
+++ b/sonar-server/src/test/java/org/sonar/server/es/SearchNodeTest.java
@@ -19,13 +19,13 @@
  */
 package org.sonar.server.es;
 
-import org.sonar.server.es.SearchNode;
-
 import org.apache.commons.io.FileUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.ClusterAdminClient;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.StrictDynamicMappingException;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -91,6 +91,30 @@ public class SearchNodeTest {
     assertThat(dataDir).exists().isDirectory();
   }
 
+  @Test(expected = StrictDynamicMappingException.class)
+  public void should_use_default_settings_for_index() throws Exception {
+    SearchNode node = new SearchNode(fs, new Settings().setProperty("sonar.es.http.port", 9200));
+    node.start();
+
+    node.client().admin().indices().prepareCreate("polop")
+      .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}")
+      .execute().actionGet();
+    node.client().admin().cluster().prepareHealth("polop").setWaitForYellowStatus().execute().actionGet();
+
+    // default "sortable" analyzer is defined for all indices
+    assertThat(node.client().admin().indices().prepareAnalyze("polop", "This Is A Wonderful Text").setAnalyzer("sortable").execute().actionGet()
+      .getTokens().get(0).getTerm()).isEqualTo("this is a wonderful text");
+
+    // strict mapping is enforced
+    try {
+      node.client().prepareIndex("polop", "type1", "666").setSource(
+        XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject()
+      ).execute().actionGet();
+    } finally {
+      node.stop();
+    }
+  }
+
   @Test
   public void should_restore_status_on_startup() throws Exception {
     ZipUtils.unzip(TestUtils.getResource(SearchNodeTest.class, "data-es-clean.zip"), dataDir);
@@ -99,8 +123,8 @@ public class SearchNodeTest {
     node.start();
 
     AdminClient admin = node.client().admin();
-    assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue();;
-    assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isEqualTo(ClusterHealthStatus.YELLOW);
+    assertThat(admin.indices().prepareExists("myindex").execute().actionGet().isExists()).isTrue();
+    assertThat(admin.cluster().prepareHealth("myindex").setWaitForYellowStatus().execute().actionGet().getStatus()).isIn(ClusterHealthStatus.GREEN, ClusterHealthStatus.YELLOW);
 
     node.stop();
   }