From d32682708852e29b65f53e4ac57c88e8ff0b93ec Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lievremont Date: Thu, 7 Nov 2013 12:06:36 +0100 Subject: SONAR-4832 Cleanup a bit experiment leftovers Introduce elasticsearch-test for ES integration tests Enhance coverage on SearchIndex Rename node in SearchNode Unplug index at startup (will be triggered by call from RegisterRule to RuleRegistry) --- sonar-server/pom.xml | 6 ++ .../java/org/sonar/server/platform/Platform.java | 4 +- .../java/org/sonar/server/rule/RuleRegistry.java | 101 +++++++++++++++++++++ .../java/org/sonar/server/search/SearchIndex.java | 22 ++--- .../java/org/sonar/server/search/SearchNode.java | 1 + .../java/org/sonar/server/startup/IndexRules.java | 96 -------------------- .../org/sonar/server/search/SearchIndexTest.java | 65 +++++++++++-- .../search/SearchIndexTest/correct_mapping1.json | 9 ++ .../search/SearchIndexTest/correct_mapping2.json | 9 ++ .../server/search/SearchIndexTest/malformed.json | 1 + 10 files changed, 196 insertions(+), 118 deletions(-) create mode 100644 sonar-server/src/main/java/org/sonar/server/rule/RuleRegistry.java delete mode 100644 sonar-server/src/main/java/org/sonar/server/startup/IndexRules.java create mode 100644 sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping1.json create mode 100644 sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping2.json create mode 100644 sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/malformed.json (limited to 'sonar-server') diff --git a/sonar-server/pom.xml b/sonar-server/pom.xml index ed96eff0aa2..e67c953a9b8 100644 --- a/sonar-server/pom.xml +++ b/sonar-server/pom.xml @@ -147,6 +147,12 @@ sonar-testing-harness test + + com.github.tlrx + elasticsearch-test + ${elasticsearch.version} + test + diff --git a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java index 777e93e08cc..c7e56a72e19 100644 --- a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java +++ b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java @@ -19,6 +19,8 @@ */ package org.sonar.server.platform; +import org.sonar.server.rule.RuleRegistry; + import org.apache.commons.configuration.BaseConfiguration; import org.slf4j.LoggerFactory; import org.sonar.api.config.EmailSettings; @@ -354,7 +356,7 @@ public final class Platform { startupContainer.addSingleton(LogServerId.class); startupContainer.addSingleton(RegisterServletFilters.class); startupContainer.addSingleton(CleanDryRunCache.class); - startupContainer.addSingleton(IndexRules.class); + startupContainer.addSingleton(RuleRegistry.class); startupContainer.startComponents(); startupContainer.getComponentByType(ServerLifecycleNotifier.class).notifyStart(); diff --git a/sonar-server/src/main/java/org/sonar/server/rule/RuleRegistry.java b/sonar-server/src/main/java/org/sonar/server/rule/RuleRegistry.java new file mode 100644 index 00000000000..de7ffa48d7f --- /dev/null +++ b/sonar-server/src/main/java/org/sonar/server/rule/RuleRegistry.java @@ -0,0 +1,101 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.rule; + +import org.elasticsearch.common.collect.Lists; +import org.elasticsearch.common.io.BytesStream; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.sonar.api.database.DatabaseSession; +import org.sonar.api.rules.Rule; +import org.sonar.api.rules.RuleParam; +import org.sonar.core.i18n.RuleI18nManager; +import org.sonar.jpa.session.DatabaseSessionFactory; +import org.sonar.server.search.SearchIndex; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; + +/** + * Fill search index with rules + * @since 4.1 + */ +public final class RuleRegistry { + + private static final String INDEX_RULES = "rules"; + private static final String TYPE_RULE = "rule"; + + private SearchIndex searchIndex; + private DatabaseSessionFactory sessionFactory; + private RuleI18nManager ruleI18nManager; + + public RuleRegistry(SearchIndex searchIndex, DatabaseSessionFactory sessionFactory, RuleI18nManager ruleI18nManager) { + this.searchIndex = searchIndex; + this.sessionFactory = sessionFactory; + this.ruleI18nManager = ruleI18nManager; + } + + public void start() { + searchIndex.addMappingFromClasspath(INDEX_RULES, TYPE_RULE, "/com/sonar/search/rule_mapping.json"); + } + + public void bulkRegisterRules() { + DatabaseSession session = sessionFactory.getSession(); + + try { + List ids = Lists.newArrayList(); + List docs = Lists.newArrayList(); + for (Rule rule: session.getResults(Rule.class)) { + ids.add(rule.getId().toString()); + XContentBuilder document = XContentFactory.jsonBuilder() + .startObject() + .field("id", rule.getId()) + .field("key", rule.ruleKey()) + .field("language", rule.getLanguage()) + .field("name", ruleI18nManager.getName(rule, Locale.getDefault())) + .field("description", ruleI18nManager.getDescription(rule.getRepositoryKey(), rule.getKey(), Locale.getDefault())) + .field("parentKey", rule.getParent() == null ? null : rule.getParent().getKey()) + .field("repositoryKey", rule.getRepositoryKey()) + .field("severity", rule.getSeverity()) + .field("status", rule.getStatus()) + .field("createdAt", rule.getCreatedAt()) + .field("updatedAt", rule.getUpdatedAt()); + if(!rule.getParams().isEmpty()) { + document.startArray("params"); + for (RuleParam param: rule.getParams()) { + document.startObject() + .field("key", param.getKey()) + .field("type", param.getType()) + .field("defaultValue", param.getDefaultValue()) + .field("description", param.getDescription()) + .endObject(); + } + document.endArray(); + } + docs.add(document.endObject()); + } + searchIndex.bulkIndex(INDEX_RULES, TYPE_RULE, ids.toArray(new String[0]), docs.toArray(new BytesStream[0])); + } catch(IOException ioe) { + throw new IllegalStateException("Unable to index rules", ioe); + } + } +} diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java index 99704461929..8935db2faba 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java @@ -20,6 +20,7 @@ package org.sonar.server.search; import org.apache.commons.io.IOUtils; +import org.elasticsearch.ElasticSearchParseException; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -29,7 +30,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.client.Requests; import org.elasticsearch.common.io.BytesStream; -import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,10 +61,6 @@ public class SearchIndex { client.prepareIndex(index, type, id).setSource(source.bytes()).execute().actionGet(); } - public void put(String index, String type, String id, BytesStream source, String parent) { - client.prepareIndex(index, type, id).setParent(parent).setSource(source.bytes()).execute().actionGet(); - } - public void bulkIndex(String index, String type, String[] ids, BytesStream[] sources) { BulkRequestBuilder builder = new BulkRequestBuilder(client); for (int i=0; i ids = Lists.newArrayList(); - List docs = Lists.newArrayList(); - for (Rule rule: session.getResults(Rule.class)) { - ids.add(rule.getId().toString()); - XContentBuilder document = XContentFactory.jsonBuilder() - .startObject() - .field("id", rule.getId()) - .field("key", rule.ruleKey()) - .field("language", rule.getLanguage()) - .field("name", ruleI18nManager.getName(rule, Locale.getDefault())) - .field("description", ruleI18nManager.getDescription(rule.getRepositoryKey(), rule.getKey(), Locale.getDefault())) - .field("parentKey", rule.getParent() == null ? null : rule.getParent().getKey()) - .field("repositoryKey", rule.getRepositoryKey()) - .field("severity", rule.getSeverity()) - .field("status", rule.getStatus()) - .field("createdAt", rule.getCreatedAt()) - .field("updatedAt", rule.getUpdatedAt()); - if(!rule.getParams().isEmpty()) { - document.startArray("params"); - for (RuleParam param: rule.getParams()) { - document.startObject() - .field("key", param.getKey()) - .field("type", param.getType()) - .field("defaultValue", param.getDefaultValue()) - .field("description", param.getDescription()) - .endObject(); - } - document.endArray(); - } - docs.add(document.endObject()); - } - searchIndex.bulkIndex(INDEX_RULES, TYPE_RULE, ids.toArray(new String[0]), docs.toArray(new BytesStream[0])); - } catch(IOException ioe) { - throw new IllegalStateException("Unable to index rules", ioe); - } - } -} diff --git a/sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java b/sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java index f10047b8ea9..0edd4146ff7 100644 --- a/sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java +++ b/sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java @@ -20,39 +20,88 @@ package org.sonar.server.search; -import org.elasticsearch.client.Client; +import com.github.tlrx.elasticsearch.test.EsSetup; +import org.junit.After; import org.junit.Before; import org.junit.Test; +import static org.fest.assertions.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class SearchIndexTest { + private EsSetup esSetup; private SearchNode searchNode; - private Client client; - private SearchIndex searchIndex; @Before public void setUp() { + esSetup = new EsSetup(); + esSetup.execute(EsSetup.deleteAll()); + searchNode = mock(SearchNode.class); - client = mock(Client.class); - when(searchNode.client()).thenReturn(client); + when(searchNode.client()).thenReturn(esSetup.client()); searchIndex = new SearchIndex(searchNode); + searchIndex.start(); + } + + @After + public void tearDown() { + esSetup.terminate(); } @Test public void should_start_and_stop_properly() { + verify(searchNode).client(); + searchIndex.stop(); + } + + @Test + public void should_create_index_when_loading_mapping_from_classpath() { + String index = "index"; + String type = "type"; + String resourcePath = "/org/sonar/server/search/SearchIndexTest/correct_mapping1.json"; + searchIndex.start(); + searchIndex.addMappingFromClasspath(index, type, resourcePath); - verify(searchNode).client(); + assertThat(esSetup.exists(index)).isTrue(); + } - searchIndex.stop(); + @Test + public void should_reuse_index_when_loading_mapping_from_classpath() { + String index = "index"; + String type1 = "type1"; + String type2 = "type2"; + String resourcePath1 = "/org/sonar/server/search/SearchIndexTest/correct_mapping1.json"; + String resourcePath2 = "/org/sonar/server/search/SearchIndexTest/correct_mapping2.json"; + + searchIndex.start(); + searchIndex.addMappingFromClasspath(index, type1, resourcePath1); + searchIndex.addMappingFromClasspath(index, type2, resourcePath2); - verify(client).close(); + assertThat(esSetup.exists(index)).isTrue(); } + + + @Test(expected = IllegalArgumentException.class) + public void should_fail_to_load_inexistent_mapping() { + String resourcePath = "/org/sonar/server/search/SearchIndexTest/inexistent.json"; + + searchIndex.start(); + searchIndex.addMappingFromClasspath("unchecked", "unchecked", resourcePath); + } + + @Test(expected = IllegalArgumentException.class) + public void should_fail_to_load_malformed_mapping() { + String resourcePath = "/org/sonar/server/search/SearchIndexTest/malformed.json"; + + searchIndex.start(); + searchIndex.addMappingFromClasspath("unchecked", "unchecked", resourcePath); + } + } diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping1.json b/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping1.json new file mode 100644 index 00000000000..67e2dddf823 --- /dev/null +++ b/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping1.json @@ -0,0 +1,9 @@ +{ + "type1": { + "properties": { + "value": { + "type": "string" + } + } + } +} diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping2.json b/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping2.json new file mode 100644 index 00000000000..d1b9448215a --- /dev/null +++ b/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/correct_mapping2.json @@ -0,0 +1,9 @@ +{ + "type2": { + "properties": { + "value": { + "type": "string" + } + } + } +} diff --git a/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/malformed.json b/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/malformed.json new file mode 100644 index 00000000000..a01ad5e86aa --- /dev/null +++ b/sonar-server/src/test/resources/org/sonar/server/search/SearchIndexTest/malformed.json @@ -0,0 +1 @@ +blarg -- cgit v1.2.3