From 8aabceea511700fb8ff95be315fdc86411b4d4fe Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Lievremont Date: Wed, 6 Nov 2013 16:28:03 +0100 Subject: [PATCH] SONAR-4832 Integrate ElasticSearch into sonar-server (with initial WIP API) --- pom.xml | 1 + sonar-application/pom.xml | 2 +- sonar-server/pom.xml | 5 + .../org/sonar/server/platform/Platform.java | 4 + .../org/sonar/server/search/SearchIndex.java | 120 ++++++++++++++++++ .../org/sonar/server/search/SearchNode.java | 108 ++++++++++++++++ .../org/sonar/server/search/SearchQuery.java | 95 ++++++++++++++ .../org/sonar/server/search/package-info.java | 23 ++++ .../sonar/server/search/SearchIndexTest.java | 58 +++++++++ .../sonar/server/search/SearchNodeTest.java | 117 +++++++++++++++++ .../sonar/server/search/SearchQueryTest.java | 54 ++++++++ 11 files changed, 586 insertions(+), 1 deletion(-) create mode 100644 sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java create mode 100644 sonar-server/src/main/java/org/sonar/server/search/SearchNode.java create mode 100644 sonar-server/src/main/java/org/sonar/server/search/SearchQuery.java create mode 100644 sonar-server/src/main/java/org/sonar/server/search/package-info.java create mode 100644 sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java create mode 100644 sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java create mode 100644 sonar-server/src/test/java/org/sonar/server/search/SearchQueryTest.java diff --git a/pom.xml b/pom.xml index 088158a9578..5e80eee267d 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ 1.0.13 1.7.5 7.0.42 + 0.90.5 UTF-8 3.0.5 2.2.0 diff --git a/sonar-application/pom.xml b/sonar-application/pom.xml index 63a9423e04a..27799dba52c 100644 --- a/sonar-application/pom.xml +++ b/sonar-application/pom.xml @@ -282,7 +282,7 @@ 55000000 - 60400000 + 75000000 ${project.build.directory}/sonarqube-${project.version}.zip diff --git a/sonar-server/pom.xml b/sonar-server/pom.xml index 293c7e7ab6b..ed96eff0aa2 100644 --- a/sonar-server/pom.xml +++ b/sonar-server/pom.xml @@ -125,6 +125,11 @@ org.jruby.rack jruby-rack + + org.elasticsearch + elasticsearch + ${elasticsearch.version} + org.apache.tomcat.embed tomcat-embed-core diff --git a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java index 5af173d2839..83da0f3c6c1 100644 --- a/sonar-server/src/main/java/org/sonar/server/platform/Platform.java +++ b/sonar-server/src/main/java/org/sonar/server/platform/Platform.java @@ -93,6 +93,8 @@ import org.sonar.server.plugins.*; import org.sonar.server.rule.RubyRuleService; import org.sonar.server.rules.ProfilesConsole; import org.sonar.server.rules.RulesConsole; +import org.sonar.server.search.SearchIndex; +import org.sonar.server.search.SearchNode; import org.sonar.server.startup.*; import org.sonar.server.technicaldebt.InternalRubyTechnicalDebtService; import org.sonar.server.technicaldebt.TechnicalDebtFormatter; @@ -210,6 +212,7 @@ public final class Platform { coreContainer.addSingleton(ThreadLocalDatabaseSessionFactory.class); coreContainer.addPicoAdapter(new DatabaseSessionProvider()); coreContainer.addSingleton(ServerMetadataPersister.class); + coreContainer.addSingleton(SearchNode.class); coreContainer.startComponents(); } @@ -219,6 +222,7 @@ public final class Platform { private void startServiceComponents() { servicesContainer = coreContainer.createChild(); + servicesContainer.addSingleton(SearchIndex.class); servicesContainer.addSingleton(HttpDownloader.class); servicesContainer.addSingleton(UriReader.class); servicesContainer.addSingleton(UpdateCenterClient.class); diff --git a/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java new file mode 100644 index 00000000000..ad8c9ee0120 --- /dev/null +++ b/sonar-server/src/main/java/org/sonar/server/search/SearchIndex.java @@ -0,0 +1,120 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search; + +import org.apache.commons.io.IOUtils; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.io.BytesStream; +import org.elasticsearch.index.query.QueryBuilders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class SearchIndex { + + private static final Logger LOG = LoggerFactory.getLogger(SearchIndex.class); + + private SearchNode searchNode; + private Client client; + + public SearchIndex(SearchNode searchNode) { + this.searchNode = searchNode; + } + + public void start() { + this.client = searchNode.client(); + } + + public void stop() { + if(client != null) { + client.close(); + } + } + + public void put(String index, String type, String id, BytesStream source) { + client.prepareIndex(index, type, id).setSource(source.bytes()).execute(); + } + + public void put(String index, String type, String id, BytesStream source, String parent) { + client.prepareIndex(index, type, id).setParent(parent).setSource(source.bytes()).execute(); + } + + public void bulkIndex(String index, String type, String[] ids, BytesStream[] sources) { + BulkRequestBuilder builder = new BulkRequestBuilder(client); + for (int i=0; i + SearchQuery.create("polop") + .field("field1", "value1") + .field("field2", "value2") + + * ...yields the following query string:
+ *
+ polop AND field1:value1 AND field2:value2 +
+ * @since 4.1 + */ +public class SearchQuery { + private String searchString; + private List indices; + private List types; + private Map fieldCriteria; + private Map termFacets; + + private SearchQuery() { + indices = Lists.newArrayList(); + types = Lists.newArrayList(); + fieldCriteria = Maps.newLinkedHashMap(); + termFacets = Maps.newHashMap(); + } + + public static SearchQuery create() { + return new SearchQuery(); + } + + public static SearchQuery create(String searchString) { + SearchQuery searchQuery = new SearchQuery(); + searchQuery.searchString = searchString; + return searchQuery; + } + + public SearchQuery index(String index) { + indices.add(index); + return this; + } + + public SearchQuery field(String fieldName, String fieldValue) { + fieldCriteria.put(fieldName, fieldValue); + return this; + } + + public SearchQuery facet(String facetName, String fieldName) { + fieldCriteria.put(facetName, fieldName); + return this; + } + + private SearchRequestBuilder addFacets(SearchRequestBuilder builder) { + for (String facetName: termFacets.keySet()) { + builder.addFacet(FacetBuilders.termsFacet(facetName).field(termFacets.get(facetName))); + } + return builder; + } + + String getQueryString() { + List criteria = Lists.newArrayList(); + if (StringUtils.isNotBlank(searchString)) { + criteria.add(searchString); + } + for (String fieldName: fieldCriteria.keySet()) { + criteria.add(String.format("%s:%s", fieldName, fieldCriteria.get(fieldName))); + } + return StringUtils.join(criteria, " AND "); + } + + SearchRequestBuilder toBuilder(Client client) { + SearchRequestBuilder builder = client.prepareSearch(indices.toArray(new String[0])).setTypes(types.toArray(new String[0])); + String queryString = getQueryString(); + if (StringUtils.isBlank(queryString)) { + builder.setQuery(QueryBuilders.matchAllQuery()); + } else { + builder.setQuery(queryString); + } + return addFacets(builder); + } +} diff --git a/sonar-server/src/main/java/org/sonar/server/search/package-info.java b/sonar-server/src/main/java/org/sonar/server/search/package-info.java new file mode 100644 index 00000000000..2f141ea121d --- /dev/null +++ b/sonar-server/src/main/java/org/sonar/server/search/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.search; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java b/sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java new file mode 100644 index 00000000000..f10047b8ea9 --- /dev/null +++ b/sonar-server/src/test/java/org/sonar/server/search/SearchIndexTest.java @@ -0,0 +1,58 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.search; + +import org.elasticsearch.client.Client; +import org.junit.Before; +import org.junit.Test; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SearchIndexTest { + + private SearchNode searchNode; + + private Client client; + + private SearchIndex searchIndex; + + @Before + public void setUp() { + searchNode = mock(SearchNode.class); + client = mock(Client.class); + when(searchNode.client()).thenReturn(client); + + searchIndex = new SearchIndex(searchNode); + } + + @Test + public void should_start_and_stop_properly() { + searchIndex.start(); + + verify(searchNode).client(); + + searchIndex.stop(); + + verify(client).close(); + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java new file mode 100644 index 00000000000..d68eb691e4a --- /dev/null +++ b/sonar-server/src/test/java/org/sonar/server/search/SearchNodeTest.java @@ -0,0 +1,117 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.search; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; +import org.junit.Before; +import org.junit.Test; +import org.sonar.api.config.Settings; +import org.sonar.api.utils.TempFolder; + +import java.io.File; + +import static org.fest.assertions.Assertions.assertThat; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SearchNodeTest { + + private TempFolder tempFolder; + private Settings settings; + private ImmutableSettings.Builder settingsBuilder; + private NodeBuilder nodeBuilder; + private SearchNode searchNode; + + @Before + public void createMocks() { + tempFolder = mock(TempFolder.class); + when(tempFolder.newDir("es")).thenReturn(mock(File.class)); + settings = mock(Settings.class); + + settingsBuilder = mock(ImmutableSettings.Builder.class); + when(settingsBuilder.put(anyString(), anyString())).thenReturn(settingsBuilder); + when(settingsBuilder.put(anyString(), anyInt())).thenReturn(settingsBuilder); + + nodeBuilder = mock(NodeBuilder.class); + when(nodeBuilder.local(anyBoolean())).thenReturn(nodeBuilder); + when(nodeBuilder.clusterName(anyString())).thenReturn(nodeBuilder); + when(nodeBuilder.settings(settingsBuilder)).thenReturn(nodeBuilder); + + searchNode = new SearchNode(tempFolder, settings, settingsBuilder, nodeBuilder); + } + + @Test(expected = IllegalStateException.class) + public void should_fail_if_not_properly_started() { + searchNode.stop(); + searchNode.client(); + } + + @Test + public void should_manage_node_without_http() { + Node node = mock(Node.class); + Client client = mock(Client.class); + + when(nodeBuilder.node()).thenReturn(node); + when(node.client()).thenReturn(client); + + searchNode.start(); + assertThat(searchNode.client()).isEqualTo(client); + searchNode.stop(); + + verify(settingsBuilder).put("http.enabled", false); + verify(nodeBuilder).node(); + verify(node).client(); + verify(node).close(); + } + + @Test + public void should_initialize_node_with_http() { + String httpHost = "httpHost"; + String httpPort = "httpPort"; + when(settings.getString("sonar.es.http.host")).thenReturn(httpHost); + when(settings.getString("sonar.es.http.port")).thenReturn(httpPort); + + searchNode.start(); + + verify(settingsBuilder).put("http.enabled", true); + verify(settingsBuilder).put("http.host", httpHost); + verify(settingsBuilder).put("http.port", httpPort); + } + + @Test + public void should_initialize_node_with_http_on_localhost() { + String httpPort = "httpPort"; + when(settings.getString("sonar.es.http.port")).thenReturn(httpPort); + + searchNode.start(); + + verify(settingsBuilder).put("http.enabled", true); + verify(settingsBuilder).put("http.host", "127.0.0.1"); + verify(settingsBuilder).put("http.port", httpPort); + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/search/SearchQueryTest.java b/sonar-server/src/test/java/org/sonar/server/search/SearchQueryTest.java new file mode 100644 index 00000000000..57266eafb09 --- /dev/null +++ b/sonar-server/src/test/java/org/sonar/server/search/SearchQueryTest.java @@ -0,0 +1,54 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2013 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search; + +import org.junit.Test; + +import static org.fest.assertions.Assertions.assertThat; + +public class SearchQueryTest { + + @Test + public void should_return_empty_query() { + assertThat(SearchQuery.create().getQueryString()).isEmpty(); + } + + @Test + public void should_handle_custom_query() { + assertThat(SearchQuery.create("polop").getQueryString()).isEqualTo("polop"); + } + + @Test + public void should_add_fields() { + assertThat(SearchQuery.create() + .field("field1", "value1") + .field("field2", "value2") + .getQueryString()).isEqualTo("field1:value1 AND field2:value2"); + } + + @Test + public void should_add_fields_to_custom_query() { + assertThat(SearchQuery.create("polop") + .field("field1", "value1") + .field("field2", "value2") + .getQueryString()).isEqualTo("polop AND field1:value1 AND field2:value2"); + } + +} -- 2.39.5