From 09440b9414e4c541140958df31cfa02959f31805 Mon Sep 17 00:00:00 2001 From: Teryk Bellahsene Date: Wed, 11 Oct 2017 17:11:52 +0200 Subject: [PATCH] SONAR-9928 Search components in ES --- .../component/index/ComponentIndex.java | 48 +++++- .../component/index/ComponentQuery.java | 83 ++++++++++ .../index/ComponentIndexSearchTest.java | 153 ++++++++++++++++++ 3 files changed, 282 insertions(+), 2 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java index 0946a08abd7..489699055c5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java @@ -23,7 +23,9 @@ import com.google.common.annotations.VisibleForTesting; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.function.Consumer; import java.util.stream.Stream; +import javax.annotation.Nullable; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -40,7 +42,10 @@ import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationB import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortOrder; import org.sonar.server.es.EsClient; +import org.sonar.server.es.SearchIdResult; +import org.sonar.server.es.SearchOptions; import org.sonar.server.es.textsearch.ComponentTextSearchFeature; import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; import org.sonar.server.es.textsearch.ComponentTextSearchQueryFactory; @@ -49,11 +54,14 @@ import org.sonar.server.permission.index.AuthorizationTypeSupport; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_KEY; +import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_LANGUAGE; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_QUALIFIER; import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_TYPE_COMPONENT; import static org.sonar.server.component.index.ComponentIndexDefinition.NAME_ANALYZERS; +import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; public class ComponentIndex { @@ -68,6 +76,31 @@ public class ComponentIndex { this.authorizationTypeSupport = authorizationTypeSupport; } + public SearchIdResult search(ComponentQuery query, SearchOptions searchOptions) { + SearchRequestBuilder requestBuilder = client + .prepareSearch(INDEX_TYPE_COMPONENT) + .setFetchSource(false) + .setFrom(searchOptions.getOffset()) + .setSize(searchOptions.getLimit()); + + BoolQueryBuilder esQuery = boolQuery(); + esQuery.filter(authorizationTypeSupport.createQueryFilter()); + setNullable(query.getQuery(), q -> { + ComponentTextSearchQuery componentTextSearchQuery = ComponentTextSearchQuery.builder() + .setQueryText(q) + .setFieldKey(FIELD_KEY) + .setFieldName(FIELD_NAME) + .build(); + esQuery.must(ComponentTextSearchQueryFactory.createQuery(componentTextSearchQuery, ComponentTextSearchFeatureRepertoire.values())); + }); + setEmptiable(query.getQualifiers(), q -> esQuery.must(termsQuery(FIELD_QUALIFIER, q))); + setNullable(query.getLanguage(), l -> esQuery.must(termsQuery(FIELD_LANGUAGE, l))); + requestBuilder.setQuery(esQuery); + requestBuilder.addSort(SORTABLE_ANALYZER.subField(FIELD_NAME), SortOrder.ASC); + + return new SearchIdResult<>(requestBuilder.get(), id -> id); + } + public ComponentIndexResults searchSuggestions(SuggestionQuery query) { return searchSuggestions(query, ComponentTextSearchFeatureRepertoire.values()); } @@ -118,8 +151,7 @@ public class ComponentIndex { .encoder("html") .preTags("") .postTags("") - .field(createHighlighterField()) - ) + .field(createHighlighterField())) .from(query.getSkip()) .size(query.getLimit()) .sort(new ScoreSortBuilder()) @@ -157,4 +189,16 @@ public class ComponentIndex { return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits()); } + + private static void setNullable(@Nullable T parameter, Consumer consumer) { + if (parameter != null) { + consumer.accept(parameter); + } + } + + private static void setEmptiable(Collection parameter, Consumer> consumer) { + if (!parameter.isEmpty()) { + consumer.accept(parameter); + } + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java new file mode 100644 index 00000000000..10a5bbb8c86 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.component.index; + +import java.util.Collection; + +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableCollection; + +public class ComponentQuery { + private final String query; + private final Collection qualifiers; + private final String language; + + private ComponentQuery(Builder builder) { + this.query = builder.query; + this.qualifiers = builder.qualifiers; + this.language = builder.language; + } + + public String getQuery() { + return query; + } + + public Collection getQualifiers() { + return qualifiers; + } + + public String getLanguage() { + return language; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String query; + private Collection qualifiers = emptySet(); + private String language; + + private Builder() { + // enforce static factory method + } + + public Builder setQuery(String query) { + this.query = query; + return this; + } + + public Builder setQualifiers(Collection qualifiers) { + this.qualifiers = unmodifiableCollection(qualifiers); + return this; + } + + public Builder setLanguage(String language) { + this.language = language; + return this; + } + + public ComponentQuery build() { + return new ComponentQuery(this); + } + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java new file mode 100644 index 00000000000..52e3b3e981d --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java @@ -0,0 +1,153 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.component.index; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.api.resources.Qualifiers; +import org.sonar.api.utils.System2; +import org.sonar.db.DbTester; +import org.sonar.db.component.ComponentDto; +import org.sonar.server.es.EsTester; +import org.sonar.server.es.SearchIdResult; +import org.sonar.server.es.SearchOptions; +import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRule; +import org.sonar.server.permission.index.AuthorizationTypeSupport; +import org.sonar.server.permission.index.PermissionIndexerTester; +import org.sonar.server.tester.UserSessionRule; + +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; +import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.db.component.ComponentTesting.newFileDto; + +public class ComponentIndexSearchTest { + @Rule + public EsTester es = new EsTester(new ComponentIndexDefinition(new MapSettings().asConfig())); + @Rule + public DbTester db = DbTester.create(System2.INSTANCE); + @Rule + public UserSessionRule userSession = UserSessionRule.standalone(); + @Rule + public ComponentTextSearchFeatureRule features = new ComponentTextSearchFeatureRule(); + + private ComponentIndexer indexer = new ComponentIndexer(db.getDbClient(), es.client()); + private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es, indexer); + + private ComponentIndex underTest = new ComponentIndex(es.client(), new AuthorizationTypeSupport(userSession)); + + @Test + public void filter_by_language() { + ComponentDto project = db.components().insertPrivateProject(); + ComponentDto javaFile = db.components().insertComponent(newFileDto(project).setLanguage("java")); + ComponentDto jsFile1 = db.components().insertComponent(newFileDto(project).setLanguage("js")); + ComponentDto jsFile2 = db.components().insertComponent(newFileDto(project).setLanguage("js")); + index(project); + + SearchIdResult result = underTest.search(ComponentQuery.builder().setLanguage("js").build(), new SearchOptions()); + + assertThat(result.getIds()).containsExactlyInAnyOrder(jsFile1.uuid(), jsFile2.uuid()); + } + + @Test + public void filter_by_name() { + ComponentDto ignoredProject = db.components().insertPrivateProject(p -> p.setName("ignored project")); + ComponentDto project = db.components().insertPrivateProject(p -> p.setName("Project Shiny name")); + index(ignoredProject, project); + + SearchIdResult result = underTest.search(ComponentQuery.builder().setQuery("shiny").build(), new SearchOptions()); + + assertThat(result.getIds()).containsExactlyInAnyOrder(project.uuid()); + } + + @Test + public void filter_by_key_with_exact_match() { + ComponentDto ignoredProject = db.components().insertPrivateProject(p -> p.setDbKey("ignored-project")); + ComponentDto project = db.components().insertPrivateProject(p -> p.setDbKey("shiny-project")); + ComponentDto anotherIgnoreProject = db.components().insertPrivateProject(p -> p.setDbKey("another-shiny-project")); + index(ignoredProject, project); + + SearchIdResult result = underTest.search(ComponentQuery.builder().setQuery("shiny-project").build(), new SearchOptions()); + + assertThat(result.getIds()).containsExactlyInAnyOrder(project.uuid()); + } + + @Test + public void filter_by_qualifier() { + ComponentDto project = db.components().insertPrivateProject(); + ComponentDto file = db.components().insertComponent(newFileDto(project)); + index(project); + + SearchIdResult result = underTest.search(ComponentQuery.builder().setQualifiers(singleton(Qualifiers.FILE)).build(), new SearchOptions()); + + assertThat(result.getIds()).containsExactlyInAnyOrder(file.uuid()); + } + + @Test + public void order_by_name_case_insensitive() { + ComponentDto project2 = db.components().insertPrivateProject(p -> p.setName("PROJECT 2")); + ComponentDto project3 = db.components().insertPrivateProject(p -> p.setName("project 3")); + ComponentDto project1 = db.components().insertPrivateProject(p -> p.setName("Project 1")); + index(project1, project2, project3); + + SearchIdResult result = underTest.search(ComponentQuery.builder().build(), new SearchOptions()); + + assertThat(result.getIds()).containsExactly(project1.uuid(), project2.uuid(), project3.uuid()); + } + + @Test + public void paginate_results() { + List projects = IntStream.range(0, 9) + .mapToObj(i -> db.components().insertPrivateProject(p -> p.setName("project " + i))) + .collect(Collectors.toList()); + index(projects.toArray(new ComponentDto[0])); + + SearchIdResult result = underTest.search(ComponentQuery.builder().build(), new SearchOptions().setPage(2, 3)); + + assertThat(result.getIds()).containsExactlyInAnyOrder(projects.get(3).uuid(), projects.get(4).uuid(), projects.get(5).uuid()); + } + + @Test + public void filter_unauthorized_components() { + ComponentDto unauthorizedProject = db.components().insertPrivateProject(); + ComponentDto project1 = db.components().insertPrivateProject(); + ComponentDto project2 = db.components().insertPrivateProject(); + indexer.indexOnStartup(emptySet()); + authorizationIndexerTester.allowOnlyAnyone(project1); + authorizationIndexerTester.allowOnlyAnyone(project2); + + SearchIdResult result = underTest.search(ComponentQuery.builder().build(), new SearchOptions()); + + assertThat(result.getIds()).containsExactlyInAnyOrder(project1.uuid(), project2.uuid()) + .doesNotContain(unauthorizedProject.uuid()); + } + + private void index(ComponentDto... components) { + indexer.indexOnStartup(emptySet()); + Arrays.stream(components).forEach(c -> authorizationIndexerTester.allowOnlyAnyone(c)); + } + +} -- 2.39.5