From 7f974fc34295da1ec4bc937d4df536e86f37306c Mon Sep 17 00:00:00 2001 From: Daniel Schwarz Date: Thu, 26 Jan 2017 10:24:58 +0100 Subject: [PATCH] SONAR-8694 make search query features independent from each other --- .../component/index/ComponentIndex.java | 47 +++++-------- .../index/ComponentIndexSearchFeature.java | 69 +++++++++++++++++++ .../sonar/server/es/DefaultIndexSettings.java | 2 + .../es/DefaultIndexSettingsElement.java | 11 +++ .../ComponentIndexSearchFeatureRule.java | 37 ++++++++++ .../component/index/ComponentIndexTest.java | 10 ++- 6 files changed, 143 insertions(+), 33 deletions(-) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexSearchFeature.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchFeatureRule.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java index 20a5a2dceb0..d7459b71f47 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java @@ -19,39 +19,31 @@ */ package org.sonar.server.component.index; +import com.google.common.annotations.VisibleForTesting; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.Set; -import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.sonar.core.util.stream.Collectors; import org.sonar.server.es.BaseIndex; -import org.sonar.server.es.DefaultIndexSettings; import org.sonar.server.es.EsClient; import org.sonar.server.user.UserSession; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_AUTHORIZATION_GROUPS; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_AUTHORIZATION_USERS; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_KEY; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_QUALIFIER; import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_COMPONENTS; import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_AUTHORIZATION; import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; -import static org.sonar.server.es.DefaultIndexSettingsElement.FUZZY_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; public class ComponentIndex extends BaseIndex { @@ -63,6 +55,11 @@ public class ComponentIndex extends BaseIndex { } public List search(ComponentIndexQuery query) { + return search(query, ComponentIndexSearchFeature.values()); + } + + @VisibleForTesting + List search(ComponentIndexQuery query, ComponentIndexSearchFeature... features) { SearchRequestBuilder request = getClient() .prepareSearch(INDEX_COMPONENTS) .setTypes(TYPE_COMPONENT) @@ -70,38 +67,28 @@ public class ComponentIndex extends BaseIndex { query.getLimit().ifPresent(request::setSize); - request.setQuery(createQuery(query)); + request.setQuery(createQuery(query, features)); - return Arrays.stream(request.get().getHits().hits()) + SearchResponse searchResponse = request.get(); + + return Arrays.stream(searchResponse.getHits().hits()) .map(SearchHit::getId) .collect(Collectors.toList()); } - private QueryBuilder createQuery(ComponentIndexQuery query) { + private QueryBuilder createQuery(ComponentIndexQuery query, ComponentIndexSearchFeature... features) { BoolQueryBuilder esQuery = boolQuery(); esQuery.filter(createAuthorizationFilter()); query.getQualifier().ifPresent(q -> esQuery.filter(termQuery(FIELD_QUALIFIER, q))); - String queryText = query.getQuery(); - - // We will truncate the search to the maximum length of nGrams in the index. - // Otherwise the search would for sure not find any results. - String truncatedQuery = StringUtils.left(queryText, DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH); - - return esQuery.must(boolQuery() - - // partial name matches - .should(matchQuery(SEARCH_GRAMS_ANALYZER.subField(FIELD_NAME), truncatedQuery)) - - // fuzzy name matches - .should(matchQuery(FUZZY_ANALYZER.subField(FIELD_NAME), queryText).fuzziness(Fuzziness.AUTO)) + BoolQueryBuilder featureQuery = boolQuery(); - // prefix matches - .should(prefixQuery(FUZZY_ANALYZER.subField(FIELD_NAME), queryText)) + Arrays.stream(features) + .map(f -> f.getQuery(query.getQuery())) + .forEach(featureQuery::should); - // exact match on the key - .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_KEY), queryText).boost(5f))); + return esQuery.must(featureQuery); } private QueryBuilder createAuthorizationFilter() { diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexSearchFeature.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexSearchFeature.java new file mode 100644 index 00000000000..fa26c366150 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexSearchFeature.java @@ -0,0 +1,69 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.component.index; + +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.query.QueryBuilder; +import org.sonar.server.es.DefaultIndexSettings; + +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; +import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_KEY; +import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME; +import static org.sonar.server.es.DefaultIndexSettingsElement.FUZZY_ANALYZER; +import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; + +public enum ComponentIndexSearchFeature { + + PARTIAL { + @Override + public QueryBuilder getQuery(String queryText) { + + // We will truncate the search to the maximum length of nGrams in the index. + // Otherwise the search would for sure not find any results. + String truncatedQuery = StringUtils.left(queryText, DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH); + + return matchQuery(SEARCH_GRAMS_ANALYZER.subField(FIELD_NAME), truncatedQuery); + + } + }, + FUZZY { + @Override + public QueryBuilder getQuery(String queryText) { + return matchQuery(FUZZY_ANALYZER.subField(FIELD_NAME), queryText).fuzziness(Fuzziness.AUTO); + } + }, + FUZZY_PREFIX { + @Override + public QueryBuilder getQuery(String queryText) { + return prefixQuery(FUZZY_ANALYZER.subField(FIELD_NAME), queryText); + } + }, + KEY { + @Override + public QueryBuilder getQuery(String queryText) { + return matchQuery(SORTABLE_ANALYZER.subField(FIELD_KEY), queryText).boost(5f); + } + }; + + public abstract QueryBuilder getQuery(String queryText); +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java index 41f7726dbd4..41c4700c987 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettings.java @@ -45,6 +45,7 @@ public class DefaultIndexSettings { public static final String CUSTOM = "custom"; public static final String KEYWORD = "keyword"; public static final String CLASSIC = "classic"; + public static final String TRUNCATE = "truncate"; public static final String SUB_FIELD_DELIMITER = "."; @@ -56,6 +57,7 @@ public class DefaultIndexSettings { public static final String PORTER_STEM = "porter_stem"; public static final String MIN_GRAM = "min_gram"; public static final String MAX_GRAM = "max_gram"; + public static final String LENGTH = "length"; private DefaultIndexSettings() { // only static stuff diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java index c057eadd214..96032a88731 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java @@ -35,6 +35,7 @@ import static org.sonar.server.es.DefaultIndexSettings.DELIMITER; import static org.sonar.server.es.DefaultIndexSettings.FILTER; import static org.sonar.server.es.DefaultIndexSettings.INDEX; import static org.sonar.server.es.DefaultIndexSettings.KEYWORD; +import static org.sonar.server.es.DefaultIndexSettings.LENGTH; import static org.sonar.server.es.DefaultIndexSettings.LOWERCASE; import static org.sonar.server.es.DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH; import static org.sonar.server.es.DefaultIndexSettings.MAX_GRAM; @@ -48,6 +49,7 @@ import static org.sonar.server.es.DefaultIndexSettings.STRING; import static org.sonar.server.es.DefaultIndexSettings.SUB_FIELD_DELIMITER; import static org.sonar.server.es.DefaultIndexSettings.TOKENIZER; import static org.sonar.server.es.DefaultIndexSettings.TRIM; +import static org.sonar.server.es.DefaultIndexSettings.TRUNCATE; import static org.sonar.server.es.DefaultIndexSettings.TYPE; import static org.sonar.server.es.DefaultIndexSettings.WHITESPACE; @@ -70,6 +72,15 @@ public enum DefaultIndexSettingsElement { set("stem_english_possessive", true); } }, + EDGE_NGRAM_FILTER(FILTER) { + + @Override + protected void setup() { + set(TYPE, "edge_ngram"); + set(MIN_GRAM, 1); + set(MAX_GRAM, 15); + } + }, // Tokenizers diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchFeatureRule.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchFeatureRule.java new file mode 100644 index 00000000000..0527752e124 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchFeatureRule.java @@ -0,0 +1,37 @@ +/* + * SonarQube + * Copyright (C) 2009-2016 SonarSource SA + * mailto:contact AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.component.index; + +import org.junit.rules.ExternalResource; + +public class ComponentIndexSearchFeatureRule extends ExternalResource { + + private ComponentIndexSearchFeature[] features; + + @Override + protected void before() throws Throwable { + features = ComponentIndexSearchFeature.values(); + } + + public ComponentIndexSearchFeature[] get() { + return features; + } + +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java index f36364715d0..42a7ec81b68 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java @@ -58,7 +58,11 @@ public class ComponentIndexTest { @Rule public UserSessionRule userSession = UserSessionRule.standalone(); - private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es); + + @Rule + public ComponentIndexSearchFeatureRule features = new ComponentIndexSearchFeatureRule(); + + protected PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es); private ComponentIndex index; private ComponentIndexer indexer; @@ -396,8 +400,8 @@ public class ComponentIndexTest { return assertSearch(new ComponentIndexQuery(query)); } - private AbstractListAssert, String> assertSearch(ComponentIndexQuery query) { - return assertThat(index.search(query)); + protected AbstractListAssert, String> assertSearch(ComponentIndexQuery query) { + return assertThat(index.search(query, features.get())); } private void assertSearchResults(String query, ComponentDto... expectedComponents) { -- 2.39.5