SONAR-9072 for api/components/suggestions use all valid search tokens

The search for suggestions should ignore tokens, that are only one character long (with warning), but use all other tokens for the search.

4 files changed, 83 insertions, 28 deletions

diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java
index cb9adf7fab6..4fc37137a3f 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java
@@ -22,6 +22,7 @@ package org.sonar.server.component.ws;
 import com.google.common.collect.ListMultimap;
 import com.google.common.html.HtmlEscapers;
 import com.google.common.io.Resources;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -48,7 +49,7 @@ import org.sonar.server.component.index.ComponentHitsPerQualifier;
 import org.sonar.server.component.index.ComponentIndex;
 import org.sonar.server.component.index.ComponentIndexQuery;
 import org.sonar.server.component.index.ComponentIndexResults;
-import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire;
+import org.sonar.server.es.DefaultIndexSettings;
 import org.sonar.server.favorite.FavoriteFinder;
 import org.sonar.server.user.UserSession;
 import org.sonarqube.ws.WsComponents.SuggestionsWsResponse;
@@ -202,6 +203,12 @@ public class SuggestionsAction implements ComponentsWsAction {
   }
 
   private SuggestionsWsResponse loadSuggestionsWithSearch(String query, int skip, int limit, Set<String> recentlyBrowsedKeys, List<String> qualifiers) {
+    if (split(query).noneMatch(token -> token.length() >= MINIMUM_NGRAM_LENGTH)) {
+      SuggestionsWsResponse.Builder queryBuilder = newBuilder();
+      getWarning(query).ifPresent(queryBuilder::setWarning);
+      return queryBuilder.build();
+    }
+
     List<ComponentDto> favorites = favoriteFinder.list();
     Set<String> favoriteKeys = favorites.stream().map(ComponentDto::getKey).collect(MoreCollectors.toSet(favorites.size()));
     ComponentIndexQuery.Builder queryBuilder = ComponentIndexQuery.builder()
@@ -230,11 +237,14 @@ public class SuggestionsAction implements ComponentsWsAction {
   }
 
   private static Optional<String> getWarning(String query) {
-    List<String> tokens = ComponentTextSearchFeatureRepertoire.split(query).collect(Collectors.toList());
-    if (tokens.stream().anyMatch(token -> token.length() < MINIMUM_NGRAM_LENGTH)) {
-      return Optional.of(SHORT_INPUT_WARNING);
-    }
-    return Optional.empty();
+    return split(query)
+      .filter(token -> token.length() < MINIMUM_NGRAM_LENGTH)
+      .findAny()
+      .map(x -> SHORT_INPUT_WARNING);
+  }
+
+  private static Stream<String> split(String query) {
+    return Arrays.stream(query.split(DefaultIndexSettings.SEARCH_TERM_TOKENIZER_PATTERN));
   }
 
   private static List<String> getQualifiers(@Nullable String more) {
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java b/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java
index 791fa07a17a..0a96cbf0e72 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java
@@ -19,7 +19,7 @@
  */
 package org.sonar.server.es.textsearch;
 
-import java.util.Arrays;
+import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -28,6 +28,7 @@ import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.server.es.DefaultIndexSettings;
 import org.sonar.server.es.DefaultIndexSettingsElement;
 import org.sonar.server.es.textsearch.ComponentTextSearchQueryFactory.ComponentTextSearchQuery;
@@ -53,28 +54,41 @@ public enum ComponentTextSearchFeatureRepertoire implements ComponentTextSearchF
   },
   PREFIX(CHANGE_ORDER_OF_RESULTS) {
     @Override
-    public QueryBuilder getQuery(ComponentTextSearchQuery query) {
-      return prefixAndPartialQuery(query.getQueryText(), query.getFieldName(), SEARCH_PREFIX_ANALYZER)
+    public Stream<QueryBuilder> getQueries(ComponentTextSearchQuery query) {
+      List<String> tokens = query.getQueryTextTokens();
+      if (tokens.isEmpty()) {
+        return Stream.empty();
+      }
+      BoolQueryBuilder queryBuilder = prefixAndPartialQuery(tokens, query.getFieldName(), SEARCH_PREFIX_ANALYZER)
         .boost(3f);
+      return Stream.of(queryBuilder);
     }
   },
   PREFIX_IGNORE_CASE(GENERATE_RESULTS) {
     @Override
-    public QueryBuilder getQuery(ComponentTextSearchQuery query) {
-      String lowerCaseQueryText = query.getQueryText().toLowerCase(Locale.getDefault());
-      return prefixAndPartialQuery(lowerCaseQueryText, query.getFieldName(), SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER)
+    public Stream<QueryBuilder> getQueries(ComponentTextSearchQuery query) {
+      List<String> tokens = query.getQueryTextTokens();
+      if (tokens.isEmpty()) {
+        return Stream.empty();
+      }
+      List<String> lowerCaseTokens = tokens.stream().map(t -> t.toLowerCase(Locale.getDefault())).collect(MoreCollectors.toList());
+      BoolQueryBuilder queryBuilder = prefixAndPartialQuery(lowerCaseTokens, query.getFieldName(), SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER)
         .boost(2f);
+      return Stream.of(queryBuilder);
     }
   },
   PARTIAL(GENERATE_RESULTS) {
     @Override
-    public QueryBuilder getQuery(ComponentTextSearchQuery query) {
-      BoolQueryBuilder queryBuilder = boolQuery();
-      split(query.getQueryText())
+    public Stream<QueryBuilder> getQueries(ComponentTextSearchQuery query) {
+      List<String> tokens = query.getQueryTextTokens();
+      if (tokens.isEmpty()) {
+        return Stream.empty();
+      }
+      BoolQueryBuilder queryBuilder = boolQuery().boost(0.5f);
+      tokens.stream()
         .map(text -> tokenQuery(text, query.getFieldName(), SEARCH_GRAMS_ANALYZER))
         .forEach(queryBuilder::must);
-      return queryBuilder
-        .boost(0.5f);
+      return Stream.of(queryBuilder);
     }
   },
   KEY(GENERATE_RESULTS) {
@@ -116,17 +130,10 @@ public enum ComponentTextSearchFeatureRepertoire implements ComponentTextSearchF
     throw new UnsupportedOperationException();
   }
 
-  public static Stream<String> split(String queryText) {
-    return Arrays.stream(
-      queryText.split(DefaultIndexSettings.SEARCH_TERM_TOKENIZER_PATTERN))
-      .filter(StringUtils::isNotEmpty);
-  }
-
-  protected BoolQueryBuilder prefixAndPartialQuery(String queryText, String originalFieldName, DefaultIndexSettingsElement analyzer) {
+  protected BoolQueryBuilder prefixAndPartialQuery(List<String> tokens, String originalFieldName, DefaultIndexSettingsElement analyzer) {
     BoolQueryBuilder queryBuilder = boolQuery();
-
     AtomicBoolean first = new AtomicBoolean(true);
-    split(queryText)
+    tokens.stream()
       .map(queryTerm -> {
 
         if (first.getAndSet(false)) {
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchQueryFactory.java b/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchQueryFactory.java
index bbd9e6aa55a..18d398e9a82 100644
--- a/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchQueryFactory.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchQueryFactory.java
@@ -22,16 +22,21 @@ package org.sonar.server.es.textsearch;
 import com.google.common.collect.ImmutableSet;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.lang.StringUtils;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.sonar.core.util.stream.MoreCollectors;
+import org.sonar.server.es.DefaultIndexSettings;
 import org.sonar.server.es.textsearch.ComponentTextSearchFeature.UseCase;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.sonar.server.es.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH;
 
 /**
  * This class is used in order to do some advanced full text search in an index on component key and component name
@@ -48,13 +53,13 @@ public class ComponentTextSearchQueryFactory {
     checkArgument(features.length > 0, "features cannot be empty");
     BoolQueryBuilder esQuery = boolQuery().must(
       createQuery(query, features, UseCase.GENERATE_RESULTS)
-        .orElseThrow(() -> new IllegalStateException("No text search features found to generate search results. Features: "+Arrays.toString(features))));
+        .orElseThrow(() -> new IllegalStateException("No text search features found to generate search results. Features: " + Arrays.toString(features))));
     createQuery(query, features, UseCase.CHANGE_ORDER_OF_RESULTS)
       .ifPresent(esQuery::should);
     return esQuery;
   }
 
-  public static Optional<QueryBuilder> createQuery(ComponentTextSearchQuery query, ComponentTextSearchFeature[] features, UseCase useCase) {
+  private static Optional<QueryBuilder> createQuery(ComponentTextSearchQuery query, ComponentTextSearchFeature[] features, UseCase useCase) {
     BoolQueryBuilder generateResults = boolQuery();
     AtomicBoolean anyFeatures = new AtomicBoolean();
     Arrays.stream(features)
@@ -70,6 +75,7 @@ public class ComponentTextSearchQueryFactory {
 
   public static class ComponentTextSearchQuery {
     private final String queryText;
+    private final List<String> queryTextTokens;
     private final String fieldKey;
     private final String fieldName;
     private final Set<String> recentlyBrowsedKeys;
@@ -77,16 +83,29 @@ public class ComponentTextSearchQueryFactory {
 
     private ComponentTextSearchQuery(Builder builder) {
       this.queryText = builder.queryText;
+      this.queryTextTokens = split(builder.queryText);
       this.fieldKey = builder.fieldKey;
       this.fieldName = builder.fieldName;
       this.recentlyBrowsedKeys = builder.recentlyBrowsedKeys;
       this.favoriteKeys = builder.favoriteKeys;
     }
 
+    private static List<String> split(String queryText) {
+      return Arrays.stream(
+        queryText.split(DefaultIndexSettings.SEARCH_TERM_TOKENIZER_PATTERN))
+        .filter(StringUtils::isNotEmpty)
+        .filter(s -> s.length() >= MINIMUM_NGRAM_LENGTH)
+        .collect(MoreCollectors.toList());
+    }
+
     public String getQueryText() {
       return queryText;
     }
 
+    public List<String> getQueryTextTokens() {
+      return queryTextTokens;
+    }
+
     public String getFieldKey() {
       return fieldKey;
     }
diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java
index 7173b5c927d..9c5e0c35a04 100644
--- a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SuggestionsActionTest.java
@@ -392,6 +392,25 @@ public class SuggestionsActionTest {
   }
 
   @Test
+  public void should_warn_about_short_inputs_but_return_results_based_on_other_terms() throws Exception {
+    ComponentDto project = db.components().insertComponent(newPrivateProjectDto(organization).setName("SonarQube"));
+
+    componentIndexer.indexOnStartup(null);
+    authorizationIndexerTester.allowOnlyAnyone(project);
+
+    SuggestionsWsResponse response = ws.newRequest()
+      .setMethod("POST")
+      .setParam(PARAM_QUERY, "Sonar Q")
+      .executeProtobuf(SuggestionsWsResponse.class);
+
+    assertThat(response.getResultsList())
+      .flatExtracting(Category::getItemsList)
+      .extracting(Suggestion::getKey)
+      .contains(project.getKey());
+    assertThat(response.getWarning()).contains(SHORT_INPUT_WARNING);
+  }
+
+  @Test
   public void should_contain_component_names() throws Exception {
     OrganizationDto organization1 = db.organizations().insert(o -> o.setKey("org-1").setName("Organization One"));