diff options
author | Sébastien Lesaint <sebastien.lesaint@sonarsource.com> | 2019-08-12 17:48:03 +0200 |
---|---|---|
committer | SonarTech <sonartech@sonarsource.com> | 2019-08-14 20:21:14 +0200 |
commit | 3814e193bf5b1dab3b58b620a4b5614dfdcf056a (patch) | |
tree | f38b3ac438edd96be92e22298d55d742c996b050 /server/sonar-server | |
parent | 2528f0d148c0f0c0f6e4022c423ac5d67dcb650a (diff) | |
download | sonarqube-3814e193bf5b1dab3b58b620a4b5614dfdcf056a.tar.gz sonarqube-3814e193bf5b1dab3b58b620a4b5614dfdcf056a.zip |
create sonar-webserver-es from sonar-server
Diffstat (limited to 'server/sonar-server')
58 files changed, 2 insertions, 11427 deletions
diff --git a/server/sonar-server/build.gradle b/server/sonar-server/build.gradle index f225dbac9ca..4084fea8809 100644 --- a/server/sonar-server/build.gradle +++ b/server/sonar-server/build.gradle @@ -52,6 +52,7 @@ dependencies { compile project(':server:sonar-server-common') compile project(':server:sonar-webserver-auth') compile project(':server:sonar-webserver-common') + compile project(':server:sonar-webserver-es') compile project(':server:sonar-webserver-ws') compile project(':sonar-core') compile project(':sonar-duplications') @@ -83,6 +84,7 @@ dependencies { testCompile project(':server:sonar-db-testing') testCompile project(path: ":server:sonar-server-common", configuration: "tests") testCompile project(path: ":server:sonar-webserver-auth", configuration: "tests") + testCompile project(path: ":server:sonar-webserver-es", configuration: "tests") testCompile project(path: ":server:sonar-webserver-ws", configuration: "tests") testCompile project(':sonar-testing-harness') diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java deleted file mode 100644 index b4feffd7863..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Consumer; -import java.util.stream.Stream; -import javax.annotation.Nullable; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters.InternalBucket; -import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits; -import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder; -import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.ScoreSortBuilder; -import org.elasticsearch.search.sort.SortOrder; -import org.sonar.api.utils.System2; -import org.sonar.server.es.EsClient; -import org.sonar.server.es.SearchIdResult; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.es.textsearch.ComponentTextSearchFeature; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; -import org.sonar.server.es.textsearch.ComponentTextSearchQueryFactory; -import org.sonar.server.es.textsearch.ComponentTextSearchQueryFactory.ComponentTextSearchQuery; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; - -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.index.query.QueryBuilders.termsQuery; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_KEY; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_LANGUAGE; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_ORGANIZATION_UUID; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_QUALIFIER; -import static org.sonar.server.component.index.ComponentIndexDefinition.NAME_ANALYZERS; -import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; -import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; -import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; - -public class ComponentIndex { - - private static final String FILTERS_AGGREGATION_NAME = "filters"; - private static final String DOCS_AGGREGATION_NAME = "docs"; - - private final EsClient client; - private final WebAuthorizationTypeSupport authorizationTypeSupport; - private final System2 system2; - - public ComponentIndex(EsClient client, WebAuthorizationTypeSupport authorizationTypeSupport, System2 system2) { - this.client = client; - this.authorizationTypeSupport = authorizationTypeSupport; - this.system2 = system2; - } - - public SearchIdResult<String> search(ComponentQuery query, SearchOptions searchOptions) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setFetchSource(false) - .setFrom(searchOptions.getOffset()) - .setSize(searchOptions.getLimit()); - - BoolQueryBuilder esQuery = boolQuery(); - esQuery.filter(authorizationTypeSupport.createQueryFilter()); - setNullable(query.getQuery(), q -> { - ComponentTextSearchQuery componentTextSearchQuery = ComponentTextSearchQuery.builder() - .setQueryText(q) - .setFieldKey(FIELD_KEY) - .setFieldName(FIELD_NAME) - .build(); - esQuery.must(ComponentTextSearchQueryFactory.createQuery(componentTextSearchQuery, ComponentTextSearchFeatureRepertoire.values())); - }); - setEmptiable(query.getQualifiers(), q -> esQuery.must(termsQuery(FIELD_QUALIFIER, q))); - setNullable(query.getLanguage(), l -> esQuery.must(termQuery(FIELD_LANGUAGE, l))); - setNullable(query.getOrganizationUuid(), o -> esQuery.must(termQuery(FIELD_ORGANIZATION_UUID, o))); - requestBuilder.setQuery(esQuery); - requestBuilder.addSort(SORTABLE_ANALYZER.subField(FIELD_NAME), SortOrder.ASC); - - return new SearchIdResult<>(requestBuilder.get(), id -> id, system2.getDefaultTimeZone()); - } - - public ComponentIndexResults searchSuggestions(SuggestionQuery query) { - return searchSuggestions(query, ComponentTextSearchFeatureRepertoire.values()); - } - - @VisibleForTesting - ComponentIndexResults searchSuggestions(SuggestionQuery query, ComponentTextSearchFeature... features) { - Collection<String> qualifiers = query.getQualifiers(); - if (qualifiers.isEmpty()) { - return ComponentIndexResults.newBuilder().build(); - } - - SearchRequestBuilder request = client - .prepareSearch(TYPE_COMPONENT.getMainType()) - .setQuery(createQuery(query, features)) - .addAggregation(createAggregation(query)) - - // the search hits are part of the aggregations - .setSize(0); - - SearchResponse response = request.get(); - - return aggregationsToQualifiers(response); - } - - private static HighlightBuilder.Field createHighlighterField() { - HighlightBuilder.Field field = new HighlightBuilder.Field(FIELD_NAME); - field.highlighterType("fvh"); - field.matchedFields( - Stream.concat( - Stream.of(FIELD_NAME), - Arrays - .stream(NAME_ANALYZERS) - .map(a -> a.subField(FIELD_NAME))) - .toArray(String[]::new)); - return field; - } - - private static FiltersAggregationBuilder createAggregation(SuggestionQuery query) { - return AggregationBuilders.filters( - FILTERS_AGGREGATION_NAME, - query.getQualifiers().stream().map(q -> new KeyedFilter(q, termQuery(FIELD_QUALIFIER, q))).toArray(KeyedFilter[]::new)) - .subAggregation(createSubAggregation(query)); - } - - private static TopHitsAggregationBuilder createSubAggregation(SuggestionQuery query) { - return AggregationBuilders.topHits(DOCS_AGGREGATION_NAME) - .highlighter(new HighlightBuilder() - .encoder("html") - .preTags("<mark>") - .postTags("</mark>") - .field(createHighlighterField())) - .from(query.getSkip()) - .size(query.getLimit()) - .sort(new ScoreSortBuilder()) - .sort(new FieldSortBuilder(ComponentIndexDefinition.FIELD_NAME)) - .fetchSource(false); - } - - private QueryBuilder createQuery(SuggestionQuery query, ComponentTextSearchFeature... features) { - BoolQueryBuilder esQuery = boolQuery(); - esQuery.filter(termQuery(FIELD_INDEX_TYPE, TYPE_COMPONENT.getName())); - esQuery.filter(authorizationTypeSupport.createQueryFilter()); - ComponentTextSearchQuery componentTextSearchQuery = ComponentTextSearchQuery.builder() - .setQueryText(query.getQuery()) - .setFieldKey(FIELD_KEY) - .setFieldName(FIELD_NAME) - .setRecentlyBrowsedKeys(query.getRecentlyBrowsedKeys()) - .setFavoriteKeys(query.getFavoriteKeys()) - .build(); - return esQuery.must(ComponentTextSearchQueryFactory.createQuery(componentTextSearchQuery, features)); - } - - private static ComponentIndexResults aggregationsToQualifiers(SearchResponse response) { - InternalFilters filtersAgg = response.getAggregations().get(FILTERS_AGGREGATION_NAME); - List<InternalBucket> buckets = filtersAgg.getBuckets(); - return ComponentIndexResults.newBuilder() - .setQualifiers( - buckets.stream().map(ComponentIndex::bucketToQualifier)) - .build(); - } - - private static ComponentHitsPerQualifier bucketToQualifier(InternalBucket bucket) { - InternalTopHits docs = bucket.getAggregations().get(DOCS_AGGREGATION_NAME); - - SearchHits hitList = docs.getHits(); - SearchHit[] hits = hitList.getHits(); - - return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits()); - } - - private static <T> void setNullable(@Nullable T parameter, Consumer<T> consumer) { - if (parameter != null) { - consumer.accept(parameter); - } - } - - private static <T> void setEmptiable(Collection<T> parameter, Consumer<Collection<T>> consumer) { - if (!parameter.isEmpty()) { - consumer.accept(parameter); - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexResults.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexResults.java deleted file mode 100644 index 83b6b2b62a0..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexResults.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Collections.emptyList; -import static java.util.Objects.requireNonNull; - -public class ComponentIndexResults { - - private final List<ComponentHitsPerQualifier> qualifiers; - - private ComponentIndexResults(Builder builder) { - this.qualifiers = requireNonNull(builder.qualifiers); - } - - public Stream<ComponentHitsPerQualifier> getQualifiers() { - return qualifiers.stream(); - } - - public boolean isEmpty() { - return qualifiers.isEmpty(); - } - - public static Builder newBuilder() { - return new Builder(); - } - - public static class Builder { - - private List<ComponentHitsPerQualifier> qualifiers = emptyList(); - - private Builder() { - } - - public Builder setQualifiers(Stream<ComponentHitsPerQualifier> qualifiers) { - this.qualifiers = qualifiers.collect(Collectors.toList()); - return this; - } - - public ComponentIndexResults build() { - return new ComponentIndexResults(this); - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java deleted file mode 100644 index 252becafc6b..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentQuery.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.Collection; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; - -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableCollection; - -public class ComponentQuery { - private final String organizationUuid; - private final String query; - private final Collection<String> qualifiers; - private final String language; - - private ComponentQuery(Builder builder) { - this.organizationUuid = builder.organizationUuid; - this.query = builder.query; - this.qualifiers = builder.qualifiers; - this.language = builder.language; - } - - @CheckForNull - public String getOrganizationUuid() { - return organizationUuid; - } - - @CheckForNull - public String getQuery() { - return query; - } - - public Collection<String> getQualifiers() { - return qualifiers; - } - - @CheckForNull - public String getLanguage() { - return language; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - private String organizationUuid; - private String query; - private Collection<String> qualifiers = emptySet(); - private String language; - - private Builder() { - // enforce static factory method - } - - public Builder setOrganization(@Nullable String organizationUuid) { - this.organizationUuid = organizationUuid; - return this; - } - - public Builder setQuery(@Nullable String query) { - this.query = query; - return this; - } - - public Builder setQualifiers(Collection<String> qualifiers) { - this.qualifiers = unmodifiableCollection(qualifiers); - return this; - } - - public Builder setLanguage(@Nullable String language) { - this.language = language; - return this; - } - - public ComponentQuery build() { - return new ComponentQuery(this); - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/package-info.java deleted file mode 100644 index 0dc30a8e4e7..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.server.component.index; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java deleted file mode 100644 index 0a45e823dbe..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.picocontainer.Startable; -import org.sonar.api.config.Configuration; -import org.sonar.api.server.ServerSide; -import org.sonar.api.utils.log.Logger; -import org.sonar.api.utils.log.Loggers; -import org.sonar.process.ProcessProperties; -import org.sonar.server.es.metadata.EsDbCompatibility; -import org.sonar.server.es.metadata.MetadataIndex; -import org.sonar.server.es.metadata.MetadataIndexDefinition; -import org.sonar.server.es.newindex.BuiltIndex; -import org.sonar.server.es.newindex.NewIndex; -import org.sonar.server.platform.db.migration.es.MigrationEsClient; - -import static org.sonar.server.es.metadata.MetadataIndexDefinition.DESCRIPTOR; -import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA; - -/** - * Creates/deletes all indices in Elasticsearch during server startup. - */ -@ServerSide -public class IndexCreator implements Startable { - - private static final Logger LOGGER = Loggers.get(IndexCreator.class); - - private final MetadataIndexDefinition metadataIndexDefinition; - private final MetadataIndex metadataIndex; - private final EsClient client; - private final MigrationEsClient migrationEsClient; - private final IndexDefinitions definitions; - private final EsDbCompatibility esDbCompatibility; - private final Configuration configuration; - - public IndexCreator(EsClient client, IndexDefinitions definitions, MetadataIndexDefinition metadataIndexDefinition, - MetadataIndex metadataIndex, MigrationEsClient migrationEsClient, EsDbCompatibility esDbCompatibility, Configuration configuration) { - this.client = client; - this.definitions = definitions; - this.metadataIndexDefinition = metadataIndexDefinition; - this.metadataIndex = metadataIndex; - this.migrationEsClient = migrationEsClient; - this.esDbCompatibility = esDbCompatibility; - this.configuration = configuration; - } - - @Override - public void start() { - // create the "metadata" index first - IndexType.IndexMainType metadataMainType = TYPE_METADATA; - if (!client.prepareIndicesExist(metadataMainType.getIndex()).get().isExists()) { - IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); - metadataIndexDefinition.define(context); - NewIndex index = context.getIndices().values().iterator().next(); - createIndex(index.build(), false); - } else { - ensureWritable(metadataMainType); - } - - checkDbCompatibility(definitions.getIndices().values()); - - // create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...) - definitions.getIndices().values().stream() - .filter(i -> !i.getMainType().equals(metadataMainType)) - .forEach(index -> { - boolean exists = client.prepareIndicesExist(index.getMainType().getIndex()).get().isExists(); - if (!exists) { - createIndex(index, true); - } else if (hasDefinitionChange(index)) { - updateIndex(index); - } else { - ensureWritable(index.getMainType()); - } - }); - } - - private void ensureWritable(IndexType.IndexMainType mainType) { - if (isReadOnly(mainType)) { - removeReadOnly(mainType); - } - } - - private boolean isReadOnly(IndexType.IndexMainType mainType) { - String indexName = mainType.getIndex().getName(); - String readOnly = client.nativeClient().admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet() - .getSetting(indexName, "index.blocks.read_only_allow_delete"); - return readOnly != null && "true".equalsIgnoreCase(readOnly); - } - - private void removeReadOnly(IndexType.IndexMainType mainType) { - LOGGER.info("Index [{}] is read-only. Making it writable...", mainType.getIndex().getName()); - - String indexName = mainType.getIndex().getName(); - Settings.Builder builder = Settings.builder(); - builder.putNull("index.blocks.read_only_allow_delete"); - client.nativeClient().admin().indices() - .updateSettings(new UpdateSettingsRequest().indices(indexName).settings(builder.build())) - .actionGet(); - } - - @Override - public void stop() { - // nothing to do - } - - private void createIndex(BuiltIndex<?> builtIndex, boolean useMetadata) { - Index index = builtIndex.getMainType().getIndex(); - LOGGER.info(String.format("Create index [%s]", index.getName())); - Settings.Builder settings = Settings.builder(); - settings.put(builtIndex.getSettings()); - if (useMetadata) { - metadataIndex.setHash(index, IndexDefinitionHash.of(builtIndex)); - metadataIndex.setInitialized(builtIndex.getMainType(), false); - builtIndex.getRelationTypes().forEach(relationType -> metadataIndex.setInitialized(relationType, false)); - } - CreateIndexResponse indexResponse = client - .prepareCreate(index) - .setSettings(settings) - .get(); - if (!indexResponse.isAcknowledged()) { - throw new IllegalStateException("Failed to create index [" + index.getName() + "]"); - } - client.waitForStatus(ClusterHealthStatus.YELLOW); - - // create types - LOGGER.info("Create type {}", builtIndex.getMainType().format()); - AcknowledgedResponse mappingResponse = client.preparePutMapping(index) - .setType(builtIndex.getMainType().getType()) - .setSource(builtIndex.getAttributes()) - .get(); - if (!mappingResponse.isAcknowledged()) { - throw new IllegalStateException("Failed to create type " + builtIndex.getMainType().getType()); - } - client.waitForStatus(ClusterHealthStatus.YELLOW); - } - - private void deleteIndex(String indexName) { - client.nativeClient().admin().indices().prepareDelete(indexName).get(); - } - - private void updateIndex(BuiltIndex<?> index) { - boolean blueGreen = configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false); - String indexName = index.getMainType().getIndex().getName(); - - if (blueGreen) { - // SonarCloud - if (migrationEsClient.getUpdatedIndices().contains(indexName)) { - LOGGER.info("Resetting definition hash of Elasticsearch index [{}]", indexName); - metadataIndex.setHash(index.getMainType().getIndex(), IndexDefinitionHash.of(index)); - } else { - throw new IllegalStateException("Blue/green deployment is not supported. Elasticsearch index [" + indexName + "] changed and needs to be dropped."); - } - } else { - // SonarQube - LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName); - deleteIndex(indexName); - createIndex(index, true); - } - } - - private boolean hasDefinitionChange(BuiltIndex<?> index) { - return metadataIndex.getHash(index.getMainType().getIndex()) - .map(hash -> { - String defHash = IndexDefinitionHash.of(index); - return !StringUtils.equals(hash, defHash); - }).orElse(true); - } - - private void checkDbCompatibility(Collection<BuiltIndex> definitions) { - List<String> existingIndices = loadExistingIndicesExceptMetadata(definitions); - if (!existingIndices.isEmpty()) { - boolean delete = false; - if (!esDbCompatibility.hasSameDbVendor()) { - LOGGER.info("Delete Elasticsearch indices (DB vendor changed)"); - delete = true; - } - if (delete) { - existingIndices.forEach(this::deleteIndex); - } - } - esDbCompatibility.markAsCompatible(); - } - - private List<String> loadExistingIndicesExceptMetadata(Collection<BuiltIndex> definitions) { - Set<String> definedNames = definitions.stream() - .map(t -> t.getMainType().getIndex().getName()) - .collect(Collectors.toSet()); - return Arrays.stream(client.nativeClient().admin().indices().prepareGetIndex().get().getIndices()) - .filter(definedNames::contains) - .filter(index -> !DESCRIPTOR.getName().equals(index)) - .collect(Collectors.toList()); - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java deleted file mode 100644 index 22b8b59b0c3..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.collect.Maps; -import java.util.Map; -import org.picocontainer.Startable; -import org.sonar.api.config.Configuration; -import org.sonar.api.server.ServerSide; -import org.sonar.server.es.newindex.BuiltIndex; -import org.sonar.server.es.newindex.NewIndex; - -/** - * This class collects definitions of all Elasticsearch indices during server startup - */ -@ServerSide -public class IndexDefinitions implements Startable { - - private final Map<String, BuiltIndex> byKey = Maps.newHashMap(); - private final IndexDefinition[] defs; - private final Configuration config; - - public IndexDefinitions(IndexDefinition[] defs, Configuration config) { - this.defs = defs; - this.config = config; - } - - public Map<String, BuiltIndex> getIndices() { - return byKey; - } - - @Override - public void start() { - // collect definitions - IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); - - if (!config.getBoolean("sonar.internal.es.disableIndexes").orElse(false)) { - for (IndexDefinition definition : defs) { - definition.define(context); - } - - for (Map.Entry<String, NewIndex> entry : context.getIndices().entrySet()) { - byKey.put(entry.getKey(), entry.getValue().build()); - } - } - } - - @Override - public void stop() { - // nothing to do - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java deleted file mode 100644 index f700fc635f5..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ListMultimap; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang.math.RandomUtils; -import org.sonar.api.Startable; -import org.sonar.api.config.Configuration; -import org.sonar.api.utils.System2; -import org.sonar.api.utils.log.Logger; -import org.sonar.api.utils.log.Loggers; -import org.sonar.api.utils.log.Profiler; -import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.es.EsQueueDto; - -import static java.lang.String.format; - -public class RecoveryIndexer implements Startable { - - private static final Logger LOGGER = Loggers.get(RecoveryIndexer.class); - private static final String LOG_PREFIX = "Elasticsearch recovery - "; - private static final String PROPERTY_INITIAL_DELAY = "sonar.search.recovery.initialDelayInMs"; - private static final String PROPERTY_DELAY = "sonar.search.recovery.delayInMs"; - private static final String PROPERTY_MIN_AGE = "sonar.search.recovery.minAgeInMs"; - private static final String PROPERTY_LOOP_LIMIT = "sonar.search.recovery.loopLimit"; - private static final long DEFAULT_DELAY_IN_MS = 5L * 60 * 1000; - private static final long DEFAULT_MIN_AGE_IN_MS = 5L * 60 * 1000; - private static final int DEFAULT_LOOP_LIMIT = 10_000; - private static final double CIRCUIT_BREAKER_IN_PERCENT = 0.7; - - private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, - new ThreadFactoryBuilder() - .setPriority(Thread.MIN_PRIORITY) - .setNameFormat("RecoveryIndexer-%d") - .build()); - private final System2 system2; - private final Configuration config; - private final DbClient dbClient; - private final Map<String, Indexer> indexersByType; - private final long minAgeInMs; - private final long loopLimit; - - public RecoveryIndexer(System2 system2, Configuration config, DbClient dbClient, ResilientIndexer... indexers) { - this.system2 = system2; - this.config = config; - this.dbClient = dbClient; - this.indexersByType = new HashMap<>(); - Arrays.stream(indexers).forEach(i -> i.getIndexTypes().forEach(indexType -> indexersByType.put(indexType.format(), new Indexer(indexType, i)))); - this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS); - this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT); - } - - private static final class Indexer { - private final IndexType indexType; - private final ResilientIndexer delegate; - - private Indexer(IndexType indexType, ResilientIndexer delegate) { - this.indexType = indexType; - this.delegate = delegate; - } - - public IndexType getIndexType() { - return indexType; - } - - public ResilientIndexer getDelegate() { - return delegate; - } - } - - @Override - public void start() { - long delayInMs = getSetting(PROPERTY_DELAY, DEFAULT_DELAY_IN_MS); - - // in the cluster mode, avoid (but not prevent!) simultaneous executions of recovery - // indexers so that a document is not handled multiple times. - long initialDelayInMs = getSetting(PROPERTY_INITIAL_DELAY, RandomUtils.nextInt(1 + (int) (delayInMs / 2))); - - executorService.scheduleAtFixedRate( - this::recover, - initialDelayInMs, - delayInMs, - TimeUnit.MILLISECONDS); - } - - @Override - public void stop() { - try { - executorService.shutdown(); - executorService.awaitTermination(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - LOGGER.error(LOG_PREFIX + "Unable to stop recovery indexer in timely fashion", e); - executorService.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - - @VisibleForTesting - void recover() { - try (DbSession dbSession = dbClient.openSession(false)) { - Profiler profiler = Profiler.create(LOGGER).start(); - long beforeDate = system2.now() - minAgeInMs; - IndexingResult result = new IndexingResult(); - - Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); - while (!items.isEmpty()) { - IndexingResult loopResult = new IndexingResult(); - - groupItemsByDocType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems))); - result.add(loopResult); - - if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) { - LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal()); - break; - } - - if (loopResult.getTotal() == 0L) { - break; - } - - items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); - } - if (result.getTotal() > 0L) { - profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures())); - } - } catch (Throwable t) { - LOGGER.error(LOG_PREFIX + "fail to recover documents", t); - } - } - - private IndexingResult doIndex(DbSession dbSession, String docType, Collection<EsQueueDto> typeItems) { - LOGGER.trace(LOG_PREFIX + "processing {} [{}]", typeItems.size(), docType); - - Indexer indexer = indexersByType.get(docType); - if (indexer == null) { - LOGGER.error(LOG_PREFIX + "ignore {} items with unsupported type [{}]", typeItems.size(), docType); - return new IndexingResult(); - } - return indexer.delegate.index(dbSession, typeItems); - } - - private static ListMultimap<String, EsQueueDto> groupItemsByDocType(Collection<EsQueueDto> items) { - return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType)); - } - - private long getSetting(String key, long defaultValue) { - long val = config.getLong(key).orElse(defaultValue); - LOGGER.debug(LOG_PREFIX + "{}={}", key, val); - return val; - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibility.java b/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibility.java deleted file mode 100644 index 7733aab7682..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibility.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.metadata; - -/** - * Checks when Elasticsearch indices must be dropped because - * of changes in database - */ -public interface EsDbCompatibility { - - /** - * Whether the effective DB vendor equals the vendor - * registered in Elasticsearch metadata. - * Return {@code false} if at least one of the values is absent - */ - boolean hasSameDbVendor(); - - /** - * Stores in Elasticsearch the metadata about database - */ - void markAsCompatible(); -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java b/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java deleted file mode 100644 index c535e81d92d..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.metadata; - -import java.util.Optional; -import org.sonar.db.DbClient; - -public class EsDbCompatibilityImpl implements EsDbCompatibility { - - private final DbClient dbClient; - private final MetadataIndex metadataIndex; - - public EsDbCompatibilityImpl(DbClient dbClient, MetadataIndex metadataIndex) { - this.dbClient = dbClient; - this.metadataIndex = metadataIndex; - } - - @Override - public boolean hasSameDbVendor() { - Optional<String> registeredDbVendor = metadataIndex.getDbVendor(); - return registeredDbVendor.isPresent() && registeredDbVendor.get().equals(getDbVendor()); - } - - @Override - public void markAsCompatible() { - if (!hasSameDbVendor()) { - metadataIndex.setDbMetadata(getDbVendor()); - } - } - - private String getDbVendor() { - return dbClient.getDatabase().getDialect().getId(); - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/es/metadata/package-info.java deleted file mode 100644 index dfcbf2d7554..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.server.es.metadata; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java deleted file mode 100644 index ea1182c26b8..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ /dev/null @@ -1,1001 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.OptionalLong; -import java.util.Set; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.indices.TermsLookup; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.HasAggregations; -import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds; -import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; -import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms; -import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.max.InternalMax; -import org.elasticsearch.search.aggregations.metrics.min.Min; -import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.joda.time.DateTimeZone; -import org.joda.time.Duration; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.Severity; -import org.sonar.api.rules.RuleType; -import org.sonar.api.utils.DateUtils; -import org.sonar.api.utils.System2; -import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.server.es.BaseDoc; -import org.sonar.server.es.EsClient; -import org.sonar.server.es.EsUtils; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.es.Sorting; -import org.sonar.server.es.StickyFacetBuilder; -import org.sonar.server.issue.index.IssueQuery.PeriodStart; -import org.sonar.server.permission.index.AuthorizationDoc; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.user.UserSession; -import org.sonar.server.view.index.ViewIndexDefinition; - -import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; -import static java.util.Arrays.stream; -import static java.util.Collections.singletonList; -import static java.util.stream.Collectors.toList; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.existsQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.index.query.QueryBuilders.termsQuery; -import static org.sonar.api.rules.RuleType.SECURITY_HOTSPOT; -import static org.sonar.api.rules.RuleType.VULNERABILITY; -import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; -import static org.sonar.server.es.BaseDoc.epochMillisToEpochSeconds; -import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; -import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; -import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNED_TO_ME; -import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNEES; -import static org.sonar.server.issue.index.IssueIndex.Facet.AUTHOR; -import static org.sonar.server.issue.index.IssueIndex.Facet.AUTHORS; -import static org.sonar.server.issue.index.IssueIndex.Facet.CREATED_AT; -import static org.sonar.server.issue.index.IssueIndex.Facet.CWE; -import static org.sonar.server.issue.index.IssueIndex.Facet.DIRECTORIES; -import static org.sonar.server.issue.index.IssueIndex.Facet.FILE_UUIDS; -import static org.sonar.server.issue.index.IssueIndex.Facet.LANGUAGES; -import static org.sonar.server.issue.index.IssueIndex.Facet.MODULE_UUIDS; -import static org.sonar.server.issue.index.IssueIndex.Facet.OWASP_TOP_10; -import static org.sonar.server.issue.index.IssueIndex.Facet.PROJECT_UUIDS; -import static org.sonar.server.issue.index.IssueIndex.Facet.RESOLUTIONS; -import static org.sonar.server.issue.index.IssueIndex.Facet.RULES; -import static org.sonar.server.issue.index.IssueIndex.Facet.SANS_TOP_25; -import static org.sonar.server.issue.index.IssueIndex.Facet.SEVERITIES; -import static org.sonar.server.issue.index.IssueIndex.Facet.SONARSOURCE_SECURITY; -import static org.sonar.server.issue.index.IssueIndex.Facet.STATUSES; -import static org.sonar.server.issue.index.IssueIndex.Facet.TAGS; -import static org.sonar.server.issue.index.IssueIndex.Facet.TYPES; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE_UUID; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_AUTHOR_LOGIN; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_BRANCH_UUID; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_COMPONENT_UUID; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_CWE; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_DIRECTORY_PATH; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_EFFORT; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_FILE_PATH; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_FUNC_CLOSED_AT; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_FUNC_UPDATED_AT; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_IS_MAIN_BRANCH; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_KEY; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_LANGUAGE; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_LINE; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_ORGANIZATION_UUID; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_OWASP_TOP_10; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_RESOLUTION; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_RULE_ID; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SANS_TOP_25; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SEVERITY; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SEVERITY_VALUE; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SONARSOURCE_SECURITY; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_STATUS; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TAGS; -import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TYPE; -import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; -import static org.sonar.server.security.SecurityStandardHelper.SANS_TOP_25_CWE_MAPPING; -import static org.sonar.server.security.SecurityStandardHelper.SANS_TOP_25_INSECURE_INTERACTION; -import static org.sonar.server.security.SecurityStandardHelper.SANS_TOP_25_POROUS_DEFENSES; -import static org.sonar.server.security.SecurityStandardHelper.SANS_TOP_25_RISKY_RESOURCE; -import static org.sonar.server.security.SecurityStandardHelper.SONARSOURCE_CWE_MAPPING; -import static org.sonar.server.security.SecurityStandardHelper.SONARSOURCE_OTHER_CWES_CATEGORY; -import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.DEPRECATED_PARAM_AUTHORS; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_ASSIGNEES; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_AUTHOR; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CREATED_AT; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CWE; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_DIRECTORIES; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_FILE_UUIDS; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_LANGUAGES; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_MODULE_UUIDS; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_OWASP_TOP_10; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_RESOLUTIONS; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_RULES; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SANS_TOP_25; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SEVERITIES; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SONARSOURCE_SECURITY; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_STATUSES; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_TAGS; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_TYPES; - -/** - * The unique entry-point to interact with Elasticsearch index "issues". - * All the requests are listed here. - */ -public class IssueIndex { - - public static final String FACET_PROJECTS = "projects"; - public static final String FACET_ASSIGNED_TO_ME = "assigned_to_me"; - - private static final int DEFAULT_FACET_SIZE = 15; - private static final int MAX_FACET_SIZE = 100; - private static final String AGG_VULNERABILITIES = "vulnerabilities"; - private static final String AGG_SEVERITIES = "severities"; - private static final String AGG_TO_REVIEW_SECURITY_HOTSPOTS = "toReviewSecurityHotspots"; - private static final String AGG_IN_REVIEW_SECURITY_HOTSPOTS = "inReviewSecurityHotspots"; - private static final String AGG_REVIEWED_SECURITY_HOTSPOTS = "reviewedSecurityHotspots"; - private static final String AGG_CWES = "cwes"; - private static final BoolQueryBuilder NON_RESOLVED_VULNERABILITIES_FILTER = boolQuery() - .filter(termQuery(FIELD_ISSUE_TYPE, VULNERABILITY.name())) - .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)); - private static final BoolQueryBuilder IN_REVIEW_HOTSPOTS_FILTER = boolQuery() - .filter(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())) - .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_IN_REVIEW)) - .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)); - private static final BoolQueryBuilder TO_REVIEW_HOTSPOTS_FILTER = boolQuery() - .filter(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())) - .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_TO_REVIEW)) - .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)); - private static final BoolQueryBuilder REVIEWED_HOTSPOTS_FILTER = boolQuery() - .filter(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())) - .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_REVIEWED)) - .filter(termQuery(FIELD_ISSUE_RESOLUTION, Issue.RESOLUTION_FIXED)); - - public enum Facet { - SEVERITIES(PARAM_SEVERITIES, FIELD_ISSUE_SEVERITY, Severity.ALL.size()), - STATUSES(PARAM_STATUSES, FIELD_ISSUE_STATUS, Issue.STATUSES.size()), - // Resolutions facet returns one more element than the number of resolutions to take into account unresolved issues - RESOLUTIONS(PARAM_RESOLUTIONS, FIELD_ISSUE_RESOLUTION, Issue.RESOLUTIONS.size() + 1), - TYPES(PARAM_TYPES, FIELD_ISSUE_TYPE, RuleType.values().length), - LANGUAGES(PARAM_LANGUAGES, FIELD_ISSUE_LANGUAGE, MAX_FACET_SIZE), - RULES(PARAM_RULES, FIELD_ISSUE_RULE_ID, MAX_FACET_SIZE), - TAGS(PARAM_TAGS, FIELD_ISSUE_TAGS, MAX_FACET_SIZE), - AUTHORS(DEPRECATED_PARAM_AUTHORS, FIELD_ISSUE_AUTHOR_LOGIN, MAX_FACET_SIZE), - AUTHOR(PARAM_AUTHOR, FIELD_ISSUE_AUTHOR_LOGIN, MAX_FACET_SIZE), - PROJECT_UUIDS(FACET_PROJECTS, FIELD_ISSUE_PROJECT_UUID, MAX_FACET_SIZE), - MODULE_UUIDS(PARAM_MODULE_UUIDS, FIELD_ISSUE_MODULE_UUID, MAX_FACET_SIZE), - FILE_UUIDS(PARAM_FILE_UUIDS, FIELD_ISSUE_COMPONENT_UUID, MAX_FACET_SIZE), - DIRECTORIES(PARAM_DIRECTORIES, FIELD_ISSUE_DIRECTORY_PATH, MAX_FACET_SIZE), - ASSIGNEES(PARAM_ASSIGNEES, FIELD_ISSUE_ASSIGNEE_UUID, MAX_FACET_SIZE), - ASSIGNED_TO_ME(FACET_ASSIGNED_TO_ME, FIELD_ISSUE_ASSIGNEE_UUID, 1), - OWASP_TOP_10(PARAM_OWASP_TOP_10, FIELD_ISSUE_OWASP_TOP_10, DEFAULT_FACET_SIZE), - SANS_TOP_25(PARAM_SANS_TOP_25, FIELD_ISSUE_SANS_TOP_25, DEFAULT_FACET_SIZE), - CWE(PARAM_CWE, FIELD_ISSUE_CWE, DEFAULT_FACET_SIZE), - CREATED_AT(PARAM_CREATED_AT, FIELD_ISSUE_FUNC_CREATED_AT, DEFAULT_FACET_SIZE), - SONARSOURCE_SECURITY(PARAM_SONARSOURCE_SECURITY, FIELD_ISSUE_SONARSOURCE_SECURITY, DEFAULT_FACET_SIZE); - - private final String name; - private final String fieldName; - private final int size; - - Facet(String name, String fieldName, int size) { - this.name = name; - this.fieldName = fieldName; - this.size = size; - } - - public String getName() { - return name; - } - - public String getFieldName() { - return fieldName; - } - - public int getSize() { - return size; - } - - public static Facet of(String name) { - return stream(values()) - .filter(f -> f.getName().equals(name)) - .reduce((a, b) -> { - throw new IllegalStateException("Multiple facets with same name: " + a + ", " + b); - }) - .orElseThrow(() -> new IllegalArgumentException(String.format("Facet name '%s' hasn't been found", name))); - } - } - - private static final String SUBSTRING_MATCH_REGEXP = ".*%s.*"; - // TODO to be documented - // TODO move to Facets ? - private static final String FACET_SUFFIX_MISSING = "_missing"; - private static final String IS_ASSIGNED_FILTER = "__isAssigned"; - private static final SumAggregationBuilder EFFORT_AGGREGATION = AggregationBuilders.sum(FACET_MODE_EFFORT).field(FIELD_ISSUE_EFFORT); - private static final BucketOrder EFFORT_AGGREGATION_ORDER = BucketOrder.aggregation(FACET_MODE_EFFORT, false); - private static final Duration TWENTY_DAYS = Duration.standardDays(20L); - private static final Duration TWENTY_WEEKS = Duration.standardDays(20L * 7L); - private static final Duration TWENTY_MONTHS = Duration.standardDays(20L * 30L); - private static final String AGG_COUNT = "count"; - private final Sorting sorting; - private final EsClient client; - private final System2 system; - private final UserSession userSession; - private final WebAuthorizationTypeSupport authorizationTypeSupport; - - public IssueIndex(EsClient client, System2 system, UserSession userSession, WebAuthorizationTypeSupport authorizationTypeSupport) { - this.client = client; - this.system = system; - this.userSession = userSession; - this.authorizationTypeSupport = authorizationTypeSupport; - - this.sorting = new Sorting(); - this.sorting.add(IssueQuery.SORT_BY_STATUS, FIELD_ISSUE_STATUS); - this.sorting.add(IssueQuery.SORT_BY_SEVERITY, FIELD_ISSUE_SEVERITY_VALUE); - this.sorting.add(IssueQuery.SORT_BY_CREATION_DATE, FIELD_ISSUE_FUNC_CREATED_AT); - this.sorting.add(IssueQuery.SORT_BY_UPDATE_DATE, FIELD_ISSUE_FUNC_UPDATED_AT); - this.sorting.add(IssueQuery.SORT_BY_CLOSE_DATE, FIELD_ISSUE_FUNC_CLOSED_AT); - this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, FIELD_ISSUE_PROJECT_UUID); - this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, FIELD_ISSUE_FILE_PATH); - this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, FIELD_ISSUE_LINE); - this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, FIELD_ISSUE_SEVERITY_VALUE).reverse(); - this.sorting.add(IssueQuery.SORT_BY_FILE_LINE, FIELD_ISSUE_KEY); - - // by default order by created date, project, file, line and issue key (in order to be deterministic when same ms) - this.sorting.addDefault(FIELD_ISSUE_FUNC_CREATED_AT).reverse(); - this.sorting.addDefault(FIELD_ISSUE_PROJECT_UUID); - this.sorting.addDefault(FIELD_ISSUE_FILE_PATH); - this.sorting.addDefault(FIELD_ISSUE_LINE); - this.sorting.addDefault(FIELD_ISSUE_KEY); - } - - public SearchResponse search(IssueQuery query, SearchOptions options) { - SearchRequestBuilder requestBuilder = client.prepareSearch(TYPE_ISSUE.getMainType()); - - configureSorting(query, requestBuilder); - configurePagination(options, requestBuilder); - configureRouting(query, options, requestBuilder); - - QueryBuilder esQuery = matchAllQuery(); - BoolQueryBuilder esFilter = boolQuery(); - Map<String, QueryBuilder> filters = createFilters(query); - for (QueryBuilder filter : filters.values()) { - if (filter != null) { - esFilter.must(filter); - } - } - if (esFilter.hasClauses()) { - requestBuilder.setQuery(boolQuery().must(esQuery).filter(esFilter)); - } else { - requestBuilder.setQuery(esQuery); - } - - configureStickyFacets(query, options, filters, esQuery, requestBuilder); - requestBuilder.addAggregation(EFFORT_AGGREGATION); - requestBuilder.setFetchSource(false); - return requestBuilder.get(); - } - - /** - * Optimization - do not send ES request to all shards when scope is restricted - * to a set of projects. Because project UUID is used for routing, the request - * can be sent to only the shards containing the specified projects. - * Note that sticky facets may involve all projects, so this optimization must be - * disabled when facets are enabled. - */ - private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequestBuilder requestBuilder) { - Collection<String> uuids = query.projectUuids(); - if (!uuids.isEmpty() && options.getFacets().isEmpty()) { - requestBuilder.setRouting(uuids.stream().map(AuthorizationDoc::idOf).toArray(String[]::new)); - } - } - - private static void configurePagination(SearchOptions options, SearchRequestBuilder esSearch) { - esSearch.setFrom(options.getOffset()).setSize(options.getLimit()); - } - - private Map<String, QueryBuilder> createFilters(IssueQuery query) { - Map<String, QueryBuilder> filters = new HashMap<>(); - filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_ISSUE.getName())); - filters.put("__authorization", createAuthorizationFilter()); - - // Issue is assigned Filter - if (BooleanUtils.isTrue(query.assigned())) { - filters.put(IS_ASSIGNED_FILTER, existsQuery(FIELD_ISSUE_ASSIGNEE_UUID)); - } else if (BooleanUtils.isFalse(query.assigned())) { - filters.put(IS_ASSIGNED_FILTER, boolQuery().mustNot(existsQuery(FIELD_ISSUE_ASSIGNEE_UUID))); - } - - // Issue is Resolved Filter - String isResolved = "__isResolved"; - if (BooleanUtils.isTrue(query.resolved())) { - filters.put(isResolved, existsQuery(FIELD_ISSUE_RESOLUTION)); - } else if (BooleanUtils.isFalse(query.resolved())) { - filters.put(isResolved, boolQuery().mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))); - } - - // Field Filters - filters.put(FIELD_ISSUE_KEY, createTermsFilter(FIELD_ISSUE_KEY, query.issueKeys())); - filters.put(FIELD_ISSUE_ASSIGNEE_UUID, createTermsFilter(FIELD_ISSUE_ASSIGNEE_UUID, query.assignees())); - filters.put(FIELD_ISSUE_LANGUAGE, createTermsFilter(FIELD_ISSUE_LANGUAGE, query.languages())); - filters.put(FIELD_ISSUE_TAGS, createTermsFilter(FIELD_ISSUE_TAGS, query.tags())); - filters.put(FIELD_ISSUE_TYPE, createTermsFilter(FIELD_ISSUE_TYPE, query.types())); - filters.put(FIELD_ISSUE_RESOLUTION, createTermsFilter(FIELD_ISSUE_RESOLUTION, query.resolutions())); - filters.put(FIELD_ISSUE_AUTHOR_LOGIN, createTermsFilter(FIELD_ISSUE_AUTHOR_LOGIN, query.authors())); - filters.put(FIELD_ISSUE_RULE_ID, createTermsFilter( - FIELD_ISSUE_RULE_ID, - query.rules().stream().map(RuleDefinitionDto::getId).collect(toList()))); - filters.put(FIELD_ISSUE_STATUS, createTermsFilter(FIELD_ISSUE_STATUS, query.statuses())); - filters.put(FIELD_ISSUE_ORGANIZATION_UUID, createTermFilter(FIELD_ISSUE_ORGANIZATION_UUID, query.organizationUuid())); - filters.put(FIELD_ISSUE_OWASP_TOP_10, createTermsFilter(FIELD_ISSUE_OWASP_TOP_10, query.owaspTop10())); - filters.put(FIELD_ISSUE_SANS_TOP_25, createTermsFilter(FIELD_ISSUE_SANS_TOP_25, query.sansTop25())); - filters.put(FIELD_ISSUE_CWE, createTermsFilter(FIELD_ISSUE_CWE, query.cwe())); - addSeverityFilter(query, filters); - filters.put(FIELD_ISSUE_SONARSOURCE_SECURITY, createTermsFilter(FIELD_ISSUE_SONARSOURCE_SECURITY, query.sonarsourceSecurity())); - - addComponentRelatedFilters(query, filters); - addDatesFilter(filters, query); - addCreatedAfterByProjectsFilter(filters, query); - return filters; - } - - private static void addSeverityFilter(IssueQuery query, Map<String, QueryBuilder> filters) { - QueryBuilder severityFieldFilter = createTermsFilter(FIELD_ISSUE_SEVERITY, query.severities()); - if (severityFieldFilter != null) { - filters.put(FIELD_ISSUE_SEVERITY, boolQuery() - .must(severityFieldFilter) - // Ignore severity of Security HotSpots - .mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name()))); - } - } - - private static void addComponentRelatedFilters(IssueQuery query, Map<String, QueryBuilder> filters) { - addCommonComponentRelatedFilters(query, filters); - if (query.viewUuids().isEmpty()) { - addBranchComponentRelatedFilters(query, filters); - } else { - addViewRelatedFilters(query, filters); - } - } - - private static void addCommonComponentRelatedFilters(IssueQuery query, Map<String, QueryBuilder> filters) { - QueryBuilder componentFilter = createTermsFilter(FIELD_ISSUE_COMPONENT_UUID, query.componentUuids()); - QueryBuilder projectFilter = createTermsFilter(FIELD_ISSUE_PROJECT_UUID, query.projectUuids()); - QueryBuilder moduleRootFilter = createTermsFilter(FIELD_ISSUE_MODULE_PATH, query.moduleRootUuids()); - QueryBuilder moduleFilter = createTermsFilter(FIELD_ISSUE_MODULE_UUID, query.moduleUuids()); - QueryBuilder directoryFilter = createTermsFilter(FIELD_ISSUE_DIRECTORY_PATH, query.directories()); - QueryBuilder fileFilter = createTermsFilter(FIELD_ISSUE_COMPONENT_UUID, query.fileUuids()); - - if (BooleanUtils.isTrue(query.onComponentOnly())) { - filters.put(FIELD_ISSUE_COMPONENT_UUID, componentFilter); - } else { - filters.put(FIELD_ISSUE_PROJECT_UUID, projectFilter); - filters.put("__module", moduleRootFilter); - filters.put(FIELD_ISSUE_MODULE_UUID, moduleFilter); - filters.put(FIELD_ISSUE_DIRECTORY_PATH, directoryFilter); - filters.put(FIELD_ISSUE_COMPONENT_UUID, fileFilter != null ? fileFilter : componentFilter); - } - } - - private static void addBranchComponentRelatedFilters(IssueQuery query, Map<String, QueryBuilder> filters) { - if (BooleanUtils.isTrue(query.onComponentOnly())) { - return; - } - QueryBuilder branchFilter = createTermFilter(FIELD_ISSUE_BRANCH_UUID, query.branchUuid()); - filters.put("__is_main_branch", createTermFilter(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(query.isMainBranch()))); - filters.put(FIELD_ISSUE_BRANCH_UUID, branchFilter); - } - - private static void addViewRelatedFilters(IssueQuery query, Map<String, QueryBuilder> filters) { - if (BooleanUtils.isTrue(query.onComponentOnly())) { - return; - } - Collection<String> viewUuids = query.viewUuids(); - String branchUuid = query.branchUuid(); - boolean onApplicationBranch = branchUuid != null && !viewUuids.isEmpty(); - if (onApplicationBranch) { - filters.put("__view", createViewFilter(singletonList(query.branchUuid()))); - } else { - filters.put("__is_main_branch", createTermFilter(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(true))); - filters.put("__view", createViewFilter(viewUuids)); - } - } - - @CheckForNull - private static QueryBuilder createViewFilter(Collection<String> viewUuids) { - if (viewUuids.isEmpty()) { - return null; - } - - BoolQueryBuilder viewsFilter = boolQuery(); - for (String viewUuid : viewUuids) { - IndexType.IndexMainType mainType = TYPE_VIEW; - viewsFilter.should(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID, - new TermsLookup( - mainType.getIndex().getName(), - mainType.getType(), - viewUuid, - ViewIndexDefinition.FIELD_PROJECTS))); - } - return viewsFilter; - } - - private static AggregationBuilder addEffortAggregationIfNeeded(IssueQuery query, AggregationBuilder aggregation) { - if (hasQueryEffortFacet(query)) { - aggregation.subAggregation(EFFORT_AGGREGATION); - } - return aggregation; - } - - private static boolean hasQueryEffortFacet(IssueQuery query) { - return FACET_MODE_EFFORT.equals(query.facetMode()); - } - - private static AggregationBuilder createSeverityFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder queryBuilder) { - String fieldName = SEVERITIES.getFieldName(); - String facetName = SEVERITIES.getName(); - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder); - BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(fieldName) - // Ignore severity of Security HotSpots - .mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name())); - FilterAggregationBuilder facetTopAggregation = stickyFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, SEVERITIES.getSize()); - return AggregationBuilders - .global(facetName) - .subAggregation(facetTopAggregation); - } - - private static AggregationBuilder createAssigneesFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder queryBuilder) { - String fieldName = ASSIGNEES.getFieldName(); - String facetName = ASSIGNEES.getName(); - - // Same as in super.stickyFacetBuilder - Map<String, QueryBuilder> assigneeFilters = Maps.newHashMap(filters); - assigneeFilters.remove(IS_ASSIGNED_FILTER); - assigneeFilters.remove(fieldName); - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, assigneeFilters, queryBuilder); - BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(fieldName); - FilterAggregationBuilder facetTopAggregation = stickyFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, ASSIGNEES.getSize()); - if (!query.assignees().isEmpty()) { - facetTopAggregation = stickyFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t, query.assignees().toArray()); - } - - // Add missing facet for unassigned issues - facetTopAggregation.subAggregation( - addEffortAggregationIfNeeded(query, AggregationBuilders - .missing(facetName + FACET_SUFFIX_MISSING) - .field(fieldName))); - - return AggregationBuilders - .global(facetName) - .subAggregation(facetTopAggregation); - } - - private static AggregationBuilder createResolutionFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder esQuery) { - String fieldName = RESOLUTIONS.getFieldName(); - String facetName = RESOLUTIONS.getName(); - - // Same as in super.stickyFacetBuilder - Map<String, QueryBuilder> resolutionFilters = Maps.newHashMap(filters); - resolutionFilters.remove("__isResolved"); - resolutionFilters.remove(fieldName); - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, resolutionFilters, esQuery); - BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(fieldName); - FilterAggregationBuilder facetTopAggregation = stickyFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, RESOLUTIONS.getSize()); - facetTopAggregation = stickyFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t); - - // Add missing facet for unresolved issues - facetTopAggregation.subAggregation( - addEffortAggregationIfNeeded(query, AggregationBuilders - .missing(facetName + FACET_SUFFIX_MISSING) - .field(fieldName))); - - return AggregationBuilders - .global(facetName) - .subAggregation(facetTopAggregation); - } - - @CheckForNull - private static QueryBuilder createTermsFilter(String field, Collection<?> values) { - return values.isEmpty() ? null : termsQuery(field, values); - } - - @CheckForNull - private static QueryBuilder createTermFilter(String field, @Nullable String value) { - return value == null ? null : termQuery(field, value); - } - - private void configureSorting(IssueQuery query, SearchRequestBuilder esRequest) { - createSortBuilders(query).forEach(esRequest::addSort); - } - - private List<FieldSortBuilder> createSortBuilders(IssueQuery query) { - String sortField = query.sort(); - if (sortField != null) { - boolean asc = BooleanUtils.isTrue(query.asc()); - return sorting.fill(sortField, asc); - } - return sorting.fillDefault(); - } - - private QueryBuilder createAuthorizationFilter() { - return authorizationTypeSupport.createQueryFilter(); - } - - private void addDatesFilter(Map<String, QueryBuilder> filters, IssueQuery query) { - PeriodStart createdAfter = query.createdAfter(); - Date createdBefore = query.createdBefore(); - - validateCreationDateBounds(createdBefore, createdAfter != null ? createdAfter.date() : null); - - if (createdAfter != null) { - filters.put("__createdAfter", QueryBuilders - .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) - .from(BaseDoc.dateToEpochSeconds(createdAfter.date()), createdAfter.inclusive())); - } - if (createdBefore != null) { - filters.put("__createdBefore", QueryBuilders - .rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT) - .lt(BaseDoc.dateToEpochSeconds(createdBefore))); - } - Date createdAt = query.createdAt(); - if (createdAt != null) { - filters.put("__createdAt", termQuery(FIELD_ISSUE_FUNC_CREATED_AT, BaseDoc.dateToEpochSeconds(createdAt))); - } - } - - private static void addCreatedAfterByProjectsFilter(Map<String, QueryBuilder> filters, IssueQuery query) { - Map<String, PeriodStart> createdAfterByProjectUuids = query.createdAfterByProjectUuids(); - BoolQueryBuilder boolQueryBuilder = boolQuery(); - createdAfterByProjectUuids.forEach((projectUuid, createdAfterDate) -> boolQueryBuilder.should(boolQuery() - .filter(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)) - .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).from(BaseDoc.dateToEpochSeconds(createdAfterDate.date()), createdAfterDate.inclusive())))); - filters.put("createdAfterByProjectUuids", boolQueryBuilder); - } - - private void validateCreationDateBounds(@Nullable Date createdBefore, @Nullable Date createdAfter) { - Preconditions.checkArgument(createdAfter == null || createdAfter.before(new Date(system.now())), - "Start bound cannot be in the future"); - Preconditions.checkArgument(createdAfter == null || createdBefore == null || createdAfter.before(createdBefore), - "Start bound cannot be larger or equal to end bound"); - } - - private void configureStickyFacets(IssueQuery query, SearchOptions options, Map<String, QueryBuilder> filters, QueryBuilder esQuery, SearchRequestBuilder esSearch) { - if (!options.getFacets().isEmpty()) { - StickyFacetBuilder stickyFacetBuilder = newStickyFacetBuilder(query, filters, esQuery); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, STATUSES); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, PROJECT_UUIDS, query.projectUuids().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, MODULE_UUIDS, query.moduleUuids().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, DIRECTORIES, query.directories().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, FILE_UUIDS, query.fileUuids().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, LANGUAGES, query.languages().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, RULES, query.rules().stream().map(RuleDefinitionDto::getId).toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, AUTHORS, query.authors().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, AUTHOR, query.authors().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, TAGS, query.tags().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, TYPES, query.types().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, OWASP_TOP_10, query.owaspTop10().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, SANS_TOP_25, query.sansTop25().toArray()); - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, CWE, query.cwe().toArray()); - if (options.getFacets().contains(PARAM_SEVERITIES)) { - esSearch.addAggregation(createSeverityFacet(query, filters, esQuery)); - } - addSimpleStickyFacetIfNeeded(options, stickyFacetBuilder, esSearch, SONARSOURCE_SECURITY, query.sonarsourceSecurity().toArray()); - if (options.getFacets().contains(PARAM_RESOLUTIONS)) { - esSearch.addAggregation(createResolutionFacet(query, filters, esQuery)); - } - if (options.getFacets().contains(PARAM_ASSIGNEES)) { - esSearch.addAggregation(createAssigneesFacet(query, filters, esQuery)); - } - if (options.getFacets().contains(PARAM_CREATED_AT)) { - getCreatedAtFacet(query, filters, esQuery).ifPresent(esSearch::addAggregation); - } - addAssignedToMeFacetIfNeeded(esSearch, options, query, filters, esQuery); - } - } - - private Optional<AggregationBuilder> getCreatedAtFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder esQuery) { - long startTime; - boolean startInclusive; - PeriodStart createdAfter = query.createdAfter(); - if (createdAfter == null) { - OptionalLong minDate = getMinCreatedAt(filters, esQuery); - if (!minDate.isPresent()) { - return Optional.empty(); - } - startTime = minDate.getAsLong(); - startInclusive = true; - } else { - startTime = createdAfter.date().getTime(); - startInclusive = createdAfter.inclusive(); - } - Date createdBefore = query.createdBefore(); - long endTime = createdBefore == null ? system.now() : createdBefore.getTime(); - - Duration timeSpan = new Duration(startTime, endTime); - DateHistogramInterval bucketSize = DateHistogramInterval.YEAR; - if (timeSpan.isShorterThan(TWENTY_DAYS)) { - bucketSize = DateHistogramInterval.DAY; - } else if (timeSpan.isShorterThan(TWENTY_WEEKS)) { - bucketSize = DateHistogramInterval.WEEK; - } else if (timeSpan.isShorterThan(TWENTY_MONTHS)) { - bucketSize = DateHistogramInterval.MONTH; - } - - AggregationBuilder dateHistogram = AggregationBuilders.dateHistogram(CREATED_AT.getName()) - .field(CREATED_AT.getFieldName()) - .dateHistogramInterval(bucketSize) - .minDocCount(0L) - .format(DateUtils.DATETIME_FORMAT) - .timeZone(DateTimeZone.forOffsetMillis(system.getDefaultTimeZone().getRawOffset())) - // ES dateHistogram bounds are inclusive while createdBefore parameter is exclusive - .extendedBounds(new ExtendedBounds(startInclusive ? startTime : (startTime + 1), endTime - 1L)); - addEffortAggregationIfNeeded(query, dateHistogram); - return Optional.of(dateHistogram); - } - - private OptionalLong getMinCreatedAt(Map<String, QueryBuilder> filters, QueryBuilder esQuery) { - String facetNameAndField = CREATED_AT.getFieldName(); - SearchRequestBuilder esRequest = client - .prepareSearch(TYPE_ISSUE.getMainType()) - .setSize(0); - BoolQueryBuilder esFilter = boolQuery(); - filters.values().stream().filter(Objects::nonNull).forEach(esFilter::must); - if (esFilter.hasClauses()) { - esRequest.setQuery(QueryBuilders.boolQuery().must(esQuery).filter(esFilter)); - } else { - esRequest.setQuery(esQuery); - } - esRequest.addAggregation(AggregationBuilders.min(facetNameAndField).field(facetNameAndField)); - Min minValue = esRequest.get().getAggregations().get(facetNameAndField); - - double actualValue = minValue.getValue(); - if (Double.isInfinite(actualValue)) { - return OptionalLong.empty(); - } - return OptionalLong.of((long) actualValue); - } - - private void addAssignedToMeFacetIfNeeded(SearchRequestBuilder builder, SearchOptions options, IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder queryBuilder) { - String uuid = userSession.getUuid(); - - if (!options.getFacets().contains(ASSIGNED_TO_ME.getName()) || StringUtils.isEmpty(uuid)) { - return; - } - - String fieldName = ASSIGNED_TO_ME.getFieldName(); - String facetName = ASSIGNED_TO_ME.getName(); - - // Same as in super.stickyFacetBuilder - StickyFacetBuilder assignedToMeFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder); - BoolQueryBuilder facetFilter = assignedToMeFacetBuilder.getStickyFacetFilter(IS_ASSIGNED_FILTER, fieldName); - - FilterAggregationBuilder facetTopAggregation = AggregationBuilders - .filter(facetName + "__filter", facetFilter) - .subAggregation(addEffortAggregationIfNeeded(query, AggregationBuilders.terms(facetName + "__terms") - .field(fieldName) - .includeExclude(new IncludeExclude(escapeSpecialRegexChars(uuid), null)))); - - builder.addAggregation( - AggregationBuilders.global(facetName) - .subAggregation(facetTopAggregation)); - } - - private static StickyFacetBuilder newStickyFacetBuilder(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder esQuery) { - if (hasQueryEffortFacet(query)) { - return new StickyFacetBuilder(esQuery, filters, EFFORT_AGGREGATION, EFFORT_AGGREGATION_ORDER); - } - return new StickyFacetBuilder(esQuery, filters); - } - - private static void addSimpleStickyFacetIfNeeded(SearchOptions options, StickyFacetBuilder stickyFacetBuilder, SearchRequestBuilder esSearch, - Facet facet, Object... selectedValues) { - if (options.getFacets().contains(facet.getName())) { - esSearch.addAggregation(stickyFacetBuilder.buildStickyFacet(facet.getFieldName(), facet.getName(), facet.getSize(), selectedValues)); - } - } - - public List<String> searchTags(IssueQuery query, @Nullable String textQuery, int size) { - Terms terms = listTermsMatching(FIELD_ISSUE_TAGS, query, textQuery, BucketOrder.key(true), size); - return EsUtils.termsKeys(terms); - } - - public Map<String, Long> countTags(IssueQuery query, int maxNumberOfTags) { - Terms terms = listTermsMatching(FIELD_ISSUE_TAGS, query, null, BucketOrder.count(false), maxNumberOfTags); - return EsUtils.termsToMap(terms); - } - - public List<String> searchAuthors(IssueQuery query, @Nullable String textQuery, int maxNumberOfAuthors) { - Terms terms = listTermsMatching(FIELD_ISSUE_AUTHOR_LOGIN, query, textQuery, BucketOrder.key(true), maxNumberOfAuthors); - return EsUtils.termsKeys(terms); - } - - private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable String textQuery, BucketOrder termsOrder, int size) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_ISSUE.getMainType()) - // Avoids returning search hits - .setSize(0); - - requestBuilder.setQuery(boolQuery().must(QueryBuilders.matchAllQuery()).filter(createBoolFilter(query))); - - TermsAggregationBuilder aggreg = AggregationBuilders.terms("_ref") - .field(fieldName) - .size(size) - .order(termsOrder) - .minDocCount(1L); - if (textQuery != null) { - aggreg.includeExclude(new IncludeExclude(format(SUBSTRING_MATCH_REGEXP, escapeSpecialRegexChars(textQuery)), null)); - } - - SearchResponse searchResponse = requestBuilder.addAggregation(aggreg).get(); - return searchResponse.getAggregations().get("_ref"); - } - - private BoolQueryBuilder createBoolFilter(IssueQuery query) { - BoolQueryBuilder boolQuery = boolQuery(); - for (QueryBuilder filter : createFilters(query).values()) { - if (filter != null) { - boolQuery.must(filter); - } - } - return boolQuery; - } - - public List<ProjectStatistics> searchProjectStatistics(List<String> projectUuids, List<Long> froms, @Nullable String assigneeUuid) { - checkState(projectUuids.size() == froms.size(), - "Expected same size for projectUuids (had size %s) and froms (had size %s)", projectUuids.size(), froms.size()); - if (projectUuids.isEmpty()) { - return Collections.emptyList(); - } - SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) - .setQuery( - boolQuery() - .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) - .filter(termQuery(FIELD_ISSUE_ASSIGNEE_UUID, assigneeUuid)) - .mustNot(termQuery(FIELD_ISSUE_TYPE, SECURITY_HOTSPOT.name()))) - .setSize(0); - IntStream.range(0, projectUuids.size()).forEach(i -> { - String projectUuid = projectUuids.get(i); - long from = froms.get(i); - request - .addAggregation(AggregationBuilders - .filter(projectUuid, boolQuery() - .filter(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)) - .filter(rangeQuery(FIELD_ISSUE_FUNC_CREATED_AT).gte(epochMillisToEpochSeconds(from)))) - .subAggregation( - AggregationBuilders.terms("branchUuid").field(FIELD_ISSUE_BRANCH_UUID) - .subAggregation( - AggregationBuilders.count(AGG_COUNT).field(FIELD_ISSUE_KEY)) - .subAggregation( - AggregationBuilders.max("maxFuncCreatedAt").field(FIELD_ISSUE_FUNC_CREATED_AT)))); - }); - SearchResponse response = request.get(); - return response.getAggregations().asList().stream() - .map(x -> (InternalFilter) x) - .flatMap(projectBucket -> ((StringTerms) projectBucket.getAggregations().get("branchUuid")).getBuckets().stream() - .flatMap(branchBucket -> { - long count = ((InternalValueCount) branchBucket.getAggregations().get(AGG_COUNT)).getValue(); - if (count < 1L) { - return Stream.empty(); - } - long lastIssueDate = (long) ((InternalMax) branchBucket.getAggregations().get("maxFuncCreatedAt")).getValue(); - return Stream.of(new ProjectStatistics(branchBucket.getKeyAsString(), count, lastIssueDate)); - })) - .collect(MoreCollectors.toList(projectUuids.size())); - } - - public List<BranchStatistics> searchBranchStatistics(String projectUuid, List<String> branchUuids) { - if (branchUuids.isEmpty()) { - return Collections.emptyList(); - } - - SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) - .setRouting(AuthorizationDoc.idOf(projectUuid)) - .setQuery( - boolQuery() - .must(termsQuery(FIELD_ISSUE_BRANCH_UUID, branchUuids)) - .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) - .must(termQuery(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(false)))) - .setSize(0) - .addAggregation(AggregationBuilders.terms("branchUuids") - .field(FIELD_ISSUE_BRANCH_UUID) - .size(branchUuids.size()) - .subAggregation(AggregationBuilders.terms("types") - .field(FIELD_ISSUE_TYPE))); - SearchResponse response = request.get(); - return ((StringTerms) response.getAggregations().get("branchUuids")).getBuckets().stream() - .map(bucket -> new BranchStatistics(bucket.getKeyAsString(), - ((StringTerms) bucket.getAggregations().get("types")).getBuckets() - .stream() - .collect(uniqueIndex(StringTerms.Bucket::getKeyAsString, InternalTerms.Bucket::getDocCount)))) - .collect(MoreCollectors.toList(branchUuids.size())); - } - - public List<SecurityStandardCategoryStatistics> getSansTop25Report(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); - Stream.of(SANS_TOP_25_INSECURE_INTERACTION, SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES) - .forEach(sansCategory -> request.addAggregation(createAggregation(FIELD_ISSUE_SANS_TOP_25, sansCategory, includeCwe, Optional.of(SANS_TOP_25_CWE_MAPPING)))); - return processSecurityReportSearchResults(request, includeCwe); - } - - public List<SecurityStandardCategoryStatistics> getSonarSourceReport(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); - Stream.concat(SONARSOURCE_CWE_MAPPING.keySet().stream(), Stream.of(SONARSOURCE_OTHER_CWES_CATEGORY)) - .forEach(sonarsourceCategory -> request.addAggregation( - createAggregation(FIELD_ISSUE_SONARSOURCE_SECURITY, sonarsourceCategory, includeCwe, Optional.of(SONARSOURCE_CWE_MAPPING)))); - return processSecurityReportSearchResults(request, includeCwe); - } - - public List<SecurityStandardCategoryStatistics> getOwaspTop10Report(String projectUuid, boolean isViewOrApp, boolean includeCwe) { - SearchRequestBuilder request = prepareNonClosedVulnerabilitiesAndHotspotSearch(projectUuid, isViewOrApp); - IntStream.rangeClosed(1, 10).mapToObj(i -> "a" + i) - .forEach(owaspCategory -> request.addAggregation(createAggregation(FIELD_ISSUE_OWASP_TOP_10, owaspCategory, includeCwe, Optional.empty()))); - return processSecurityReportSearchResults(request, includeCwe); - } - - private static AggregationBuilder createAggregation(String categoryField, String category, boolean includeCwe, Optional<Map<String, Set<String>>> categoryToCwesMap) { - return addSecurityReportSubAggregations(AggregationBuilders - .filter(category, boolQuery() - .filter(termQuery(categoryField, category))), - includeCwe, categoryToCwesMap.map(m -> m.get(category))); - } - - private static List<SecurityStandardCategoryStatistics> processSecurityReportSearchResults(SearchRequestBuilder request, boolean includeCwe) { - SearchResponse response = request.get(); - return response.getAggregations().asList().stream() - .map(c -> processSecurityReportIssueSearchResults((InternalFilter) c, includeCwe)) - .collect(MoreCollectors.toList()); - } - - private static SecurityStandardCategoryStatistics processSecurityReportIssueSearchResults(InternalFilter categoryBucket, boolean includeCwe) { - List<SecurityStandardCategoryStatistics> children = new ArrayList<>(); - if (includeCwe) { - Stream<StringTerms.Bucket> stream = ((StringTerms) categoryBucket.getAggregations().get(AGG_CWES)).getBuckets().stream(); - children = stream.map(cweBucket -> processSecurityReportCategorySearchResults(cweBucket, cweBucket.getKeyAsString(), null)).collect(toList()); - } - - return processSecurityReportCategorySearchResults(categoryBucket, categoryBucket.getName(), children); - } - - private static SecurityStandardCategoryStatistics processSecurityReportCategorySearchResults(HasAggregations categoryBucket, String categoryName, - @Nullable List<SecurityStandardCategoryStatistics> children) { - List<StringTerms.Bucket> severityBuckets = ((StringTerms) ((InternalFilter) categoryBucket.getAggregations().get(AGG_VULNERABILITIES)).getAggregations().get(AGG_SEVERITIES)) - .getBuckets(); - long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((InternalValueCount) b.getAggregations().get(AGG_COUNT)).getValue()).sum(); - // Worst severity having at least one issue - OptionalInt severityRating = severityBuckets.stream() - .filter(b -> ((InternalValueCount) b.getAggregations().get(AGG_COUNT)).getValue() != 0) - .mapToInt(b -> Severity.ALL.indexOf(b.getKeyAsString()) + 1) - .max(); - - long toReviewSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get(AGG_TO_REVIEW_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) - .getValue(); - long inReviewSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get(AGG_IN_REVIEW_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) - .getValue(); - long reviewedSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get(AGG_REVIEWED_SECURITY_HOTSPOTS)).getAggregations().get(AGG_COUNT)) - .getValue(); - - return new SecurityStandardCategoryStatistics(categoryName, vulnerabilities, severityRating, inReviewSecurityHotspots, toReviewSecurityHotspots, - reviewedSecurityHotspots, children); - } - - private static AggregationBuilder addSecurityReportSubAggregations(AggregationBuilder categoriesAggs, boolean includeCwe, Optional<Set<String>> cwesInCategory) { - AggregationBuilder aggregationBuilder = addSecurityReportIssueCountAggregations(categoriesAggs); - if (includeCwe) { - final TermsAggregationBuilder cwesAgg = AggregationBuilders.terms(AGG_CWES) - .field(FIELD_ISSUE_CWE) - // 100 should be enough to display all CWEs. If not, the UI will be broken anyway - .size(MAX_FACET_SIZE); - cwesInCategory.ifPresent(set -> { - cwesAgg.includeExclude(new IncludeExclude(set.toArray(new String[0]), new String[0])); - }); - categoriesAggs - .subAggregation(addSecurityReportIssueCountAggregations(cwesAgg)); - } - return aggregationBuilder; - } - - private static AggregationBuilder addSecurityReportIssueCountAggregations(AggregationBuilder categoryAggs) { - return categoryAggs - .subAggregation( - AggregationBuilders.filter(AGG_VULNERABILITIES, NON_RESOLVED_VULNERABILITIES_FILTER) - .subAggregation( - AggregationBuilders.terms(AGG_SEVERITIES).field(FIELD_ISSUE_SEVERITY) - .subAggregation( - AggregationBuilders.count(AGG_COUNT).field(FIELD_ISSUE_KEY)))) - .subAggregation(AggregationBuilders.filter(AGG_TO_REVIEW_SECURITY_HOTSPOTS, TO_REVIEW_HOTSPOTS_FILTER) - .subAggregation( - AggregationBuilders.count(AGG_COUNT).field(FIELD_ISSUE_KEY))) - .subAggregation(AggregationBuilders.filter(AGG_IN_REVIEW_SECURITY_HOTSPOTS, IN_REVIEW_HOTSPOTS_FILTER) - .subAggregation( - AggregationBuilders.count(AGG_COUNT).field(FIELD_ISSUE_KEY))) - .subAggregation(AggregationBuilders.filter(AGG_REVIEWED_SECURITY_HOTSPOTS, REVIEWED_HOTSPOTS_FILTER) - .subAggregation( - AggregationBuilders.count(AGG_COUNT).field(FIELD_ISSUE_KEY))); - } - - private SearchRequestBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) { - BoolQueryBuilder componentFilter = boolQuery(); - if (isViewOrApp) { - IndexType.IndexMainType mainType = TYPE_VIEW; - componentFilter.filter(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID, - new TermsLookup( - mainType.getIndex().getName(), - mainType.getType(), - projectUuid, - ViewIndexDefinition.FIELD_PROJECTS))); - } else { - componentFilter.filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectUuid)); - } - return client.prepareSearch(TYPE_ISSUE.getMainType()) - .setQuery( - componentFilter - .should(NON_RESOLVED_VULNERABILITIES_FILTER) - .should(TO_REVIEW_HOTSPOTS_FILTER) - .should(IN_REVIEW_HOTSPOTS_FILTER) - .should(REVIEWED_HOTSPOTS_FILTER) - .minimumShouldMatch(1)) - .setSize(0); - } - -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueQuery.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueQuery.java deleted file mode 100644 index 9109b3fb6f4..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueQuery.java +++ /dev/null @@ -1,553 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import com.google.common.collect.ImmutableSet; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.Map; -import java.util.Set; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; -import org.apache.commons.lang.builder.ReflectionToStringBuilder; -import org.sonar.db.rule.RuleDefinitionDto; - -import static com.google.common.base.Preconditions.checkArgument; -import static org.sonar.server.es.SearchOptions.MAX_LIMIT; - -/** - * @since 3.6 - */ -public class IssueQuery { - - public static final String SORT_BY_CREATION_DATE = "CREATION_DATE"; - public static final String SORT_BY_UPDATE_DATE = "UPDATE_DATE"; - public static final String SORT_BY_CLOSE_DATE = "CLOSE_DATE"; - /** - * @deprecated since 7.2, it's no more possible to sort by assignee - */ - @Deprecated - public static final String SORT_BY_ASSIGNEE = "ASSIGNEE"; - public static final String SORT_BY_SEVERITY = "SEVERITY"; - public static final String SORT_BY_STATUS = "STATUS"; - - /** - * Sort by project, file path then line id - */ - public static final String SORT_BY_FILE_LINE = "FILE_LINE"; - - public static final Set<String> SORTS = ImmutableSet.of(SORT_BY_CREATION_DATE, SORT_BY_UPDATE_DATE, SORT_BY_CLOSE_DATE, SORT_BY_ASSIGNEE, SORT_BY_SEVERITY, - SORT_BY_STATUS, SORT_BY_FILE_LINE); - - private final Collection<String> issueKeys; - private final Collection<String> severities; - private final Collection<String> statuses; - private final Collection<String> resolutions; - private final Collection<String> components; - private final Collection<String> modules; - private final Collection<String> moduleRoots; - private final Collection<String> projects; - private final Collection<String> directories; - private final Collection<String> files; - private final Collection<String> views; - private final Collection<RuleDefinitionDto> rules; - private final Collection<String> assignees; - private final Collection<String> authors; - private final Collection<String> languages; - private final Collection<String> tags; - private final Collection<String> types; - private final Collection<String> owaspTop10; - private final Collection<String> sansTop25; - private final Collection<String> cwe; - private final Collection<String> sonarsourceSecurity; - private final Map<String, PeriodStart> createdAfterByProjectUuids; - private final Boolean onComponentOnly; - private final Boolean assigned; - private final Boolean resolved; - private final Date createdAt; - private final PeriodStart createdAfter; - private final Date createdBefore; - private final String sort; - private final Boolean asc; - private final String facetMode; - private final String organizationUuid; - private final String branchUuid; - private final boolean mainBranch; - - private IssueQuery(Builder builder) { - this.issueKeys = defaultCollection(builder.issueKeys); - this.severities = defaultCollection(builder.severities); - this.statuses = defaultCollection(builder.statuses); - this.resolutions = defaultCollection(builder.resolutions); - this.components = defaultCollection(builder.components); - this.modules = defaultCollection(builder.modules); - this.moduleRoots = defaultCollection(builder.moduleRoots); - this.projects = defaultCollection(builder.projects); - this.directories = defaultCollection(builder.directories); - this.files = defaultCollection(builder.files); - this.views = defaultCollection(builder.views); - this.rules = defaultCollection(builder.rules); - this.assignees = defaultCollection(builder.assigneeUuids); - this.authors = defaultCollection(builder.authors); - this.languages = defaultCollection(builder.languages); - this.tags = defaultCollection(builder.tags); - this.types = defaultCollection(builder.types); - this.owaspTop10 = defaultCollection(builder.owaspTop10); - this.sansTop25 = defaultCollection(builder.sansTop25); - this.cwe = defaultCollection(builder.cwe); - this.sonarsourceSecurity = defaultCollection(builder.sonarsourceSecurity); - this.createdAfterByProjectUuids = defaultMap(builder.createdAfterByProjectUuids); - this.onComponentOnly = builder.onComponentOnly; - this.assigned = builder.assigned; - this.resolved = builder.resolved; - this.createdAt = builder.createdAt; - this.createdAfter = builder.createdAfter; - this.createdBefore = builder.createdBefore; - this.sort = builder.sort; - this.asc = builder.asc; - this.facetMode = builder.facetMode; - this.organizationUuid = builder.organizationUuid; - this.branchUuid = builder.branchUuid; - this.mainBranch = builder.mainBranch; - } - - public Collection<String> issueKeys() { - return issueKeys; - } - - public Collection<String> severities() { - return severities; - } - - public Collection<String> statuses() { - return statuses; - } - - public Collection<String> resolutions() { - return resolutions; - } - - public Collection<String> componentUuids() { - return components; - } - - public Collection<String> moduleUuids() { - return modules; - } - - public Collection<String> moduleRootUuids() { - return moduleRoots; - } - - public Collection<String> projectUuids() { - return projects; - } - - public Collection<String> directories() { - return directories; - } - - public Collection<String> fileUuids() { - return files; - } - - public Collection<String> viewUuids() { - return views; - } - - public Collection<RuleDefinitionDto> rules() { - return rules; - } - - public Collection<String> assignees() { - return assignees; - } - - public Collection<String> authors() { - return authors; - } - - public Collection<String> languages() { - return languages; - } - - public Collection<String> tags() { - return tags; - } - - public Collection<String> types() { - return types; - } - - public Collection<String> owaspTop10() { - return owaspTop10; - } - - public Collection<String> sansTop25() { - return sansTop25; - } - - public Collection<String> cwe() { - return cwe; - } - - public Collection<String> sonarsourceSecurity() { - return sonarsourceSecurity; - } - - public Map<String, PeriodStart> createdAfterByProjectUuids() { - return createdAfterByProjectUuids; - } - - @CheckForNull - public Boolean onComponentOnly() { - return onComponentOnly; - } - - @CheckForNull - public Boolean assigned() { - return assigned; - } - - @CheckForNull - public Boolean resolved() { - return resolved; - } - - @CheckForNull - public PeriodStart createdAfter() { - return createdAfter; - } - - @CheckForNull - public Date createdAt() { - return createdAt == null ? null : new Date(createdAt.getTime()); - } - - @CheckForNull - public Date createdBefore() { - return createdBefore == null ? null : new Date(createdBefore.getTime()); - } - - @CheckForNull - public String sort() { - return sort; - } - - @CheckForNull - public Boolean asc() { - return asc; - } - - @CheckForNull - public String organizationUuid() { - return organizationUuid; - } - - @CheckForNull - public String branchUuid() { - return branchUuid; - } - - public boolean isMainBranch() { - return mainBranch; - } - - public String facetMode() { - return facetMode; - } - - @Override - public String toString() { - return ReflectionToStringBuilder.toString(this); - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - private Collection<String> issueKeys; - private Collection<String> severities; - private Collection<String> statuses; - private Collection<String> resolutions; - private Collection<String> components; - private Collection<String> modules; - private Collection<String> moduleRoots; - private Collection<String> projects; - private Collection<String> directories; - private Collection<String> files; - private Collection<String> views; - private Collection<RuleDefinitionDto> rules; - private Collection<String> assigneeUuids; - private Collection<String> authors; - private Collection<String> languages; - private Collection<String> tags; - private Collection<String> types; - private Collection<String> owaspTop10; - private Collection<String> sansTop25; - private Collection<String> cwe; - private Collection<String> sonarsourceSecurity; - private Map<String, PeriodStart> createdAfterByProjectUuids; - private Boolean onComponentOnly = false; - private Boolean assigned = null; - private Boolean resolved = null; - private Date createdAt; - private PeriodStart createdAfter; - private Date createdBefore; - private String sort; - private Boolean asc = false; - private String facetMode; - private String organizationUuid; - private String branchUuid; - private boolean mainBranch = true; - - private Builder() { - - } - - public Builder issueKeys(@Nullable Collection<String> l) { - this.issueKeys = l; - return this; - } - - public Builder severities(@Nullable Collection<String> l) { - this.severities = l; - return this; - } - - public Builder statuses(@Nullable Collection<String> l) { - this.statuses = l; - return this; - } - - public Builder resolutions(@Nullable Collection<String> l) { - this.resolutions = l; - return this; - } - - public Builder componentUuids(@Nullable Collection<String> l) { - this.components = l; - return this; - } - - public Builder moduleUuids(@Nullable Collection<String> l) { - this.modules = l; - return this; - } - - public Builder moduleRootUuids(@Nullable Collection<String> l) { - this.moduleRoots = l; - return this; - } - - public Builder projectUuids(@Nullable Collection<String> l) { - this.projects = l; - return this; - } - - public Builder directories(@Nullable Collection<String> l) { - this.directories = l; - return this; - } - - public Builder fileUuids(@Nullable Collection<String> l) { - this.files = l; - return this; - } - - public Builder viewUuids(@Nullable Collection<String> l) { - this.views = l; - return this; - } - - public Builder rules(@Nullable Collection<RuleDefinitionDto> rules) { - this.rules = rules; - return this; - } - - public Builder assigneeUuids(@Nullable Collection<String> l) { - this.assigneeUuids = l; - return this; - } - - public Builder authors(@Nullable Collection<String> l) { - this.authors = l; - return this; - } - - public Builder languages(@Nullable Collection<String> l) { - this.languages = l; - return this; - } - - public Builder tags(@Nullable Collection<String> t) { - this.tags = t; - return this; - } - - public Builder types(@Nullable Collection<String> t) { - this.types = t; - return this; - } - - public Builder owaspTop10(@Nullable Collection<String> o) { - this.owaspTop10 = o; - return this; - } - - public Builder sansTop25(@Nullable Collection<String> s) { - this.sansTop25 = s; - return this; - } - - public Builder cwe(@Nullable Collection<String> cwe) { - this.cwe = cwe; - return this; - } - - public Builder sonarsourceSecurity(@Nullable Collection<String> sonarsourceSecurity) { - this.sonarsourceSecurity = sonarsourceSecurity; - return this; - } - - public Builder createdAfterByProjectUuids(@Nullable Map<String, PeriodStart> createdAfterByProjectUuids) { - this.createdAfterByProjectUuids = createdAfterByProjectUuids; - return this; - } - - /** - * If true, it will return only issues on the passed component(s) - * If false, it will return all issues on the passed component(s) and their descendants - */ - public Builder onComponentOnly(@Nullable Boolean b) { - this.onComponentOnly = b; - return this; - } - - /** - * If true, it will return all issues assigned to someone - * If false, it will return all issues not assigned to someone - */ - public Builder assigned(@Nullable Boolean b) { - this.assigned = b; - return this; - } - - /** - * If true, it will return all resolved issues - * If false, it will return all none resolved issues - */ - public Builder resolved(@Nullable Boolean resolved) { - this.resolved = resolved; - return this; - } - - public Builder createdAt(@Nullable Date d) { - this.createdAt = d == null ? null : new Date(d.getTime()); - return this; - } - - public Builder createdAfter(@Nullable Date d) { - this.createdAfter(d, true); - return this; - } - - public Builder createdAfter(@Nullable Date d, boolean inclusive) { - this.createdAfter = d == null ? null : new PeriodStart(new Date(d.getTime()), inclusive); - return this; - } - - public Builder createdBefore(@Nullable Date d) { - this.createdBefore = d == null ? null : new Date(d.getTime()); - return this; - } - - public Builder sort(@Nullable String s) { - if (s != null && !SORTS.contains(s)) { - throw new IllegalArgumentException("Bad sort field: " + s); - } - this.sort = s; - return this; - } - - public Builder asc(@Nullable Boolean asc) { - this.asc = asc; - return this; - } - - public IssueQuery build() { - if (issueKeys != null) { - checkArgument(issueKeys.size() <= MAX_LIMIT, "Number of issue keys must be less than " + MAX_LIMIT + " (got " + issueKeys.size() + ")"); - } - return new IssueQuery(this); - } - - public Builder facetMode(String facetMode) { - this.facetMode = facetMode; - return this; - } - - public Builder organizationUuid(String s) { - this.organizationUuid = s; - return this; - } - - public Builder branchUuid(@Nullable String s) { - this.branchUuid = s; - return this; - } - - public Builder mainBranch(boolean mainBranch) { - this.mainBranch = mainBranch; - return this; - } - } - - private static <T> Collection<T> defaultCollection(@Nullable Collection<T> c) { - return c == null ? Collections.emptyList() : Collections.unmodifiableCollection(c); - } - - private static <K, V> Map<K, V> defaultMap(@Nullable Map<K, V> map) { - return map == null ? Collections.emptyMap() : Collections.unmodifiableMap(map); - } - - public static class PeriodStart { - private final Date date; - private final boolean inclusive; - - public PeriodStart(Date date, boolean inclusive) { - this.date = date; - this.inclusive = inclusive; - - } - - public Date date() { - return date; - } - - public boolean inclusive() { - return inclusive; - } - - } - -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java deleted file mode 100644 index 7aacc93e2c7..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import com.google.common.base.Joiner; -import com.google.common.base.Strings; -import java.time.Clock; -import java.time.OffsetDateTime; -import java.time.Period; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import javax.annotation.CheckForNull; -import javax.annotation.Nullable; -import org.apache.commons.lang.BooleanUtils; -import org.sonar.api.resources.Qualifiers; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.server.ServerSide; -import org.sonar.api.web.UserRole; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.SnapshotDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.server.issue.SearchRequest; -import org.sonar.server.issue.index.IssueQuery.PeriodStart; -import org.sonar.server.user.UserSession; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.Collections2.transform; -import static java.lang.String.format; -import static java.util.Collections.singleton; -import static java.util.Collections.singletonList; -import static org.sonar.api.utils.DateUtils.longToDate; -import static org.sonar.api.utils.DateUtils.parseDateOrDateTime; -import static org.sonar.api.utils.DateUtils.parseEndingDateOrDateTime; -import static org.sonar.api.utils.DateUtils.parseStartingDateOrDateTime; -import static org.sonar.core.util.stream.MoreCollectors.toHashSet; -import static org.sonar.core.util.stream.MoreCollectors.toList; -import static org.sonar.core.util.stream.MoreCollectors.toSet; -import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_COMPONENT_KEYS; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_COMPONENT_UUIDS; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CREATED_AFTER; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CREATED_IN_LAST; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_SINCE_LEAK_PERIOD; - -/** - * This component is used to create an IssueQuery, in order to transform the component and component roots keys into uuid. - */ -@ServerSide -public class IssueQueryFactory { - - public static final String UNKNOWN = "<UNKNOWN>"; - private static final ComponentDto UNKNOWN_COMPONENT = new ComponentDto().setUuid(UNKNOWN).setProjectUuid(UNKNOWN); - - private final DbClient dbClient; - private final Clock clock; - private final UserSession userSession; - - public IssueQueryFactory(DbClient dbClient, Clock clock, UserSession userSession) { - this.dbClient = dbClient; - this.clock = clock; - this.userSession = userSession; - } - - public IssueQuery create(SearchRequest request) { - try (DbSession dbSession = dbClient.openSession(false)) { - IssueQuery.Builder builder = IssueQuery.builder() - .issueKeys(request.getIssues()) - .severities(request.getSeverities()) - .statuses(request.getStatuses()) - .resolutions(request.getResolutions()) - .resolved(request.getResolved()) - .rules(ruleKeysToRuleId(dbSession, request.getRules())) - .assigneeUuids(request.getAssigneeUuids()) - .authors(request.getAuthors()) - .languages(request.getLanguages()) - .tags(request.getTags()) - .types(request.getTypes()) - .owaspTop10(request.getOwaspTop10()) - .sansTop25(request.getSansTop25()) - .cwe(request.getCwe()) - .sonarsourceSecurity(request.getSonarsourceSecurity()) - .assigned(request.getAssigned()) - .createdAt(parseDateOrDateTime(request.getCreatedAt())) - .createdBefore(parseEndingDateOrDateTime(request.getCreatedBefore())) - .facetMode(request.getFacetMode()) - .organizationUuid(convertOrganizationKeyToUuid(dbSession, request.getOrganization())); - - List<ComponentDto> allComponents = new ArrayList<>(); - boolean effectiveOnComponentOnly = mergeDeprecatedComponentParameters(dbSession, request, allComponents); - addComponentParameters(builder, dbSession, effectiveOnComponentOnly, allComponents, request); - - setCreatedAfterFromRequest(dbSession, builder, request, allComponents); - String sort = request.getSort(); - if (!Strings.isNullOrEmpty(sort)) { - builder.sort(sort); - builder.asc(request.getAsc()); - } - return builder.build(); - } - } - - private void setCreatedAfterFromDates(IssueQuery.Builder builder, @Nullable Date createdAfter, @Nullable String createdInLast, boolean createdAfterInclusive) { - checkArgument(createdAfter == null || createdInLast == null, format("Parameters %s and %s cannot be set simultaneously", PARAM_CREATED_AFTER, PARAM_CREATED_IN_LAST)); - - Date actualCreatedAfter = createdAfter; - if (createdInLast != null) { - actualCreatedAfter = Date.from( - OffsetDateTime.now(clock) - .minus(Period.parse("P" + createdInLast.toUpperCase(Locale.ENGLISH))) - .toInstant()); - } - builder.createdAfter(actualCreatedAfter, createdAfterInclusive); - } - - @CheckForNull - private String convertOrganizationKeyToUuid(DbSession dbSession, @Nullable String organizationKey) { - if (organizationKey == null) { - return null; - } - Optional<OrganizationDto> organization = dbClient.organizationDao().selectByKey(dbSession, organizationKey); - return organization.map(OrganizationDto::getUuid).orElse(UNKNOWN); - } - - private void setCreatedAfterFromRequest(DbSession dbSession, IssueQuery.Builder builder, SearchRequest request, List<ComponentDto> componentUuids) { - Date createdAfter = parseStartingDateOrDateTime(request.getCreatedAfter()); - String createdInLast = request.getCreatedInLast(); - - if (request.getSinceLeakPeriod() == null || !request.getSinceLeakPeriod()) { - setCreatedAfterFromDates(builder, createdAfter, createdInLast, true); - } else { - checkArgument(createdAfter == null, "Parameters '%s' and '%s' cannot be set simultaneously", PARAM_CREATED_AFTER, PARAM_SINCE_LEAK_PERIOD); - checkArgument(componentUuids.size() == 1, "One and only one component must be provided when searching since leak period"); - ComponentDto component = componentUuids.iterator().next(); - Date createdAfterFromSnapshot = findCreatedAfterFromComponentUuid(dbSession, component); - setCreatedAfterFromDates(builder, createdAfterFromSnapshot, createdInLast, false); - } - } - - @CheckForNull - private Date findCreatedAfterFromComponentUuid(DbSession dbSession, ComponentDto component) { - Optional<SnapshotDto> snapshot = dbClient.snapshotDao().selectLastAnalysisByComponentUuid(dbSession, component.uuid()); - return snapshot.map(s -> longToDate(s.getPeriodDate())).orElse(null); - } - - private boolean mergeDeprecatedComponentParameters(DbSession session, SearchRequest request, List<ComponentDto> allComponents) { - Boolean onComponentOnly = request.getOnComponentOnly(); - Collection<String> components = request.getComponents(); - Collection<String> componentUuids = request.getComponentUuids(); - Collection<String> componentKeys = request.getComponentKeys(); - Collection<String> componentRootUuids = request.getComponentRootUuids(); - Collection<String> componentRoots = request.getComponentRoots(); - String branch = request.getBranch(); - String pullRequest = request.getPullRequest(); - - boolean effectiveOnComponentOnly = false; - - checkArgument(atMostOneNonNullElement(components, componentUuids, componentKeys, componentRootUuids, componentRoots), - "At most one of the following parameters can be provided: %s and %s", PARAM_COMPONENT_KEYS, PARAM_COMPONENT_UUIDS); - - if (componentRootUuids != null) { - allComponents.addAll(getComponentsFromUuids(session, componentRootUuids)); - } else if (componentRoots != null) { - allComponents.addAll(getComponentsFromKeys(session, componentRoots, branch, pullRequest)); - } else if (components != null) { - allComponents.addAll(getComponentsFromKeys(session, components, branch, pullRequest)); - effectiveOnComponentOnly = true; - } else if (componentUuids != null) { - allComponents.addAll(getComponentsFromUuids(session, componentUuids)); - effectiveOnComponentOnly = BooleanUtils.isTrue(onComponentOnly); - } else if (componentKeys != null) { - allComponents.addAll(getComponentsFromKeys(session, componentKeys, branch, pullRequest)); - effectiveOnComponentOnly = BooleanUtils.isTrue(onComponentOnly); - } - - return effectiveOnComponentOnly; - } - - private static boolean atMostOneNonNullElement(Object... objects) { - return Arrays.stream(objects) - .filter(Objects::nonNull) - .count() <= 1; - } - - private void addComponentParameters(IssueQuery.Builder builder, DbSession session, boolean onComponentOnly, List<ComponentDto> components, SearchRequest request) { - builder.onComponentOnly(onComponentOnly); - if (onComponentOnly) { - builder.componentUuids(components.stream().map(ComponentDto::uuid).collect(toList())); - setBranch(builder, components.get(0), request.getBranch(), request.getPullRequest()); - return; - } - - List<String> projectKeys = request.getProjectKeys(); - if (projectKeys != null) { - List<ComponentDto> projects = getComponentsFromKeys(session, projectKeys, request.getBranch(), request.getPullRequest()); - builder.projectUuids(projects.stream().map(IssueQueryFactory::toProjectUuid).collect(toList())); - setBranch(builder, projects.get(0), request.getBranch(), request.getPullRequest()); - } - builder.moduleUuids(request.getModuleUuids()); - builder.directories(request.getDirectories()); - builder.fileUuids(request.getFileUuids()); - - addComponentsBasedOnQualifier(builder, session, components, request); - } - - private void addComponentsBasedOnQualifier(IssueQuery.Builder builder, DbSession dbSession, List<ComponentDto> components, SearchRequest request) { - if (components.isEmpty()) { - return; - } - if (components.stream().map(ComponentDto::uuid).anyMatch(uuid -> uuid.equals(UNKNOWN))) { - builder.componentUuids(singleton(UNKNOWN)); - return; - } - - Set<String> qualifiers = components.stream().map(ComponentDto::qualifier).collect(toHashSet()); - checkArgument(qualifiers.size() == 1, "All components must have the same qualifier, found %s", String.join(",", qualifiers)); - - setBranch(builder, components.get(0), request.getBranch(), request.getPullRequest()); - String qualifier = qualifiers.iterator().next(); - switch (qualifier) { - case Qualifiers.VIEW: - case Qualifiers.SUBVIEW: - addViewsOrSubViews(builder, components); - break; - case Qualifiers.APP: - addApplications(builder, dbSession, components, request); - addProjectUuidsForApplication(builder, dbSession, request); - break; - case Qualifiers.PROJECT: - builder.projectUuids(components.stream().map(IssueQueryFactory::toProjectUuid).collect(toList())); - break; - case Qualifiers.MODULE: - builder.moduleRootUuids(components.stream().map(ComponentDto::uuid).collect(toList())); - break; - case Qualifiers.DIRECTORY: - addDirectories(builder, components); - break; - case Qualifiers.FILE: - case Qualifiers.UNIT_TEST_FILE: - builder.fileUuids(components.stream().map(ComponentDto::uuid).collect(toList())); - break; - default: - throw new IllegalArgumentException("Unable to set search root context for components " + Joiner.on(',').join(components)); - } - } - - private void addProjectUuidsForApplication(IssueQuery.Builder builder, DbSession session, SearchRequest request) { - List<String> projectKeys = request.getProjectKeys(); - if (projectKeys != null) { - // On application, branch should only be applied on the application, not on projects - List<ComponentDto> projects = getComponentsFromKeys(session, projectKeys, null, null); - builder.projectUuids(projects.stream().map(ComponentDto::uuid).collect(toList())); - } - } - - private void addViewsOrSubViews(IssueQuery.Builder builder, Collection<ComponentDto> viewOrSubViewUuids) { - List<String> filteredViewUuids = viewOrSubViewUuids.stream() - .filter(uuid -> userSession.hasComponentPermission(UserRole.USER, uuid)) - .map(ComponentDto::uuid) - .collect(Collectors.toList()); - if (filteredViewUuids.isEmpty()) { - filteredViewUuids.add(UNKNOWN); - } - builder.viewUuids(filteredViewUuids); - } - - private void addApplications(IssueQuery.Builder builder, DbSession dbSession, List<ComponentDto> applications, SearchRequest request) { - Set<String> authorizedApplicationUuids = applications.stream() - .filter(app -> userSession.hasComponentPermission(UserRole.USER, app)) - .map(ComponentDto::uuid) - .collect(toSet()); - - builder.viewUuids(authorizedApplicationUuids.isEmpty() ? singleton(UNKNOWN) : authorizedApplicationUuids); - addCreatedAfterByProjects(builder, dbSession, request, authorizedApplicationUuids); - } - - private void addCreatedAfterByProjects(IssueQuery.Builder builder, DbSession dbSession, SearchRequest request, Set<String> applicationUuids) { - if (request.getSinceLeakPeriod() == null || !request.getSinceLeakPeriod()) { - return; - } - - Set<String> projectUuids = applicationUuids.stream() - .flatMap(app -> dbClient.componentDao().selectProjectsFromView(dbSession, app, app).stream()) - .collect(toSet()); - - Map<String, PeriodStart> leakByProjects = dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, projectUuids) - .stream() - .filter(s -> s.getPeriodDate() != null) - .collect(uniqueIndex(SnapshotDto::getComponentUuid, s -> new PeriodStart(longToDate(s.getPeriodDate()), false))); - builder.createdAfterByProjectUuids(leakByProjects); - } - - private static void addDirectories(IssueQuery.Builder builder, List<ComponentDto> directories) { - Collection<String> directoryModuleUuids = new HashSet<>(); - Collection<String> directoryPaths = new HashSet<>(); - for (ComponentDto directory : directories) { - directoryModuleUuids.add(directory.moduleUuid()); - directoryPaths.add(directory.path()); - } - builder.moduleUuids(directoryModuleUuids); - builder.directories(directoryPaths); - } - - private List<ComponentDto> getComponentsFromKeys(DbSession dbSession, Collection<String> componentKeys, @Nullable String branch, @Nullable String pullRequest) { - List<ComponentDto> componentDtos; - if (branch != null) { - componentDtos = dbClient.componentDao().selectByKeysAndBranch(dbSession, componentKeys, branch); - } else if (pullRequest != null) { - componentDtos = dbClient.componentDao().selectByKeysAndPullRequest(dbSession, componentKeys, pullRequest); - } else { - componentDtos = dbClient.componentDao().selectByKeys(dbSession, componentKeys); - } - if (!componentKeys.isEmpty() && componentDtos.isEmpty()) { - return singletonList(UNKNOWN_COMPONENT); - } - return componentDtos; - } - - private List<ComponentDto> getComponentsFromUuids(DbSession dbSession, Collection<String> componentUuids) { - List<ComponentDto> componentDtos = dbClient.componentDao().selectByUuids(dbSession, componentUuids); - if (!componentUuids.isEmpty() && componentDtos.isEmpty()) { - return singletonList(UNKNOWN_COMPONENT); - } - return componentDtos; - } - - @CheckForNull - private Collection<RuleDefinitionDto> ruleKeysToRuleId(DbSession dbSession, @Nullable Collection<String> rules) { - if (rules != null) { - return dbClient.ruleDao().selectDefinitionByKeys(dbSession, transform(rules, RuleKey::parse)); - } - return Collections.emptyList(); - } - - private static String toProjectUuid(ComponentDto componentDto) { - String mainBranchProjectUuid = componentDto.getMainBranchProjectUuid(); - return mainBranchProjectUuid == null ? componentDto.projectUuid() : mainBranchProjectUuid; - } - - private static void setBranch(IssueQuery.Builder builder, ComponentDto component, @Nullable String branch, @Nullable String pullRequest) { - builder.branchUuid(branch == null && pullRequest == null ? null : component.projectUuid()); - builder.mainBranch(UNKNOWN_COMPONENT.equals(component) - || (branch == null && pullRequest == null) - || (branch != null && !branch.equals(component.getBranch())) - || (pullRequest != null && !pullRequest.equals(component.getPullRequest()))); - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/package-info.java deleted file mode 100644 index 5bbf5661d41..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.server.issue.index; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java deleted file mode 100644 index e60cf06a1db..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ /dev/null @@ -1,461 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.measure.index; - -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Multimap; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import javax.annotation.Nullable; -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; -import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; -import org.elasticsearch.search.aggregations.bucket.filter.Filter; -import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; -import org.elasticsearch.search.aggregations.bucket.nested.Nested; -import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.sum.Sum; -import org.elasticsearch.search.sort.FieldSortBuilder; -import org.elasticsearch.search.sort.NestedSortBuilder; -import org.sonar.api.measures.Metric; -import org.sonar.api.server.ServerSide; -import org.sonar.api.utils.System2; -import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.server.es.EsClient; -import org.sonar.server.es.SearchIdResult; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.es.StickyFacetBuilder; -import org.sonar.server.es.newindex.DefaultIndexSettingsElement; -import org.sonar.server.measure.index.ProjectMeasuresQuery.MetricCriterion; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Collections.emptyList; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; -import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.index.query.QueryBuilders.termsQuery; -import static org.elasticsearch.search.aggregations.AggregationBuilders.filters; -import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; -import static org.elasticsearch.search.sort.SortOrder.ASC; -import static org.elasticsearch.search.sort.SortOrder.DESC; -import static org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY; -import static org.sonar.api.measures.CoreMetrics.COVERAGE_KEY; -import static org.sonar.api.measures.CoreMetrics.DUPLICATED_LINES_DENSITY_KEY; -import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; -import static org.sonar.api.measures.CoreMetrics.NEW_COVERAGE_KEY; -import static org.sonar.api.measures.CoreMetrics.NEW_DUPLICATED_LINES_DENSITY_KEY; -import static org.sonar.api.measures.CoreMetrics.NEW_LINES_KEY; -import static org.sonar.api.measures.CoreMetrics.NEW_MAINTAINABILITY_RATING_KEY; -import static org.sonar.api.measures.CoreMetrics.NEW_RELIABILITY_RATING_KEY; -import static org.sonar.api.measures.CoreMetrics.NEW_SECURITY_RATING_KEY; -import static org.sonar.api.measures.CoreMetrics.RELIABILITY_RATING_KEY; -import static org.sonar.api.measures.CoreMetrics.SECURITY_RATING_KEY; -import static org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY; -import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; -import static org.sonar.server.es.EsUtils.termsToMap; -import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; -import static org.sonar.server.measure.index.ProjectMeasuresDoc.QUALITY_GATE_STATUS; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_LANGUAGES; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_LANGUAGE_DISTRIBUTION; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ORGANIZATION_UUID; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; -import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_LAST_ANALYSIS_DATE; -import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_NAME; -import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_LANGUAGES; -import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_TAGS; -import static org.sonarqube.ws.client.project.ProjectsWsParameters.MAX_PAGE_SIZE; - -@ServerSide -public class ProjectMeasuresIndex { - - public static final List<String> SUPPORTED_FACETS = ImmutableList.of( - NCLOC_KEY, - NEW_LINES_KEY, - DUPLICATED_LINES_DENSITY_KEY, - NEW_DUPLICATED_LINES_DENSITY_KEY, - COVERAGE_KEY, - NEW_COVERAGE_KEY, - SQALE_RATING_KEY, - NEW_MAINTAINABILITY_RATING_KEY, - RELIABILITY_RATING_KEY, - NEW_RELIABILITY_RATING_KEY, - SECURITY_RATING_KEY, - NEW_SECURITY_RATING_KEY, - ALERT_STATUS_KEY, - FILTER_LANGUAGES, - FILTER_TAGS); - - private static final Double[] LINES_THRESHOLDS = new Double[] {1_000d, 10_000d, 100_000d, 500_000d}; - private static final Double[] COVERAGE_THRESHOLDS = new Double[] {30d, 50d, 70d, 80d}; - private static final Double[] DUPLICATIONS_THRESHOLDS = new Double[] {3d, 5d, 10d, 20d}; - - private static final String FIELD_MEASURES_KEY = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY; - private static final String FIELD_MEASURES_VALUE = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE; - private static final String FIELD_DISTRIB_LANGUAGE = FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "." + ProjectMeasuresIndexDefinition.FIELD_DISTRIB_LANGUAGE; - private static final String FIELD_DISTRIB_NCLOC = FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "." + ProjectMeasuresIndexDefinition.FIELD_DISTRIB_NCLOC; - - private static final Map<String, FacetSetter> FACET_FACTORIES = ImmutableMap.<String, FacetSetter>builder() - .put(NCLOC_KEY, (esSearch, query, facetBuilder) -> addRangeFacet(esSearch, NCLOC_KEY, facetBuilder, LINES_THRESHOLDS)) - .put(NEW_LINES_KEY, (esSearch, query, facetBuilder) -> addRangeFacet(esSearch, NEW_LINES_KEY, facetBuilder, LINES_THRESHOLDS)) - .put(DUPLICATED_LINES_DENSITY_KEY, - (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, DUPLICATED_LINES_DENSITY_KEY, facetBuilder, DUPLICATIONS_THRESHOLDS)) - .put(NEW_DUPLICATED_LINES_DENSITY_KEY, - (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, NEW_DUPLICATED_LINES_DENSITY_KEY, facetBuilder, DUPLICATIONS_THRESHOLDS)) - .put(COVERAGE_KEY, (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, COVERAGE_KEY, facetBuilder, COVERAGE_THRESHOLDS)) - .put(NEW_COVERAGE_KEY, (esSearch, query, facetBuilder) -> addRangeFacetIncludingNoData(esSearch, NEW_COVERAGE_KEY, facetBuilder, COVERAGE_THRESHOLDS)) - .put(SQALE_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, SQALE_RATING_KEY, facetBuilder)) - .put(NEW_MAINTAINABILITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, NEW_MAINTAINABILITY_RATING_KEY, facetBuilder)) - .put(RELIABILITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, RELIABILITY_RATING_KEY, facetBuilder)) - .put(NEW_RELIABILITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, NEW_RELIABILITY_RATING_KEY, facetBuilder)) - .put(SECURITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, SECURITY_RATING_KEY, facetBuilder)) - .put(NEW_SECURITY_RATING_KEY, (esSearch, query, facetBuilder) -> addRatingFacet(esSearch, NEW_SECURITY_RATING_KEY, facetBuilder)) - .put(ALERT_STATUS_KEY, (esSearch, query, facetBuilder) -> esSearch.addAggregation(createStickyFacet(ALERT_STATUS_KEY, facetBuilder, createQualityGateFacet(query)))) - .put(FILTER_LANGUAGES, ProjectMeasuresIndex::addLanguagesFacet) - .put(FIELD_TAGS, ProjectMeasuresIndex::addTagsFacet) - .build(); - - private final EsClient client; - private final WebAuthorizationTypeSupport authorizationTypeSupport; - private final System2 system2; - - public ProjectMeasuresIndex(EsClient client, WebAuthorizationTypeSupport authorizationTypeSupport, System2 system2) { - this.client = client; - this.authorizationTypeSupport = authorizationTypeSupport; - this.system2 = system2; - } - - public SearchIdResult<String> search(ProjectMeasuresQuery query, SearchOptions searchOptions) { - SearchRequestBuilder requestBuilder = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setFetchSource(false) - .setFrom(searchOptions.getOffset()) - .setSize(searchOptions.getLimit()); - - BoolQueryBuilder esFilter = boolQuery(); - Map<String, QueryBuilder> filters = createFilters(query); - filters.values().forEach(esFilter::must); - requestBuilder.setQuery(esFilter); - - addFacets(requestBuilder, searchOptions, filters, query); - addSort(query, requestBuilder); - return new SearchIdResult<>(requestBuilder.get(), id -> id, system2.getDefaultTimeZone()); - } - - public ProjectMeasuresStatistics searchTelemetryStatistics() { - SearchRequestBuilder request = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setFetchSource(false) - .setSize(0); - - BoolQueryBuilder esFilter = boolQuery(); - request.setQuery(esFilter); - request.addAggregation(AggregationBuilders.terms(FIELD_LANGUAGES) - .field(FIELD_LANGUAGES) - .size(MAX_PAGE_SIZE) - .minDocCount(1) - .order(BucketOrder.count(false))); - request.addAggregation(AggregationBuilders.nested(FIELD_NCLOC_LANGUAGE_DISTRIBUTION, FIELD_NCLOC_LANGUAGE_DISTRIBUTION) - .subAggregation(AggregationBuilders.terms(FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "_terms") - .field(FIELD_DISTRIB_LANGUAGE) - .size(MAX_PAGE_SIZE) - .minDocCount(1) - .order(BucketOrder.count(false)) - .subAggregation(sum(FIELD_DISTRIB_NCLOC).field(FIELD_DISTRIB_NCLOC)))); - - request.addAggregation(AggregationBuilders.nested(NCLOC_KEY, FIELD_MEASURES) - .subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_KEY, NCLOC_KEY)) - .subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_VALUE)))); - - ProjectMeasuresStatistics.Builder statistics = ProjectMeasuresStatistics.builder(); - - SearchResponse response = request.get(); - statistics.setProjectCount(response.getHits().getTotalHits()); - Stream.of(NCLOC_KEY) - .map(metric -> (Nested) response.getAggregations().get(metric)) - .map(nested -> (Filter) nested.getAggregations().get(nested.getName() + "_filter")) - .map(filter -> (Sum) filter.getAggregations().get(filter.getName() + "_sum")) - .forEach(sum -> { - String metric = sum.getName().replace("_filter_sum", ""); - long value = Math.round(sum.getValue()); - statistics.setSum(metric, value); - }); - statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES))); - Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_DISTRIB_NCLOC)).getValue()); - Map<String, Long> nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_LANGUAGE_DISTRIBUTION)) - .map(nested -> (Terms) nested.getAggregations().get(nested.getName() + "_terms")) - .flatMap(terms -> terms.getBuckets().stream()) - .collect(MoreCollectors.uniqueIndex(Bucket::getKeyAsString, bucketToNcloc)); - statistics.setNclocByLanguage(nclocByLanguage); - - return statistics.build(); - } - - private static void addSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder) { - String sort = query.getSort(); - if (SORT_BY_NAME.equals(sort)) { - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), query.isAsc() ? ASC : DESC); - } else if (SORT_BY_LAST_ANALYSIS_DATE.equals(sort)) { - requestBuilder.addSort(FIELD_ANALYSED_AT, query.isAsc() ? ASC : DESC); - } else if (ALERT_STATUS_KEY.equals(sort)) { - requestBuilder.addSort(FIELD_QUALITY_GATE_STATUS, query.isAsc() ? ASC : DESC); - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); - } else { - addMetricSort(query, requestBuilder, sort); - requestBuilder.addSort(DefaultIndexSettingsElement.SORTABLE_ANALYZER.subField(FIELD_NAME), ASC); - } - // last sort is by key in order to be deterministic when same value - requestBuilder.addSort(FIELD_KEY, ASC); - } - - private static void addMetricSort(ProjectMeasuresQuery query, SearchRequestBuilder requestBuilder, String sort) { - requestBuilder.addSort( - new FieldSortBuilder(FIELD_MEASURES_VALUE) - .setNestedSort( - new NestedSortBuilder(FIELD_MEASURES) - .setFilter(termQuery(FIELD_MEASURES_KEY, sort))) - .order(query.isAsc() ? ASC : DESC)); - } - - private static void addRangeFacet(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder, Double... thresholds) { - esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder, createRangeFacet(metricKey, thresholds))); - } - - private static void addRangeFacetIncludingNoData(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder, Double... thresholds) { - esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder, - AggregationBuilders.filter("combined_" + metricKey, matchAllQuery()) - .subAggregation(createRangeFacet(metricKey, thresholds)) - .subAggregation(createNoDataFacet(metricKey)))); - } - - private static void addRatingFacet(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder) { - esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder, createRatingFacet(metricKey))); - } - - private static void addLanguagesFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder) { - esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_LANGUAGES, FILTER_LANGUAGES, query.getLanguages().map(Set::toArray).orElseGet(() -> new Object[] {}))); - } - - private static void addTagsFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder) { - esSearch.addAggregation(facetBuilder.buildStickyFacet(FIELD_TAGS, FILTER_TAGS, query.getTags().map(Set::toArray).orElseGet(() -> new Object[] {}))); - } - - private static void addFacets(SearchRequestBuilder esSearch, SearchOptions options, Map<String, QueryBuilder> filters, ProjectMeasuresQuery query) { - StickyFacetBuilder facetBuilder = new StickyFacetBuilder(matchAllQuery(), filters); - options.getFacets().stream() - .filter(FACET_FACTORIES::containsKey) - .map(FACET_FACTORIES::get) - .forEach(factory -> factory.addFacet(esSearch, query, facetBuilder)); - } - - private static AbstractAggregationBuilder createStickyFacet(String facetKey, StickyFacetBuilder facetBuilder, AbstractAggregationBuilder aggregationBuilder) { - BoolQueryBuilder facetFilter = facetBuilder.getStickyFacetFilter(facetKey); - return AggregationBuilders - .global(facetKey) - .subAggregation( - AggregationBuilders - .filter("facet_filter_" + facetKey, facetFilter) - .subAggregation(aggregationBuilder)); - } - - private static AbstractAggregationBuilder createRangeFacet(String metricKey, Double... thresholds) { - RangeAggregationBuilder rangeAgg = AggregationBuilders.range(metricKey) - .field(FIELD_MEASURES_VALUE); - final int lastIndex = thresholds.length - 1; - IntStream.range(0, thresholds.length) - .forEach(i -> { - if (i == 0) { - rangeAgg.addUnboundedTo(thresholds[0]); - rangeAgg.addRange(thresholds[0], thresholds[1]); - } else if (i == lastIndex) { - rangeAgg.addUnboundedFrom(thresholds[lastIndex]); - } else { - rangeAgg.addRange(thresholds[i], thresholds[i + 1]); - } - }); - - return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES) - .subAggregation( - AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey)) - .subAggregation(rangeAgg)); - } - - private static AbstractAggregationBuilder createNoDataFacet(String metricKey) { - return AggregationBuilders.filter( - "no_data_" + metricKey, - boolQuery().mustNot(nestedQuery(FIELD_MEASURES, termQuery(FIELD_MEASURES_KEY, metricKey), ScoreMode.Avg))); - } - - private static AbstractAggregationBuilder createRatingFacet(String metricKey) { - return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES) - .subAggregation( - AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey)) - .subAggregation(filters(metricKey, - new KeyedFilter("1", termQuery(FIELD_MEASURES_VALUE, 1d)), - new KeyedFilter("2", termQuery(FIELD_MEASURES_VALUE, 2d)), - new KeyedFilter("3", termQuery(FIELD_MEASURES_VALUE, 3d)), - new KeyedFilter("4", termQuery(FIELD_MEASURES_VALUE, 4d)), - new KeyedFilter("5", termQuery(FIELD_MEASURES_VALUE, 5d))))); - } - - private static AbstractAggregationBuilder createQualityGateFacet(ProjectMeasuresQuery projectMeasuresQuery) { - return filters( - ALERT_STATUS_KEY, - QUALITY_GATE_STATUS - .entrySet() - .stream() - .filter(qgs -> !(projectMeasuresQuery.isIgnoreWarning() && qgs.getKey().equals(Metric.Level.WARN.name()))) - .map(entry -> new KeyedFilter(entry.getKey(), termQuery(FIELD_QUALITY_GATE_STATUS, entry.getValue()))) - .toArray(KeyedFilter[]::new)); - } - - private Map<String, QueryBuilder> createFilters(ProjectMeasuresQuery query) { - Map<String, QueryBuilder> filters = new HashMap<>(); - filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())); - if (!query.isIgnoreAuthorization()) { - filters.put("__authorization", authorizationTypeSupport.createQueryFilter()); - } - Multimap<String, MetricCriterion> metricCriterionMultimap = ArrayListMultimap.create(); - query.getMetricCriteria().forEach(metricCriterion -> metricCriterionMultimap.put(metricCriterion.getMetricKey(), metricCriterion)); - metricCriterionMultimap.asMap().forEach((key, value) -> { - BoolQueryBuilder metricFilters = boolQuery(); - value - .stream() - .map(ProjectMeasuresIndex::toQuery) - .forEach(metricFilters::must); - filters.put(key, metricFilters); - }); - - query.getQualityGateStatus() - .ifPresent(qualityGateStatus -> filters.put(ALERT_STATUS_KEY, termQuery(FIELD_QUALITY_GATE_STATUS, QUALITY_GATE_STATUS.get(qualityGateStatus.name())))); - - query.getProjectUuids() - .ifPresent(projectUuids -> filters.put("ids", termsQuery("_id", projectUuids))); - - query.getLanguages() - .ifPresent(languages -> filters.put(FILTER_LANGUAGES, termsQuery(FIELD_LANGUAGES, languages))); - - query.getOrganizationUuid() - .ifPresent(organizationUuid -> filters.put(FIELD_ORGANIZATION_UUID, termQuery(FIELD_ORGANIZATION_UUID, organizationUuid))); - - query.getTags() - .ifPresent(tags -> filters.put(FIELD_TAGS, termsQuery(FIELD_TAGS, tags))); - - query.getQueryText() - .map(ProjectsTextSearchQueryFactory::createQuery) - .ifPresent(queryBuilder -> filters.put("textQuery", queryBuilder)); - return filters; - } - - private static QueryBuilder toQuery(MetricCriterion criterion) { - if (criterion.isNoData()) { - return boolQuery().mustNot( - nestedQuery( - FIELD_MEASURES, - termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey()), - ScoreMode.Avg)); - } - return nestedQuery( - FIELD_MEASURES, - boolQuery() - .filter(termQuery(FIELD_MEASURES_KEY, criterion.getMetricKey())) - .filter(toValueQuery(criterion)), - ScoreMode.Avg); - } - - private static QueryBuilder toValueQuery(MetricCriterion criterion) { - String fieldName = FIELD_MEASURES_VALUE; - - switch (criterion.getOperator()) { - case GT: - return rangeQuery(fieldName).gt(criterion.getValue()); - case GTE: - return rangeQuery(fieldName).gte(criterion.getValue()); - case LT: - return rangeQuery(fieldName).lt(criterion.getValue()); - case LTE: - return rangeQuery(fieldName).lte(criterion.getValue()); - case EQ: - return termQuery(fieldName, criterion.getValue()); - default: - throw new IllegalStateException("Metric criteria non supported: " + criterion.getOperator().name()); - } - } - - public List<String> searchTags(@Nullable String textQuery, int size) { - int maxPageSize = 500; - checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize); - if (size <= 0) { - return emptyList(); - } - - TermsAggregationBuilder tagFacet = AggregationBuilders.terms(FIELD_TAGS) - .field(FIELD_TAGS) - .size(size) - .minDocCount(1) - .order(BucketOrder.key(true)); - if (textQuery != null) { - tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null)); - } - - SearchRequestBuilder searchQuery = client - .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) - .setQuery(authorizationTypeSupport.createQueryFilter()) - .setFetchSource(false) - .setSize(0) - .addAggregation(tagFacet); - - Terms aggregation = searchQuery.get().getAggregations().get(FIELD_TAGS); - - return aggregation.getBuckets().stream() - .map(Bucket::getKeyAsString) - .collect(MoreCollectors.toList()); - } - - @FunctionalInterface - private interface FacetSetter { - void addFacet(SearchRequestBuilder esSearch, ProjectMeasuresQuery query, StickyFacetBuilder facetBuilder); - } - -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresQuery.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresQuery.java deleted file mode 100644 index 2074358fcae..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresQuery.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.measure.index; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import javax.annotation.Nullable; -import org.sonar.api.measures.Metric; - -import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; -import static java.util.Arrays.stream; -import static java.util.Objects.requireNonNull; - -public class ProjectMeasuresQuery { - - public static final String SORT_BY_NAME = "name"; - public static final String SORT_BY_LAST_ANALYSIS_DATE = "analysisDate"; - - private List<MetricCriterion> metricCriteria = new ArrayList<>(); - private Metric.Level qualityGateStatus; - private String organizationUuid; - private Set<String> projectUuids; - private Set<String> languages; - private Set<String> tags; - private String sort = SORT_BY_NAME; - private boolean asc = true; - private String queryText; - private boolean ignoreAuthorization; - private boolean ignoreWarning; - - public ProjectMeasuresQuery addMetricCriterion(MetricCriterion metricCriterion) { - this.metricCriteria.add(metricCriterion); - return this; - } - - public List<MetricCriterion> getMetricCriteria() { - return metricCriteria; - } - - public ProjectMeasuresQuery setQualityGateStatus(Metric.Level qualityGateStatus) { - this.qualityGateStatus = requireNonNull(qualityGateStatus); - return this; - } - - public Optional<Metric.Level> getQualityGateStatus() { - return Optional.ofNullable(qualityGateStatus); - } - - public ProjectMeasuresQuery setOrganizationUuid(@Nullable String organizationUuid) { - this.organizationUuid = organizationUuid; - return this; - } - - public Optional<String> getOrganizationUuid() { - return Optional.ofNullable(organizationUuid); - } - - public ProjectMeasuresQuery setProjectUuids(@Nullable Set<String> projectUuids) { - this.projectUuids = projectUuids; - return this; - } - - public Optional<Set<String>> getProjectUuids() { - return Optional.ofNullable(projectUuids); - } - - public ProjectMeasuresQuery setLanguages(@Nullable Set<String> languages) { - this.languages = languages; - return this; - } - - public Optional<Set<String>> getLanguages() { - return Optional.ofNullable(languages); - } - - public ProjectMeasuresQuery setTags(@Nullable Set<String> tags) { - this.tags = tags; - return this; - } - - public Optional<Set<String>> getTags() { - return Optional.ofNullable(tags); - } - - public Optional<String> getQueryText() { - return Optional.ofNullable(queryText); - } - - public ProjectMeasuresQuery setQueryText(@Nullable String queryText) { - this.queryText = queryText; - return this; - } - - public String getSort() { - return sort; - } - - public ProjectMeasuresQuery setSort(String sort) { - this.sort = requireNonNull(sort, "Sort cannot be null"); - return this; - } - - public boolean isAsc() { - return asc; - } - - public ProjectMeasuresQuery setAsc(boolean asc) { - this.asc = asc; - return this; - } - - public boolean isIgnoreAuthorization() { - return ignoreAuthorization; - } - - public ProjectMeasuresQuery setIgnoreAuthorization(boolean ignoreAuthorization) { - this.ignoreAuthorization = ignoreAuthorization; - return this; - } - - public boolean isIgnoreWarning() { - return ignoreWarning; - } - - public ProjectMeasuresQuery setIgnoreWarning(boolean ignoreWarning) { - this.ignoreWarning = ignoreWarning; - return this; - } - - public static class MetricCriterion { - private final String metricKey; - private final Operator operator; - @Nullable - private final Double value; - - private MetricCriterion(String metricKey, @Nullable Operator operator, @Nullable Double value) { - this.metricKey = metricKey; - this.operator = operator; - this.value = value; - } - - public String getMetricKey() { - return metricKey; - } - - public Operator getOperator() { - checkDataAvailable(); - return operator; - } - - public double getValue() { - checkDataAvailable(); - return value; - } - - public boolean isNoData() { - return value == null; - } - - public static MetricCriterion createNoData(String metricKey) { - return new MetricCriterion(requireNonNull(metricKey), null, null); - } - - public static MetricCriterion create(String metricKey, Operator operator, double value) { - return new MetricCriterion(requireNonNull(metricKey), requireNonNull(operator), value); - } - - private void checkDataAvailable() { - checkState(!isNoData(), "The criterion for metric %s has no data", metricKey); - } - } - - public enum Operator { - LT("<"), LTE("<="), GT(">"), GTE(">="), EQ("="), IN("in"); - - String value; - - Operator(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - public static Operator getByValue(String value) { - return stream(Operator.values()) - .filter(operator -> operator.getValue().equalsIgnoreCase(value)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException(format("Unknown operator '%s'", value))); - } - } - -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsEsModule.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsEsModule.java deleted file mode 100644 index 4517de975e7..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsEsModule.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.measure.index; - -import org.sonar.core.platform.Module; - -public class ProjectsEsModule extends Module { - @Override - protected void configureModule() { - add( - ProjectMeasuresIndexDefinition.class, - ProjectMeasuresIndex.class, - ProjectMeasuresIndexer.class); - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsTextSearchQueryFactory.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsTextSearchQueryFactory.java deleted file mode 100644 index 63a487a0a37..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsTextSearchQueryFactory.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.measure.index; - -import java.util.Arrays; -import java.util.Locale; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Stream; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.sonar.server.es.newindex.DefaultIndexSettings; - -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; -import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME; - -/** - * This class is used in order to do some advanced full text search on projects key and name - */ -class ProjectsTextSearchQueryFactory { - - private ProjectsTextSearchQueryFactory() { - // Only static methods - } - - static QueryBuilder createQuery(String queryText) { - BoolQueryBuilder featureQuery = boolQuery(); - Arrays.stream(ComponentTextSearchFeature.values()) - .map(f -> f.getQuery(queryText)) - .forEach(featureQuery::should); - return featureQuery; - } - - private enum ComponentTextSearchFeature { - - EXACT_IGNORE_CASE { - @Override - QueryBuilder getQuery(String queryText) { - return matchQuery(SORTABLE_ANALYZER.subField(FIELD_NAME), queryText) - .boost(2.5f); - } - }, - PREFIX { - @Override - QueryBuilder getQuery(String queryText) { - return prefixAndPartialQuery(queryText, FIELD_NAME, FIELD_NAME) - .boost(2f); - } - }, - PREFIX_IGNORE_CASE { - @Override - QueryBuilder getQuery(String queryText) { - String lowerCaseQueryText = queryText.toLowerCase(Locale.ENGLISH); - return prefixAndPartialQuery(lowerCaseQueryText, SORTABLE_ANALYZER.subField(FIELD_NAME), FIELD_NAME) - .boost(3f); - } - }, - PARTIAL { - @Override - QueryBuilder getQuery(String queryText) { - BoolQueryBuilder queryBuilder = boolQuery(); - split(queryText) - .map(text -> partialTermQuery(text, FIELD_NAME)) - .forEach(queryBuilder::must); - return queryBuilder - .boost(0.5f); - } - }, - KEY { - @Override - QueryBuilder getQuery(String queryText) { - return matchQuery(SORTABLE_ANALYZER.subField(FIELD_KEY), queryText) - .boost(50f); - } - }; - - abstract QueryBuilder getQuery(String queryText); - - protected Stream<String> split(String queryText) { - return Arrays.stream( - queryText.split(DefaultIndexSettings.SEARCH_TERM_TOKENIZER_PATTERN)) - .filter(StringUtils::isNotEmpty); - } - - protected BoolQueryBuilder prefixAndPartialQuery(String queryText, String fieldName, String originalFieldName) { - BoolQueryBuilder queryBuilder = boolQuery(); - AtomicBoolean first = new AtomicBoolean(true); - split(queryText) - .map(queryTerm -> { - if (first.getAndSet(false)) { - return prefixQuery(fieldName, queryTerm); - } - return partialTermQuery(queryTerm, originalFieldName); - }) - .forEach(queryBuilder::must); - return queryBuilder; - } - - protected MatchQueryBuilder partialTermQuery(String queryTerm, String fieldName) { - // We will truncate the search to the maximum length of nGrams in the index. - // Otherwise the search would for sure not find any results. - String truncatedQuery = StringUtils.left(queryTerm, DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH); - return matchQuery(SEARCH_GRAMS_ANALYZER.subField(fieldName), truncatedQuery); - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/package-info.java deleted file mode 100644 index 28d89c96888..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.server.measure.index; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java deleted file mode 100644 index f509a15b099..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableSet; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.es.EsQueueDto; -import org.sonar.server.es.BulkIndexer; -import org.sonar.server.es.BulkIndexer.Size; -import org.sonar.server.es.EsClient; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.IndexingResult; -import org.sonar.server.es.OneToOneResilientIndexingListener; -import org.sonar.server.es.ProjectIndexer; - -import static java.util.Collections.emptyList; -import static org.sonar.core.util.stream.MoreCollectors.toArrayList; - -/** - * Populates the types "authorization" of each index requiring project - * authorization. - */ -public class PermissionIndexer implements ProjectIndexer { - - private final DbClient dbClient; - private final EsClient esClient; - private final Collection<AuthorizationScope> authorizationScopes; - private final Map<String, IndexType> indexTypeByFormat; - - public PermissionIndexer(DbClient dbClient, EsClient esClient, NeedAuthorizationIndexer... needAuthorizationIndexers) { - this(dbClient, esClient, Arrays.stream(needAuthorizationIndexers) - .map(NeedAuthorizationIndexer::getAuthorizationScope) - .collect(MoreCollectors.toList(needAuthorizationIndexers.length))); - } - - @VisibleForTesting - public PermissionIndexer(DbClient dbClient, EsClient esClient, Collection<AuthorizationScope> authorizationScopes) { - this.dbClient = dbClient; - this.esClient = esClient; - this.authorizationScopes = authorizationScopes; - this.indexTypeByFormat = authorizationScopes.stream() - .map(AuthorizationScope::getIndexType) - .collect(MoreCollectors.uniqueIndex(IndexType.IndexMainType::format, t -> t, authorizationScopes.size())); - } - - @Override - public Set<IndexType> getIndexTypes() { - return ImmutableSet.copyOf(indexTypeByFormat.values()); - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - // TODO do not load everything in memory. Db rows should be scrolled. - List<IndexPermissions> authorizations = getAllAuthorizations(); - Stream<AuthorizationScope> scopes = getScopes(uninitializedIndexTypes); - index(authorizations, scopes, Size.LARGE); - } - - @VisibleForTesting - void index(List<IndexPermissions> authorizations) { - index(authorizations, authorizationScopes.stream(), Size.REGULAR); - } - - @Override - public void indexOnAnalysis(String branchUuid) { - // nothing to do, permissions don't change during an analysis - } - - @Override - public Collection<EsQueueDto> prepareForRecovery(DbSession dbSession, Collection<String> projectUuids, ProjectIndexer.Cause cause) { - switch (cause) { - case MEASURE_CHANGE: - case PROJECT_KEY_UPDATE: - case PROJECT_TAGS_UPDATE: - // nothing to change. Measures, project key and tags are not part of this index - return emptyList(); - - case PROJECT_CREATION: - case PROJECT_DELETION: - case PERMISSION_CHANGE: - return insertIntoEsQueue(dbSession, projectUuids); - - default: - // defensive case - throw new IllegalStateException("Unsupported cause: " + cause); - } - } - - private Collection<EsQueueDto> insertIntoEsQueue(DbSession dbSession, Collection<String> projectUuids) { - List<EsQueueDto> items = indexTypeByFormat.values().stream() - .flatMap(indexType -> projectUuids.stream().map(projectUuid -> EsQueueDto.create(indexType.format(), AuthorizationDoc.idOf(projectUuid), null, projectUuid))) - .collect(toArrayList()); - - dbClient.esQueueDao().insert(dbSession, items); - return items; - } - - private void index(Collection<IndexPermissions> authorizations, Stream<AuthorizationScope> scopes, Size bulkSize) { - if (authorizations.isEmpty()) { - return; - } - - // index each authorization in each scope - scopes.forEach(scope -> { - IndexType indexType = scope.getIndexType(); - - BulkIndexer bulkIndexer = new BulkIndexer(esClient, indexType, bulkSize); - bulkIndexer.start(); - - authorizations.stream() - .filter(scope.getProjectPredicate()) - .map(dto -> AuthorizationDoc.fromDto(indexType, dto).toIndexRequest()) - .forEach(bulkIndexer::add); - - bulkIndexer.stop(); - }); - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - IndexingResult result = new IndexingResult(); - - List<BulkIndexer> bulkIndexers = items.stream() - .map(EsQueueDto::getDocType) - .distinct() - .map(indexTypeByFormat::get) - .filter(Objects::nonNull) - .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) - .collect(Collectors.toList()); - - if (bulkIndexers.isEmpty()) { - return result; - } - - bulkIndexers.forEach(BulkIndexer::start); - - PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); - Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId) - .map(AuthorizationDoc::projectUuidOf) - .collect(MoreCollectors.toHashSet()); - permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> { - remainingProjectUuids.remove(p.getProjectUuid()); - bulkIndexers.forEach(bi -> bi.add(AuthorizationDoc.fromDto(bi.getIndexType(), p).toIndexRequest())); - }); - - // the remaining references on projects that don't exist in db. They must - // be deleted from index. - remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> { - String authorizationDocId = AuthorizationDoc.idOf(projectUuid); - bi.addDeletion(bi.getIndexType(), authorizationDocId, authorizationDocId); - })); - - bulkIndexers.forEach(b -> result.add(b.stop())); - - return result; - } - - private Stream<AuthorizationScope> getScopes(Set<IndexType> indexTypes) { - return authorizationScopes.stream() - .filter(scope -> indexTypes.contains(scope.getIndexType())); - } - - private List<IndexPermissions> getAllAuthorizations() { - try (DbSession dbSession = dbClient.openSession(false)) { - return new PermissionIndexerDao().selectAll(dbClient, dbSession); - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java deleted file mode 100644 index a819ea71d4a..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import com.google.common.collect.ImmutableList; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang.StringUtils; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; - -import static org.apache.commons.lang.StringUtils.repeat; -import static org.sonar.db.DatabaseUtils.executeLargeInputs; - -/** - * No streaming because of union of joins -> no need to use ResultSetIterator - */ -public class PermissionIndexerDao { - - private enum RowKind { - USER, GROUP, ANYONE, NONE - } - - private static final String SQL_TEMPLATE = "SELECT " + - " project_authorization.kind as kind, " + - " project_authorization.project as project, " + - " project_authorization.user_id as user_id, " + - " project_authorization.group_id as group_id, " + - " project_authorization.qualifier as qualifier " + - "FROM ( " + - - // users - - " SELECT '" + RowKind.USER + "' as kind," + - " projects.uuid AS project, " + - " projects.qualifier AS qualifier, " + - " user_roles.user_id AS user_id, " + - " NULL AS group_id " + - " FROM projects " + - " INNER JOIN user_roles ON user_roles.resource_id = projects.id AND user_roles.role = 'user' " + - " WHERE " + - " (projects.qualifier = 'TRK' " + - " or projects.qualifier = 'VW' " + - " or projects.qualifier = 'APP') " + - " AND projects.copy_component_uuid is NULL " + - " {projectsCondition} " + - " UNION " + - - // groups - - " SELECT '" + RowKind.GROUP + "' as kind," + - " projects.uuid AS project, " + - " projects.qualifier AS qualifier, " + - " NULL AS user_id, " + - " groups.id AS group_id " + - " FROM projects " + - " INNER JOIN group_roles ON group_roles.resource_id = projects.id AND group_roles.role = 'user' " + - " INNER JOIN groups ON groups.id = group_roles.group_id " + - " WHERE " + - " (projects.qualifier = 'TRK' " + - " or projects.qualifier = 'VW' " + - " or projects.qualifier = 'APP') " + - " AND projects.copy_component_uuid is NULL " + - " {projectsCondition} " + - " AND group_id IS NOT NULL " + - " UNION " + - - // public projects are accessible to any one - - " SELECT '" + RowKind.ANYONE + "' as kind," + - " projects.uuid AS project, " + - " projects.qualifier AS qualifier, " + - " NULL AS user_id, " + - " NULL AS group_id " + - " FROM projects " + - " WHERE " + - " (projects.qualifier = 'TRK' " + - " or projects.qualifier = 'VW' " + - " or projects.qualifier = 'APP') " + - " AND projects.copy_component_uuid is NULL " + - " AND projects.private = ? " + - " {projectsCondition} " + - " UNION " + - - // private project is returned when no authorization - " SELECT '" + RowKind.NONE + "' as kind," + - " projects.uuid AS project, " + - " projects.qualifier AS qualifier, " + - " NULL AS user_id, " + - " NULL AS group_id " + - " FROM projects " + - " WHERE " + - " (projects.qualifier = 'TRK' " + - " or projects.qualifier = 'VW' " + - " or projects.qualifier = 'APP') " + - " AND projects.copy_component_uuid is NULL " + - " AND projects.private = ? " + - " {projectsCondition} " + - - " ) project_authorization"; - - List<IndexPermissions> selectAll(DbClient dbClient, DbSession session) { - return doSelectByProjects(dbClient, session, Collections.emptyList()); - } - - List<IndexPermissions> selectByUuids(DbClient dbClient, DbSession session, Collection<String> projectOrViewUuids) { - return executeLargeInputs(projectOrViewUuids, subProjectOrViewUuids -> doSelectByProjects(dbClient, session, subProjectOrViewUuids)); - } - - private static List<IndexPermissions> doSelectByProjects(DbClient dbClient, DbSession session, List<String> projectUuids) { - try { - Map<String, IndexPermissions> dtosByProjectUuid = new HashMap<>(); - try (PreparedStatement stmt = createStatement(dbClient, session, projectUuids); - ResultSet rs = stmt.executeQuery()) { - while (rs.next()) { - processRow(rs, dtosByProjectUuid); - } - return ImmutableList.copyOf(dtosByProjectUuid.values()); - } - } catch (SQLException e) { - throw new IllegalStateException("Fail to select authorizations", e); - } - } - - private static PreparedStatement createStatement(DbClient dbClient, DbSession session, List<String> projectUuids) throws SQLException { - String sql; - if (projectUuids.isEmpty()) { - sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", ""); - } else { - sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", " AND projects.uuid in (" + repeat("?", ", ", projectUuids.size()) + ")"); - } - PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); - int index = 1; - // query for RowKind.USER - index = populateProjectUuidPlaceholders(stmt, projectUuids, index); - // query for RowKind.GROUP - index = populateProjectUuidPlaceholders(stmt, projectUuids, index); - // query for RowKind.ANYONE - index = setPrivateProjectPlaceHolder(stmt, index, false); - index = populateProjectUuidPlaceholders(stmt, projectUuids, index); - // query for RowKind.NONE - index = setPrivateProjectPlaceHolder(stmt, index, true); - populateProjectUuidPlaceholders(stmt, projectUuids, index); - return stmt; - } - - private static int populateProjectUuidPlaceholders(PreparedStatement stmt, List<String> projectUuids, int index) throws SQLException { - int newIndex = index; - for (String projectUuid : projectUuids) { - stmt.setString(newIndex, projectUuid); - newIndex++; - } - return newIndex; - } - - private static int setPrivateProjectPlaceHolder(PreparedStatement stmt, int index, boolean isPrivate) throws SQLException { - int newIndex = index; - stmt.setBoolean(newIndex, isPrivate); - newIndex++; - return newIndex; - } - - private static void processRow(ResultSet rs, Map<String, IndexPermissions> dtosByProjectUuid) throws SQLException { - RowKind rowKind = RowKind.valueOf(rs.getString(1)); - String projectUuid = rs.getString(2); - - IndexPermissions dto = dtosByProjectUuid.get(projectUuid); - if (dto == null) { - String qualifier = rs.getString(5); - dto = new IndexPermissions(projectUuid, qualifier); - dtosByProjectUuid.put(projectUuid, dto); - } - switch (rowKind) { - case NONE: - break; - case USER: - dto.addUserId(rs.getInt(3)); - break; - case GROUP: - dto.addGroupId(rs.getInt(4)); - break; - case ANYONE: - dto.allowAnyone(); - break; - } - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/WebAuthorizationTypeSupport.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/WebAuthorizationTypeSupport.java deleted file mode 100644 index eb4e2a53bfe..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/WebAuthorizationTypeSupport.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import java.util.Optional; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.join.query.JoinQueryBuilders; -import org.sonar.api.server.ServerSide; -import org.sonar.db.user.GroupDto; -import org.sonar.server.user.UserSession; - -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_ALLOW_ANYONE; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_GROUP_IDS; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_USER_IDS; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; - -@ServerSide -public class WebAuthorizationTypeSupport { - - private final UserSession userSession; - - public WebAuthorizationTypeSupport(UserSession userSession) { - this.userSession = userSession; - } - - /** - * Build a filter to restrict query to the documents on which - * user has read access. - */ - public QueryBuilder createQueryFilter() { - if (userSession.isRoot()) { - return QueryBuilders.matchAllQuery(); - } - - Integer userId = userSession.getUserId(); - BoolQueryBuilder filter = boolQuery(); - - // anyone - filter.should(QueryBuilders.termQuery(FIELD_ALLOW_ANYONE, true)); - - // users - Optional.ofNullable(userId) - .map(Integer::longValue) - .ifPresent(id -> filter.should(termQuery(FIELD_USER_IDS, id))); - - // groups - userSession.getGroups() - .stream() - .map(GroupDto::getId) - .forEach(groupId -> filter.should(termQuery(FIELD_GROUP_IDS, groupId))); - - return JoinQueryBuilders.hasParentQuery( - TYPE_AUTHORIZATION, - QueryBuilders.boolQuery().filter(filter), - false); - } -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/package-info.java deleted file mode 100644 index db70696c0f3..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -@ParametersAreNonnullByDefault -package org.sonar.server.permission.index; - -import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexCombinationTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexCombinationTest.java deleted file mode 100644 index fcd2576ede2..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexCombinationTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.stream.IntStream; -import org.junit.Test; -import org.sonar.api.resources.Qualifiers; -import org.sonar.db.component.ComponentDto; - -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; - -public class ComponentIndexCombinationTest extends ComponentIndexTest { - - @Test - public void return_empty_list_if_no_fields_match_query() { - indexProject("struts", "Apache Struts"); - - assertThat(index.searchSuggestions(SuggestionQuery.builder().setQuery("missing").build()).isEmpty()).isTrue(); - } - - @Test - public void should_not_return_components_that_do_not_match_at_all() { - indexProject("banana", "Banana Project 1"); - - assertNoSearchResults("Apple"); - } - - @Test - public void filter_results_by_qualifier() { - ComponentDto project = indexProject("struts", "Apache Struts"); - indexFile(project, "src/main/java/StrutsManager.java", "StrutsManager.java"); - - assertSearchResults(SuggestionQuery.builder().setQuery("struts").setQualifiers(singletonList(Qualifiers.PROJECT)).build(), project); - } - - @Test - public void should_limit_the_number_of_results() { - IntStream.rangeClosed(0, 10).forEach(i -> indexProject("sonarqube" + i, "SonarQube" + i)); - - assertSearch(SuggestionQuery.builder().setQuery("sonarqube").setLimit(5).setQualifiers(singletonList(Qualifiers.PROJECT)).build()).hasSize(5); - } - - @Test - public void should_not_support_wildcards() { - indexProject("theKey", "the name"); - - assertNoSearchResults("*t*"); - assertNoSearchResults("th?Key"); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureExactTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureExactTest.java deleted file mode 100644 index d352b6c718a..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureExactTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.Collections; -import org.junit.Before; -import org.junit.Test; -import org.sonar.db.component.ComponentDto; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.sonar.api.resources.Qualifiers.PROJECT; - -public class ComponentIndexFeatureExactTest extends ComponentIndexTest { - - @Before - public void before() { - features.set(query -> matchAllQuery(), ComponentTextSearchFeatureRepertoire.EXACT_IGNORE_CASE); - } - - @Test - public void scoring_cares_about_exact_matches() { - ComponentDto project1 = indexProject("project1", "LongNameLongNameLongNameLongNameSonarQube"); - ComponentDto project2 = indexProject("project2", "LongNameLongNameLongNameLongNameSonarQubeX"); - - SuggestionQuery query1 = SuggestionQuery.builder() - .setQuery("LongNameLongNameLongNameLongNameSonarQube") - .setQualifiers(Collections.singletonList(PROJECT)) - .build(); - assertSearch(query1).containsExactly(uuids(project1, project2)); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureFavoriteTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureFavoriteTest.java deleted file mode 100644 index 34a3683e207..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureFavoriteTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import org.junit.Before; -import org.junit.Test; -import org.sonar.db.component.ComponentDto; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -import static com.google.common.collect.ImmutableSet.of; -import static java.util.Collections.singletonList; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_KEY; - -public class ComponentIndexFeatureFavoriteTest extends ComponentIndexTest { - - @Before - public void before() { - features.set(q -> matchAllQuery(), ComponentTextSearchFeatureRepertoire.FAVORITE); - } - - @Test - public void scoring_cares_about_favorites() { - ComponentDto project1 = indexProject("sonarqube", "SonarQube"); - ComponentDto project2 = indexProject("recent", "SonarQube Recently"); - - SuggestionQuery query1 = SuggestionQuery.builder() - .setQuery("SonarQube") - .setQualifiers(singletonList(PROJECT)) - .setFavoriteKeys(of(project1.getDbKey())) - .build(); - assertSearch(query1).containsExactly(uuids(project1, project2)); - - SuggestionQuery query2 = SuggestionQuery.builder() - .setQuery("SonarQube") - .setQualifiers(singletonList(PROJECT)) - .setFavoriteKeys(of(project2.getDbKey())) - .build(); - assertSearch(query2).containsExactly(uuids(project2, project1)); - } - - @Test - public void irrelevant_favorites_are_not_returned() { - features.set(q -> termQuery(FIELD_KEY, "non-existing-value"), ComponentTextSearchFeatureRepertoire.FAVORITE); - ComponentDto project1 = indexProject("foo", "foo"); - - SuggestionQuery query1 = SuggestionQuery.builder() - .setQuery("bar") - .setQualifiers(singletonList(PROJECT)) - .setFavoriteKeys(of(project1.getDbKey())) - .build(); - assertSearch(query1).isEmpty(); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureKeyTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureKeyTest.java deleted file mode 100644 index 50663625573..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureKeyTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import org.junit.Before; -import org.junit.Test; -import org.sonar.db.component.ComponentDto; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -public class ComponentIndexFeatureKeyTest extends ComponentIndexTest { - - @Before - public void before() { - features.set(ComponentTextSearchFeatureRepertoire.KEY); - } - - @Test - public void should_search_projects_by_exact_case_insensitive_key() { - ComponentDto project1 = indexProject("keyOne", "Project One"); - indexProject("keyTwo", "Project Two"); - - assertSearchResults("keyOne", project1); - assertSearchResults("keyone", project1); - assertSearchResults("KEYone", project1); - } - - @Test - public void should_search_project_with_dot_in_key() { - ComponentDto project = indexProject("org.sonarqube", "SonarQube"); - - assertSearchResults("org.sonarqube", project); - assertNoSearchResults("orgsonarqube"); - } - - @Test - public void should_search_project_with_dash_in_key() { - ComponentDto project = indexProject("org-sonarqube", "SonarQube"); - - assertSearchResults("org-sonarqube", project); - assertNoSearchResults("orgsonarqube"); - } - - @Test - public void should_search_project_with_colon_in_key() { - ComponentDto project = indexProject("org:sonarqube", "Quality Product"); - - assertSearchResults("org:sonarqube", project); - assertNoSearchResults("orgsonarqube"); - assertNoSearchResults("org-sonarqube"); - assertNoSearchResults("org_sonarqube"); - } - - @Test - public void should_search_project_with_all_special_characters_in_key() { - ComponentDto project = indexProject("org.sonarqube:sonar-sérvèr_ç", "SonarQube"); - - assertSearchResults("org.sonarqube:sonar-sérvèr_ç", project); - } - - @Test - public void should_not_return_results_when_searching_by_partial_key() { - indexProject("theKey", "some name"); - - assertNoSearchResults("theke"); - assertNoSearchResults("hekey"); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeaturePartialTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeaturePartialTest.java deleted file mode 100644 index 8f243f367c5..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeaturePartialTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import org.junit.Before; -import org.junit.Test; -import org.sonar.db.component.ComponentDto; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -public class ComponentIndexFeaturePartialTest extends ComponentIndexTest { - - @Before - public void before() { - features.set(ComponentTextSearchFeatureRepertoire.PARTIAL); - } - - @Test - public void search_projects_by_exact_name() { - ComponentDto struts = indexProject("struts", "Apache Struts"); - indexProject("sonarqube", "SonarQube"); - - assertSearchResults("Apache Struts", struts); - assertSearchResults("APACHE STRUTS", struts); - assertSearchResults("APACHE struTS", struts); - } - - @Test - public void search_file_with_long_name() { - ComponentDto project = indexProject("struts", "Apache Struts"); - ComponentDto file1 = indexFile(project, "src/main/java/DefaultRubyComponentServiceTestManagerFactory.java", "DefaultRubyComponentServiceTestManagerFactory.java"); - - assertSearchResults("DefaultRubyComponentServiceTestManagerFactory", file1); - assertSearchResults("DefaultRubyComponentServiceTestManagerFactory.java", file1); - assertSearchResults("RubyComponentServiceTestManager", file1); - assertSearchResults("te", file1); - } - - @Test - public void should_search_by_name_with_two_characters() { - ComponentDto project = indexProject("struts", "Apache Struts"); - - assertSearchResults("st", project); - assertSearchResults("tr", project); - } - - @Test - public void search_projects_by_partial_name() { - ComponentDto struts = indexProject("struts", "Apache Struts"); - - assertSearchResults("truts", struts); - assertSearchResults("pache", struts); - assertSearchResults("apach", struts); - assertSearchResults("che stru", struts); - } - - @Test - public void search_projects_and_files_by_partial_name() { - ComponentDto project = indexProject("struts", "Apache Struts"); - ComponentDto file1 = indexFile(project, "src/main/java/StrutsManager.java", "StrutsManager.java"); - indexFile(project, "src/main/java/Foo.java", "Foo.java"); - - assertSearchResults("struts", project, file1); - assertSearchResults("Struts", project, file1); - assertSearchResults("StrutsManager", file1); - assertSearchResults("STRUTSMAN", file1); - assertSearchResults("utsManag", file1); - } - - @Test - public void should_find_file_by_file_extension() { - ComponentDto project = indexProject("struts", "Apache Struts"); - ComponentDto file1 = indexFile(project, "src/main/java/StrutsManager.java", "StrutsManager.java"); - ComponentDto file2 = indexFile(project, "src/main/java/Foo.java", "Foo.java"); - - assertSearchResults(".java", file1, file2); - assertSearchResults("manager.java", file1); - - // do not match - assertNoSearchResults("somethingStrutsManager.java"); - } - - @Test - public void should_search_for_word_and_suffix() { - assertFileMatches("plugin java", "AbstractPluginFactory.java"); - } - - @Test - public void should_search_for_word_and_suffix_in_any_order() { - assertFileMatches("java plugin", "AbstractPluginFactory.java"); - } - - @Test - public void should_search_for_two_words() { - assertFileMatches("abstract factory", "AbstractPluginFactory.java"); - } - - @Test - public void should_search_for_two_words_in_any_order() { - assertFileMatches("factory abstract", "AbstractPluginFactory.java"); - } - - @Test - public void should_require_at_least_one_matching_word() { - assertNoFileMatches("monitor object", "AbstractPluginFactory.java"); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeaturePrefixTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeaturePrefixTest.java deleted file mode 100644 index 0ad8be8907c..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeaturePrefixTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import org.junit.Before; -import org.junit.Test; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -public class ComponentIndexFeaturePrefixTest extends ComponentIndexTest { - - @Before - public void before() { - features.set(ComponentTextSearchFeatureRepertoire.PREFIX, ComponentTextSearchFeatureRepertoire.PREFIX_IGNORE_CASE); - } - - @Test - public void should_find_prefix() { - assertResultOrder("comp", "component"); - } - - @Test - public void should_find_exact_match() { - assertResultOrder("component.js", "component.js"); - } - - @Test - public void should_not_find_partially() { - assertNoFileMatches("component.js", "my_component.js"); - } - - @Test - public void should_be_able_to_ignore_case() { - features.set(ComponentTextSearchFeatureRepertoire.PREFIX_IGNORE_CASE); - assertResultOrder("cOmPoNeNt.Js", "CoMpOnEnT.jS"); - } - - @Test - public void should_prefer_matching_case() { - assertResultOrder("cOmPoNeNt.Js", "cOmPoNeNt.Js", "CoMpOnEnT.jS"); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureRecentlyBrowsedTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureRecentlyBrowsedTest.java deleted file mode 100644 index aa2870cce86..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexFeatureRecentlyBrowsedTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.Collections; -import org.junit.Before; -import org.junit.Test; -import org.sonar.db.component.ComponentDto; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -import static com.google.common.collect.ImmutableSet.of; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.sonar.api.resources.Qualifiers.PROJECT; - -public class ComponentIndexFeatureRecentlyBrowsedTest extends ComponentIndexTest { - - @Before - public void before() { - features.set(query -> matchAllQuery(), ComponentTextSearchFeatureRepertoire.RECENTLY_BROWSED); - } - - @Test - public void scoring_cares_about_recently_browsed() { - ComponentDto project1 = indexProject("sonarqube", "SonarQube"); - ComponentDto project2 = indexProject("recent", "SonarQube Recently"); - - SuggestionQuery query1 = SuggestionQuery.builder() - .setQuery("SonarQube") - .setQualifiers(Collections.singletonList(PROJECT)) - .setRecentlyBrowsedKeys(of(project1.getDbKey())) - .build(); - assertSearch(query1).containsExactly(uuids(project1, project2)); - - SuggestionQuery query2 = SuggestionQuery.builder() - .setQuery("SonarQube") - .setQualifiers(Collections.singletonList(PROJECT)) - .setRecentlyBrowsedKeys(of(project2.getDbKey())) - .build(); - assertSearch(query2).containsExactly(uuids(project2, project1)); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexHighlightTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexHighlightTest.java deleted file mode 100644 index 317430009b1..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexHighlightTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.Collections; -import java.util.Optional; -import java.util.stream.Stream; -import org.junit.Test; -import org.sonar.api.resources.Qualifiers; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ComponentIndexHighlightTest extends ComponentIndexTest { - - @Test - public void should_escape_html() { - assertHighlighting("quick< brown fox", "brown", "quick< <mark>brown</mark> fox"); - } - - @Test - public void should_highlight_partial_name() { - assertHighlighting("quickbrownfox", "brown", "quick<mark>brown</mark>fox"); - } - - @Test - public void should_highlight_prefix() { - assertHighlighting("quickbrownfox", "quick", "<mark>quick</mark>brownfox"); - } - - @Test - public void should_highlight_suffix() { - assertHighlighting("quickbrownfox", "fox", "quickbrown<mark>fox</mark>"); - } - - @Test - public void should_highlight_multiple_words() { - assertHighlighting("quickbrownfox", "fox bro", "quick<mark>bro</mark>wn<mark>fox</mark>"); - } - - @Test - public void should_highlight_multiple_connected_words() { - assertHighlighting("quickbrownfox", "fox brown", "quick<mark>brownfox</mark>"); - } - - private void assertHighlighting(String fileName, String search, String expectedHighlighting) { - indexFile(fileName); - - SuggestionQuery query = SuggestionQuery.builder() - .setQuery(search) - .setQualifiers(Collections.singletonList(Qualifiers.FILE)) - .build(); - Stream<ComponentHitsPerQualifier> results = index.searchSuggestions(query, features.get()).getQualifiers(); - - assertThat(results).flatExtracting(ComponentHitsPerQualifier::getHits) - .extracting(ComponentHit::getHighlightedText) - .extracting(Optional::get) - .containsExactly(expectedHighlighting); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexLoginTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexLoginTest.java deleted file mode 100644 index 89b8fbb25fd..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexLoginTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import org.junit.Test; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.UserDto; - -import static org.sonar.db.user.GroupTesting.newGroupDto; -import static org.sonar.db.user.UserTesting.newUserDto; - -public class ComponentIndexLoginTest extends ComponentIndexTest { - - @Test - public void should_filter_unauthorized_results() { - indexer.index(newProject("sonarqube", "Quality Product")); - - // do not give any permissions to that project - - assertNoSearchResults("sonarqube"); - assertNoSearchResults("Quality Product"); - } - - @Test - public void should_find_project_for_which_the_user_has_direct_permission() { - UserDto user = newUserDto(); - userSession.logIn(user); - - ComponentDto project = newProject("sonarqube", "Quality Product"); - indexer.index(project); - - assertNoSearchResults("sonarqube"); - - // give the user explicit access - authorizationIndexerTester.allowOnlyUser(project, user); - assertSearchResults("sonarqube", project); - } - - @Test - public void should_find_project_for_which_the_user_has_indirect_permission_through_group() { - GroupDto group = newGroupDto(); - userSession.logIn().setGroups(group); - - ComponentDto project = newProject("sonarqube", "Quality Product"); - indexer.index(project); - - assertNoSearchResults("sonarqube"); - - // give the user implicit access (though group) - authorizationIndexerTester.allowOnlyGroup(project, group); - assertSearchResults("sonarqube", project); - } - - @Test - public void do_not_check_permissions_when_logged_in_user_is_root() { - userSession.logIn().setRoot(); - ComponentDto project = newProject("sonarqube", "Quality Product"); - indexer.index(project); - // do not give any permissions to that project - - assertSearchResults("sonarqube", project); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexMultipleWordsTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexMultipleWordsTest.java deleted file mode 100644 index bb91e5ec0d6..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexMultipleWordsTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import org.junit.Test; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -public class ComponentIndexMultipleWordsTest extends ComponentIndexTest { - - @Test - public void should_find_perfect_match() { - assertResultOrder("struts java", - "Struts.java"); - } - - @Test - public void should_find_partial_match() { - features.set(ComponentTextSearchFeatureRepertoire.PARTIAL); - assertResultOrder("struts java", - "Xstrutsx.Xjavax"); - } - - @Test - public void should_find_partial_match_prefix_word1() { - assertResultOrder("struts java", - "MyStruts.java"); - } - - @Test - public void should_find_partial_match_suffix_word1() { - assertResultOrder("struts java", - "StrutsObject.java"); - } - - @Test - public void should_find_partial_match_prefix_word2() { - assertResultOrder("struts java", - "MyStruts.xjava"); - } - - @Test - public void should_find_partial_match_suffix_word2() { - assertResultOrder("struts java", - "MyStruts.javax"); - } - - @Test - public void should_find_partial_match_prefix_and_suffix_everywhere() { - assertResultOrder("struts java", - "MyStrutsObject.xjavax"); - } - - @Test - public void should_find_subset_of_document_terms() { - assertResultOrder("struts java", - "Some.Struts.Class.java.old"); - } - - @Test - public void should_require_all_words_to_match() { - assertNoFileMatches("struts java", - "Struts"); - } - - @Test - public void should_ignore_empty_words() { - assertFileMatches(" struts \n \n\n", - "Struts"); - } - - @Test - public void should_require_all_words_to_match_for_partial() { - features.set(ComponentTextSearchFeatureRepertoire.PARTIAL); - assertNoFileMatches("struts java", - "Struts"); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java deleted file mode 100644 index 4a9213de8da..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexScoreTest.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import com.google.common.collect.ImmutableSet; -import org.junit.Test; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentTesting; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRepertoire; - -import static java.util.Arrays.asList; -import static org.sonar.api.resources.Qualifiers.FILE; -import static org.sonar.api.resources.Qualifiers.MODULE; -import static org.sonar.api.resources.Qualifiers.PROJECT; - -public class ComponentIndexScoreTest extends ComponentIndexTest { - - @Test - public void should_prefer_components_without_prefix() { - assertResultOrder("File.java", - "File.java", - "MyFile.java"); - } - - @Test - public void should_prefer_components_without_suffix() { - assertResultOrder("File", - "File", - "Filex"); - } - - @Test - public void should_prefer_key_matching_over_name_matching() { - ComponentDto project1 = indexProject("quality", "SonarQube"); - ComponentDto project2 = indexProject("sonarqube", "Quality Product"); - - assertExactResults("sonarqube", project2, project1); - } - - @Test - public void should_prefer_prefix_matching_over_partial_matching() { - assertResultOrder("corem", - "CoreMetrics.java", - "ScoreMatrix.java"); - } - - @Test - public void should_prefer_case_sensitive_prefix() { - assertResultOrder("caSe", - "caSeBla.java", - "CaseBla.java"); - } - - @Test - public void scoring_prefix_with_multiple_words() { - assertResultOrder("index java", - "IndexSomething.java", - "MyIndex.java"); - } - - @Test - public void scoring_prefix_with_multiple_words_and_case() { - assertResultOrder("Index JAVA", - "IndexSomething.java", - "index_java.js"); - } - - @Test - public void scoring_long_items() { - assertResultOrder("ThisIsAVeryLongNameToSearchForAndItExceeds15Characters.java", - "ThisIsAVeryLongNameToSearchForAndItExceeds15Characters.java", - "ThisIsAVeryLongNameToSearchForAndItEndsDifferently.java"); - } - - @Test - public void scoring_perfect_match() { - assertResultOrder("SonarQube", - "SonarQube", - "SonarQube SCM Git"); - } - - @Test - public void scoring_perfect_match_dispite_case_changes() { - assertResultOrder("sonarqube", - "SonarQube", - "SonarQube SCM Git"); - } - - @Test - public void scoring_perfect_match_with_matching_case_higher_than_without_matching_case() { - assertResultOrder("sonarqube", - "sonarqube", - "SonarQube"); - } - - @Test - public void should_prefer_favorite_over_recently_browsed() { - ComponentDto file1 = db.components().insertPrivateProject(c -> c.setName("File1")); - index(file1); - - ComponentDto file2 = db.components().insertPrivateProject(c -> c.setName("File2")); - index(file2); - - assertSearch(SuggestionQuery.builder() - .setQuery("File") - .setQualifiers(asList(PROJECT, MODULE, FILE)) - .setRecentlyBrowsedKeys(ImmutableSet.of(file1.getDbKey())) - .setFavoriteKeys(ImmutableSet.of(file2.getDbKey())) - .build()).containsExactly(uuids(file2, file1)); - - assertSearch(SuggestionQuery.builder() - .setQuery("File") - .setQualifiers(asList(PROJECT, MODULE, FILE)) - .setRecentlyBrowsedKeys(ImmutableSet.of(file2.getDbKey())) - .setFavoriteKeys(ImmutableSet.of(file1.getDbKey())) - .build()).containsExactly(uuids(file1, file2)); - } - - @Test - public void do_not_match_wrong_file_extension() { - ComponentDto file1 = indexFile("MyClass.java"); - ComponentDto file2 = indexFile("ClassExample.java"); - ComponentDto file3 = indexFile("Class.java"); - indexFile("Class.cs"); - indexFile("Class.js"); - indexFile("Class.rb"); - - assertExactResults("Class java", file3, file2, file1); - } - - @Test - public void if_relevancy_is_equal_fall_back_to_alphabetical_ordering() { - assertResultOrder("sonarqube", - "sonarqubeA", - "sonarqubeB"); - } - - @Test - public void scoring_test_DbTester() { - features.set(ComponentTextSearchFeatureRepertoire.PARTIAL); - - ComponentDto project = indexProject("key-1", "Quality Product"); - - index(ComponentTesting.newFileDto(project) - .setName("DbTester.java") - .setDbKey("java/org/example/DbTester.java") - .setUuid("UUID-DbTester")); - - index(ComponentTesting.newFileDto(project) - .setName("WebhookDbTesting.java") - .setDbKey("java/org/example/WebhookDbTesting.java") - .setUuid("UUID-WebhookDbTesting")); - - assertSearch("dbt").containsExactly( - - "UUID-DbTester", - "UUID-WebhookDbTesting" - - ); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java deleted file mode 100644 index 25a16027af0..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexSearchTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.resources.Qualifiers; -import org.sonar.api.utils.System2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.SearchIdResult; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRule; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Collections.emptySet; -import static java.util.Collections.singleton; -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.db.component.ComponentTesting.newFileDto; - -public class ComponentIndexSearchTest { - @Rule - public EsTester es = EsTester.create(); - @Rule - public DbTester db = DbTester.create(System2.INSTANCE); - @Rule - public UserSessionRule userSession = UserSessionRule.standalone(); - @Rule - public ComponentTextSearchFeatureRule features = new ComponentTextSearchFeatureRule(); - - private ComponentIndexer indexer = new ComponentIndexer(db.getDbClient(), es.client()); - private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es, indexer); - - private ComponentIndex underTest = new ComponentIndex(es.client(), new WebAuthorizationTypeSupport(userSession), System2.INSTANCE); - - @Test - public void filter_by_language() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto javaFile = db.components().insertComponent(newFileDto(project).setLanguage("java")); - ComponentDto jsFile1 = db.components().insertComponent(newFileDto(project).setLanguage("js")); - ComponentDto jsFile2 = db.components().insertComponent(newFileDto(project).setLanguage("js")); - index(project); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().setLanguage("js").build(), new SearchOptions()); - - assertThat(result.getIds()).containsExactlyInAnyOrder(jsFile1.uuid(), jsFile2.uuid()); - } - - @Test - public void filter_by_name() { - ComponentDto ignoredProject = db.components().insertPrivateProject(p -> p.setName("ignored project")); - ComponentDto project = db.components().insertPrivateProject(p -> p.setName("Project Shiny name")); - index(ignoredProject, project); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().setQuery("shiny").build(), new SearchOptions()); - - assertThat(result.getIds()).containsExactlyInAnyOrder(project.uuid()); - } - - @Test - public void filter_by_key_with_exact_match() { - ComponentDto ignoredProject = db.components().insertPrivateProject(p -> p.setDbKey("ignored-project")); - ComponentDto project = db.components().insertPrivateProject(p -> p.setDbKey("shiny-project")); - ComponentDto anotherIgnoreProject = db.components().insertPrivateProject(p -> p.setDbKey("another-shiny-project")); - index(ignoredProject, project); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().setQuery("shiny-project").build(), new SearchOptions()); - - assertThat(result.getIds()).containsExactlyInAnyOrder(project.uuid()); - } - - @Test - public void filter_by_qualifier() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project)); - index(project); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().setQualifiers(singleton(Qualifiers.FILE)).build(), new SearchOptions()); - - assertThat(result.getIds()).containsExactlyInAnyOrder(file.uuid()); - } - - @Test - public void filter_by_organization() { - OrganizationDto organization = db.organizations().insert(); - OrganizationDto anotherOrganization = db.organizations().insert(); - ComponentDto project = db.components().insertPrivateProject(organization); - ComponentDto anotherProject = db.components().insertPrivateProject(anotherOrganization); - index(project, anotherProject); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().setOrganization(organization.getUuid()).build(), new SearchOptions()); - - assertThat(result.getIds()).containsExactlyInAnyOrder(project.uuid()); - } - - @Test - public void order_by_name_case_insensitive() { - ComponentDto project2 = db.components().insertPrivateProject(p -> p.setName("PROJECT 2")); - ComponentDto project3 = db.components().insertPrivateProject(p -> p.setName("project 3")); - ComponentDto project1 = db.components().insertPrivateProject(p -> p.setName("Project 1")); - index(project1, project2, project3); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().build(), new SearchOptions()); - - assertThat(result.getIds()).containsExactly(project1.uuid(), project2.uuid(), project3.uuid()); - } - - @Test - public void paginate_results() { - List<ComponentDto> projects = IntStream.range(0, 9) - .mapToObj(i -> db.components().insertPrivateProject(p -> p.setName("project " + i))) - .collect(Collectors.toList()); - index(projects.toArray(new ComponentDto[0])); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().build(), new SearchOptions().setPage(2, 3)); - - assertThat(result.getIds()).containsExactlyInAnyOrder(projects.get(3).uuid(), projects.get(4).uuid(), projects.get(5).uuid()); - } - - @Test - public void filter_unauthorized_components() { - ComponentDto unauthorizedProject = db.components().insertPrivateProject(); - ComponentDto project1 = db.components().insertPrivateProject(); - ComponentDto project2 = db.components().insertPrivateProject(); - indexer.indexOnStartup(emptySet()); - authorizationIndexerTester.allowOnlyAnyone(project1); - authorizationIndexerTester.allowOnlyAnyone(project2); - - SearchIdResult<String> result = underTest.search(ComponentQuery.builder().build(), new SearchOptions()); - - assertThat(result.getIds()).containsExactlyInAnyOrder(project1.uuid(), project2.uuid()) - .doesNotContain(unauthorizedProject.uuid()); - } - - private void index(ComponentDto... components) { - indexer.indexOnStartup(emptySet()); - Arrays.stream(components).forEach(c -> authorizationIndexerTester.allowOnlyAnyone(c)); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java deleted file mode 100644 index c625ad61f4a..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/component/index/ComponentIndexTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.component.index; - -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; -import org.assertj.core.api.ListAssert; -import org.junit.Before; -import org.junit.Rule; -import org.sonar.api.utils.System2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentTesting; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.organization.OrganizationTesting; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.textsearch.ComponentTextSearchFeatureRule; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.api.resources.Qualifiers.FILE; -import static org.sonar.api.resources.Qualifiers.MODULE; -import static org.sonar.api.resources.Qualifiers.PROJECT; - -public abstract class ComponentIndexTest { - - @Rule - public EsTester es = EsTester.create(); - @Rule - public DbTester db = DbTester.create(System2.INSTANCE); - @Rule - public UserSessionRule userSession = UserSessionRule.standalone(); - - @Rule - public ComponentTextSearchFeatureRule features = new ComponentTextSearchFeatureRule(); - - protected ComponentIndexer indexer = new ComponentIndexer(db.getDbClient(), es.client()); - protected ComponentIndex index = new ComponentIndex(es.client(), new WebAuthorizationTypeSupport(userSession), System2.INSTANCE); - protected PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(es, indexer); - private OrganizationDto organization; - - @Before - public void setUp() { - organization = OrganizationTesting.newOrganizationDto(); - } - - protected void assertFileMatches(String query, String... fileNames) { - ComponentDto[] files = Arrays.stream(fileNames) - .map(this::indexFile) - .toArray(ComponentDto[]::new); - assertSearch(query).containsExactlyInAnyOrder(uuids(files)); - } - - protected void assertNoFileMatches(String query, String... fileNames) { - Arrays.stream(fileNames) - .forEach(this::indexFile); - assertSearch(query).isEmpty(); - } - - protected void assertResultOrder(String query, String... resultsInOrder) { - ComponentDto project = indexProject("key-1", "Quality Product"); - List<ComponentDto> files = Arrays.stream(resultsInOrder) - .map(r -> ComponentTesting.newFileDto(project).setName(r)) - .peek(f -> f.setUuid(f.uuid() + "_" + f.name().replaceAll("[^a-zA-Z0-9]", ""))) - .collect(Collectors.toList()); - - // index them, but not in the expected order - files.stream() - .sorted(Comparator.comparing(ComponentDto::uuid).reversed()) - .forEach(this::index); - - assertExactResults(query, files.toArray(new ComponentDto[0])); - } - - protected ListAssert<String> assertSearch(String query) { - return assertSearch(SuggestionQuery.builder().setQuery(query).setQualifiers(asList(PROJECT, MODULE, FILE)).build()); - } - - protected ListAssert<String> assertSearch(SuggestionQuery query) { - return (ListAssert<String>)assertThat(index.searchSuggestions(query, features.get()).getQualifiers()) - .flatExtracting(ComponentHitsPerQualifier::getHits) - .extracting(ComponentHit::getUuid); - } - - protected void assertSearchResults(String query, ComponentDto... expectedComponents) { - assertSearchResults(SuggestionQuery.builder().setQuery(query).setQualifiers(asList(PROJECT, MODULE, FILE)).build(), expectedComponents); - } - - protected void assertSearchResults(SuggestionQuery query, ComponentDto... expectedComponents) { - assertSearch(query).containsOnly(uuids(expectedComponents)); - } - - protected void assertExactResults(String query, ComponentDto... expectedComponents) { - assertSearch(query).containsExactly(uuids(expectedComponents)); - } - - protected void assertNoSearchResults(String query) { - assertSearchResults(query); - } - - protected ComponentDto indexProject(String key, String name) { - return index( - ComponentTesting.newPrivateProjectDto(organization, "UUID_" + key) - .setDbKey(key) - .setName(name)); - } - - protected ComponentDto newProject(String key, String name) { - return ComponentTesting.newPrivateProjectDto(organization, "UUID_" + key) - .setDbKey(key) - .setName(name); - } - - protected ComponentDto indexFile(String fileName) { - ComponentDto project = indexProject("key-1", "SonarQube"); - return indexFile(project, "src/main/java/" + fileName, fileName); - } - - protected ComponentDto indexFile(ComponentDto project, String fileKey, String fileName) { - return index( - ComponentTesting.newFileDto(project) - .setDbKey(fileKey) - .setName(fileName)); - } - - protected ComponentDto index(ComponentDto dto) { - indexer.index(dto); - authorizationIndexerTester.allowOnlyAnyone(dto); - return dto; - } - - protected static String[] uuids(ComponentDto... expectedComponents) { - return Arrays.stream(expectedComponents).map(ComponentDto::uuid).toArray(String[]::new); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java deleted file mode 100644 index b2b26d951a7..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; -import java.util.Map; -import java.util.function.Consumer; -import javax.annotation.CheckForNull; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.Settings; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.config.internal.MapSettings; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.server.es.IndexType.IndexMainType; -import org.sonar.server.es.metadata.MetadataIndex; -import org.sonar.server.es.metadata.MetadataIndexDefinition; -import org.sonar.server.es.metadata.MetadataIndexImpl; -import org.sonar.server.es.newindex.NewRegularIndex; -import org.sonar.server.es.newindex.SettingsConfiguration; -import org.sonar.server.platform.db.migration.es.MigrationEsClient; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.sonar.server.es.IndexType.main; -import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; - -public class IndexCreatorTest { - - private static final SettingsConfiguration SETTINGS_CONFIGURATION = newBuilder(new MapSettings().asConfig()).build(); - private static final String LOG_DB_VENDOR_CHANGED = "Delete Elasticsearch indices (DB vendor changed)"; - private static final String LOG_DB_SCHEMA_CHANGED = "Delete Elasticsearch indices (DB schema changed)"; - - @Rule - public LogTester logTester = new LogTester(); - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Rule - public EsTester es = EsTester.createCustom(); - - private MetadataIndexDefinition metadataIndexDefinition = new MetadataIndexDefinition(new MapSettings().asConfig()); - private MetadataIndex metadataIndex = new MetadataIndexImpl(es.client()); - private TestEsDbCompatibility esDbCompatibility = new TestEsDbCompatibility(); - private MapSettings settings = new MapSettings(); - private MigrationEsClient migrationEsClient = mock(MigrationEsClient.class); - - @Test - public void create_index() { - IndexCreator underTest = run(new FakeIndexDefinition()); - - // check that index is created with related mapping - verifyFakeIndexV1(); - - // of course do not delete indices on stop - underTest.stop(); - assertThat(mappings()).isNotEmpty(); - } - - @Test - public void creation_of_new_index_is_supported_in_blue_green_deployment() { - enableBlueGreenDeployment(); - - run(new FakeIndexDefinition()); - - verifyFakeIndexV1(); - } - - @Test - public void recreate_index_on_definition_changes() { - // v1 - run(new FakeIndexDefinition()); - - IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake"); - String id = "1"; - es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); - assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); - - // v2 - run(new FakeIndexDefinitionV2()); - - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("fakes").get("fake"); - assertThat(countMappingFields(mapping)).isEqualTo(3); - assertThat(field(mapping, "updatedAt").get("type")).isEqualTo("date"); - assertThat(field(mapping, "newField").get("type")).isEqualTo("integer"); - - assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isFalse(); - } - - @Test - public void fail_to_recreate_index_on_definition_changes_if_blue_green_deployment() { - enableBlueGreenDeployment(); - - // v1 - run(new FakeIndexDefinition()); - - // v2 - expectedException.expect(IllegalStateException.class); - expectedException.expectMessage("Blue/green deployment is not supported. Elasticsearch index [fakes] changed and needs to be dropped."); - - run(new FakeIndexDefinitionV2()); - } - - @Test - public void reset_definition_hash_if_change_applied_during_migration_of_blue_green_deployment() { - enableBlueGreenDeployment(); - - // v1 - run(new FakeIndexDefinition()); - - // v2 - when(migrationEsClient.getUpdatedIndices()).thenReturn(Sets.newHashSet(FakeIndexDefinition.INDEX_TYPE.getIndex().getName())); - run(new FakeIndexDefinitionV2()); - // index has not been dropped-and-recreated - verifyFakeIndexV1(); - } - - @Test - public void mark_all_non_existing_index_types_as_uninitialized() { - Index fakesIndex = Index.simple("fakes"); - Index fakersIndex = Index.simple("fakers"); - run(context -> { - context.create(fakesIndex, SETTINGS_CONFIGURATION) - .createTypeMapping(IndexType.main(fakesIndex, "fake")); - context.create(fakersIndex, SETTINGS_CONFIGURATION) - .createTypeMapping(IndexType.main(fakersIndex, "faker")); - }); - - assertThat(metadataIndex.getHash(fakesIndex)).isNotEmpty(); - assertThat(metadataIndex.getHash(fakersIndex)).isNotEmpty(); - assertThat(metadataIndex.getInitialized(main(fakesIndex, "fake"))).isFalse(); - assertThat(metadataIndex.getInitialized(main(fakersIndex, "faker"))).isFalse(); - } - - @Test - public void delete_existing_indices_if_db_vendor_changed() { - testDeleteOnDbChange(LOG_DB_VENDOR_CHANGED, - c -> c.setHasSameDbVendor(false)); - } - - @Test - public void do_not_check_db_compatibility_on_fresh_es() { - // supposed to be ignored - esDbCompatibility.setHasSameDbVendor(false); - - run(new FakeIndexDefinition()); - - assertThat(logTester.logs(LoggerLevel.INFO)) - .doesNotContain(LOG_DB_VENDOR_CHANGED) - .doesNotContain(LOG_DB_SCHEMA_CHANGED) - .contains("Create type fakes/fake") - .contains("Create type metadatas/metadata"); - } - - @Test - public void start_makes_metadata_index_read_write_if_read_only() { - run(new FakeIndexDefinition()); - - IndexMainType mainType = MetadataIndexDefinition.TYPE_METADATA; - makeReadOnly(mainType); - - run(new FakeIndexDefinition()); - - assertThat(isNotReadOnly(mainType)).isTrue(); - } - - @Test - public void start_makes_index_read_write_if_read_only() { - FakeIndexDefinition fakeIndexDefinition = new FakeIndexDefinition(); - IndexMainType fakeIndexMainType= FakeIndexDefinition.INDEX_TYPE.getMainType(); - run(fakeIndexDefinition); - - IndexMainType mainType = MetadataIndexDefinition.TYPE_METADATA; - makeReadOnly(mainType); - makeReadOnly(fakeIndexMainType); - - run(fakeIndexDefinition); - - assertThat(isNotReadOnly(mainType)).isTrue(); - assertThat(isNotReadOnly(fakeIndexMainType)).isTrue(); - } - - private boolean isNotReadOnly(IndexMainType mainType) { - String indexName = mainType.getIndex().getName(); - String readOnly = es.client().nativeClient().admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet() - .getSetting(indexName, "index.blocks.read_only_allow_delete"); - return readOnly == null; - } - - private void makeReadOnly(IndexMainType mainType) { - Settings.Builder builder = Settings.builder(); - builder.put("index.blocks.read_only_allow_delete", "true"); - es.client().nativeClient().admin().indices() - .updateSettings(new UpdateSettingsRequest().indices(mainType.getIndex().getName()).settings(builder.build())) - .actionGet(); - } - - private void enableBlueGreenDeployment() { - settings.setProperty("sonar.blueGreenEnabled", "true"); - } - - private void testDeleteOnDbChange(String expectedLog, Consumer<TestEsDbCompatibility> afterFirstStart) { - run(new FakeIndexDefinition()); - assertThat(logTester.logs(LoggerLevel.INFO)) - .doesNotContain(expectedLog) - .contains("Create type fakes/fake") - .contains("Create type metadatas/metadata"); - putFakeDocument(); - assertThat(es.countDocuments(FakeIndexDefinition.INDEX_TYPE)).isEqualTo(1); - - afterFirstStart.accept(esDbCompatibility); - logTester.clear(); - run(new FakeIndexDefinition()); - - assertThat(logTester.logs(LoggerLevel.INFO)) - .contains(expectedLog) - .contains("Create type fakes/fake") - // keep existing metadata - .doesNotContain("Create type metadatas/metadata"); - // index has been dropped and re-created - assertThat(es.countDocuments(FakeIndexDefinition.INDEX_TYPE)).isEqualTo(0); - } - - private ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings() { - return es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings(); - } - - @CheckForNull - @SuppressWarnings("unchecked") - private Map<String, Object> field(MappingMetaData mapping, String field) { - Map<String, Object> props = (Map<String, Object>) mapping.getSourceAsMap().get("properties"); - return (Map<String, Object>) props.get(field); - } - - private int countMappingFields(MappingMetaData mapping) { - return ((Map) mapping.getSourceAsMap().get("properties")).size(); - } - - private IndexCreator run(IndexDefinition... definitions) { - IndexDefinitions defs = new IndexDefinitions(definitions, new MapSettings().asConfig()); - defs.start(); - IndexCreator creator = new IndexCreator(es.client(), defs, metadataIndexDefinition, metadataIndex, migrationEsClient, esDbCompatibility, settings.asConfig()); - creator.start(); - return creator; - } - - private void putFakeDocument() { - es.putDocuments(FakeIndexDefinition.INDEX_TYPE, ImmutableMap.of("key", "foo")); - } - - private void verifyFakeIndexV1() { - ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); - MappingMetaData mapping = mappings.get("fakes").get("fake"); - assertThat(mapping.type()).isEqualTo("fake"); - assertThat(mapping.getSourceAsMap()).isNotEmpty(); - assertThat(countMappingFields(mapping)).isEqualTo(2); - assertThat(field(mapping, "updatedAt").get("type")).isEqualTo("date"); - } - - private static class FakeIndexDefinition implements IndexDefinition { - static final IndexMainType INDEX_TYPE = main(Index.simple("fakes"), "fake"); - - @Override - public void define(IndexDefinitionContext context) { - Index index = Index.simple("fakes"); - NewRegularIndex newIndex = context.create(index, SETTINGS_CONFIGURATION); - newIndex.createTypeMapping(IndexType.main(index, "fake")) - .keywordFieldBuilder("key").build() - .createDateTimeField("updatedAt"); - } - } - - private static class FakeIndexDefinitionV2 implements IndexDefinition { - @Override - public void define(IndexDefinitionContext context) { - Index index = Index.simple("fakes"); - NewRegularIndex newIndex = context.create(index, SETTINGS_CONFIGURATION); - newIndex.createTypeMapping(IndexType.main(index, "fake")) - .keywordFieldBuilder("key").build() - .createDateTimeField("updatedAt") - .createIntegerField("newField"); - } - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java deleted file mode 100644 index 8f9bf8b324f..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java +++ /dev/null @@ -1,505 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.collect.ImmutableSet; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.IntStream; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.DisableOnDebug; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; -import org.sonar.api.config.Configuration; -import org.sonar.api.config.internal.MapSettings; -import org.sonar.api.utils.MessageException; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.api.utils.log.LogTester; -import org.sonar.api.utils.log.LoggerLevel; -import org.sonar.db.DbSession; -import org.sonar.db.DbTester; -import org.sonar.db.es.EsQueueDto; -import org.sonar.server.es.IndexType.IndexMainType; -import org.sonar.server.rule.index.RuleIndexer; -import org.sonar.server.user.index.UserIndexer; - -import static java.util.stream.IntStream.rangeClosed; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.spy; -import static org.sonar.api.utils.log.LoggerLevel.ERROR; -import static org.sonar.api.utils.log.LoggerLevel.INFO; -import static org.sonar.api.utils.log.LoggerLevel.TRACE; - -public class RecoveryIndexerTest { - - private static final long PAST = 1_000L; - private static final IndexMainType FOO_TYPE = IndexType.main(Index.simple("foos"), "foo"); - - private TestSystem2 system2 = new TestSystem2().setNow(PAST); - private MapSettings emptySettings = new MapSettings(); - - @Rule - public EsTester es = EsTester.createCustom(); - @Rule - public DbTester db = DbTester.create(system2); - @Rule - public LogTester logTester = new LogTester().setLevel(TRACE); - @Rule - public TestRule safeguard = new DisableOnDebug(Timeout.builder().withTimeout(60, TimeUnit.SECONDS).withLookingForStuckThread(true).build()); - - private RecoveryIndexer underTest; - - @After - public void tearDown() { - if (underTest != null) { - underTest.stop(); - } - } - - @Test - public void display_default_configuration_at_startup() { - underTest = newRecoveryIndexer(emptySettings.asConfig()); - - underTest.start(); - underTest.stop(); - - assertThat(logTester.logs(LoggerLevel.DEBUG)).contains( - "Elasticsearch recovery - sonar.search.recovery.delayInMs=300000", - "Elasticsearch recovery - sonar.search.recovery.minAgeInMs=300000"); - } - - @Test - public void start_triggers_recovery_run_at_fixed_rate() throws Exception { - MapSettings settings = new MapSettings() - .setProperty("sonar.search.recovery.initialDelayInMs", "0") - .setProperty("sonar.search.recovery.delayInMs", "1"); - underTest = spy(new RecoveryIndexer(system2, settings.asConfig(), db.getDbClient())); - AtomicInteger calls = new AtomicInteger(0); - doAnswer(invocation -> { - calls.incrementAndGet(); - return null; - }).when(underTest).recover(); - - underTest.start(); - - // wait for 2 runs - while (calls.get() < 2) { - Thread.sleep(1L); - } - } - - @Test - public void successfully_recover_indexing_requests() { - EsQueueDto item1a = insertItem(FOO_TYPE, "f1"); - EsQueueDto item1b = insertItem(FOO_TYPE, "f2"); - IndexMainType type2 = IndexType.main(Index.simple("bars"), "bar"); - EsQueueDto item2 = insertItem(type2, "b1"); - - SuccessfulFakeIndexer indexer1 = new SuccessfulFakeIndexer(FOO_TYPE); - SuccessfulFakeIndexer indexer2 = new SuccessfulFakeIndexer(type2); - advanceInTime(); - - underTest = newRecoveryIndexer(indexer1, indexer2); - underTest.recover(); - - assertThatQueueHasSize(0); - assertThatLogsContain(INFO, "Elasticsearch recovery - 3 documents processed [0 failures]"); - - assertThat(indexer1.called).hasSize(1); - assertThat(indexer1.called.get(0)) - .extracting(EsQueueDto::getUuid) - .containsExactlyInAnyOrder(item1a.getUuid(), item1b.getUuid()); - assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 2 [foos/foo]"); - - assertThat(indexer2.called).hasSize(1); - assertThat(indexer2.called.get(0)) - .extracting(EsQueueDto::getUuid) - .containsExactlyInAnyOrder(item2.getUuid()); - assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 1 [bars/bar]"); - } - - @Test - public void recent_records_are_not_recovered() { - EsQueueDto item = insertItem(FOO_TYPE, "f1"); - - SuccessfulFakeIndexer indexer = new SuccessfulFakeIndexer(FOO_TYPE); - // do not advance in time - - underTest = newRecoveryIndexer(indexer); - underTest.recover(); - - assertThatQueueHasSize(1); - assertThat(indexer.called).isEmpty(); - - assertThatLogsDoNotContain(TRACE, "Elasticsearch recovery - processing 2 [foos/foo]"); - assertThatLogsDoNotContain(INFO, "documents processed"); - } - - @Test - public void do_nothing_if_queue_is_empty() { - underTest = newRecoveryIndexer(); - - underTest.recover(); - - assertThatNoLogsFromRecovery(INFO); - assertThatNoLogsFromRecovery(ERROR); - assertThatQueueHasSize(0); - } - - @Test - public void hard_failures_are_logged_and_do_not_stop_recovery_scheduling() throws Exception { - insertItem(FOO_TYPE, "f1"); - - HardFailingFakeIndexer indexer = new HardFailingFakeIndexer(FOO_TYPE); - advanceInTime(); - - underTest = newRecoveryIndexer(indexer); - underTest.start(); - - // all runs fail, but they are still scheduled - // -> waiting for 2 runs - while (indexer.called.size() < 2) { - Thread.sleep(1L); - } - - underTest.stop(); - - // No rows treated - assertThatQueueHasSize(1); - assertThatLogsContain(ERROR, "Elasticsearch recovery - fail to recover documents"); - } - - @Test - public void soft_failures_are_logged_and_do_not_stop_recovery_scheduling() throws Exception { - insertItem(FOO_TYPE, "f1"); - - SoftFailingFakeIndexer indexer = new SoftFailingFakeIndexer(FOO_TYPE); - advanceInTime(); - - underTest = newRecoveryIndexer(indexer); - underTest.start(); - - // all runs fail, but they are still scheduled - // -> waiting for 2 runs - while (indexer.called.size() < 2) { - Thread.sleep(1L); - } - - underTest.stop(); - - // No rows treated - assertThatQueueHasSize(1); - assertThatLogsContain(INFO, "Elasticsearch recovery - 1 documents processed [1 failures]"); - } - - @Test - public void unsupported_types_are_kept_in_queue_for_manual_fix_operation() { - insertItem(FOO_TYPE, "f1"); - - ResilientIndexer indexer = new SuccessfulFakeIndexer(IndexType.main(Index.simple("bars"), "bar")); - advanceInTime(); - - underTest = newRecoveryIndexer(indexer); - underTest.recover(); - - assertThatQueueHasSize(1); - assertThatLogsContain(ERROR, "Elasticsearch recovery - ignore 1 items with unsupported type [foos/foo]"); - } - - @Test - public void stop_run_if_too_many_failures() { - IntStream.range(0, 10).forEach(i -> insertItem(FOO_TYPE, "" + i)); - advanceInTime(); - - // 10 docs to process, by groups of 3. - // The first group successfully recovers only 1 docs --> above 30% of failures --> stop run - PartiallyFailingIndexer indexer = new PartiallyFailingIndexer(FOO_TYPE, 1); - MapSettings settings = new MapSettings() - .setProperty("sonar.search.recovery.loopLimit", "3"); - underTest = newRecoveryIndexer(settings.asConfig(), indexer); - underTest.recover(); - - assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [2/3 documents], waiting for next run"); - assertThatQueueHasSize(9); - - // The indexer must have been called once and only once. - assertThat(indexer.called).hasSize(3); - } - - @Test - public void do_not_stop_run_if_success_rate_is_greater_than_circuit_breaker() { - IntStream.range(0, 10).forEach(i -> insertItem(FOO_TYPE, "" + i)); - advanceInTime(); - - // 10 docs to process, by groups of 5. - // Each group successfully recovers 4 docs --> below 30% of failures --> continue run - PartiallyFailingIndexer indexer = new PartiallyFailingIndexer(FOO_TYPE, 4, 4, 2); - MapSettings settings = new MapSettings() - .setProperty("sonar.search.recovery.loopLimit", "5"); - underTest = newRecoveryIndexer(settings.asConfig(), indexer); - underTest.recover(); - - assertThatLogsDoNotContain(ERROR, "too many failures"); - assertThatQueueHasSize(0); - assertThat(indexer.indexed).hasSize(10); - assertThat(indexer.called).hasSize(10 + 2 /* retries */); - } - - @Test - public void failing_always_on_same_document_does_not_generate_infinite_loop() { - EsQueueDto buggy = insertItem(FOO_TYPE, "buggy"); - IntStream.range(0, 10).forEach(i -> insertItem(FOO_TYPE, "" + i)); - advanceInTime(); - - FailingAlwaysOnSameElementIndexer indexer = new FailingAlwaysOnSameElementIndexer(FOO_TYPE, buggy); - underTest = newRecoveryIndexer(indexer); - underTest.recover(); - - assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [1/1 documents], waiting for next run"); - assertThatQueueHasSize(1); - } - - @Test - public void recover_multiple_times_the_same_document() { - EsQueueDto item1 = insertItem(FOO_TYPE, "f1"); - EsQueueDto item2 = insertItem(FOO_TYPE, item1.getDocId()); - EsQueueDto item3 = insertItem(FOO_TYPE, item1.getDocId()); - advanceInTime(); - - SuccessfulFakeIndexer indexer = new SuccessfulFakeIndexer(FOO_TYPE); - underTest = newRecoveryIndexer(indexer); - underTest.recover(); - - assertThatQueueHasSize(0); - assertThat(indexer.called).hasSize(1); - assertThat(indexer.called.get(0)).extracting(EsQueueDto::getUuid) - .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid(), item3.getUuid()); - - assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 3 [foos/foo]"); - assertThatLogsContain(INFO, "Elasticsearch recovery - 3 documents processed [0 failures]"); - } - - private class FailingAlwaysOnSameElementIndexer implements ResilientIndexer { - private final IndexType indexType; - private final EsQueueDto failing; - - FailingAlwaysOnSameElementIndexer(IndexType indexType, EsQueueDto failing) { - this.indexType = indexType; - this.failing = failing; - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - IndexingResult result = new IndexingResult(); - items.forEach(item -> { - result.incrementRequests(); - if (!item.getUuid().equals(failing.getUuid())) { - result.incrementSuccess(); - db.getDbClient().esQueueDao().delete(dbSession, item); - dbSession.commit(); - } - }); - return result; - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(indexType); - } - } - - private class PartiallyFailingIndexer implements ResilientIndexer { - private final IndexType indexType; - private final List<EsQueueDto> called = new ArrayList<>(); - private final List<EsQueueDto> indexed = new ArrayList<>(); - private final Iterator<Integer> successfulReturns; - - PartiallyFailingIndexer(IndexType indexType, int... successfulReturns) { - this.indexType = indexType; - this.successfulReturns = IntStream.of(successfulReturns).iterator(); - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - called.addAll(items); - int success = successfulReturns.next(); - IndexingResult result = new IndexingResult(); - items.stream().limit(success).forEach(i -> { - db.getDbClient().esQueueDao().delete(dbSession, i); - result.incrementSuccess(); - indexed.add(i); - }); - - rangeClosed(1, items.size()).forEach(i -> result.incrementRequests()); - dbSession.commit(); - return result; - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(indexType); - } - } - - private void advanceInTime() { - system2.setNow(system2.now() + 100_000_000L); - } - - private void assertThatLogsContain(LoggerLevel loggerLevel, String message) { - assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains(message)).isNotEmpty(); - } - - private void assertThatLogsDoNotContain(LoggerLevel loggerLevel, String message) { - assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains(message)).isEmpty(); - } - - private void assertThatNoLogsFromRecovery(LoggerLevel loggerLevel) { - assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains("Elasticsearch recovery - ")).isEmpty(); - } - - private void assertThatQueueHasSize(int number) { - assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isEqualTo(number); - } - - private RecoveryIndexer newRecoveryIndexer() { - UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client()); - RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient()); - return newRecoveryIndexer(userIndexer, ruleIndexer); - } - - private RecoveryIndexer newRecoveryIndexer(ResilientIndexer... indexers) { - MapSettings settings = new MapSettings() - .setProperty("sonar.search.recovery.initialDelayInMs", "0") - .setProperty("sonar.search.recovery.delayInMs", "1") - .setProperty("sonar.search.recovery.minAgeInMs", "1"); - return newRecoveryIndexer(settings.asConfig(), indexers); - } - - private RecoveryIndexer newRecoveryIndexer(Configuration config, ResilientIndexer... indexers) { - return new RecoveryIndexer(system2, config, db.getDbClient(), indexers); - } - - private EsQueueDto insertItem(IndexType indexType, String docUuid) { - EsQueueDto item = EsQueueDto.create(indexType.format(), docUuid); - db.getDbClient().esQueueDao().insert(db.getSession(), item); - db.commit(); - return item; - } - - private class SuccessfulFakeIndexer implements ResilientIndexer { - private final Set<IndexType> types; - private final List<Collection<EsQueueDto>> called = new ArrayList<>(); - - private SuccessfulFakeIndexer(IndexType type) { - this.types = ImmutableSet.of(type); - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<IndexType> getIndexTypes() { - return types; - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - called.add(items); - IndexingResult result = new IndexingResult(); - items.forEach(i -> result.incrementSuccess().incrementRequests()); - db.getDbClient().esQueueDao().delete(dbSession, items); - dbSession.commit(); - return result; - } - } - - private class HardFailingFakeIndexer implements ResilientIndexer { - private final Set<IndexType> types; - private final List<Collection<EsQueueDto>> called = new ArrayList<>(); - - private HardFailingFakeIndexer(IndexType type) { - this.types = ImmutableSet.of(type); - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<IndexType> getIndexTypes() { - return types; - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - called.add(items); - // MessageException is used just to reduce noise in test logs - throw MessageException.of("BOOM"); - } - } - - private class SoftFailingFakeIndexer implements ResilientIndexer { - private final Set<IndexType> types; - private final List<Collection<EsQueueDto>> called = new ArrayList<>(); - - private SoftFailingFakeIndexer(IndexType type) { - this.types = ImmutableSet.of(type); - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<IndexType> getIndexTypes() { - return types; - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - called.add(items); - IndexingResult result = new IndexingResult(); - items.forEach(i -> result.incrementRequests()); - return result; - } - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/TestEsDbCompatibility.java b/server/sonar-server/src/test/java/org/sonar/server/es/TestEsDbCompatibility.java deleted file mode 100644 index 5b8cc1ee7f2..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/es/TestEsDbCompatibility.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import org.sonar.server.es.metadata.EsDbCompatibility; - -public class TestEsDbCompatibility implements EsDbCompatibility { - - private boolean hasSameDbVendor = true; - - public TestEsDbCompatibility setHasSameDbVendor(boolean b) { - this.hasSameDbVendor = b; - return this; - } - - @Override - public boolean hasSameDbVendor() { - return hasSameDbVendor; - } - - @Override - public void markAsCompatible() { - - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java deleted file mode 100644 index 3ba78fc31aa..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es.metadata; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import javax.annotation.CheckForNull; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mockito; -import org.sonar.db.DbClient; -import org.sonar.server.es.Index; -import org.sonar.server.es.IndexType; - -import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class EsDbCompatibilityImplTest { - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - private DbClient dbClient = mock(DbClient.class, Mockito.RETURNS_DEEP_STUBS); - private MetadataIndex metadataIndex = spy(new TestMetadataIndex()); - private EsDbCompatibilityImpl underTest = new EsDbCompatibilityImpl(dbClient, metadataIndex); - - @Test - public void hasSameDbVendor_is_true_if_values_match() { - prepareDb("mssql"); - prepareEs("mssql"); - - assertThat(underTest.hasSameDbVendor()).isTrue(); - } - - @Test - public void hasSameDbVendor_is_false_if_values_dont_match() { - prepareDb("mssql"); - prepareEs("postgres"); - - assertThat(underTest.hasSameDbVendor()).isFalse(); - } - - @Test - public void hasSameDbVendor_is_false_if_value_is_absent_from_es() { - prepareDb("mssql"); - - assertThat(underTest.hasSameDbVendor()).isFalse(); - } - - @Test - public void markAsCompatible_db_metadata_in_es() { - prepareDb("mssql"); - - underTest.markAsCompatible(); - - assertThat(metadataIndex.getDbVendor()).hasValue("mssql"); - } - - @Test - public void markAsCompatible_updates_db_metadata_in_es() { - prepareEs("mssql"); - prepareDb("postgres"); - - underTest.markAsCompatible(); - - assertThat(metadataIndex.getDbVendor()).hasValue("postgres"); - } - - @Test - public void markAsCompatible_marks_es_as_compatible_with_db() { - prepareDb("postgres"); - - underTest.markAsCompatible(); - - assertThat(underTest.hasSameDbVendor()).isTrue(); - } - - @Test - public void markAsCompatible_has_no_effect_if_vendor_is_the_same() { - String vendor = randomAlphabetic(12); - prepareEs(vendor); - prepareDb(vendor); - - underTest.markAsCompatible(); - - assertThat(underTest.hasSameDbVendor()).isTrue(); - verify(metadataIndex, times(0)).setDbMetadata(anyString()); - } - - private void prepareDb(String dbVendor) { - when(dbClient.getDatabase().getDialect().getId()).thenReturn(dbVendor); - } - - private void prepareEs(String dbVendor) { - metadataIndex.setDbMetadata(dbVendor); - // reset spy to not perturbate assertions on spy from verified code - reset(metadataIndex); - } - - private static class TestMetadataIndex implements MetadataIndex { - private final Map<Index, String> hashes = new HashMap<>(); - private final Map<IndexType, Boolean> initializeds = new HashMap<>(); - @CheckForNull - private String dbVendor = null; - - @Override - public Optional<String> getHash(Index index) { - return Optional.ofNullable(hashes.get(index)); - } - - @Override - public void setHash(Index index, String hash) { - hashes.put(index, hash); - } - - @Override - public boolean getInitialized(IndexType indexType) { - return initializeds.getOrDefault(indexType, false); - } - - @Override - public void setInitialized(IndexType indexType, boolean initialized) { - initializeds.put(indexType, initialized); - } - - @Override - public Optional<String> getDbVendor() { - return Optional.ofNullable(dbVendor); - } - - @Override - public void setDbMetadata(String vendor) { - this.dbVendor = vendor; - } - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java deleted file mode 100644 index e46f938784c..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import java.util.Map; -import java.util.TimeZone; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.rule.Severity; -import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentTesting; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Facets; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.issue.IssueDocTesting; -import org.sonar.server.issue.index.IssueQuery.Builder; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.Collections.singletonList; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; -import static org.sonar.api.issue.Issue.STATUS_CLOSED; -import static org.sonar.api.issue.Issue.STATUS_OPEN; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.utils.DateUtils.parseDateTime; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT; - -public class IssueIndexDebtTest { - - private System2 system2 = new TestSystem2().setNow(1_500_000_000_000L).setDefaultTimeZone(TimeZone.getTimeZone("GMT-01:00")); - - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - @Rule - public DbTester db = DbTester.create(system2); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void facets_on_projects() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project = ComponentTesting.newPrivateProjectDto(organizationDto, "ABCD"); - ComponentDto project2 = ComponentTesting.newPrivateProjectDto(organizationDto, "EFGH"); - - indexIssues( - IssueDocTesting.newDoc("I1", ComponentTesting.newFileDto(project, null)).setEffort(10L), - IssueDocTesting.newDoc("I2", ComponentTesting.newFileDto(project, null)).setEffort(10L), - IssueDocTesting.newDoc("I3", ComponentTesting.newFileDto(project2, null)).setEffort(10L)); - - Facets facets = search("projects"); - assertThat(facets.getNames()).containsOnly("projects", FACET_MODE_EFFORT); - assertThat(facets.get("projects")).containsOnly(entry("ABCD", 20L), entry("EFGH", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 30L)); - } - - @Test - public void facets_on_components() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto(), "A"); - ComponentDto file1 = ComponentTesting.newFileDto(project, null, "ABCD"); - ComponentDto file2 = ComponentTesting.newFileDto(project, null, "BCDE"); - ComponentDto file3 = ComponentTesting.newFileDto(project, null, "CDEF"); - - indexIssues( - IssueDocTesting.newDoc("I1", project).setEffort(10L), - IssueDocTesting.newDoc("I2", file1).setEffort(10L), - IssueDocTesting.newDoc("I3", file2).setEffort(10L), - IssueDocTesting.newDoc("I4", file2).setEffort(10L), - IssueDocTesting.newDoc("I5", file3).setEffort(10L)); - - Facets facets = search("fileUuids"); - assertThat(facets.getNames()).containsOnly("fileUuids", FACET_MODE_EFFORT); - assertThat(facets.get("fileUuids")) - .containsOnly(entry("A", 10L), entry("ABCD", 10L), entry("BCDE", 20L), entry("CDEF", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 50L)); - } - - @Test - public void facets_on_directories() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file1 = ComponentTesting.newFileDto(project, null).setPath("src/main/xoo/F1.xoo"); - ComponentDto file2 = ComponentTesting.newFileDto(project, null).setPath("F2.xoo"); - - indexIssues( - IssueDocTesting.newDoc("I1", file1).setDirectoryPath("/src/main/xoo").setEffort(10L), - IssueDocTesting.newDoc("I2", file2).setDirectoryPath("/").setEffort(10L)); - - Facets facets = search("directories"); - assertThat(facets.getNames()).containsOnly("directories", FACET_MODE_EFFORT); - assertThat(facets.get("directories")).containsOnly(entry("/src/main/xoo", 10L), entry("/", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 20L)); - } - - @Test - public void facets_on_severities() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = ComponentTesting.newFileDto(project, null); - - indexIssues( - IssueDocTesting.newDoc("I1", file).setSeverity(Severity.INFO).setEffort(10L), - IssueDocTesting.newDoc("I2", file).setSeverity(Severity.INFO).setEffort(10L), - IssueDocTesting.newDoc("I3", file).setSeverity(Severity.MAJOR).setEffort(10L)); - - Facets facets = search("severities"); - assertThat(facets.getNames()).containsOnly("severities", FACET_MODE_EFFORT); - assertThat(facets.get("severities")).containsOnly(entry("INFO", 20L), entry("MAJOR", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 30L)); - } - - @Test - public void facets_on_statuses() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = ComponentTesting.newFileDto(project, null); - - indexIssues( - IssueDocTesting.newDoc("I1", file).setStatus(STATUS_CLOSED).setEffort(10L), - IssueDocTesting.newDoc("I2", file).setStatus(STATUS_CLOSED).setEffort(10L), - IssueDocTesting.newDoc("I3", file).setStatus(STATUS_OPEN).setEffort(10L)); - - Facets facets = search("statuses"); - assertThat(facets.getNames()).containsOnly("statuses", FACET_MODE_EFFORT); - assertThat(facets.get("statuses")).containsOnly(entry("CLOSED", 20L), entry("OPEN", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 30L)); - } - - @Test - public void facets_on_resolutions() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = ComponentTesting.newFileDto(project, null); - - indexIssues( - IssueDocTesting.newDoc("I1", file).setResolution(Issue.RESOLUTION_FALSE_POSITIVE).setEffort(10L), - IssueDocTesting.newDoc("I2", file).setResolution(Issue.RESOLUTION_FALSE_POSITIVE).setEffort(10L), - IssueDocTesting.newDoc("I3", file).setResolution(Issue.RESOLUTION_FIXED).setEffort(10L)); - - Facets facets = search("resolutions"); - assertThat(facets.getNames()).containsOnly("resolutions", FACET_MODE_EFFORT); - assertThat(facets.get("resolutions")).containsOnly(entry("FALSE-POSITIVE", 20L), entry("FIXED", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 30L)); - } - - @Test - public void facets_on_languages() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = ComponentTesting.newFileDto(project, null); - RuleKey ruleKey = RuleKey.of("repo", "X1"); - - indexIssues(IssueDocTesting.newDoc("I1", file).setLanguage("xoo").setEffort(10L)); - - Facets facets = search("languages"); - assertThat(facets.getNames()).containsOnly("languages", FACET_MODE_EFFORT); - assertThat(facets.get("languages")).containsOnly(entry("xoo", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 10L)); - } - - @Test - public void facets_on_assignees() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = ComponentTesting.newFileDto(project, null); - - indexIssues( - IssueDocTesting.newDoc("I1", file).setAssigneeUuid("uuid-steph").setEffort(10L), - IssueDocTesting.newDoc("I2", file).setAssigneeUuid("uuid-simon").setEffort(10L), - IssueDocTesting.newDoc("I3", file).setAssigneeUuid("uuid-simon").setEffort(10L), - IssueDocTesting.newDoc("I4", file).setAssigneeUuid(null).setEffort(10L)); - - Facets facets = new Facets(underTest.search(newQueryBuilder().build(), new SearchOptions().addFacets(asList("assignees"))), system2.getDefaultTimeZone()); - assertThat(facets.getNames()).containsOnly("assignees", FACET_MODE_EFFORT); - assertThat(facets.get("assignees")).containsOnly(entry("uuid-steph", 10L), entry("uuid-simon", 20L), entry("", 10L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 40L)); - } - - @Test - public void facets_on_authors() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = ComponentTesting.newFileDto(project, null); - - indexIssues( - IssueDocTesting.newDoc("I1", file).setAuthorLogin("steph").setEffort(10L), - IssueDocTesting.newDoc("I2", file).setAuthorLogin("simon").setEffort(10L), - IssueDocTesting.newDoc("I3", file).setAuthorLogin("simon").setEffort(10L), - IssueDocTesting.newDoc("I4", file).setAuthorLogin(null).setEffort(10L)); - - Facets facets = new Facets(underTest.search(newQueryBuilder().build(), new SearchOptions().addFacets(asList("authors"))), system2.getDefaultTimeZone()); - assertThat(facets.getNames()).containsOnly("authors", FACET_MODE_EFFORT); - assertThat(facets.get("authors")).containsOnly(entry("steph", 10L), entry("simon", 20L)); - assertThat(facets.get(FACET_MODE_EFFORT)).containsOnly(entry("total", 40L)); - } - - @Test - public void facet_on_created_at() { - SearchOptions searchOptions = fixtureForCreatedAtFacet(); - - Builder query = newQueryBuilder().createdBefore(parseDateTime("2016-01-01T00:00:00+0100")); - Map<String, Long> createdAt = new Facets(underTest.search(query.build(), searchOptions), system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).containsOnly( - entry("2011-01-01", 10L), - entry("2012-01-01", 0L), - entry("2013-01-01", 0L), - entry("2014-01-01", 50L), - entry("2015-01-01", 10L)); - } - - private SearchOptions fixtureForCreatedAtFacet() { - ComponentDto project = ComponentTesting.newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = ComponentTesting.newFileDto(project, null); - - IssueDoc issue0 = IssueDocTesting.newDoc("ISSUE0", file).setEffort(10L).setFuncCreationDate(parseDateTime("2011-04-25T01:05:13+0100")); - IssueDoc issue1 = IssueDocTesting.newDoc("I1", file).setEffort(10L).setFuncCreationDate(parseDateTime("2014-09-01T12:34:56+0100")); - IssueDoc issue2 = IssueDocTesting.newDoc("I2", file).setEffort(10L).setFuncCreationDate(parseDateTime("2014-09-01T23:46:00+0100")); - IssueDoc issue3 = IssueDocTesting.newDoc("I3", file).setEffort(10L).setFuncCreationDate(parseDateTime("2014-09-02T12:34:56+0100")); - IssueDoc issue4 = IssueDocTesting.newDoc("I4", file).setEffort(10L).setFuncCreationDate(parseDateTime("2014-09-05T12:34:56+0100")); - IssueDoc issue5 = IssueDocTesting.newDoc("I5", file).setEffort(10L).setFuncCreationDate(parseDateTime("2014-09-20T12:34:56+0100")); - IssueDoc issue6 = IssueDocTesting.newDoc("I6", file).setEffort(10L).setFuncCreationDate(parseDateTime("2015-01-18T12:34:56+0100")); - - indexIssues(issue0, issue1, issue2, issue3, issue4, issue5, issue6); - - return new SearchOptions().addFacets("createdAt"); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } - - private Facets search(String additionalFacet) { - return new Facets(underTest.search(newQueryBuilder().build(), new SearchOptions().addFacets(singletonList(additionalFacet))), system2.getDefaultTimeZone()); - } - - private Builder newQueryBuilder() { - return IssueQuery.builder().facetMode(FACET_MODE_EFFORT); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexFacetsTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexFacetsTest.java deleted file mode 100644 index e72f44a18b7..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexFacetsTest.java +++ /dev/null @@ -1,540 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import java.util.Map; -import org.elasticsearch.action.search.SearchResponse; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Facets; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.Collections.singletonList; -import static java.util.TimeZone.getTimeZone; -import static java.util.stream.Collectors.toList; -import static java.util.stream.IntStream.rangeClosed; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; -import static org.junit.rules.ExpectedException.none; -import static org.sonar.api.issue.Issue.RESOLUTION_FALSE_POSITIVE; -import static org.sonar.api.issue.Issue.RESOLUTION_FIXED; -import static org.sonar.api.issue.Issue.RESOLUTION_REMOVED; -import static org.sonar.api.issue.Issue.RESOLUTION_WONT_FIX; -import static org.sonar.api.issue.Issue.STATUS_CLOSED; -import static org.sonar.api.issue.Issue.STATUS_CONFIRMED; -import static org.sonar.api.issue.Issue.STATUS_OPEN; -import static org.sonar.api.issue.Issue.STATUS_REOPENED; -import static org.sonar.api.issue.Issue.STATUS_RESOLVED; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.rule.Severity.BLOCKER; -import static org.sonar.api.rule.Severity.CRITICAL; -import static org.sonar.api.rule.Severity.INFO; -import static org.sonar.api.rule.Severity.MAJOR; -import static org.sonar.api.rule.Severity.MINOR; -import static org.sonar.api.utils.DateUtils.parseDateTime; -import static org.sonar.db.component.ComponentTesting.newDirectory; -import static org.sonar.db.component.ComponentTesting.newFileDto; -import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonar.db.rule.RuleTesting.newRule; -import static org.sonar.server.issue.IssueDocTesting.newDoc; - -public class IssueIndexFacetsTest { - - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - @Rule - public ExpectedException expectedException = none(); - private System2 system2 = new TestSystem2().setNow(1_500_000_000_000L).setDefaultTimeZone(getTimeZone("GMT-01:00")); - @Rule - public DbTester db = DbTester.create(system2); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void facet_on_projectUuids() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organizationDto, "ABCD"); - ComponentDto project2 = newPrivateProjectDto(organizationDto, "EFGH"); - - indexIssues( - newDoc("I1", newFileDto(project, null)), - newDoc("I2", newFileDto(project, null)), - newDoc("I3", newFileDto(project2, null))); - - assertThatFacetHasExactly(IssueQuery.builder(), "projects", entry("ABCD", 2L), entry("EFGH", 1L)); - } - - @Test - public void facet_on_projectUuids_return_100_entries_plus_selected_values() { - OrganizationDto organizationDto = newOrganizationDto(); - indexIssues(rangeClosed(1, 110).mapToObj(i -> newDoc(newPrivateProjectDto(organizationDto, "a" + i))).toArray(IssueDoc[]::new)); - IssueDoc issue1 = newDoc(newPrivateProjectDto(organizationDto, "project1")); - IssueDoc issue2 = newDoc(newPrivateProjectDto(organizationDto, "project2")); - indexIssues(issue1, issue2); - - assertThatFacetHasSize(IssueQuery.builder().build(), "projects", 100); - assertThatFacetHasSize(IssueQuery.builder().projectUuids(asList(issue1.projectUuid(), issue2.projectUuid())).build(), "projects", 102); - } - - @Test - public void facets_on_files() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto(), "A"); - ComponentDto file1 = newFileDto(project, null, "ABCD"); - ComponentDto file2 = newFileDto(project, null, "BCDE"); - ComponentDto file3 = newFileDto(project, null, "CDEF"); - - indexIssues( - newDoc("I1", project), - newDoc("I2", file1), - newDoc("I3", file2), - newDoc("I4", file2), - newDoc("I5", file3)); - - assertThatFacetHasOnly(IssueQuery.builder(), "fileUuids", entry("A", 1L), entry("ABCD", 1L), entry("BCDE", 2L), entry("CDEF", 1L)); - } - - @Test - public void facet_on_files_return_100_entries_plus_selected_values() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organizationDto); - indexIssues(rangeClosed(1, 110).mapToObj(i -> newDoc(newFileDto(project, null, "a" + i))).toArray(IssueDoc[]::new)); - IssueDoc issue1 = newDoc(newFileDto(project, null, "file1")); - IssueDoc issue2 = newDoc(newFileDto(project, null, "file2")); - indexIssues(issue1, issue2); - - assertThatFacetHasSize(IssueQuery.builder().build(), "fileUuids", 100); - assertThatFacetHasSize(IssueQuery.builder().fileUuids(asList(issue1.componentUuid(), issue2.componentUuid())).build(), "fileUuids", 102); - } - - @Test - public void facets_on_directories() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file1 = newFileDto(project, null).setPath("src/main/xoo/F1.xoo"); - ComponentDto file2 = newFileDto(project, null).setPath("F2.xoo"); - - indexIssues( - newDoc("I1", file1).setDirectoryPath("/src/main/xoo"), - newDoc("I2", file2).setDirectoryPath("/")); - - assertThatFacetHasOnly(IssueQuery.builder(), "directories", entry("/src/main/xoo", 1L), entry("/", 1L)); - } - - @Test - public void facet_on_directories_return_100_entries_plus_selected_values() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organizationDto); - indexIssues(rangeClosed(1, 110).mapToObj(i -> newDoc(newFileDto(project, newDirectory(project, "dir" + i))).setDirectoryPath("a" + i)).toArray(IssueDoc[]::new)); - IssueDoc issue1 = newDoc(newFileDto(project, newDirectory(project, "path1"))).setDirectoryPath("directory1"); - IssueDoc issue2 = newDoc(newFileDto(project, newDirectory(project, "path2"))).setDirectoryPath("directory2"); - indexIssues(issue1, issue2); - - assertThatFacetHasSize(IssueQuery.builder().build(), "directories", 100); - assertThatFacetHasSize(IssueQuery.builder().directories(asList(issue1.directoryPath(), issue2.directoryPath())).build(), "directories", 102); - } - - @Test - public void facets_on_severities() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setSeverity(INFO), - newDoc("I2", file).setSeverity(INFO), - newDoc("I3", file).setSeverity(MAJOR)); - - assertThatFacetHasOnly(IssueQuery.builder(), "severities", entry("INFO", 2L), entry("MAJOR", 1L)); - } - - @Test - public void facet_on_severities_return_5_entries_max() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I2", file).setSeverity(INFO), - newDoc("I1", file).setSeverity(MINOR), - newDoc("I3", file).setSeverity(MAJOR), - newDoc("I4", file).setSeverity(CRITICAL), - newDoc("I5", file).setSeverity(BLOCKER), - newDoc("I6", file).setSeverity(MAJOR)); - - assertThatFacetHasSize(IssueQuery.builder().build(), "severities", 5); - } - - @Test - public void facets_on_statuses() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setStatus(STATUS_CLOSED), - newDoc("I2", file).setStatus(STATUS_CLOSED), - newDoc("I3", file).setStatus(STATUS_OPEN)); - - assertThatFacetHasOnly(IssueQuery.builder(), "statuses", entry("CLOSED", 2L), entry("OPEN", 1L)); - } - - @Test - public void facet_on_statuses_return_5_entries_max() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setStatus(STATUS_OPEN), - newDoc("I2", file).setStatus(STATUS_CONFIRMED), - newDoc("I3", file).setStatus(STATUS_REOPENED), - newDoc("I4", file).setStatus(STATUS_RESOLVED), - newDoc("I5", file).setStatus(STATUS_CLOSED), - newDoc("I6", file).setStatus(STATUS_OPEN)); - - assertThatFacetHasSize(IssueQuery.builder().build(), "statuses", 5); - } - - @Test - public void facets_on_resolutions() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setResolution(RESOLUTION_FALSE_POSITIVE), - newDoc("I2", file).setResolution(RESOLUTION_FALSE_POSITIVE), - newDoc("I3", file).setResolution(RESOLUTION_FIXED)); - - assertThatFacetHasOnly(IssueQuery.builder(), "resolutions", entry("FALSE-POSITIVE", 2L), entry("FIXED", 1L)); - } - - @Test - public void facets_on_resolutions_return_5_entries_max() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setResolution(RESOLUTION_FIXED), - newDoc("I2", file).setResolution(RESOLUTION_FALSE_POSITIVE), - newDoc("I3", file).setResolution(RESOLUTION_REMOVED), - newDoc("I4", file).setResolution(RESOLUTION_WONT_FIX), - newDoc("I5", file).setResolution(null)); - - assertThatFacetHasSize(IssueQuery.builder().build(), "resolutions", 5); - } - - @Test - public void facets_on_languages() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - RuleDefinitionDto ruleDefinitionDto = newRule(); - db.rules().insert(ruleDefinitionDto); - - indexIssues(newDoc("I1", file).setRuleId(ruleDefinitionDto.getId()).setLanguage("xoo")); - - assertThatFacetHasOnly(IssueQuery.builder(), "languages", entry("xoo", 1L)); - } - - @Test - public void facets_on_languages_return_100_entries_plus_selected_values() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organizationDto); - indexIssues(rangeClosed(1, 100).mapToObj(i -> newDoc(newFileDto(project, null)).setLanguage("a" + i)).toArray(IssueDoc[]::new)); - IssueDoc issue1 = newDoc(newFileDto(project, null)).setLanguage("language1"); - IssueDoc issue2 = newDoc(newFileDto(project, null)).setLanguage("language2"); - indexIssues(issue1, issue2); - - assertThatFacetHasSize(IssueQuery.builder().build(), "languages", 100); - assertThatFacetHasSize(IssueQuery.builder().languages(asList(issue1.language(), issue2.language())).build(), "languages", 102); - } - - @Test - public void facets_on_assignees() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setAssigneeUuid("steph-uuid"), - newDoc("I2", file).setAssigneeUuid("marcel-uuid"), - newDoc("I3", file).setAssigneeUuid("marcel-uuid"), - newDoc("I4", file).setAssigneeUuid(null)); - - assertThatFacetHasOnly(IssueQuery.builder(), "assignees", entry("steph-uuid", 1L), entry("marcel-uuid", 2L), entry("", 1L)); - } - - @Test - public void facets_on_assignees_return_only_100_entries_plus_selected_values() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organizationDto); - indexIssues(rangeClosed(1, 110).mapToObj(i -> newDoc(newFileDto(project, null)).setAssigneeUuid("a" + i)).toArray(IssueDoc[]::new)); - IssueDoc issue1 = newDoc(newFileDto(project, null)).setAssigneeUuid("user1"); - IssueDoc issue2 = newDoc(newFileDto(project, null)).setAssigneeUuid("user2"); - indexIssues(issue1, issue2); - - assertThatFacetHasSize(IssueQuery.builder().build(), "assignees", 100); - assertThatFacetHasSize(IssueQuery.builder().assigneeUuids(asList(issue1.assigneeUuid(), issue2.assigneeUuid())).build(), "assignees", 102); - } - - @Test - public void facets_on_assignees_supports_dashes() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setAssigneeUuid("j-b-uuid"), - newDoc("I2", file).setAssigneeUuid("marcel-uuid"), - newDoc("I3", file).setAssigneeUuid("marcel-uuid"), - newDoc("I4", file).setAssigneeUuid(null)); - - assertThatFacetHasOnly(IssueQuery.builder().assigneeUuids(singletonList("j-b")), - "assignees", entry("j-b-uuid", 1L), entry("marcel-uuid", 2L), entry("", 1L)); - } - - @Test - public void facets_on_author() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setAuthorLogin("steph"), - newDoc("I2", file).setAuthorLogin("marcel"), - newDoc("I3", file).setAuthorLogin("marcel"), - newDoc("I4", file).setAuthorLogin(null)); - - assertThatFacetHasOnly(IssueQuery.builder(), "author", entry("steph", 1L), entry("marcel", 2L)); - } - - @Test - public void facets_on_deprecated_authors() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setAuthorLogin("steph"), - newDoc("I2", file).setAuthorLogin("marcel"), - newDoc("I3", file).setAuthorLogin("marcel"), - newDoc("I4", file).setAuthorLogin(null)); - - assertThatFacetHasOnly(IssueQuery.builder(), "authors", entry("steph", 1L), entry("marcel", 2L)); - } - - @Test - public void facets_on_authors_return_100_entries_plus_selected_values() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organizationDto); - indexIssues(rangeClosed(1, 110).mapToObj(i -> newDoc(newFileDto(project, null)).setAuthorLogin("a" + i)).toArray(IssueDoc[]::new)); - IssueDoc issue1 = newDoc(newFileDto(project, null)).setAuthorLogin("user1"); - IssueDoc issue2 = newDoc(newFileDto(project, null)).setAuthorLogin("user2"); - indexIssues(issue1, issue2); - - assertThatFacetHasSize(IssueQuery.builder().build(), "authors", 100); - assertThatFacetHasSize(IssueQuery.builder().authors(asList(issue1.authorLogin(), issue2.authorLogin())).build(), "authors", 102); - } - - @Test - public void facet_on_created_at_with_less_than_20_days() { - SearchOptions options = fixtureForCreatedAtFacet(); - - IssueQuery query = IssueQuery.builder() - .createdAfter(parseDateTime("2014-09-01T00:00:00+0100")) - .createdBefore(parseDateTime("2014-09-08T00:00:00+0100")) - .build(); - SearchResponse result = underTest.search(query, options); - Map<String, Long> buckets = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(buckets).containsOnly( - entry("2014-08-31", 0L), - entry("2014-09-01", 2L), - entry("2014-09-02", 1L), - entry("2014-09-03", 0L), - entry("2014-09-04", 0L), - entry("2014-09-05", 1L), - entry("2014-09-06", 0L), - entry("2014-09-07", 0L)); - } - - @Test - public void facet_on_created_at_with_less_than_20_weeks() { - SearchOptions options = fixtureForCreatedAtFacet(); - - SearchResponse result = underTest.search(IssueQuery.builder() - .createdAfter(parseDateTime("2014-09-01T00:00:00+0100")) - .createdBefore(parseDateTime("2014-09-21T00:00:00+0100")).build(), - options); - Map<String, Long> createdAt = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).containsOnly( - entry("2014-08-25", 0L), - entry("2014-09-01", 4L), - entry("2014-09-08", 0L), - entry("2014-09-15", 1L)); - } - - @Test - public void facet_on_created_at_with_less_than_20_months() { - SearchOptions options = fixtureForCreatedAtFacet(); - - SearchResponse result = underTest.search(IssueQuery.builder() - .createdAfter(parseDateTime("2014-09-01T00:00:00+0100")) - .createdBefore(parseDateTime("2015-01-19T00:00:00+0100")).build(), - options); - Map<String, Long> createdAt = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).containsOnly( - entry("2014-08-01", 0L), - entry("2014-09-01", 5L), - entry("2014-10-01", 0L), - entry("2014-11-01", 0L), - entry("2014-12-01", 0L), - entry("2015-01-01", 1L)); - } - - @Test - public void facet_on_created_at_with_more_than_20_months() { - SearchOptions options = fixtureForCreatedAtFacet(); - - SearchResponse result = underTest.search(IssueQuery.builder() - .createdAfter(parseDateTime("2011-01-01T00:00:00+0100")) - .createdBefore(parseDateTime("2016-01-01T00:00:00+0100")).build(), - options); - Map<String, Long> createdAt = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).containsOnly( - entry("2010-01-01", 0L), - entry("2011-01-01", 1L), - entry("2012-01-01", 0L), - entry("2013-01-01", 0L), - entry("2014-01-01", 5L), - entry("2015-01-01", 1L)); - } - - @Test - public void facet_on_created_at_with_one_day() { - SearchOptions options = fixtureForCreatedAtFacet(); - - SearchResponse result = underTest.search(IssueQuery.builder() - .createdAfter(parseDateTime("2014-09-01T00:00:00-0100")) - .createdBefore(parseDateTime("2014-09-02T00:00:00-0100")).build(), - options); - Map<String, Long> createdAt = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).containsOnly( - entry("2014-09-01", 2L)); - } - - @Test - public void facet_on_created_at_with_bounds_outside_of_data() { - SearchOptions options = fixtureForCreatedAtFacet(); - - SearchResponse result = underTest.search(IssueQuery.builder() - .createdAfter(parseDateTime("2009-01-01T00:00:00+0100")) - .createdBefore(parseDateTime("2016-01-01T00:00:00+0100")) - .build(), options); - Map<String, Long> createdAt = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).containsOnly( - entry("2008-01-01", 0L), - entry("2009-01-01", 0L), - entry("2010-01-01", 0L), - entry("2011-01-01", 1L), - entry("2012-01-01", 0L), - entry("2013-01-01", 0L), - entry("2014-01-01", 5L), - entry("2015-01-01", 1L)); - } - - @Test - public void facet_on_created_at_without_start_bound() { - SearchOptions searchOptions = fixtureForCreatedAtFacet(); - - SearchResponse result = underTest.search(IssueQuery.builder() - .createdBefore(parseDateTime("2016-01-01T00:00:00+0100")).build(), - searchOptions); - Map<String, Long> createdAt = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).containsOnly( - entry("2011-01-01", 1L), - entry("2012-01-01", 0L), - entry("2013-01-01", 0L), - entry("2014-01-01", 5L), - entry("2015-01-01", 1L)); - } - - @Test - public void facet_on_created_at_without_issues() { - SearchOptions searchOptions = new SearchOptions().addFacets("createdAt"); - - SearchResponse result = underTest.search(IssueQuery.builder().build(), searchOptions); - Map<String, Long> createdAt = new Facets(result, system2.getDefaultTimeZone()).get("createdAt"); - assertThat(createdAt).isNull(); - } - - private SearchOptions fixtureForCreatedAtFacet() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - IssueDoc issue0 = newDoc("ISSUE0", file).setFuncCreationDate(parseDateTime("2011-04-25T00:05:13+0000")); - IssueDoc issue1 = newDoc("I1", file).setFuncCreationDate(parseDateTime("2014-09-01T12:34:56+0100")); - IssueDoc issue2 = newDoc("I2", file).setFuncCreationDate(parseDateTime("2014-09-01T10:46:00-1200")); - IssueDoc issue3 = newDoc("I3", file).setFuncCreationDate(parseDateTime("2014-09-02T23:34:56+1200")); - IssueDoc issue4 = newDoc("I4", file).setFuncCreationDate(parseDateTime("2014-09-05T12:34:56+0100")); - IssueDoc issue5 = newDoc("I5", file).setFuncCreationDate(parseDateTime("2014-09-20T12:34:56+0100")); - IssueDoc issue6 = newDoc("I6", file).setFuncCreationDate(parseDateTime("2015-01-18T12:34:56+0100")); - - indexIssues(issue0, issue1, issue2, issue3, issue4, issue5, issue6); - - return new SearchOptions().addFacets("createdAt"); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } - - @SafeVarargs - private final void assertThatFacetHasExactly(IssueQuery.Builder query, String facet, Map.Entry<String, Long>... expectedEntries) { - SearchResponse result = underTest.search(query.build(), new SearchOptions().addFacets(singletonList(facet))); - Facets facets = new Facets(result, system2.getDefaultTimeZone()); - assertThat(facets.getNames()).containsOnly(facet, "effort"); - assertThat(facets.get(facet)).containsExactly(expectedEntries); - } - - @SafeVarargs - private final void assertThatFacetHasOnly(IssueQuery.Builder query, String facet, Map.Entry<String, Long>... expectedEntries) { - SearchResponse result = underTest.search(query.build(), new SearchOptions().addFacets(singletonList(facet))); - Facets facets = new Facets(result, system2.getDefaultTimeZone()); - assertThat(facets.getNames()).containsOnly(facet, "effort"); - assertThat(facets.get(facet)).containsOnly(expectedEntries); - } - - private void assertThatFacetHasSize(IssueQuery issueQuery, String facet, int expectedSize) { - SearchResponse result = underTest.search(issueQuery, new SearchOptions().addFacets(singletonList(facet))); - Facets facets = new Facets(result, system2.getDefaultTimeZone()); - assertThat(facets.get(facet)).hasSize(expectedSize); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java deleted file mode 100644 index 7bdb1edfdae..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java +++ /dev/null @@ -1,775 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import com.google.common.collect.ImmutableMap; -import java.util.Arrays; -import java.util.Date; -import java.util.List; -import java.util.stream.Collectors; -import org.assertj.core.api.Fail; -import org.elasticsearch.search.SearchHit; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.Severity; -import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; -import org.sonar.server.view.index.ViewDoc; -import org.sonar.server.view.index.ViewIndexer; - -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static java.util.TimeZone.getTimeZone; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.rules.ExpectedException.none; -import static org.sonar.api.resources.Qualifiers.APP; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.utils.DateUtils.addDays; -import static org.sonar.api.utils.DateUtils.parseDate; -import static org.sonar.api.utils.DateUtils.parseDateTime; -import static org.sonar.db.component.ComponentTesting.newFileDto; -import static org.sonar.db.component.ComponentTesting.newModuleDto; -import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonar.db.rule.RuleTesting.newRule; -import static org.sonar.server.issue.IssueDocTesting.newDoc; - -public class IssueIndexFiltersTest { - - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - @Rule - public ExpectedException expectedException = none(); - private System2 system2 = new TestSystem2().setNow(1_500_000_000_000L).setDefaultTimeZone(getTimeZone("GMT-01:00")); - @Rule - public DbTester db = DbTester.create(system2); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); - private ViewIndexer viewIndexer = new ViewIndexer(db.getDbClient(), es.client()); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void filter_by_keys() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - - indexIssues( - newDoc("I1", newFileDto(project, null)), - newDoc("I2", newFileDto(project, null))); - - assertThatSearchReturnsOnly(IssueQuery.builder().issueKeys(asList("I1", "I2")), "I1", "I2"); - assertThatSearchReturnsOnly(IssueQuery.builder().issueKeys(singletonList("I1")), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().issueKeys(asList("I3", "I4"))); - } - - @Test - public void filter_by_projects() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto module = newModuleDto(project); - ComponentDto subModule = newModuleDto(module); - - indexIssues( - newDoc("I1", project), - newDoc("I2", newFileDto(project, null)), - newDoc("I3", module), - newDoc("I4", newFileDto(module, null)), - newDoc("I5", subModule), - newDoc("I6", newFileDto(subModule, null))); - - assertThatSearchReturnsOnly(IssueQuery.builder().projectUuids(singletonList(project.uuid())), "I1", "I2", "I3", "I4", "I5", "I6"); - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList("unknown"))); - } - - @Test - public void filter_by_modules() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto module = newModuleDto(project); - ComponentDto subModule = newModuleDto(module); - ComponentDto file = newFileDto(subModule, null); - - indexIssues( - newDoc("I3", module), - newDoc("I5", subModule), - newDoc("I2", file)); - - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList(project.uuid())).moduleUuids(singletonList(file.uuid()))); - assertThatSearchReturnsOnly(IssueQuery.builder().projectUuids(singletonList(project.uuid())).moduleUuids(singletonList(module.uuid())), "I3"); - assertThatSearchReturnsOnly(IssueQuery.builder().projectUuids(singletonList(project.uuid())).moduleUuids(singletonList(subModule.uuid())), "I2", "I5"); - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList(project.uuid())).moduleUuids(singletonList(project.uuid()))); - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList(project.uuid())).moduleUuids(singletonList("unknown"))); - } - - @Test - public void filter_by_components_on_contextualized_search() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto module = newModuleDto(project); - ComponentDto subModule = newModuleDto(module); - ComponentDto file1 = newFileDto(project, null); - ComponentDto file2 = newFileDto(module, null); - ComponentDto file3 = newFileDto(subModule, null); - String view = "ABCD"; - indexView(view, asList(project.uuid())); - - indexIssues( - newDoc("I1", project), - newDoc("I2", file1), - newDoc("I3", module), - newDoc("I4", file2), - newDoc("I5", subModule), - newDoc("I6", file3)); - - assertThatSearchReturnsOnly(IssueQuery.builder().fileUuids(asList(file1.uuid(), file2.uuid(), file3.uuid())), "I2", "I4", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().fileUuids(singletonList(file1.uuid())), "I2"); - assertThatSearchReturnsOnly(IssueQuery.builder().moduleRootUuids(singletonList(subModule.uuid())), "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().moduleRootUuids(singletonList(module.uuid())), "I3", "I4", "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().projectUuids(singletonList(project.uuid())), "I1", "I2", "I3", "I4", "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(view)), "I1", "I2", "I3", "I4", "I5", "I6"); - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList("unknown"))); - } - - @Test - public void filter_by_components_on_non_contextualized_search() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto(), "project"); - ComponentDto file1 = newFileDto(project, null, "file1"); - ComponentDto module = newModuleDto(project).setUuid("module"); - ComponentDto file2 = newFileDto(module, null, "file2"); - ComponentDto subModule = newModuleDto(module).setUuid("subModule"); - ComponentDto file3 = newFileDto(subModule, null, "file3"); - String view = "ABCD"; - indexView(view, asList(project.uuid())); - - indexIssues( - newDoc("I1", project), - newDoc("I2", file1), - newDoc("I3", module), - newDoc("I4", file2), - newDoc("I5", subModule), - newDoc("I6", file3)); - - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList("unknown"))); - assertThatSearchReturnsOnly(IssueQuery.builder().projectUuids(singletonList(project.uuid())), "I1", "I2", "I3", "I4", "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(view)), "I1", "I2", "I3", "I4", "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().moduleUuids(singletonList(module.uuid())), "I3", "I4"); - assertThatSearchReturnsOnly(IssueQuery.builder().moduleUuids(singletonList(subModule.uuid())), "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().fileUuids(singletonList(file1.uuid())), "I2"); - assertThatSearchReturnsOnly(IssueQuery.builder().fileUuids(asList(file1.uuid(), file2.uuid(), file3.uuid())), "I2", "I4", "I6"); - } - - @Test - public void filter_by_directories() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file1 = newFileDto(project, null).setPath("src/main/xoo/F1.xoo"); - ComponentDto file2 = newFileDto(project, null).setPath("F2.xoo"); - - indexIssues( - newDoc("I1", file1).setDirectoryPath("/src/main/xoo"), - newDoc("I2", file2).setDirectoryPath("/")); - - assertThatSearchReturnsOnly(IssueQuery.builder().directories(singletonList("/src/main/xoo")), "I1"); - assertThatSearchReturnsOnly(IssueQuery.builder().directories(singletonList("/")), "I2"); - assertThatSearchReturnsEmpty(IssueQuery.builder().directories(singletonList("unknown"))); - } - - @Test - public void filter_by_portfolios() { - ComponentDto portfolio1 = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto portfolio2 = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto project1 = db.components().insertPrivateProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project1)); - ComponentDto project2 = db.components().insertPrivateProject(); - - IssueDoc issueOnProject1 = newDoc(project1); - IssueDoc issueOnFile = newDoc(file); - IssueDoc issueOnProject2 = newDoc(project2); - - indexIssues(issueOnProject1, issueOnFile, issueOnProject2); - indexView(portfolio1.uuid(), singletonList(project1.uuid())); - indexView(portfolio2.uuid(), singletonList(project2.uuid())); - - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(portfolio1.uuid())), issueOnProject1.key(), issueOnFile.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(portfolio2.uuid())), issueOnProject2.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(asList(portfolio1.uuid(), portfolio2.uuid())), issueOnProject1.key(), issueOnFile.key(), issueOnProject2.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(portfolio1.uuid())).projectUuids(singletonList(project1.uuid())), issueOnProject1.key(), - issueOnFile.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(portfolio1.uuid())).fileUuids(singletonList(file.uuid())), issueOnFile.key()); - assertThatSearchReturnsEmpty(IssueQuery.builder().viewUuids(singletonList("unknown"))); - } - - @Test - public void filter_by_portfolios_not_having_projects() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project1 = newPrivateProjectDto(organizationDto); - ComponentDto file1 = newFileDto(project1, null); - indexIssues(newDoc("I2", file1)); - String view1 = "ABCD"; - indexView(view1, emptyList()); - - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(view1))); - } - - @Test - public void do_not_return_issues_from_project_branch_when_filtering_by_portfolios() { - ComponentDto portfolio = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto project = db.components().insertMainBranch(); - ComponentDto projectBranch = db.components().insertProjectBranch(project); - ComponentDto fileOnProjectBranch = db.components().insertComponent(newFileDto(projectBranch)); - indexView(portfolio.uuid(), singletonList(project.uuid())); - - IssueDoc issueOnProject = newDoc(project); - IssueDoc issueOnProjectBranch = newDoc(projectBranch); - IssueDoc issueOnFileOnProjectBranch = newDoc(fileOnProjectBranch); - indexIssues(issueOnProject, issueOnFileOnProjectBranch, issueOnProjectBranch); - - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(portfolio.uuid())), issueOnProject.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(portfolio.uuid())).projectUuids(singletonList(project.uuid())), - issueOnProject.key()); - assertThatSearchReturnsEmpty(IssueQuery.builder().viewUuids(singletonList(portfolio.uuid())).projectUuids(singletonList(projectBranch.uuid()))); - } - - @Test - public void filter_one_issue_by_project_and_branch() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto branch = db.components().insertProjectBranch(project); - ComponentDto anotherbBranch = db.components().insertProjectBranch(project); - - IssueDoc issueOnProject = newDoc(project); - IssueDoc issueOnBranch = newDoc(branch); - IssueDoc issueOnAnotherBranch = newDoc(anotherbBranch); - indexIssues(issueOnProject, issueOnBranch, issueOnAnotherBranch); - - assertThatSearchReturnsOnly(IssueQuery.builder().branchUuid(branch.uuid()).mainBranch(false), issueOnBranch.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().componentUuids(singletonList(branch.uuid())).branchUuid(branch.uuid()).mainBranch(false), issueOnBranch.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().projectUuids(singletonList(project.uuid())).branchUuid(branch.uuid()).mainBranch(false), issueOnBranch.key()); - assertThatSearchReturnsOnly( - IssueQuery.builder().componentUuids(singletonList(branch.uuid())).projectUuids(singletonList(project.uuid())).branchUuid(branch.uuid()).mainBranch(false), - issueOnBranch.key()); - assertThatSearchReturnsEmpty(IssueQuery.builder().branchUuid("unknown")); - } - - @Test - public void issues_from_branch_component_children() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto projectModule = db.components().insertComponent(newModuleDto(project)); - ComponentDto projectFile = db.components().insertComponent(newFileDto(projectModule)); - ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setKey("my_branch")); - ComponentDto branchModule = db.components().insertComponent(newModuleDto(branch)); - ComponentDto branchFile = db.components().insertComponent(newFileDto(branchModule)); - - indexIssues( - newDoc("I1", project), - newDoc("I2", projectFile), - newDoc("I3", projectModule), - newDoc("I4", branch), - newDoc("I5", branchModule), - newDoc("I6", branchFile)); - - assertThatSearchReturnsOnly(IssueQuery.builder().branchUuid(branch.uuid()).mainBranch(false), "I4", "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().moduleUuids(singletonList(branchModule.uuid())).branchUuid(branch.uuid()).mainBranch(false), "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().fileUuids(singletonList(branchFile.uuid())).branchUuid(branch.uuid()).mainBranch(false), "I6"); - assertThatSearchReturnsEmpty(IssueQuery.builder().fileUuids(singletonList(branchFile.uuid())).mainBranch(false).branchUuid("unknown")); - } - - @Test - public void issues_from_main_branch() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto branch = db.components().insertProjectBranch(project); - - IssueDoc issueOnProject = newDoc(project); - IssueDoc issueOnBranch = newDoc(branch); - indexIssues(issueOnProject, issueOnBranch); - - assertThatSearchReturnsOnly(IssueQuery.builder().branchUuid(project.uuid()).mainBranch(true), issueOnProject.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().componentUuids(singletonList(project.uuid())).branchUuid(project.uuid()).mainBranch(true), issueOnProject.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().projectUuids(singletonList(project.uuid())).branchUuid(project.uuid()).mainBranch(true), issueOnProject.key()); - assertThatSearchReturnsOnly( - IssueQuery.builder().componentUuids(singletonList(project.uuid())).projectUuids(singletonList(project.uuid())).branchUuid(project.uuid()).mainBranch(true), - issueOnProject.key()); - } - - @Test - public void branch_issues_are_ignored_when_no_branch_param() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto branch = db.components().insertProjectBranch(project, b -> b.setKey("my_branch")); - - IssueDoc projectIssue = newDoc(project); - IssueDoc branchIssue = newDoc(branch); - indexIssues(projectIssue, branchIssue); - - assertThatSearchReturnsOnly(IssueQuery.builder(), projectIssue.key()); - } - - @Test - public void filter_by_main_application() { - ComponentDto application1 = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto application2 = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto project1 = db.components().insertPrivateProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project1)); - ComponentDto project2 = db.components().insertPrivateProject(); - indexView(application1.uuid(), singletonList(project1.uuid())); - indexView(application2.uuid(), singletonList(project2.uuid())); - - IssueDoc issueOnProject1 = newDoc(project1); - IssueDoc issueOnFile = newDoc(file); - IssueDoc issueOnProject2 = newDoc(project2); - indexIssues(issueOnProject1, issueOnFile, issueOnProject2); - - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(application1.uuid())), issueOnProject1.key(), issueOnFile.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(application2.uuid())), issueOnProject2.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(asList(application1.uuid(), application2.uuid())), issueOnProject1.key(), issueOnFile.key(), issueOnProject2.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(application1.uuid())).projectUuids(singletonList(project1.uuid())), issueOnProject1.key(), - issueOnFile.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(application1.uuid())).fileUuids(singletonList(file.uuid())), issueOnFile.key()); - assertThatSearchReturnsEmpty(IssueQuery.builder().viewUuids(singletonList("unknown"))); - } - - @Test - public void filter_by_application_branch() { - ComponentDto application = db.components().insertMainBranch(c -> c.setQualifier(APP)); - ComponentDto branch1 = db.components().insertProjectBranch(application); - ComponentDto branch2 = db.components().insertProjectBranch(application); - ComponentDto project1 = db.components().insertPrivateProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project1)); - ComponentDto project2 = db.components().insertPrivateProject(); - indexView(branch1.uuid(), singletonList(project1.uuid())); - indexView(branch2.uuid(), singletonList(project2.uuid())); - - IssueDoc issueOnProject1 = newDoc(project1); - IssueDoc issueOnFile = newDoc(file); - IssueDoc issueOnProject2 = newDoc(project2); - indexIssues(issueOnProject1, issueOnFile, issueOnProject2); - - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(branch1.uuid())).branchUuid(branch1.uuid()).mainBranch(false), - issueOnProject1.key(), issueOnFile.key()); - assertThatSearchReturnsOnly( - IssueQuery.builder().viewUuids(singletonList(branch1.uuid())).projectUuids(singletonList(project1.uuid())).branchUuid(branch1.uuid()).mainBranch(false), - issueOnProject1.key(), issueOnFile.key()); - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(branch1.uuid())).fileUuids(singletonList(file.uuid())).branchUuid(branch1.uuid()).mainBranch(false), - issueOnFile.key()); - assertThatSearchReturnsEmpty(IssueQuery.builder().branchUuid("unknown")); - } - - @Test - public void filter_by_application_branch_having_project_branches() { - ComponentDto application = db.components().insertMainBranch(c -> c.setQualifier(APP).setDbKey("app")); - ComponentDto applicationBranch1 = db.components().insertProjectBranch(application, a -> a.setKey("app-branch1")); - ComponentDto applicationBranch2 = db.components().insertProjectBranch(application, a -> a.setKey("app-branch2")); - ComponentDto project1 = db.components().insertPrivateProject(p -> p.setDbKey("prj1")); - ComponentDto project1Branch1 = db.components().insertProjectBranch(project1); - ComponentDto fileOnProject1Branch1 = db.components().insertComponent(newFileDto(project1Branch1)); - ComponentDto project1Branch2 = db.components().insertProjectBranch(project1); - ComponentDto project2 = db.components().insertPrivateProject(p -> p.setDbKey("prj2")); - indexView(applicationBranch1.uuid(), asList(project1Branch1.uuid(), project2.uuid())); - indexView(applicationBranch2.uuid(), singletonList(project1Branch2.uuid())); - - IssueDoc issueOnProject1 = newDoc(project1); - IssueDoc issueOnProject1Branch1 = newDoc(project1Branch1); - IssueDoc issueOnFileOnProject1Branch1 = newDoc(fileOnProject1Branch1); - IssueDoc issueOnProject1Branch2 = newDoc(project1Branch2); - IssueDoc issueOnProject2 = newDoc(project2); - indexIssues(issueOnProject1, issueOnProject1Branch1, issueOnFileOnProject1Branch1, issueOnProject1Branch2, issueOnProject2); - - assertThatSearchReturnsOnly(IssueQuery.builder().viewUuids(singletonList(applicationBranch1.uuid())).branchUuid(applicationBranch1.uuid()).mainBranch(false), - issueOnProject1Branch1.key(), issueOnFileOnProject1Branch1.key(), issueOnProject2.key()); - assertThatSearchReturnsOnly( - IssueQuery.builder().viewUuids(singletonList(applicationBranch1.uuid())).projectUuids(singletonList(project1.uuid())).branchUuid(applicationBranch1.uuid()).mainBranch(false), - issueOnProject1Branch1.key(), issueOnFileOnProject1Branch1.key()); - assertThatSearchReturnsOnly( - IssueQuery.builder().viewUuids(singletonList(applicationBranch1.uuid())).fileUuids(singletonList(fileOnProject1Branch1.uuid())).branchUuid(applicationBranch1.uuid()) - .mainBranch(false), - issueOnFileOnProject1Branch1.key()); - assertThatSearchReturnsEmpty( - IssueQuery.builder().viewUuids(singletonList(applicationBranch1.uuid())).projectUuids(singletonList("unknown")).branchUuid(applicationBranch1.uuid()).mainBranch(false)); - } - - @Test - public void filter_by_created_after_by_projects() { - Date now = new Date(); - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project1 = newPrivateProjectDto(organizationDto); - IssueDoc project1Issue1 = newDoc(project1).setFuncCreationDate(addDays(now, -10)); - IssueDoc project1Issue2 = newDoc(project1).setFuncCreationDate(addDays(now, -20)); - ComponentDto project2 = newPrivateProjectDto(organizationDto); - IssueDoc project2Issue1 = newDoc(project2).setFuncCreationDate(addDays(now, -15)); - IssueDoc project2Issue2 = newDoc(project2).setFuncCreationDate(addDays(now, -30)); - indexIssues(project1Issue1, project1Issue2, project2Issue1, project2Issue2); - - // Search for issues of project 1 having less than 15 days - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfterByProjectUuids(ImmutableMap.of(project1.uuid(), new IssueQuery.PeriodStart(addDays(now, -15), true))), - project1Issue1.key()); - - // Search for issues of project 1 having less than 14 days and project 2 having less then 25 days - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfterByProjectUuids(ImmutableMap.of( - project1.uuid(), new IssueQuery.PeriodStart(addDays(now, -14), true), - project2.uuid(), new IssueQuery.PeriodStart(addDays(now, -25), true))), - project1Issue1.key(), project2Issue1.key()); - - // Search for issues of project 1 having less than 30 days - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfterByProjectUuids(ImmutableMap.of( - project1.uuid(), new IssueQuery.PeriodStart(addDays(now, -30), true))), - project1Issue1.key(), project1Issue2.key()); - - // Search for issues of project 1 and project 2 having less than 5 days - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfterByProjectUuids(ImmutableMap.of( - project1.uuid(), new IssueQuery.PeriodStart(addDays(now, -5), true), - project2.uuid(), new IssueQuery.PeriodStart(addDays(now, -5), true)))); - } - - @Test - public void filter_by_severities() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setSeverity(Severity.INFO), - newDoc("I2", file).setSeverity(Severity.MAJOR)); - - assertThatSearchReturnsOnly(IssueQuery.builder().severities(asList(Severity.INFO, Severity.MAJOR)), "I1", "I2"); - assertThatSearchReturnsOnly(IssueQuery.builder().severities(singletonList(Severity.INFO)), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().severities(singletonList(Severity.BLOCKER))); - } - - @Test - public void filter_by_statuses() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setStatus(Issue.STATUS_CLOSED), - newDoc("I2", file).setStatus(Issue.STATUS_OPEN)); - - assertThatSearchReturnsOnly(IssueQuery.builder().statuses(asList(Issue.STATUS_CLOSED, Issue.STATUS_OPEN)), "I1", "I2"); - assertThatSearchReturnsOnly(IssueQuery.builder().statuses(singletonList(Issue.STATUS_CLOSED)), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().statuses(singletonList(Issue.STATUS_CONFIRMED))); - } - - @Test - public void filter_by_resolutions() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setResolution(Issue.RESOLUTION_FALSE_POSITIVE), - newDoc("I2", file).setResolution(Issue.RESOLUTION_FIXED)); - - assertThatSearchReturnsOnly(IssueQuery.builder().resolutions(asList(Issue.RESOLUTION_FALSE_POSITIVE, Issue.RESOLUTION_FIXED)), "I1", "I2"); - assertThatSearchReturnsOnly(IssueQuery.builder().resolutions(singletonList(Issue.RESOLUTION_FALSE_POSITIVE)), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().resolutions(singletonList(Issue.RESOLUTION_REMOVED))); - } - - @Test - public void filter_by_resolved() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setStatus(Issue.STATUS_CLOSED).setResolution(Issue.RESOLUTION_FIXED), - newDoc("I2", file).setStatus(Issue.STATUS_OPEN).setResolution(null), - newDoc("I3", file).setStatus(Issue.STATUS_OPEN).setResolution(null)); - - assertThatSearchReturnsOnly(IssueQuery.builder().resolved(true), "I1"); - assertThatSearchReturnsOnly(IssueQuery.builder().resolved(false), "I2", "I3"); - assertThatSearchReturnsOnly(IssueQuery.builder().resolved(null), "I1", "I2", "I3"); - } - - @Test - public void filter_by_rules() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - RuleDefinitionDto ruleDefinitionDto = newRule(); - db.rules().insert(ruleDefinitionDto); - - indexIssues(newDoc("I1", file).setRuleId(ruleDefinitionDto.getId())); - - assertThatSearchReturnsOnly(IssueQuery.builder().rules(singletonList(ruleDefinitionDto)), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().rules(singletonList(new RuleDefinitionDto().setId(-1)))); - } - - @Test - public void filter_by_languages() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - RuleDefinitionDto ruleDefinitionDto = newRule(); - db.rules().insert(ruleDefinitionDto); - - indexIssues(newDoc("I1", file).setRuleId(ruleDefinitionDto.getId()).setLanguage("xoo")); - - assertThatSearchReturnsOnly(IssueQuery.builder().languages(singletonList("xoo")), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().languages(singletonList("unknown"))); - } - - @Test - public void filter_by_assignees() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setAssigneeUuid("steph-uuid"), - newDoc("I2", file).setAssigneeUuid("marcel-uuid"), - newDoc("I3", file).setAssigneeUuid(null)); - - assertThatSearchReturnsOnly(IssueQuery.builder().assigneeUuids(singletonList("steph-uuid")), "I1"); - assertThatSearchReturnsOnly(IssueQuery.builder().assigneeUuids(asList("steph-uuid", "marcel-uuid")), "I1", "I2"); - assertThatSearchReturnsEmpty(IssueQuery.builder().assigneeUuids(singletonList("unknown"))); - } - - @Test - public void filter_by_assigned() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setAssigneeUuid("steph-uuid"), - newDoc("I2", file).setAssigneeUuid(null), - newDoc("I3", file).setAssigneeUuid(null)); - - assertThatSearchReturnsOnly(IssueQuery.builder().assigned(true), "I1"); - assertThatSearchReturnsOnly(IssueQuery.builder().assigned(false), "I2", "I3"); - assertThatSearchReturnsOnly(IssueQuery.builder().assigned(null), "I1", "I2", "I3"); - } - - @Test - public void filter_by_authors() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setAuthorLogin("steph"), - newDoc("I2", file).setAuthorLogin("marcel"), - newDoc("I3", file).setAssigneeUuid(null)); - - assertThatSearchReturnsOnly(IssueQuery.builder().authors(singletonList("steph")), "I1"); - assertThatSearchReturnsOnly(IssueQuery.builder().authors(asList("steph", "marcel")), "I1", "I2"); - assertThatSearchReturnsEmpty(IssueQuery.builder().authors(singletonList("unknown"))); - } - - @Test - public void filter_by_created_after() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setFuncCreationDate(parseDate("2014-09-20")), - newDoc("I2", file).setFuncCreationDate(parseDate("2014-09-23"))); - - assertThatSearchReturnsOnly(IssueQuery.builder().createdAfter(parseDate("2014-09-19")), "I1", "I2"); - // Lower bound is included - assertThatSearchReturnsOnly(IssueQuery.builder().createdAfter(parseDate("2014-09-20")), "I1", "I2"); - assertThatSearchReturnsOnly(IssueQuery.builder().createdAfter(parseDate("2014-09-21")), "I2"); - assertThatSearchReturnsEmpty(IssueQuery.builder().createdAfter(parseDate("2014-09-25"))); - } - - @Test - public void filter_by_created_before() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setFuncCreationDate(parseDate("2014-09-20")), - newDoc("I2", file).setFuncCreationDate(parseDate("2014-09-23"))); - - assertThatSearchReturnsEmpty(IssueQuery.builder().createdBefore(parseDate("2014-09-19"))); - // Upper bound is excluded - assertThatSearchReturnsEmpty(IssueQuery.builder().createdBefore(parseDate("2014-09-20"))); - assertThatSearchReturnsOnly(IssueQuery.builder().createdBefore(parseDate("2014-09-21")), "I1"); - assertThatSearchReturnsOnly(IssueQuery.builder().createdBefore(parseDate("2014-09-25")), "I1", "I2"); - } - - @Test - public void filter_by_created_after_and_before() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setFuncCreationDate(parseDate("2014-09-20")), - newDoc("I2", file).setFuncCreationDate(parseDate("2014-09-23"))); - - // 19 < createdAt < 25 - assertThatSearchReturnsOnly(IssueQuery.builder().createdAfter(parseDate("2014-09-19")).createdBefore(parseDate("2014-09-25")), - "I1", "I2"); - - // 20 < createdAt < 25: excludes first issue - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfter(parseDate("2014-09-20")).createdBefore(parseDate("2014-09-25")), "I1", "I2"); - - // 21 < createdAt < 25 - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfter(parseDate("2014-09-21")).createdBefore(parseDate("2014-09-25")), "I2"); - - // 21 < createdAt < 24 - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfter(parseDate("2014-09-21")).createdBefore(parseDate("2014-09-24")), "I2"); - - // 21 < createdAt < 23: excludes second issue - assertThatSearchReturnsEmpty(IssueQuery.builder() - .createdAfter(parseDate("2014-09-21")).createdBefore(parseDate("2014-09-23"))); - - // 19 < createdAt < 21: only first issue - assertThatSearchReturnsOnly(IssueQuery.builder() - .createdAfter(parseDate("2014-09-19")).createdBefore(parseDate("2014-09-21")), "I1"); - - // 20 < createdAt < 20: exception - expectedException.expect(IllegalArgumentException.class); - underTest.search(IssueQuery.builder() - .createdAfter(parseDate("2014-09-20")).createdBefore(parseDate("2014-09-20")) - .build(), new SearchOptions()); - } - - @Test - public void filter_by_created_after_and_before_take_into_account_timezone() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setFuncCreationDate(parseDateTime("2014-09-20T00:00:00+0100")), - newDoc("I2", file).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100"))); - - assertThatSearchReturnsOnly(IssueQuery.builder().createdAfter(parseDateTime("2014-09-19T23:00:00+0000")).createdBefore(parseDateTime("2014-09-22T23:00:01+0000")), - "I1", "I2"); - - assertThatSearchReturnsEmpty(IssueQuery.builder().createdAfter(parseDateTime("2014-09-19T23:00:01+0000")).createdBefore(parseDateTime("2014-09-22T23:00:00+0000"))); - } - - @Test - public void filter_by_created_before_must_be_lower_than_after() { - try { - underTest.search(IssueQuery.builder().createdAfter(parseDate("2014-09-20")).createdBefore(parseDate("2014-09-19")).build(), - new SearchOptions()); - Fail.failBecauseExceptionWasNotThrown(IllegalArgumentException.class); - } catch (IllegalArgumentException exception) { - assertThat(exception.getMessage()).isEqualTo("Start bound cannot be larger or equal to end bound"); - } - } - - @Test - public void fail_if_created_before_equals_created_after() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Start bound cannot be larger or equal to end bound"); - - underTest.search(IssueQuery.builder().createdAfter(parseDate("2014-09-20")).createdBefore(parseDate("2014-09-20")).build(), new SearchOptions()); - } - - @Test - public void filter_by_created_after_must_not_be_in_future() { - try { - underTest.search(IssueQuery.builder().createdAfter(new Date(Long.MAX_VALUE)).build(), new SearchOptions()); - Fail.failBecauseExceptionWasNotThrown(IllegalArgumentException.class); - } catch (IllegalArgumentException exception) { - assertThat(exception.getMessage()).isEqualTo("Start bound cannot be in the future"); - } - } - - @Test - public void filter_by_created_at() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues(newDoc("I1", file).setFuncCreationDate(parseDate("2014-09-20"))); - - assertThatSearchReturnsOnly(IssueQuery.builder().createdAt(parseDate("2014-09-20")), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().createdAt(parseDate("2014-09-21"))); - } - - @Test - public void filter_by_organization() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto projectInOrg1 = newPrivateProjectDto(org1); - OrganizationDto org2 = newOrganizationDto(); - ComponentDto projectInOrg2 = newPrivateProjectDto(org2); - - indexIssues(newDoc("issueInOrg1", projectInOrg1), newDoc("issue1InOrg2", projectInOrg2), newDoc("issue2InOrg2", projectInOrg2)); - - verifyOrganizationFilter(org1.getUuid(), "issueInOrg1"); - verifyOrganizationFilter(org2.getUuid(), "issue1InOrg2", "issue2InOrg2"); - verifyOrganizationFilter("does_not_exist"); - } - - @Test - public void filter_by_organization_and_project() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto projectInOrg1 = newPrivateProjectDto(org1); - OrganizationDto org2 = newOrganizationDto(); - ComponentDto projectInOrg2 = newPrivateProjectDto(org2); - - indexIssues(newDoc("issueInOrg1", projectInOrg1), newDoc("issue1InOrg2", projectInOrg2), newDoc("issue2InOrg2", projectInOrg2)); - - // no conflict - IssueQuery.Builder query = IssueQuery.builder().organizationUuid(org1.getUuid()).projectUuids(singletonList(projectInOrg1.uuid())); - assertThatSearchReturnsOnly(query, "issueInOrg1"); - - // conflict - query = IssueQuery.builder().organizationUuid(org1.getUuid()).projectUuids(singletonList(projectInOrg2.uuid())); - assertThatSearchReturnsEmpty(query); - } - - private void verifyOrganizationFilter(String organizationUuid, String... expectedIssueKeys) { - IssueQuery.Builder query = IssueQuery.builder().organizationUuid(organizationUuid); - assertThatSearchReturnsOnly(query, expectedIssueKeys); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } - - private void indexView(String viewUuid, List<String> projects) { - viewIndexer.index(new ViewDoc().setUuid(viewUuid).setProjects(projects)); - } - - /** - * Execute the search request and return the document ids of results. - */ - private List<String> searchAndReturnKeys(IssueQuery.Builder query) { - return Arrays.stream(underTest.search(query.build(), new SearchOptions()).getHits().getHits()) - .map(SearchHit::getId) - .collect(Collectors.toList()); - } - - private void assertThatSearchReturnsOnly(IssueQuery.Builder query, String... expectedIssueKeys) { - List<String> keys = searchAndReturnKeys(query); - assertThat(keys).containsExactlyInAnyOrder(expectedIssueKeys); - } - - private void assertThatSearchReturnsEmpty(IssueQuery.Builder query) { - List<String> keys = searchAndReturnKeys(query); - assertThat(keys).isEmpty(); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java deleted file mode 100644 index f751e19972e..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import java.util.Date; -import java.util.List; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.issue.Issue; -import org.sonar.api.utils.System2; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static java.util.stream.Collectors.toList; -import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; -import static org.mockito.Mockito.mock; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.db.component.ComponentTesting.newBranchDto; -import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.db.component.ComponentTesting.newProjectBranch; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonar.server.issue.IssueDocTesting.newDoc; - -public class IssueIndexProjectStatisticsTest { - - private System2 system2 = mock(System2.class); - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), null, new IssueIteratorFactory(null)); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void searchProjectStatistics_returns_empty_list_if_no_input() { - List<ProjectStatistics> result = underTest.searchProjectStatistics(emptyList(), emptyList(), "unknownUser"); - assertThat(result).isEmpty(); - } - - @Test - public void searchProjectStatistics_returns_empty_list_if_the_input_does_not_match_anything() { - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList("unknownProjectUuid"), singletonList(1_111_234_567_890L), "unknownUser"); - assertThat(result).isEmpty(); - } - - @Test - public void searchProjectStatistics_returns_something() { - OrganizationDto organization = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organization); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues(newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userUuid); - - assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); - } - - @Test - public void searchProjectStatistics_does_not_return_results_if_assignee_does_not_match() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org1); - String user1Uuid = randomAlphanumeric(40); - String user2Uuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues(newDoc("issue1", project).setAssigneeUuid(user1Uuid).setFuncCreationDate(new Date(from + 1L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), user2Uuid); - - assertThat(result).isEmpty(); - } - - @Test - public void searchProjectStatistics_returns_results_if_assignee_matches() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org1); - String user1Uuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues(newDoc("issue1", project).setAssigneeUuid(user1Uuid).setFuncCreationDate(new Date(from + 1L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), user1Uuid); - - assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); - } - - @Test - public void searchProjectStatistics_returns_results_if_functional_date_is_strictly_after_from_date() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org1); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues(newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userUuid); - - assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); - } - - @Test - public void searchProjectStatistics_does_not_return_results_if_functional_date_is_same_as_from_date() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org1); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues(newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userUuid); - - assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); - } - - @Test - public void searchProjectStatistics_does_not_return_resolved_issues() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org1); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues( - newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)).setResolution(Issue.RESOLUTION_FALSE_POSITIVE), - newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)).setResolution(Issue.RESOLUTION_FIXED), - newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)).setResolution(Issue.RESOLUTION_REMOVED), - newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)).setResolution(Issue.RESOLUTION_WONT_FIX)); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userUuid); - - assertThat(result).isEmpty(); - } - - @Test - public void searchProjectStatistics_does_not_return_results_if_functional_date_is_before_from_date() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org1); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues(newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from - 1000L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userUuid); - - assertThat(result).isEmpty(); - } - - @Test - public void searchProjectStatistics_returns_issue_count() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org1); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues( - newDoc("issue1", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)), - newDoc("issue2", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)), - newDoc("issue3", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userUuid); - - assertThat(result).extracting(ProjectStatistics::getIssueCount).containsExactly(3L); - } - - @Test - public void searchProjectStatistics_returns_issue_count_for_multiple_projects() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project1 = newPrivateProjectDto(org1); - ComponentDto project2 = newPrivateProjectDto(org1); - ComponentDto project3 = newPrivateProjectDto(org1); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues( - newDoc("issue1", project1).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)), - newDoc("issue2", project1).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)), - newDoc("issue3", project1).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)), - - newDoc("issue4", project3).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)), - newDoc("issue5", project3).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics( - asList(project1.uuid(), project2.uuid(), project3.uuid()), - asList(from, from, from), - userUuid); - - assertThat(result) - .extracting(ProjectStatistics::getProjectUuid, ProjectStatistics::getIssueCount) - .containsExactlyInAnyOrder( - tuple(project1.uuid(), 3L), - tuple(project3.uuid(), 2L)); - } - - @Test - public void searchProjectStatistics_returns_max_date_for_multiple_projects() { - OrganizationDto org1 = newOrganizationDto(); - ComponentDto project1 = newPrivateProjectDto(org1); - ComponentDto project2 = newPrivateProjectDto(org1); - ComponentDto project3 = newPrivateProjectDto(org1); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_000L; - indexIssues( - newDoc("issue1", project1).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1_000L)), - newDoc("issue2", project1).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 2_000L)), - newDoc("issue3", project1).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 3_000L)), - - newDoc("issue4", project3).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 4_000L)), - newDoc("issue5", project3).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 5_000L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics( - asList(project1.uuid(), project2.uuid(), project3.uuid()), - asList(from, from, from), - userUuid); - - assertThat(result) - .extracting(ProjectStatistics::getProjectUuid, ProjectStatistics::getLastIssueDate) - .containsExactlyInAnyOrder( - tuple(project1.uuid(), from + 3_000L), - tuple(project3.uuid(), from + 5_000L)); - } - - @Test - public void searchProjectStatistics_return_branch_issues() { - OrganizationDto organization = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(organization); - ComponentDto branch = newProjectBranch(project, newBranchDto(project).setKey("branch")); - String userUuid = randomAlphanumeric(40); - long from = 1_111_234_567_890L; - indexIssues( - newDoc("issue1", branch).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L)), - newDoc("issue2", branch).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 2L)), - newDoc("issue3", project).setAssigneeUuid(userUuid).setFuncCreationDate(new Date(from + 1L))); - - List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userUuid); - - assertThat(result) - .extracting(ProjectStatistics::getIssueCount, ProjectStatistics::getProjectUuid, ProjectStatistics::getLastIssueDate) - .containsExactly( - tuple(2L, branch.uuid(), from + 2L), - tuple(1L, project.uuid(), from + 1L)); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java deleted file mode 100644 index 46ba1e80a12..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.rule.Severity; -import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.Facets; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.Collections.singletonList; -import static java.util.TimeZone.getTimeZone; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; -import static org.junit.rules.ExpectedException.none; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.rule.Severity.INFO; -import static org.sonar.api.rule.Severity.MAJOR; -import static org.sonar.api.rules.RuleType.BUG; -import static org.sonar.api.rules.RuleType.CODE_SMELL; -import static org.sonar.api.rules.RuleType.SECURITY_HOTSPOT; -import static org.sonar.api.rules.RuleType.VULNERABILITY; -import static org.sonar.db.component.ComponentTesting.newFileDto; -import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonar.server.issue.IssueDocTesting.newDoc; - -public class IssueIndexSecurityHotspotsTest { - - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - @Rule - public ExpectedException expectedException = none(); - private System2 system2 = new TestSystem2().setNow(1_500_000_000_000L).setDefaultTimeZone(getTimeZone("GMT-01:00")); - @Rule - public DbTester db = DbTester.create(system2); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void filter_by_security_hotspots_type() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setType(BUG), - newDoc("I2", file).setType(CODE_SMELL), - newDoc("I3", file).setType(VULNERABILITY), - newDoc("I4", file).setType(VULNERABILITY), - newDoc("I5", file).setType(SECURITY_HOTSPOT), - newDoc("I6", file).setType(SECURITY_HOTSPOT)); - - assertThatSearchReturnsOnly(IssueQuery.builder().types(singletonList(SECURITY_HOTSPOT.name())), "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder().types(asList(SECURITY_HOTSPOT.name(), VULNERABILITY.name())), "I3", "I4", "I5", "I6"); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2", "I3", "I4", "I5", "I6"); - } - - @Test - public void filter_by_severities_ignore_hotspots() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setSeverity(Severity.INFO).setType(BUG), - newDoc("I2", file).setSeverity(Severity.MAJOR).setType(CODE_SMELL), - newDoc("I3", file).setSeverity(Severity.MAJOR).setType(VULNERABILITY), - newDoc("I4", file).setSeverity(Severity.CRITICAL).setType(VULNERABILITY), - // This entry should be ignored - newDoc("I5", file).setSeverity(Severity.MAJOR).setType(SECURITY_HOTSPOT)); - - assertThatSearchReturnsOnly(IssueQuery.builder().severities(asList(Severity.INFO, Severity.MAJOR)), "I1", "I2", "I3"); - assertThatSearchReturnsOnly(IssueQuery.builder().severities(asList(Severity.INFO, Severity.MAJOR)).types(singletonList(VULNERABILITY.name())), "I3"); - assertThatSearchReturnsEmpty(IssueQuery.builder().severities(singletonList(Severity.MAJOR)).types(singletonList(SECURITY_HOTSPOT.name()))); - } - - @Test - public void facet_on_severities_ignore_hotspots() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setSeverity(INFO).setType(BUG), - newDoc("I2", file).setSeverity(INFO).setType(CODE_SMELL), - newDoc("I3", file).setSeverity(INFO).setType(VULNERABILITY), - newDoc("I4", file).setSeverity(MAJOR).setType(VULNERABILITY), - // These 2 entries should be ignored - newDoc("I5", file).setSeverity(INFO).setType(SECURITY_HOTSPOT), - newDoc("I6", file).setSeverity(MAJOR).setType(SECURITY_HOTSPOT)); - - assertThatFacetHasOnly(IssueQuery.builder(), "severities", entry("INFO", 3L), entry("MAJOR", 1L)); - assertThatFacetHasOnly(IssueQuery.builder().types(singletonList(VULNERABILITY.name())), "severities", entry("INFO", 1L), entry("MAJOR", 1L)); - assertThatFacetHasOnly(IssueQuery.builder().types(asList(BUG.name(), CODE_SMELL.name(), VULNERABILITY.name())), "severities", entry("INFO", 3L), entry("MAJOR", 1L)); - assertThatFacetHasOnly(IssueQuery.builder().types(singletonList(SECURITY_HOTSPOT.name())), "severities"); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } - - @SafeVarargs - private final void assertThatFacetHasOnly(IssueQuery.Builder query, String facet, Map.Entry<String, Long>... expectedEntries) { - SearchResponse result = underTest.search(query.build(), new SearchOptions().addFacets(singletonList(facet))); - Facets facets = new Facets(result, system2.getDefaultTimeZone()); - assertThat(facets.getNames()).containsOnly(facet, "effort"); - assertThat(facets.get(facet)).containsOnly(expectedEntries); - } - - private void assertThatSearchReturnsOnly(IssueQuery.Builder query, String... expectedIssueKeys) { - List<String> keys = searchAndReturnKeys(query); - assertThat(keys).containsExactlyInAnyOrder(expectedIssueKeys); - } - - private void assertThatSearchReturnsEmpty(IssueQuery.Builder query) { - List<String> keys = searchAndReturnKeys(query); - assertThat(keys).isEmpty(); - } - - private List<String> searchAndReturnKeys(IssueQuery.Builder query) { - return Arrays.stream(underTest.search(query.build(), new SearchOptions()).getHits().getHits()) - .map(SearchHit::getId) - .collect(Collectors.toList()); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java deleted file mode 100644 index facee47775b..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityReportsTest.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import java.util.List; -import java.util.Map; -import java.util.OptionalInt; -import java.util.stream.Collectors; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.Severity; -import org.sonar.api.rules.RuleType; -import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; -import org.sonar.server.view.index.ViewDoc; -import org.sonar.server.view.index.ViewIndexer; - -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.Collections.singletonList; -import static java.util.TimeZone.getTimeZone; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.tuple; -import static org.junit.rules.ExpectedException.none; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonar.server.issue.IssueDocTesting.newDoc; -import static org.sonar.server.security.SecurityStandardHelper.SANS_TOP_25_INSECURE_INTERACTION; -import static org.sonar.server.security.SecurityStandardHelper.SANS_TOP_25_POROUS_DEFENSES; -import static org.sonar.server.security.SecurityStandardHelper.SANS_TOP_25_RISKY_RESOURCE; -import static org.sonar.server.security.SecurityStandardHelper.UNKNOWN_STANDARD; - -public class IssueIndexSecurityReportsTest { - - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - @Rule - public ExpectedException expectedException = none(); - private System2 system2 = new TestSystem2().setNow(1_500_000_000_000L).setDefaultTimeZone(getTimeZone("GMT-01:00")); - @Rule - public DbTester db = DbTester.create(system2); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); - private ViewIndexer viewIndexer = new ViewIndexer(db.getDbClient(), es.client()); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void getOwaspTop10Report_dont_count_vulnerabilities_from_other_projects() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - ComponentDto another = newPrivateProjectDto(org); - indexIssues( - newDoc("anotherProject", another).setOwaspTop10(singletonList("a1")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.CRITICAL), - newDoc("openvul1", project).setOwaspTop10(singletonList("a1")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.MAJOR)); - - List<SecurityStandardCategoryStatistics> owaspTop10Report = underTest.getOwaspTop10Report(project.uuid(), false, false); - assertThat(owaspTop10Report) - .extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating) - .contains( - tuple("a1", 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */)); - - } - - @Test - public void getOwaspTop10Report_dont_count_closed_vulnerabilities() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - newDoc("openvul1", project).setOwaspTop10(asList("a1")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.MAJOR), - newDoc("notopenvul", project).setOwaspTop10(asList("a1")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_CLOSED).setResolution(Issue.RESOLUTION_FIXED) - .setSeverity(Severity.BLOCKER)); - - List<SecurityStandardCategoryStatistics> owaspTop10Report = underTest.getOwaspTop10Report(project.uuid(), false, false); - assertThat(owaspTop10Report) - .extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating) - .contains( - tuple("a1", 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */)); - } - - @Test - public void getOwaspTop10Report_dont_count_old_vulnerabilities() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - // Previous vulnerabilities in projects that are not reanalyzed will have no owasp nor cwe attributes (not even 'unknown') - newDoc("openvulNotReindexed", project).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.MAJOR)); - - List<SecurityStandardCategoryStatistics> owaspTop10Report = underTest.getOwaspTop10Report(project.uuid(), false, false); - assertThat(owaspTop10Report) - .extracting(SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating) - .containsOnly( - tuple(0L, OptionalInt.empty())); - } - - @Test - public void getOwaspTop10Report_dont_count_hotspots_from_other_projects() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - ComponentDto another = newPrivateProjectDto(org); - indexIssues( - newDoc("openhotspot1", project).setOwaspTop10(asList("a1")).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("anotherProject", another).setOwaspTop10(asList("a1")).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW)); - - List<SecurityStandardCategoryStatistics> owaspTop10Report = underTest.getOwaspTop10Report(project.uuid(), false, false); - assertThat(owaspTop10Report) - .extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getToReviewSecurityHotspots) - .contains( - tuple("a1", 1L /* openhotspot1 */)); - } - - @Test - public void getOwaspTop10Report_dont_count_closed_hotspots() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - newDoc("openhotspot1", project).setOwaspTop10(asList("a1")).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("closedHotspot", project).setOwaspTop10(asList("a1")).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_CLOSED) - .setResolution(Issue.RESOLUTION_FIXED)); - - List<SecurityStandardCategoryStatistics> owaspTop10Report = underTest.getOwaspTop10Report(project.uuid(), false, false); - assertThat(owaspTop10Report) - .extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getToReviewSecurityHotspots) - .contains( - tuple("a1", 1L /* openhotspot1 */)); - } - - @Test - public void getOwaspTop10Report_aggregation_no_cwe() { - List<SecurityStandardCategoryStatistics> owaspTop10Report = indexIssuesAndAssertOwaspReport(false); - - assertThat(owaspTop10Report).allMatch(category -> category.getChildren().isEmpty()); - } - - @Test - public void getOwaspTop10Report_aggregation_with_cwe() { - List<SecurityStandardCategoryStatistics> owaspTop10Report = indexIssuesAndAssertOwaspReport(true); - - Map<String, List<SecurityStandardCategoryStatistics>> cweByOwasp = owaspTop10Report.stream() - .collect(Collectors.toMap(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getChildren)); - - assertThat(cweByOwasp.get("a1")).extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating, SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, - SecurityStandardCategoryStatistics::getInReviewSecurityHotspots, SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) - .containsExactlyInAnyOrder( - tuple("123", 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */, 0L, 0L, 0L), - tuple("456", 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */, 0L, 0L, 0L), - tuple("unknown", 0L, OptionalInt.empty(), 1L /* openhotspot1 */, 0L, 0L)); - assertThat(cweByOwasp.get("a3")).extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating, SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, - SecurityStandardCategoryStatistics::getInReviewSecurityHotspots, SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) - .containsExactlyInAnyOrder( - tuple("123", 2L /* openvul1, openvul2 */, OptionalInt.of(3)/* MAJOR = C */, 0L, 0L, 0L), - tuple("456", 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */, 0L, 1L /* toReviewHotspot */, 0L), - tuple("unknown", 0L, OptionalInt.empty(), 1L /* openhotspot1 */, 0L, 0L)); - } - - private List<SecurityStandardCategoryStatistics> indexIssuesAndAssertOwaspReport(boolean includeCwe) { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - newDoc("openvul1", project).setOwaspTop10(asList("a1", "a3")).setCwe(asList("123", "456")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.MAJOR), - newDoc("openvul2", project).setOwaspTop10(asList("a3", "a6")).setCwe(asList("123")).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED).setSeverity(Severity.MINOR), - newDoc("notowaspvul", project).setOwaspTop10(singletonList(UNKNOWN_STANDARD)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.CRITICAL), - newDoc("toreviewhotspot1", project).setOwaspTop10(asList("a1", "a3")).setCwe(singletonList(UNKNOWN_STANDARD)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("toreviewhotspot2", project).setOwaspTop10(asList("a3", "a6")).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("inreviewhotspot", project).setOwaspTop10(asList("a5", "a3")).setCwe(asList("456")).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_IN_REVIEW), - newDoc("reviewedHotspot", project).setOwaspTop10(asList("a3", "a8")).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_REVIEWED).setResolution(Issue.RESOLUTION_FIXED), - newDoc("notowasphotspot", project).setOwaspTop10(singletonList(UNKNOWN_STANDARD)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW)); - - List<SecurityStandardCategoryStatistics> owaspTop10Report = underTest.getOwaspTop10Report(project.uuid(), false, includeCwe); - assertThat(owaspTop10Report) - .extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating, SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, - SecurityStandardCategoryStatistics::getInReviewSecurityHotspots, SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) - .containsExactlyInAnyOrder( - tuple("a1", 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */, 1L /* toreviewhotspot1 */, 0L, 0L), - tuple("a2", 0L, OptionalInt.empty(), 0L, 0L, 0L), - tuple("a3", 2L /* openvul1,openvul2 */, OptionalInt.of(3)/* MAJOR = C */, 2L/* toreviewhotspot1,toreviewhotspot2 */, 1L /* inReviewHotspot */, 1L /* reviewedHotspot */), - tuple("a4", 0L, OptionalInt.empty(), 0L, 0L, 0L), - tuple("a5", 0L, OptionalInt.empty(), 0L, 1L/* inReviewHotspot */, 0L), - tuple("a6", 1L /* openvul2 */, OptionalInt.of(2) /* MINOR = B */, 1L /* toreviewhotspot2 */, 0L, 0L), - tuple("a7", 0L, OptionalInt.empty(), 0L, 0L, 0L), - tuple("a8", 0L, OptionalInt.empty(), 0L, 0L, 1L /* reviewedHotspot */), - tuple("a9", 0L, OptionalInt.empty(), 0L, 0L, 0L), - tuple("a10", 0L, OptionalInt.empty(), 0L, 0L, 0L)); - return owaspTop10Report; - } - - @Test - public void getSansTop25Report_aggregation() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - newDoc("openvul1", project).setSansTop25(asList(SANS_TOP_25_INSECURE_INTERACTION, SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN) - .setSeverity(Severity.MAJOR), - newDoc("openvul2", project).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED) - .setSeverity(Severity.MINOR), - newDoc("notopenvul", project).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_CLOSED) - .setResolution(Issue.RESOLUTION_FIXED) - .setSeverity(Severity.BLOCKER), - newDoc("notsansvul", project).setSansTop25(singletonList(UNKNOWN_STANDARD)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.CRITICAL), - newDoc("toreviewhotspot1", project).setSansTop25(asList(SANS_TOP_25_INSECURE_INTERACTION, SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("toreviewhotspot2", project).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("inReviewHotspot", project).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_IN_REVIEW), - newDoc("reviewedHotspot", project).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_REVIEWED).setResolution(Issue.RESOLUTION_FIXED), - newDoc("notowasphotspot", project).setSansTop25(singletonList(UNKNOWN_STANDARD)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW)); - - List<SecurityStandardCategoryStatistics> sansTop25Report = underTest.getSansTop25Report(project.uuid(), false, false); - assertThat(sansTop25Report) - .extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating, SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, - SecurityStandardCategoryStatistics::getInReviewSecurityHotspots, SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) - .containsExactlyInAnyOrder( - tuple(SANS_TOP_25_INSECURE_INTERACTION, 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */, 1L /* toreviewhotspot1 */, 0L, 0L), - tuple(SANS_TOP_25_RISKY_RESOURCE, 2L /* openvul1,openvul2 */, OptionalInt.of(3)/* MAJOR = C */, 2L/* toreviewhotspot1,toreviewhotspot2 */, 1L /* inReviewHotspot */,1L /* reviewedHotspot */), - tuple(SANS_TOP_25_POROUS_DEFENSES, 1L /* openvul2 */, OptionalInt.of(2)/* MINOR = B */, 1L/* openhotspot2 */, 0L, 0L)); - - assertThat(sansTop25Report).allMatch(category -> category.getChildren().isEmpty()); - } - - @Test - public void getSansTop25Report_aggregation_on_portfolio() { - ComponentDto portfolio1 = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto portfolio2 = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto project1 = db.components().insertPrivateProject(); - ComponentDto project2 = db.components().insertPrivateProject(); - - indexIssues( - newDoc("openvul1", project1).setSansTop25(asList(SANS_TOP_25_INSECURE_INTERACTION, SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN) - .setSeverity(Severity.MAJOR), - newDoc("openvul2", project2).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_REOPENED) - .setSeverity(Severity.MINOR), - newDoc("notopenvul", project1).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_CLOSED) - .setResolution(Issue.RESOLUTION_FIXED) - .setSeverity(Severity.BLOCKER), - newDoc("notsansvul", project2).setSansTop25(singletonList(UNKNOWN_STANDARD)).setType(RuleType.VULNERABILITY).setStatus(Issue.STATUS_OPEN).setSeverity(Severity.CRITICAL), - newDoc("toreviewhotspot1", project1).setSansTop25(asList(SANS_TOP_25_INSECURE_INTERACTION, SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("toreviewhotspot2", project2).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE, SANS_TOP_25_POROUS_DEFENSES)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW), - newDoc("inReviewHotspot", project1).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_IN_REVIEW), - newDoc("reviewedHotspot", project2).setSansTop25(asList(SANS_TOP_25_RISKY_RESOURCE)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_REVIEWED).setResolution(Issue.RESOLUTION_FIXED), - newDoc("notowasphotspot", project1).setSansTop25(singletonList(UNKNOWN_STANDARD)).setType(RuleType.SECURITY_HOTSPOT).setStatus(Issue.STATUS_TO_REVIEW)); - - indexView(portfolio1.uuid(), singletonList(project1.uuid())); - indexView(portfolio2.uuid(), singletonList(project2.uuid())); - - List<SecurityStandardCategoryStatistics> sansTop25Report = underTest.getSansTop25Report(portfolio1.uuid(), true, false); - assertThat(sansTop25Report) - .extracting(SecurityStandardCategoryStatistics::getCategory, SecurityStandardCategoryStatistics::getVulnerabilities, - SecurityStandardCategoryStatistics::getVulnerabiliyRating, SecurityStandardCategoryStatistics::getToReviewSecurityHotspots, - SecurityStandardCategoryStatistics::getInReviewSecurityHotspots, SecurityStandardCategoryStatistics::getReviewedSecurityHotspots) - .containsExactlyInAnyOrder( - tuple(SANS_TOP_25_INSECURE_INTERACTION, 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */, 1L /* toreviewhotspot1 */, 0L, 0L), - tuple(SANS_TOP_25_RISKY_RESOURCE, 1L /* openvul1 */, OptionalInt.of(3)/* MAJOR = C */, 1L/* toreviewhotspot1 */, 1L /* inReviewHotspot */, 0L), - tuple(SANS_TOP_25_POROUS_DEFENSES, 0L, OptionalInt.empty(), 0L, 0L, 0L)); - - assertThat(sansTop25Report).allMatch(category -> category.getChildren().isEmpty()); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } - - private void indexView(String viewUuid, List<String> projects) { - viewIndexer.index(new ViewDoc().setUuid(viewUuid).setProjects(projects)); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSortTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSortTest.java deleted file mode 100644 index e5c1ec193dd..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexSortTest.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.Severity; -import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.TimeZone.getTimeZone; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.rules.ExpectedException.none; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.utils.DateUtils.parseDateTime; -import static org.sonar.db.component.ComponentTesting.newFileDto; -import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonar.server.issue.IssueDocTesting.newDoc; - -public class IssueIndexSortTest { - - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - @Rule - public ExpectedException expectedException = none(); - private System2 system2 = new TestSystem2().setNow(1_500_000_000_000L).setDefaultTimeZone(getTimeZone("GMT-01:00")); - @Rule - public DbTester db = DbTester.create(system2); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void sort_by_status() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setStatus(Issue.STATUS_OPEN), - newDoc("I2", file).setStatus(Issue.STATUS_CLOSED), - newDoc("I3", file).setStatus(Issue.STATUS_REOPENED)); - - IssueQuery.Builder query = IssueQuery.builder().sort(IssueQuery.SORT_BY_STATUS).asc(true); - assertThatSearchReturnsOnly(query, "I2", "I1", "I3"); - - query = IssueQuery.builder().sort(IssueQuery.SORT_BY_STATUS).asc(false); - assertThatSearchReturnsOnly(query, "I3", "I1", "I2"); - } - - @Test - public void sort_by_severity() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setSeverity(Severity.BLOCKER), - newDoc("I2", file).setSeverity(Severity.INFO), - newDoc("I3", file).setSeverity(Severity.MINOR), - newDoc("I4", file).setSeverity(Severity.CRITICAL), - newDoc("I5", file).setSeverity(Severity.MAJOR)); - - IssueQuery.Builder query = IssueQuery.builder().sort(IssueQuery.SORT_BY_SEVERITY).asc(true); - assertThatSearchReturnsOnly(query, "I2", "I3", "I5", "I4", "I1"); - - query = IssueQuery.builder().sort(IssueQuery.SORT_BY_SEVERITY).asc(false); - assertThatSearchReturnsOnly(query, "I1", "I4", "I5", "I3", "I2"); - } - - @Test - public void sort_by_creation_date() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100")), - newDoc("I2", file).setFuncCreationDate(parseDateTime("2014-09-24T00:00:00+0100"))); - - IssueQuery.Builder query = IssueQuery.builder().sort(IssueQuery.SORT_BY_CREATION_DATE).asc(true); - SearchResponse result = underTest.search(query.build(), new SearchOptions()); - assertThatSearchReturnsOnly(query, "I1", "I2"); - - query = IssueQuery.builder().sort(IssueQuery.SORT_BY_CREATION_DATE).asc(false); - assertThatSearchReturnsOnly(query, "I2", "I1"); - } - - @Test - public void sort_by_update_date() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setFuncUpdateDate(parseDateTime("2014-09-23T00:00:00+0100")), - newDoc("I2", file).setFuncUpdateDate(parseDateTime("2014-09-24T00:00:00+0100"))); - - IssueQuery.Builder query = IssueQuery.builder().sort(IssueQuery.SORT_BY_UPDATE_DATE).asc(true); - SearchResponse result = underTest.search(query.build(), new SearchOptions()); - assertThatSearchReturnsOnly(query, "I1", "I2"); - - query = IssueQuery.builder().sort(IssueQuery.SORT_BY_UPDATE_DATE).asc(false); - assertThatSearchReturnsOnly(query, "I2", "I1"); - } - - @Test - public void sort_by_close_date() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - - indexIssues( - newDoc("I1", file).setFuncCloseDate(parseDateTime("2014-09-23T00:00:00+0100")), - newDoc("I2", file).setFuncCloseDate(parseDateTime("2014-09-24T00:00:00+0100")), - newDoc("I3", file).setFuncCloseDate(null)); - - IssueQuery.Builder query = IssueQuery.builder().sort(IssueQuery.SORT_BY_CLOSE_DATE).asc(true); - SearchResponse result = underTest.search(query.build(), new SearchOptions()); - assertThatSearchReturnsOnly(query, "I3", "I1", "I2"); - - query = IssueQuery.builder().sort(IssueQuery.SORT_BY_CLOSE_DATE).asc(false); - assertThatSearchReturnsOnly(query, "I2", "I1", "I3"); - } - - @Test - public void sort_by_file_and_line() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file1 = newFileDto(project, null, "F1").setPath("src/main/xoo/org/sonar/samples/File.xoo"); - ComponentDto file2 = newFileDto(project, null, "F2").setPath("src/main/xoo/org/sonar/samples/File2.xoo"); - - indexIssues( - // file F1 - newDoc("F1_2", file1).setLine(20), - newDoc("F1_1", file1).setLine(null), - newDoc("F1_3", file1).setLine(25), - - // file F2 - newDoc("F2_1", file2).setLine(9), - newDoc("F2_2", file2).setLine(109), - // two issues on the same line -> sort by key - newDoc("F2_3", file2).setLine(109)); - - // ascending sort -> F1 then F2. Line "0" first. - IssueQuery.Builder query = IssueQuery.builder().sort(IssueQuery.SORT_BY_FILE_LINE).asc(true); - assertThatSearchReturnsOnly(query, "F1_1", "F1_2", "F1_3", "F2_1", "F2_2", "F2_3"); - - // descending sort -> F2 then F1 - query = IssueQuery.builder().sort(IssueQuery.SORT_BY_FILE_LINE).asc(false); - assertThatSearchReturnsOnly(query, "F2_3", "F2_2", "F2_1", "F1_3", "F1_2", "F1_1"); - } - - @Test - public void default_sort_is_by_creation_date_then_project_then_file_then_line_then_issue_key() { - OrganizationDto organizationDto = newOrganizationDto(); - ComponentDto project1 = newPrivateProjectDto(organizationDto, "P1"); - ComponentDto file1 = newFileDto(project1, null, "F1").setPath("src/main/xoo/org/sonar/samples/File.xoo"); - ComponentDto file2 = newFileDto(project1, null, "F2").setPath("src/main/xoo/org/sonar/samples/File2.xoo"); - - ComponentDto project2 = newPrivateProjectDto(organizationDto, "P2"); - ComponentDto file3 = newFileDto(project2, null, "F3").setPath("src/main/xoo/org/sonar/samples/File3.xoo"); - - indexIssues( - // file F1 from project P1 - newDoc("F1_1", file1).setLine(20).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100")), - newDoc("F1_2", file1).setLine(null).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100")), - newDoc("F1_3", file1).setLine(25).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100")), - - // file F2 from project P1 - newDoc("F2_1", file2).setLine(9).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100")), - newDoc("F2_2", file2).setLine(109).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100")), - // two issues on the same line -> sort by key - newDoc("F2_3", file2).setLine(109).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100")), - - // file F3 from project P2 - newDoc("F3_1", file3).setLine(20).setFuncCreationDate(parseDateTime("2014-09-24T00:00:00+0100")), - newDoc("F3_2", file3).setLine(20).setFuncCreationDate(parseDateTime("2014-09-23T00:00:00+0100"))); - - assertThatSearchReturnsOnly(IssueQuery.builder(), "F3_1", "F1_2", "F1_1", "F1_3", "F2_1", "F2_2", "F2_3", "F3_2"); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } - - /** - * Execute the search request and return the document ids of results. - */ - private List<String> searchAndReturnKeys(IssueQuery.Builder query) { - return Arrays.stream(underTest.search(query.build(), new SearchOptions()).getHits().getHits()) - .map(SearchHit::getId) - .collect(Collectors.toList()); - } - - private void assertThatSearchReturnsOnly(IssueQuery.Builder query, String... expectedIssueKeys) { - List<String> keys = searchAndReturnKeys(query); - assertThat(keys).containsExactlyInAnyOrder(expectedIssueKeys); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java deleted file mode 100644 index e34a915f808..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ /dev/null @@ -1,379 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterators; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.assertj.core.groups.Tuple; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.search.SearchHit; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.issue.Issue; -import org.sonar.api.utils.System2; -import org.sonar.api.impl.utils.TestSystem2; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.UserDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.SearchOptions; -import org.sonar.server.permission.index.IndexPermissions; -import org.sonar.server.permission.index.PermissionIndexerTester; -import org.sonar.server.permission.index.WebAuthorizationTypeSupport; -import org.sonar.server.rule.index.RuleIndexer; -import org.sonar.server.tester.UserSessionRule; - -import static com.google.common.collect.ImmutableSortedSet.of; -import static java.util.Arrays.asList; -import static java.util.Arrays.stream; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static java.util.TimeZone.getTimeZone; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; -import static org.assertj.core.api.Assertions.tuple; -import static org.junit.rules.ExpectedException.none; -import static org.sonar.api.issue.Issue.RESOLUTION_FIXED; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.rules.RuleType.BUG; -import static org.sonar.api.rules.RuleType.CODE_SMELL; -import static org.sonar.api.rules.RuleType.VULNERABILITY; -import static org.sonar.db.component.ComponentTesting.newFileDto; -import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; -import static org.sonar.db.user.GroupTesting.newGroupDto; -import static org.sonar.db.user.UserTesting.newUserDto; -import static org.sonar.server.issue.IssueDocTesting.newDoc; - -public class IssueIndexTest { - - @Rule - public EsTester es = EsTester.create(); - @Rule - public UserSessionRule userSessionRule = UserSessionRule.standalone(); - @Rule - public ExpectedException expectedException = none(); - private System2 system2 = new TestSystem2().setNow(1_500_000_000_000L).setDefaultTimeZone(getTimeZone("GMT-01:00")); - @Rule - public DbTester db = DbTester.create(system2); - - private IssueIndexer issueIndexer = new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())); - private RuleIndexer ruleIndexer = new RuleIndexer(es.client(), db.getDbClient()); - private PermissionIndexerTester authorizationIndexer = new PermissionIndexerTester(es, issueIndexer); - - private IssueIndex underTest = new IssueIndex(es.client(), system2, userSessionRule, new WebAuthorizationTypeSupport(userSessionRule)); - - @Test - public void paging() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - for (int i = 0; i < 12; i++) { - indexIssues(newDoc("I" + i, file)); - } - - IssueQuery.Builder query = IssueQuery.builder(); - // There are 12 issues in total, with 10 issues per page, the page 2 should only contain 2 elements - SearchResponse result = underTest.search(query.build(), new SearchOptions().setPage(2, 10)); - assertThat(result.getHits().getHits()).hasSize(2); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); - - result = underTest.search(IssueQuery.builder().build(), new SearchOptions().setOffset(0).setLimit(5)); - assertThat(result.getHits().getHits()).hasSize(5); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); - - result = underTest.search(IssueQuery.builder().build(), new SearchOptions().setOffset(2).setLimit(0)); - assertThat(result.getHits().getHits()).hasSize(10); - assertThat(result.getHits().getTotalHits()).isEqualTo(12); - } - - @Test - public void search_with_max_limit() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - List<IssueDoc> issues = new ArrayList<>(); - for (int i = 0; i < 500; i++) { - String key = "I" + i; - issues.add(newDoc(key, file)); - } - indexIssues(issues.toArray(new IssueDoc[] {})); - - IssueQuery.Builder query = IssueQuery.builder(); - SearchResponse result = underTest.search(query.build(), new SearchOptions().setLimit(Integer.MAX_VALUE)); - assertThat(result.getHits().getHits()).hasSize(SearchOptions.MAX_LIMIT); - } - - @Test - public void authorized_issues_on_groups() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project1 = newPrivateProjectDto(org); - ComponentDto project2 = newPrivateProjectDto(org); - ComponentDto project3 = newPrivateProjectDto(org); - ComponentDto file1 = newFileDto(project1, null); - ComponentDto file2 = newFileDto(project2, null); - ComponentDto file3 = newFileDto(project3, null); - GroupDto group1 = newGroupDto(); - GroupDto group2 = newGroupDto(); - - // project1 can be seen by group1 - indexIssue(newDoc("I1", file1)); - authorizationIndexer.allowOnlyGroup(project1, group1); - // project2 can be seen by group2 - indexIssue(newDoc("I2", file2)); - authorizationIndexer.allowOnlyGroup(project2, group2); - // project3 can be seen by nobody but root - indexIssue(newDoc("I3", file3)); - - userSessionRule.logIn().setGroups(group1); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I1"); - - userSessionRule.logIn().setGroups(group2); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I2"); - - userSessionRule.logIn().setGroups(group1, group2); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2"); - - GroupDto otherGroup = newGroupDto(); - userSessionRule.logIn().setGroups(otherGroup); - assertThatSearchReturnsEmpty(IssueQuery.builder()); - - userSessionRule.logIn().setGroups(group1, group2); - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList(project3.uuid()))); - - userSessionRule.setRoot(); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2", "I3"); - } - - @Test - public void authorized_issues_on_user() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project1 = newPrivateProjectDto(org); - ComponentDto project2 = newPrivateProjectDto(org); - ComponentDto project3 = newPrivateProjectDto(org); - ComponentDto file1 = newFileDto(project1, null); - ComponentDto file2 = newFileDto(project2, null); - ComponentDto file3 = newFileDto(project3, null); - UserDto user1 = newUserDto(); - UserDto user2 = newUserDto(); - - // project1 can be seen by john, project2 by max, project3 cannot be seen by anyone - indexIssue(newDoc("I1", file1)); - authorizationIndexer.allowOnlyUser(project1, user1); - indexIssue(newDoc("I2", file2)); - authorizationIndexer.allowOnlyUser(project2, user2); - indexIssue(newDoc("I3", file3)); - - userSessionRule.logIn(user1); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I1"); - assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(asList(project3.getDbKey()))); - - userSessionRule.logIn(user2); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I2"); - - // another user - userSessionRule.logIn(newUserDto()); - assertThatSearchReturnsEmpty(IssueQuery.builder()); - - userSessionRule.setRoot(); - assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2", "I3"); - } - - @Test - public void root_user_is_authorized_to_access_all_issues() { - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - indexIssue(newDoc("I1", project)); - userSessionRule.logIn().setRoot(); - - assertThatSearchReturnsOnly(IssueQuery.builder(), "I1"); - } - - @Test - public void list_tags() { - RuleDefinitionDto r1 = db.rules().insert(); - RuleDefinitionDto r2 = db.rules().insert(); - ruleIndexer.commitAndIndex(db.getSession(), asList(r1.getId(), r2.getId())); - - OrganizationDto org = db.organizations().insert(); - OrganizationDto anotherOrg = db.organizations().insert(); - ComponentDto project = newPrivateProjectDto(newOrganizationDto()); - ComponentDto file = newFileDto(project, null); - indexIssues( - newDoc("I42", file).setOrganizationUuid(anotherOrg.getUuid()).setRuleId(r1.getId()).setTags(of("another")), - newDoc("I1", file).setOrganizationUuid(org.getUuid()).setRuleId(r1.getId()).setTags(of("convention", "java8", "bug")), - newDoc("I2", file).setOrganizationUuid(org.getUuid()).setRuleId(r1.getId()).setTags(of("convention", "bug")), - newDoc("I3", file).setOrganizationUuid(org.getUuid()).setRuleId(r2.getId()), - newDoc("I4", file).setOrganizationUuid(org.getUuid()).setRuleId(r1.getId()).setTags(of("convention"))); - - assertThat(underTest.searchTags(IssueQuery.builder().organizationUuid(org.getUuid()).build(), null, 100)).containsOnly("convention", "java8", "bug"); - assertThat(underTest.searchTags(IssueQuery.builder().organizationUuid(org.getUuid()).build(), null, 2)).containsOnly("bug", "convention"); - assertThat(underTest.searchTags(IssueQuery.builder().organizationUuid(org.getUuid()).build(), "vent", 100)).containsOnly("convention"); - assertThat(underTest.searchTags(IssueQuery.builder().organizationUuid(org.getUuid()).build(), null, 1)).containsOnly("bug"); - assertThat(underTest.searchTags(IssueQuery.builder().organizationUuid(org.getUuid()).build(), null, 100)).containsOnly("convention", "java8", "bug"); - assertThat(underTest.searchTags(IssueQuery.builder().organizationUuid(org.getUuid()).build(), "invalidRegexp[", 100)).isEmpty(); - assertThat(underTest.searchTags(IssueQuery.builder().build(), null, 100)).containsExactlyInAnyOrder("another", "convention", "java8", "bug"); - } - - @Test - public void list_authors() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - newDoc("issue1", project).setAuthorLogin("luke.skywalker"), - newDoc("issue2", project).setAuthorLogin("luke@skywalker.name"), - newDoc("issue3", project).setAuthorLogin(null), - newDoc("issue4", project).setAuthorLogin("anakin@skywalker.name")); - IssueQuery query = IssueQuery.builder().build(); - - assertThat(underTest.searchAuthors(query, null, 5)).containsExactly("anakin@skywalker.name", "luke.skywalker", "luke@skywalker.name"); - assertThat(underTest.searchAuthors(query, null, 2)).containsExactly("anakin@skywalker.name", "luke.skywalker"); - assertThat(underTest.searchAuthors(query, "uke", 5)).containsExactly("luke.skywalker", "luke@skywalker.name"); - assertThat(underTest.searchAuthors(query, null, 1)).containsExactly("anakin@skywalker.name"); - assertThat(underTest.searchAuthors(query, null, Integer.MAX_VALUE)).containsExactly("anakin@skywalker.name", "luke.skywalker", "luke@skywalker.name"); - } - - @Test - public void list_authors_escapes_regexp_special_characters() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - newDoc("issue1", project).setAuthorLogin("name++")); - IssueQuery query = IssueQuery.builder().build(); - - assertThat(underTest.searchAuthors(query, "invalidRegexp[", 5)).isEmpty(); - assertThat(underTest.searchAuthors(query, "nam+", 5)).isEmpty(); - assertThat(underTest.searchAuthors(query, "name+", 5)).containsExactly("name++"); - assertThat(underTest.searchAuthors(query, ".*", 5)).isEmpty(); - } - - @Test - public void countTags() { - OrganizationDto org = newOrganizationDto(); - ComponentDto project = newPrivateProjectDto(org); - indexIssues( - newDoc("issue1", project).setTags(ImmutableSet.of("convention", "java8", "bug")), - newDoc("issue2", project).setTags(ImmutableSet.of("convention", "bug")), - newDoc("issue3", project).setTags(emptyList()), - newDoc("issue4", project).setTags(ImmutableSet.of("convention", "java8", "bug")).setResolution(Issue.RESOLUTION_FIXED), - newDoc("issue5", project).setTags(ImmutableSet.of("convention"))); - - assertThat(underTest.countTags(projectQuery(project.uuid()), 5)).containsOnly(entry("convention", 3L), entry("bug", 2L), entry("java8", 1L)); - assertThat(underTest.countTags(projectQuery(project.uuid()), 2)).contains(entry("convention", 3L), entry("bug", 2L)).doesNotContainEntry("java8", 1L); - assertThat(underTest.countTags(projectQuery("other"), 10)).isEmpty(); - } - - @Test - public void searchBranchStatistics() { - ComponentDto project = db.components().insertMainBranch(); - ComponentDto branch1 = db.components().insertProjectBranch(project); - ComponentDto branch2 = db.components().insertProjectBranch(project); - ComponentDto branch3 = db.components().insertProjectBranch(project); - ComponentDto fileOnBranch3 = db.components().insertComponent(newFileDto(branch3)); - indexIssues(newDoc(project), - newDoc(branch1).setType(BUG).setResolution(null), newDoc(branch1).setType(VULNERABILITY).setResolution(null), newDoc(branch1).setType(CODE_SMELL).setResolution(null), - newDoc(branch1).setType(CODE_SMELL).setResolution(RESOLUTION_FIXED), - newDoc(branch3).setType(CODE_SMELL).setResolution(null), newDoc(branch3).setType(CODE_SMELL).setResolution(null), - newDoc(fileOnBranch3).setType(CODE_SMELL).setResolution(null), newDoc(fileOnBranch3).setType(CODE_SMELL).setResolution(RESOLUTION_FIXED)); - - List<BranchStatistics> branchStatistics = underTest.searchBranchStatistics(project.uuid(), asList(branch1.uuid(), branch2.uuid(), branch3.uuid())); - - assertThat(branchStatistics).extracting(BranchStatistics::getBranchUuid, BranchStatistics::getBugs, BranchStatistics::getVulnerabilities, BranchStatistics::getCodeSmells) - .containsExactlyInAnyOrder( - tuple(branch1.uuid(), 1L, 1L, 1L), - tuple(branch3.uuid(), 0L, 0L, 3L)); - } - - @Test - public void searchBranchStatistics_on_many_branches() { - ComponentDto project = db.components().insertMainBranch(); - List<String> branchUuids = new ArrayList<>(); - List<Tuple> expectedResult = new ArrayList<>(); - IntStream.range(0, 15).forEach(i -> { - ComponentDto branch = db.components().insertProjectBranch(project); - addIssues(branch, 1 + i, 2 + i, 3 + i); - expectedResult.add(tuple(branch.uuid(), 1L + i, 2L + i, 3L + i)); - branchUuids.add(branch.uuid()); - }); - - List<BranchStatistics> branchStatistics = underTest.searchBranchStatistics(project.uuid(), branchUuids); - - assertThat(branchStatistics) - .extracting(BranchStatistics::getBranchUuid, BranchStatistics::getBugs, BranchStatistics::getVulnerabilities, BranchStatistics::getCodeSmells) - .hasSize(15) - .containsAll(expectedResult); - } - - @Test - public void searchBranchStatistics_on_empty_list() { - ComponentDto project = db.components().insertMainBranch(); - - assertThat(underTest.searchBranchStatistics(project.uuid(), emptyList())).isEmpty(); - assertThat(underTest.searchBranchStatistics(project.uuid(), singletonList("unknown"))).isEmpty(); - } - - private void addIssues(ComponentDto component, int bugs, int vulnerabilities, int codeSmelles) { - List<IssueDoc> issues = new ArrayList<>(); - IntStream.range(0, bugs).forEach(b -> issues.add(newDoc(component).setType(BUG).setResolution(null))); - IntStream.range(0, vulnerabilities).forEach(v -> issues.add(newDoc(component).setType(VULNERABILITY).setResolution(null))); - IntStream.range(0, codeSmelles).forEach(c -> issues.add(newDoc(component).setType(CODE_SMELL).setResolution(null))); - indexIssues(issues.toArray(new IssueDoc[issues.size()])); - } - - private IssueQuery projectQuery(String projectUuid) { - return IssueQuery.builder().projectUuids(singletonList(projectUuid)).resolved(false).build(); - } - - private void indexIssues(IssueDoc... issues) { - issueIndexer.index(asList(issues).iterator()); - authorizationIndexer.allow(stream(issues).map(issue -> new IndexPermissions(issue.projectUuid(), PROJECT).allowAnyone()).collect(toList())); - } - - private void indexIssue(IssueDoc issue) { - issueIndexer.index(Iterators.singletonIterator(issue)); - } - - /** - * Execute the search request and return the document ids of results. - */ - private List<String> searchAndReturnKeys(IssueQuery.Builder query) { - return Arrays.stream(underTest.search(query.build(), new SearchOptions()).getHits().getHits()) - .map(SearchHit::getId) - .collect(Collectors.toList()); - } - - private void assertThatSearchReturnsOnly(IssueQuery.Builder query, String... expectedIssueKeys) { - List<String> keys = searchAndReturnKeys(query); - assertThat(keys).containsExactlyInAnyOrder(expectedIssueKeys); - } - - private void assertThatSearchReturnsEmpty(IssueQuery.Builder query) { - List<String> keys = searchAndReturnKeys(query); - assertThat(keys).isEmpty(); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java deleted file mode 100644 index 2a321a15480..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java +++ /dev/null @@ -1,583 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import java.time.Clock; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.resources.Qualifiers; -import org.sonar.api.rule.RuleKey; -import org.sonar.api.utils.DateUtils; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.SnapshotDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.rule.RuleDbTester; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.db.user.UserDto; -import org.sonar.server.issue.SearchRequest; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.sonar.api.resources.Qualifiers.APP; -import static org.sonar.api.utils.DateUtils.addDays; -import static org.sonar.api.web.UserRole.USER; -import static org.sonar.db.component.ComponentTesting.newDirectory; -import static org.sonar.db.component.ComponentTesting.newFileDto; -import static org.sonar.db.component.ComponentTesting.newModuleDto; -import static org.sonar.db.component.ComponentTesting.newProjectCopy; -import static org.sonar.db.component.ComponentTesting.newSubView; -import static org.sonar.db.rule.RuleTesting.newRule; - -public class IssueQueryFactoryTest { - - @Rule - public UserSessionRule userSession = UserSessionRule.standalone(); - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Rule - public DbTester db = DbTester.create(); - - private RuleDbTester ruleDbTester = new RuleDbTester(db); - - private Clock clock = mock(Clock.class); - private IssueQueryFactory underTest = new IssueQueryFactory(db.getDbClient(), clock, userSession); - - @Test - public void create_from_parameters() { - UserDto user = db.users().insertUser(u -> u.setLogin("joanna")); - OrganizationDto organization = db.organizations().insert(); - ComponentDto project = db.components().insertPrivateProject(organization); - ComponentDto module = db.components().insertComponent(newModuleDto(project)); - ComponentDto file = db.components().insertComponent(newFileDto(project)); - - RuleDefinitionDto rule1 = ruleDbTester.insert(); - RuleDefinitionDto rule2 = ruleDbTester.insert(); - newRule(RuleKey.of("findbugs", "NullReference")); - SearchRequest request = new SearchRequest() - .setIssues(asList("anIssueKey")) - .setSeverities(asList("MAJOR", "MINOR")) - .setStatuses(asList("CLOSED")) - .setResolutions(asList("FALSE-POSITIVE")) - .setResolved(true) - .setProjectKeys(asList(project.getDbKey())) - .setModuleUuids(asList(module.uuid())) - .setDirectories(asList("aDirPath")) - .setFileUuids(asList(file.uuid())) - .setAssigneesUuid(asList(user.getUuid())) - .setLanguages(asList("xoo")) - .setTags(asList("tag1", "tag2")) - .setOrganization(organization.getKey()) - .setAssigned(true) - .setCreatedAfter("2013-04-16T09:08:24+0200") - .setCreatedBefore("2013-04-17T09:08:24+0200") - .setRules(asList(rule1.getKey().toString(), rule2.getKey().toString())) - .setSort("CREATION_DATE") - .setAsc(true); - - IssueQuery query = underTest.create(request); - - assertThat(query.issueKeys()).containsOnly("anIssueKey"); - assertThat(query.severities()).containsOnly("MAJOR", "MINOR"); - assertThat(query.statuses()).containsOnly("CLOSED"); - assertThat(query.resolutions()).containsOnly("FALSE-POSITIVE"); - assertThat(query.resolved()).isTrue(); - assertThat(query.projectUuids()).containsOnly(project.uuid()); - assertThat(query.moduleUuids()).containsOnly(module.uuid()); - assertThat(query.fileUuids()).containsOnly(file.uuid()); - assertThat(query.assignees()).containsOnly(user.getUuid()); - assertThat(query.languages()).containsOnly("xoo"); - assertThat(query.tags()).containsOnly("tag1", "tag2"); - assertThat(query.organizationUuid()).isEqualTo(organization.getUuid()); - assertThat(query.onComponentOnly()).isFalse(); - assertThat(query.assigned()).isTrue(); - assertThat(query.rules()).hasSize(2); - assertThat(query.directories()).containsOnly("aDirPath"); - assertThat(query.createdAfter().date()).isEqualTo(DateUtils.parseDateTime("2013-04-16T09:08:24+0200")); - assertThat(query.createdAfter().inclusive()).isTrue(); - assertThat(query.createdBefore()).isEqualTo(DateUtils.parseDateTime("2013-04-17T09:08:24+0200")); - assertThat(query.sort()).isEqualTo(IssueQuery.SORT_BY_CREATION_DATE); - assertThat(query.asc()).isTrue(); - } - - @Test - public void leak_period_start_date_is_exclusive() { - long leakPeriodStart = addDays(new Date(), -14).getTime(); - - ComponentDto project = db.components().insertPublicProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project)); - - SnapshotDto analysis = db.components().insertSnapshot(project, s -> s.setPeriodDate(leakPeriodStart)); - - SearchRequest request = new SearchRequest() - .setComponentUuids(Collections.singletonList(file.uuid())) - .setOnComponentOnly(true) - .setSinceLeakPeriod(true); - - IssueQuery query = underTest.create(request); - - assertThat(query.componentUuids()).containsOnly(file.uuid()); - assertThat(query.createdAfter().date()).isEqualTo(new Date(leakPeriodStart)); - assertThat(query.createdAfter().inclusive()).isFalse(); - - } - - @Test - public void dates_are_inclusive() { - SearchRequest request = new SearchRequest() - .setCreatedAfter("2013-04-16") - .setCreatedBefore("2013-04-17"); - - IssueQuery query = underTest.create(request); - - assertThat(query.createdAfter().date()).isEqualTo(DateUtils.parseDate("2013-04-16")); - assertThat(query.createdAfter().inclusive()).isTrue(); - assertThat(query.createdBefore()).isEqualTo(DateUtils.parseDate("2013-04-18")); - } - - @Test - public void creation_date_support_localdate() { - SearchRequest request = new SearchRequest() - .setCreatedAt("2013-04-16"); - - IssueQuery query = underTest.create(request); - - assertThat(query.createdAt()).isEqualTo(DateUtils.parseDate("2013-04-16")); - } - - @Test - public void creation_date_support_zoneddatetime() { - SearchRequest request = new SearchRequest() - .setCreatedAt("2013-04-16T09:08:24+0200"); - - IssueQuery query = underTest.create(request); - - assertThat(query.createdAt()).isEqualTo(DateUtils.parseDateTime("2013-04-16T09:08:24+0200")); - } - - @Test - public void add_unknown_when_no_component_found() { - SearchRequest request = new SearchRequest() - .setComponentKeys(asList("does_not_exist")); - - IssueQuery query = underTest.create(request); - - assertThat(query.componentUuids()).containsOnly("<UNKNOWN>"); - } - - @Test - public void query_without_any_parameter() { - SearchRequest request = new SearchRequest(); - - IssueQuery query = underTest.create(request); - - assertThat(query.componentUuids()).isEmpty(); - assertThat(query.projectUuids()).isEmpty(); - assertThat(query.moduleUuids()).isEmpty(); - assertThat(query.moduleRootUuids()).isEmpty(); - assertThat(query.directories()).isEmpty(); - assertThat(query.fileUuids()).isEmpty(); - assertThat(query.viewUuids()).isEmpty(); - assertThat(query.organizationUuid()).isNull(); - assertThat(query.branchUuid()).isNull(); - } - - @Test - public void fail_if_components_and_components_uuid_params_are_set_at_the_same_time() { - SearchRequest request = new SearchRequest() - .setComponentKeys(singletonList("foo")) - .setComponentUuids(singletonList("bar")); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("At most one of the following parameters can be provided: componentKeys and componentUuids"); - - underTest.create(request); - } - - @Test - public void fail_if_both_componentRoots_and_componentRootUuids_params_are_set() { - SearchRequest request = new SearchRequest() - .setComponentRoots(singletonList("foo")) - .setComponentRootUuids(singletonList("bar")); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("At most one of the following parameters can be provided: componentKeys and componentUuids"); - - underTest.create(request); - } - - @Test - public void fail_if_componentRoots_references_components_with_different_qualifier() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project)); - SearchRequest request = new SearchRequest() - .setComponentRoots(asList(project.getDbKey(), file.getDbKey())); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("All components must have the same qualifier, found FIL,TRK"); - - underTest.create(request); - } - - @Test - public void param_componentRootUuids_enables_search_in_view_tree_if_user_has_permission_on_view() { - ComponentDto view = db.components().insertView(); - SearchRequest request = new SearchRequest() - .setComponentRootUuids(asList(view.uuid())); - userSession.registerComponents(view); - - IssueQuery query = underTest.create(request); - - assertThat(query.viewUuids()).containsOnly(view.uuid()); - assertThat(query.onComponentOnly()).isFalse(); - } - - @Test - public void application_search_project_issues() { - ComponentDto project1 = db.components().insertPublicProject(); - ComponentDto project2 = db.components().insertPublicProject(); - ComponentDto application = db.components().insertApplication(db.getDefaultOrganization()); - db.components().insertComponents(newProjectCopy("PC1", project1, application)); - db.components().insertComponents(newProjectCopy("PC2", project2, application)); - userSession.registerComponents(application, project1, project2); - - IssueQuery result = underTest.create(new SearchRequest().setComponentUuids(singletonList(application.uuid()))); - - assertThat(result.viewUuids()).containsExactlyInAnyOrder(application.uuid()); - } - - @Test - public void application_search_project_issues_on_leak() { - Date now = new Date(); - ComponentDto project1 = db.components().insertPublicProject(); - SnapshotDto analysis1 = db.components().insertSnapshot(project1, s -> s.setPeriodDate(addDays(now, -14).getTime())); - ComponentDto project2 = db.components().insertPublicProject(); - SnapshotDto analysis2 = db.components().insertSnapshot(project2, s -> s.setPeriodDate(null)); - ComponentDto project3 = db.components().insertPublicProject(); - ComponentDto application = db.components().insertApplication(db.getDefaultOrganization()); - db.components().insertComponents(newProjectCopy("PC1", project1, application)); - db.components().insertComponents(newProjectCopy("PC2", project2, application)); - db.components().insertComponents(newProjectCopy("PC3", project3, application)); - userSession.registerComponents(application, project1, project2, project3); - - IssueQuery result = underTest.create(new SearchRequest() - .setComponentUuids(singletonList(application.uuid())) - .setSinceLeakPeriod(true)); - - assertThat(result.createdAfterByProjectUuids()).hasSize(1); - assertThat(result.createdAfterByProjectUuids().get(project1.uuid()).date().getTime()).isEqualTo(analysis1.getPeriodDate()); - assertThat(result.createdAfterByProjectUuids().get(project1.uuid()).inclusive()).isFalse(); - assertThat(result.viewUuids()).containsExactlyInAnyOrder(application.uuid()); - } - - @Test - public void return_empty_results_if_not_allowed_to_search_for_subview() { - ComponentDto view = db.components().insertView(); - ComponentDto subView = db.components().insertComponent(newSubView(view)); - SearchRequest request = new SearchRequest() - .setComponentRootUuids(asList(subView.uuid())); - - IssueQuery query = underTest.create(request); - - assertThat(query.viewUuids()).containsOnly("<UNKNOWN>"); - } - - @Test - public void param_componentUuids_enables_search_on_project_tree_by_default() { - ComponentDto project = db.components().insertPrivateProject(); - SearchRequest request = new SearchRequest() - .setComponentUuids(asList(project.uuid())); - - IssueQuery query = underTest.create(request); - assertThat(query.projectUuids()).containsExactly(project.uuid()); - assertThat(query.onComponentOnly()).isFalse(); - } - - @Test - public void onComponentOnly_restricts_search_to_specified_componentKeys() { - ComponentDto project = db.components().insertPrivateProject(); - SearchRequest request = new SearchRequest() - .setComponentKeys(asList(project.getDbKey())) - .setOnComponentOnly(true); - - IssueQuery query = underTest.create(request); - - assertThat(query.projectUuids()).isEmpty(); - assertThat(query.componentUuids()).containsExactly(project.uuid()); - assertThat(query.onComponentOnly()).isTrue(); - } - - @Test - public void should_search_in_tree_with_module_uuid() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto module = db.components().insertComponent(newModuleDto(project)); - SearchRequest request = new SearchRequest() - .setComponentUuids(asList(module.uuid())); - - IssueQuery query = underTest.create(request); - assertThat(query.moduleRootUuids()).containsExactly(module.uuid()); - assertThat(query.onComponentOnly()).isFalse(); - } - - @Test - public void param_componentUuids_enables_search_in_directory_tree() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto dir = db.components().insertComponent(newDirectory(project, "src/main/java/foo")); - SearchRequest request = new SearchRequest() - .setComponentUuids(asList(dir.uuid())); - - IssueQuery query = underTest.create(request); - - assertThat(query.moduleUuids()).containsOnly(dir.moduleUuid()); - assertThat(query.directories()).containsOnly(dir.path()); - assertThat(query.onComponentOnly()).isFalse(); - } - - @Test - public void param_componentUuids_enables_search_by_file() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project)); - SearchRequest request = new SearchRequest() - .setComponentUuids(asList(file.uuid())); - - IssueQuery query = underTest.create(request); - - assertThat(query.fileUuids()).containsExactly(file.uuid()); - } - - @Test - public void param_componentUuids_enables_search_by_test_file() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto file = db.components().insertComponent(newFileDto(project).setQualifier(Qualifiers.UNIT_TEST_FILE)); - SearchRequest request = new SearchRequest() - .setComponentUuids(asList(file.uuid())); - - IssueQuery query = underTest.create(request); - - assertThat(query.fileUuids()).containsExactly(file.uuid()); - } - - @Test - public void search_issue_from_branch() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto branch = db.components().insertProjectBranch(project); - - assertThat(underTest.create(new SearchRequest() - .setProjectKeys(singletonList(branch.getKey())) - .setBranch(branch.getBranch()))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.projectUuids()), IssueQuery::isMainBranch) - .containsOnly(branch.uuid(), singletonList(project.uuid()), false); - - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(branch.getKey())) - .setBranch(branch.getBranch()))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.projectUuids()), IssueQuery::isMainBranch) - .containsOnly(branch.uuid(), singletonList(project.uuid()), false); - } - - @Test - public void search_file_issue_from_branch() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto branch = db.components().insertProjectBranch(project); - ComponentDto file = db.components().insertComponent(newFileDto(branch)); - - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(file.getKey())) - .setBranch(branch.getBranch()))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.fileUuids()), IssueQuery::isMainBranch) - .containsOnly(branch.uuid(), singletonList(file.uuid()), false); - - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(branch.getKey())) - .setFileUuids(singletonList(file.uuid())) - .setBranch(branch.getBranch()))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.fileUuids()), IssueQuery::isMainBranch) - .containsOnly(branch.uuid(), singletonList(file.uuid()), false); - - assertThat(underTest.create(new SearchRequest() - .setProjectKeys(singletonList(branch.getKey())) - .setFileUuids(singletonList(file.uuid())) - .setBranch(branch.getBranch()))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.fileUuids()), IssueQuery::isMainBranch) - .containsOnly(branch.uuid(), singletonList(file.uuid()), false); - } - - @Test - public void search_issue_on_component_only_from_branch() { - ComponentDto project = db.components().insertPrivateProject(); - ComponentDto branch = db.components().insertProjectBranch(project); - ComponentDto file = db.components().insertComponent(newFileDto(branch)); - - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(file.getKey())) - .setBranch(branch.getBranch()) - .setOnComponentOnly(true))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.componentUuids()), IssueQuery::isMainBranch) - .containsOnly(branch.uuid(), singletonList(file.uuid()), false); - } - - @Test - public void search_issues_from_main_branch() { - ComponentDto project = db.components().insertMainBranch(); - ComponentDto branch = db.components().insertProjectBranch(project); - - assertThat(underTest.create(new SearchRequest() - .setProjectKeys(singletonList(project.getKey())) - .setBranch("master"))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.projectUuids()), IssueQuery::isMainBranch) - .containsOnly(project.uuid(), singletonList(project.uuid()), true); - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(project.getKey())) - .setBranch("master"))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.projectUuids()), IssueQuery::isMainBranch) - .containsOnly(project.uuid(), singletonList(project.uuid()), true); - } - - @Test - public void search_by_application_key() { - ComponentDto application = db.components().insertPrivateApplication(db.getDefaultOrganization()); - ComponentDto project1 = db.components().insertPrivateProject(); - ComponentDto project2 = db.components().insertPrivateProject(); - db.components().insertComponents(newProjectCopy(project1, application)); - db.components().insertComponents(newProjectCopy(project2, application)); - userSession.addProjectPermission(USER, application); - - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(application.getKey()))) - .viewUuids()).containsExactly(application.uuid()); - } - - @Test - public void search_by_application_key_and_branch() { - ComponentDto application = db.components().insertMainBranch(c -> c.setQualifier(APP).setDbKey("app")); - ComponentDto applicationBranch1 = db.components().insertProjectBranch(application, a -> a.setKey("app-branch1")); - ComponentDto applicationBranch2 = db.components().insertProjectBranch(application, a -> a.setKey("app-branch2")); - ComponentDto project1 = db.components().insertPrivateProject(p -> p.setDbKey("prj1")); - ComponentDto project1Branch1 = db.components().insertProjectBranch(project1); - ComponentDto fileOnProject1Branch1 = db.components().insertComponent(newFileDto(project1Branch1)); - ComponentDto project1Branch2 = db.components().insertProjectBranch(project1); - ComponentDto project2 = db.components().insertPrivateProject(p -> p.setDbKey("prj2")); - db.components().insertComponents(newProjectCopy(project1Branch1, applicationBranch1)); - db.components().insertComponents(newProjectCopy(project2, applicationBranch1)); - db.components().insertComponents(newProjectCopy(project1Branch2, applicationBranch2)); - - // Search on applicationBranch1 - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(applicationBranch1.getKey())) - .setBranch(applicationBranch1.getBranch()))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.projectUuids()), IssueQuery::isMainBranch) - .containsOnly(applicationBranch1.uuid(), Collections.emptyList(), false); - - // Search on project1Branch1 - assertThat(underTest.create(new SearchRequest() - .setComponentKeys(singletonList(applicationBranch1.getKey())) - .setProjectKeys(singletonList(project1.getKey())) - .setBranch(applicationBranch1.getBranch()))) - .extracting(IssueQuery::branchUuid, query -> new ArrayList<>(query.projectUuids()), IssueQuery::isMainBranch) - .containsOnly(applicationBranch1.uuid(), singletonList(project1.uuid()), false); - } - - @Test - public void fail_if_created_after_and_created_since_are_both_set() { - SearchRequest request = new SearchRequest() - .setCreatedAfter("2013-07-25T07:35:00+0100") - .setCreatedInLast("palap"); - - try { - underTest.create(request); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalArgumentException.class).hasMessage("Parameters createdAfter and createdInLast cannot be set simultaneously"); - } - } - - @Test - public void set_created_after_from_created_since() { - Date now = DateUtils.parseDateTime("2013-07-25T07:35:00+0100"); - when(clock.instant()).thenReturn(now.toInstant()); - when(clock.getZone()).thenReturn(ZoneOffset.UTC); - SearchRequest request = new SearchRequest() - .setCreatedInLast("1y2m3w4d"); - assertThat(underTest.create(request).createdAfter().date()).isEqualTo(DateUtils.parseDateTime("2012-04-30T07:35:00+0100")); - assertThat(underTest.create(request).createdAfter().inclusive()).isTrue(); - - } - - @Test - public void fail_if_since_leak_period_and_created_after_set_at_the_same_time() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Parameters 'createdAfter' and 'sinceLeakPeriod' cannot be set simultaneously"); - - underTest.create(new SearchRequest() - .setSinceLeakPeriod(true) - .setCreatedAfter("2013-07-25T07:35:00+0100")); - } - - @Test - public void fail_if_no_component_provided_with_since_leak_period() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("One and only one component must be provided when searching since leak period"); - - underTest.create(new SearchRequest().setSinceLeakPeriod(true)); - } - - @Test - public void fail_if_several_components_provided_with_since_leak_period() { - ComponentDto project1 = db.components().insertPrivateProject(); - ComponentDto project2 = db.components().insertPrivateProject(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("One and only one component must be provided when searching since leak period"); - - underTest.create(new SearchRequest() - .setSinceLeakPeriod(true) - .setComponentKeys(asList(project1.getKey(), project2.getKey()))); - } - - @Test - public void fail_if_date_is_not_formatted_correctly() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("'unknown-date' cannot be parsed as either a date or date+time"); - - underTest.create(new SearchRequest() - .setCreatedAfter("unknown-date")); - } - - @Test - public void return_empty_results_if_organization_with_specified_key_does_not_exist() { - SearchRequest request = new SearchRequest() - .setOrganization("does_not_exist"); - - IssueQuery query = underTest.create(request); - - assertThat(query.organizationUuid()).isEqualTo("<UNKNOWN>"); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueQueryTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueQueryTest.java deleted file mode 100644 index b8dccce0af7..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueQueryTest.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.issue.index; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import java.util.Date; -import org.junit.Test; -import org.sonar.api.issue.Issue; -import org.sonar.api.rule.Severity; -import org.sonar.db.rule.RuleDefinitionDto; -import org.sonar.server.issue.index.IssueQuery.PeriodStart; - -import static com.google.common.collect.Lists.newArrayList; -import static org.apache.commons.lang.math.RandomUtils.nextInt; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; - -public class IssueQueryTest { - - @Test - public void build_query() { - RuleDefinitionDto rule = new RuleDefinitionDto().setId(nextInt(1000)); - PeriodStart filterDate = new IssueQuery.PeriodStart(new Date(10_000_000_000L), false); - IssueQuery query = IssueQuery.builder() - .issueKeys(newArrayList("ABCDE")) - .severities(newArrayList(Severity.BLOCKER)) - .statuses(Lists.newArrayList(Issue.STATUS_RESOLVED)) - .resolutions(newArrayList(Issue.RESOLUTION_FALSE_POSITIVE)) - .projectUuids(newArrayList("PROJECT")) - .componentUuids(newArrayList("org/struts/Action.java")) - .moduleUuids(newArrayList("org.struts:core")) - .rules(newArrayList(rule)) - .assigneeUuids(newArrayList("gargantua")) - .languages(newArrayList("xoo")) - .tags(newArrayList("tag1", "tag2")) - .types(newArrayList("RELIABILITY", "SECURITY")) - .owaspTop10(newArrayList("a1", "a2")) - .sansTop25(newArrayList("insecure-interaction", "porous-defenses")) - .cwe(newArrayList("12", "125")) - .organizationUuid("orga") - .branchUuid("my_branch") - .createdAfterByProjectUuids(ImmutableMap.of("PROJECT", filterDate)) - .assigned(true) - .createdAfter(new Date()) - .createdBefore(new Date()) - .createdAt(new Date()) - .resolved(true) - .sort(IssueQuery.SORT_BY_CREATION_DATE) - .asc(true) - .build(); - assertThat(query.issueKeys()).containsOnly("ABCDE"); - assertThat(query.severities()).containsOnly(Severity.BLOCKER); - assertThat(query.statuses()).containsOnly(Issue.STATUS_RESOLVED); - assertThat(query.resolutions()).containsOnly(Issue.RESOLUTION_FALSE_POSITIVE); - assertThat(query.projectUuids()).containsOnly("PROJECT"); - assertThat(query.componentUuids()).containsOnly("org/struts/Action.java"); - assertThat(query.moduleUuids()).containsOnly("org.struts:core"); - assertThat(query.assignees()).containsOnly("gargantua"); - assertThat(query.languages()).containsOnly("xoo"); - assertThat(query.tags()).containsOnly("tag1", "tag2"); - assertThat(query.types()).containsOnly("RELIABILITY", "SECURITY"); - assertThat(query.owaspTop10()).containsOnly("a1", "a2"); - assertThat(query.sansTop25()).containsOnly("insecure-interaction", "porous-defenses"); - assertThat(query.cwe()).containsOnly("12", "125"); - assertThat(query.organizationUuid()).isEqualTo("orga"); - assertThat(query.branchUuid()).isEqualTo("my_branch"); - assertThat(query.createdAfterByProjectUuids()).containsOnly(entry("PROJECT", filterDate)); - assertThat(query.assigned()).isTrue(); - assertThat(query.rules()).containsOnly(rule); - assertThat(query.createdAfter()).isNotNull(); - assertThat(query.createdBefore()).isNotNull(); - assertThat(query.createdAt()).isNotNull(); - assertThat(query.resolved()).isTrue(); - assertThat(query.sort()).isEqualTo(IssueQuery.SORT_BY_CREATION_DATE); - assertThat(query.asc()).isTrue(); - } - - @Test - public void build_query_without_dates() { - IssueQuery query = IssueQuery.builder() - .issueKeys(newArrayList("ABCDE")) - .createdAfter(null) - .createdBefore(null) - .createdAt(null) - .build(); - - assertThat(query.issueKeys()).containsOnly("ABCDE"); - assertThat(query.createdAfter()).isNull(); - assertThat(query.createdBefore()).isNull(); - assertThat(query.createdAt()).isNull(); - } - - @Test - public void throw_exception_if_sort_is_not_valid() { - try { - IssueQuery.builder() - .sort("UNKNOWN") - .build(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalArgumentException.class).hasMessage("Bad sort field: UNKNOWN"); - } - } - - @Test - public void collection_params_should_not_be_null_but_empty() { - IssueQuery query = IssueQuery.builder() - .issueKeys(null) - .projectUuids(null) - .componentUuids(null) - .moduleUuids(null) - .statuses(null) - .assigneeUuids(null) - .resolutions(null) - .rules(null) - .severities(null) - .languages(null) - .tags(null) - .types(null) - .owaspTop10(null) - .sansTop25(null) - .cwe(null) - .createdAfterByProjectUuids(null) - .build(); - assertThat(query.issueKeys()).isEmpty(); - assertThat(query.projectUuids()).isEmpty(); - assertThat(query.componentUuids()).isEmpty(); - assertThat(query.moduleUuids()).isEmpty(); - assertThat(query.statuses()).isEmpty(); - assertThat(query.assignees()).isEmpty(); - assertThat(query.resolutions()).isEmpty(); - assertThat(query.rules()).isEmpty(); - assertThat(query.severities()).isEmpty(); - assertThat(query.languages()).isEmpty(); - assertThat(query.tags()).isEmpty(); - assertThat(query.types()).isEmpty(); - assertThat(query.owaspTop10()).isEmpty(); - assertThat(query.sansTop25()).isEmpty(); - assertThat(query.cwe()).isEmpty(); - assertThat(query.createdAfterByProjectUuids()).isEmpty(); - } - - @Test - public void test_default_query() { - IssueQuery query = IssueQuery.builder().build(); - assertThat(query.issueKeys()).isEmpty(); - assertThat(query.projectUuids()).isEmpty(); - assertThat(query.componentUuids()).isEmpty(); - assertThat(query.moduleUuids()).isEmpty(); - assertThat(query.statuses()).isEmpty(); - assertThat(query.assignees()).isEmpty(); - assertThat(query.resolutions()).isEmpty(); - assertThat(query.rules()).isEmpty(); - assertThat(query.severities()).isEmpty(); - assertThat(query.languages()).isEmpty(); - assertThat(query.tags()).isEmpty(); - assertThat(query.types()).isEmpty(); - assertThat(query.organizationUuid()).isNull(); - assertThat(query.branchUuid()).isNull(); - assertThat(query.assigned()).isNull(); - assertThat(query.createdAfter()).isNull(); - assertThat(query.createdBefore()).isNull(); - assertThat(query.resolved()).isNull(); - assertThat(query.sort()).isNull(); - assertThat(query.createdAfterByProjectUuids()).isEmpty(); - } - - @Test - public void should_accept_null_sort() { - IssueQuery query = IssueQuery.builder().sort(null).build(); - assertThat(query.sort()).isNull(); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndex.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndex.java deleted file mode 100644 index 22deb1a535b..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndex.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import java.util.Arrays; -import java.util.List; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHits; -import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.server.es.EsClient; - -import static org.sonar.server.permission.index.FooIndexDefinition.DESCRIPTOR; -import static org.sonar.server.permission.index.FooIndexDefinition.TYPE_AUTHORIZATION; - -public class FooIndex { - - private final EsClient esClient; - private final WebAuthorizationTypeSupport authorizationTypeSupport; - - public FooIndex(EsClient esClient, WebAuthorizationTypeSupport authorizationTypeSupport) { - this.esClient = esClient; - this.authorizationTypeSupport = authorizationTypeSupport; - } - - public boolean hasAccessToProject(String projectUuid) { - SearchHits hits = esClient.prepareSearch(DESCRIPTOR) - .setTypes(TYPE_AUTHORIZATION.getType()) - .setQuery(QueryBuilders.boolQuery() - .must(QueryBuilders.termQuery(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) - .filter(authorizationTypeSupport.createQueryFilter())) - .get() - .getHits(); - List<String> names = Arrays.stream(hits.getHits()) - .map(h -> h.getSourceAsMap().get(FooIndexDefinition.FIELD_NAME).toString()) - .collect(MoreCollectors.toList()); - return names.size() == 2 && names.contains("bar") && names.contains("baz"); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexDefinition.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexDefinition.java deleted file mode 100644 index 2dfe93c4c29..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexDefinition.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import org.sonar.api.config.internal.MapSettings; -import org.sonar.server.es.Index; -import org.sonar.server.es.IndexDefinition; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.newindex.NewAuthorizedIndex; - -import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; -import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; - -public class FooIndexDefinition implements IndexDefinition { - - public static final Index DESCRIPTOR = Index.withRelations("foos"); - public static final String FOO_TYPE = "foo"; - public static final IndexType.IndexMainType TYPE_AUTHORIZATION = IndexType.main(DESCRIPTOR, IndexAuthorizationConstants.TYPE_AUTHORIZATION); - public static final IndexType.IndexRelationType TYPE_FOO = IndexType.relation(TYPE_AUTHORIZATION, FOO_TYPE); - public static final String FIELD_NAME = "name"; - public static final String FIELD_PROJECT_UUID = "projectUuid"; - - @Override - public void define(IndexDefinitionContext context) { - NewAuthorizedIndex newIndex = context.createWithAuthorization( - DESCRIPTOR, - newBuilder(new MapSettings().asConfig()) - .setRefreshInterval(MANUAL_REFRESH_INTERVAL) - .build()); - - newIndex.createTypeMapping(TYPE_FOO) - .keywordFieldBuilder(FIELD_NAME).build() - .keywordFieldBuilder(FIELD_PROJECT_UUID).build(); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java deleted file mode 100644 index fd9e79f31bf..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import com.google.common.collect.ImmutableSet; -import java.util.Collection; -import java.util.Set; -import org.sonar.db.DbSession; -import org.sonar.db.es.EsQueueDto; -import org.sonar.server.es.BaseDoc; -import org.sonar.server.es.EsClient; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.IndexingResult; -import org.sonar.server.es.ProjectIndexer; - -import static org.sonar.server.permission.index.FooIndexDefinition.TYPE_FOO; - -public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer { - - private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_FOO, p -> true); - - private final EsClient esClient; - - public FooIndexer(EsClient esClient) { - this.esClient = esClient; - } - - @Override - public AuthorizationScope getAuthorizationScope() { - return AUTHORIZATION_SCOPE; - } - - @Override - public void indexOnAnalysis(String branchUuid) { - addToIndex(branchUuid, "bar"); - addToIndex(branchUuid, "baz"); - } - - @Override - public Collection<EsQueueDto> prepareForRecovery(DbSession dbSession, Collection<String> projectUuids, Cause cause) { - throw new UnsupportedOperationException(); - } - - private void addToIndex(String projectUuid, String name) { - FooDoc fooDoc = new FooDoc(projectUuid, name); - esClient.prepareIndex(TYPE_FOO) - .setId(fooDoc.getId()) - .setRouting(fooDoc.getRouting().orElse(null)) - .setSource(fooDoc.getFields()) - .get(); - } - - private static final class FooDoc extends BaseDoc { - private final String projectUuid; - private final String name; - - private FooDoc(String projectUuid, String name) { - super(TYPE_FOO); - this.projectUuid = projectUuid; - this.name = name; - setField(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid); - setField(FooIndexDefinition.FIELD_NAME, name); - setParent(AuthorizationDoc.idOf(projectUuid)); - } - - @Override - public String getId() { - return projectUuid + "_" + name; - } - - } - - @Override - public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { - throw new UnsupportedOperationException(); - } - - @Override - public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(TYPE_FOO); - } - - @Override - public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { - throw new UnsupportedOperationException(); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerDaoTest.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerDaoTest.java deleted file mode 100644 index da06a0659c5..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerDaoTest.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import org.assertj.core.api.Assertions; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.api.utils.System2; -import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.db.DbClient; -import org.sonar.db.DbSession; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.component.ComponentTesting; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.permission.GroupPermissionDto; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.UserDbTester; -import org.sonar.db.user.UserDto; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.api.resources.Qualifiers.APP; -import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.api.resources.Qualifiers.VIEW; -import static org.sonar.api.web.UserRole.ADMIN; -import static org.sonar.api.web.UserRole.USER; - -public class PermissionIndexerDaoTest { - - @Rule - public DbTester dbTester = DbTester.create(System2.INSTANCE); - - private DbClient dbClient = dbTester.getDbClient(); - private DbSession dbSession = dbTester.getSession(); - - private ComponentDbTester componentDbTester = new ComponentDbTester(dbTester); - private UserDbTester userDbTester = new UserDbTester(dbTester); - - private OrganizationDto organization; - private ComponentDto publicProject; - private ComponentDto privateProject1; - private ComponentDto privateProject2; - private ComponentDto view1; - private ComponentDto view2; - private ComponentDto application; - private UserDto user1; - private UserDto user2; - private GroupDto group; - - private PermissionIndexerDao underTest = new PermissionIndexerDao(); - - @Before - public void setUp() { - organization = dbTester.organizations().insert(); - publicProject = componentDbTester.insertPublicProject(organization); - privateProject1 = componentDbTester.insertPrivateProject(organization); - privateProject2 = componentDbTester.insertPrivateProject(organization); - view1 = componentDbTester.insertView(organization); - view2 = componentDbTester.insertView(organization); - application = componentDbTester.insertApplication(organization); - user1 = userDbTester.insertUser(); - user2 = userDbTester.insertUser(); - group = userDbTester.insertGroup(organization); - } - - @Test - public void select_all() { - insertTestDataForProjectsAndViews(); - - Collection<IndexPermissions> dtos = underTest.selectAll(dbClient, dbSession); - Assertions.assertThat(dtos).hasSize(6); - - IndexPermissions publicProjectAuthorization = getByProjectUuid(publicProject.uuid(), dtos); - isPublic(publicProjectAuthorization, PROJECT); - - IndexPermissions view1Authorization = getByProjectUuid(view1.uuid(), dtos); - isPublic(view1Authorization, VIEW); - - IndexPermissions applicationAuthorization = getByProjectUuid(application.uuid(), dtos); - isPublic(applicationAuthorization, APP); - - IndexPermissions privateProject1Authorization = getByProjectUuid(privateProject1.uuid(), dtos); - assertThat(privateProject1Authorization.getGroupIds()).containsOnly(group.getId()); - assertThat(privateProject1Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject1Authorization.getUserIds()).containsOnly(user1.getId(), user2.getId()); - assertThat(privateProject1Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions privateProject2Authorization = getByProjectUuid(privateProject2.uuid(), dtos); - assertThat(privateProject2Authorization.getGroupIds()).isEmpty(); - assertThat(privateProject2Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject2Authorization.getUserIds()).containsOnly(user1.getId()); - assertThat(privateProject2Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions view2Authorization = getByProjectUuid(view2.uuid(), dtos); - isPublic(view2Authorization, VIEW); - } - - @Test - public void selectByUuids() { - insertTestDataForProjectsAndViews(); - - Map<String, IndexPermissions> dtos = underTest - .selectByUuids(dbClient, dbSession, asList(publicProject.uuid(), privateProject1.uuid(), privateProject2.uuid(), view1.uuid(), view2.uuid(), application.uuid())) - .stream() - .collect(MoreCollectors.uniqueIndex(IndexPermissions::getProjectUuid, Function.identity())); - Assertions.assertThat(dtos).hasSize(6); - - IndexPermissions publicProjectAuthorization = dtos.get(publicProject.uuid()); - isPublic(publicProjectAuthorization, PROJECT); - - IndexPermissions view1Authorization = dtos.get(view1.uuid()); - isPublic(view1Authorization, VIEW); - - IndexPermissions applicationAuthorization = dtos.get(application.uuid()); - isPublic(applicationAuthorization, APP); - - IndexPermissions privateProject1Authorization = dtos.get(privateProject1.uuid()); - assertThat(privateProject1Authorization.getGroupIds()).containsOnly(group.getId()); - assertThat(privateProject1Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject1Authorization.getUserIds()).containsOnly(user1.getId(), user2.getId()); - assertThat(privateProject1Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions privateProject2Authorization = dtos.get(privateProject2.uuid()); - assertThat(privateProject2Authorization.getGroupIds()).isEmpty(); - assertThat(privateProject2Authorization.isAllowAnyone()).isFalse(); - assertThat(privateProject2Authorization.getUserIds()).containsOnly(user1.getId()); - assertThat(privateProject2Authorization.getQualifier()).isEqualTo(PROJECT); - - IndexPermissions view2Authorization = dtos.get(view2.uuid()); - isPublic(view2Authorization, VIEW); - } - - @Test - public void selectByUuids_returns_empty_list_when_project_does_not_exist() { - insertTestDataForProjectsAndViews(); - - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, asList("missing")); - Assertions.assertThat(dtos).isEmpty(); - } - - @Test - public void select_by_projects_with_high_number_of_projects() { - List<String> projectUuids = new ArrayList<>(); - for (int i = 0; i < 350; i++) { - ComponentDto project = ComponentTesting.newPrivateProjectDto(organization, Integer.toString(i)); - dbClient.componentDao().insert(dbSession, project); - projectUuids.add(project.uuid()); - GroupPermissionDto dto = new GroupPermissionDto() - .setOrganizationUuid(group.getOrganizationUuid()) - .setGroupId(group.getId()) - .setRole(USER) - .setResourceId(project.getId()); - dbClient.groupPermissionDao().insert(dbSession, dto); - } - dbSession.commit(); - - assertThat(underTest.selectByUuids(dbClient, dbSession, projectUuids)) - .hasSize(350) - .extracting(IndexPermissions::getProjectUuid) - .containsAll(projectUuids); - } - - @Test - public void return_private_project_without_any_permission_when_no_permission_in_DB() { - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(privateProject1.uuid())); - - // no permissions - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupIds()).isEmpty(); - assertThat(dto.getUserIds()).isEmpty(); - assertThat(dto.isAllowAnyone()).isFalse(); - assertThat(dto.getProjectUuid()).isEqualTo(privateProject1.uuid()); - assertThat(dto.getQualifier()).isEqualTo(privateProject1.qualifier()); - } - - @Test - public void return_public_project_with_only_AllowAnyone_true_when_no_permission_in_DB() { - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(publicProject.uuid())); - - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupIds()).isEmpty(); - assertThat(dto.getUserIds()).isEmpty(); - assertThat(dto.isAllowAnyone()).isTrue(); - assertThat(dto.getProjectUuid()).isEqualTo(publicProject.uuid()); - assertThat(dto.getQualifier()).isEqualTo(publicProject.qualifier()); - } - - @Test - public void return_private_project_with_AllowAnyone_false_and_user_id_when_user_is_granted_USER_permission_directly() { - dbTester.users().insertProjectPermissionOnUser(user1, USER, privateProject1); - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(privateProject1.uuid())); - - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupIds()).isEmpty(); - assertThat(dto.getUserIds()).containsOnly(user1.getId()); - assertThat(dto.isAllowAnyone()).isFalse(); - assertThat(dto.getProjectUuid()).isEqualTo(privateProject1.uuid()); - assertThat(dto.getQualifier()).isEqualTo(privateProject1.qualifier()); - } - - @Test - public void return_private_project_with_AllowAnyone_false_and_group_id_but_not_user_id_when_user_is_granted_USER_permission_through_group() { - dbTester.users().insertMember(group, user1); - dbTester.users().insertProjectPermissionOnGroup(group, USER, privateProject1); - List<IndexPermissions> dtos = underTest.selectByUuids(dbClient, dbSession, singletonList(privateProject1.uuid())); - - Assertions.assertThat(dtos).hasSize(1); - IndexPermissions dto = dtos.get(0); - assertThat(dto.getGroupIds()).containsOnly(group.getId()); - assertThat(dto.getUserIds()).isEmpty(); - assertThat(dto.isAllowAnyone()).isFalse(); - assertThat(dto.getProjectUuid()).isEqualTo(privateProject1.uuid()); - assertThat(dto.getQualifier()).isEqualTo(privateProject1.qualifier()); - } - - private void isPublic(IndexPermissions view1Authorization, String qualifier) { - assertThat(view1Authorization.getGroupIds()).isEmpty(); - assertThat(view1Authorization.isAllowAnyone()).isTrue(); - assertThat(view1Authorization.getUserIds()).isEmpty(); - assertThat(view1Authorization.getQualifier()).isEqualTo(qualifier); - } - - private static IndexPermissions getByProjectUuid(String projectUuid, Collection<IndexPermissions> dtos) { - return dtos.stream().filter(dto -> dto.getProjectUuid().equals(projectUuid)).findFirst().orElseThrow(IllegalArgumentException::new); - } - - private void insertTestDataForProjectsAndViews() { - // user1 has USER access on both private projects - userDbTester.insertProjectPermissionOnUser(user1, ADMIN, publicProject); - userDbTester.insertProjectPermissionOnUser(user1, USER, privateProject1); - userDbTester.insertProjectPermissionOnUser(user1, USER, privateProject2); - userDbTester.insertProjectPermissionOnUser(user1, ADMIN, view1); - userDbTester.insertProjectPermissionOnUser(user1, ADMIN, application); - - // user2 has USER access on privateProject1 only - userDbTester.insertProjectPermissionOnUser(user2, USER, privateProject1); - userDbTester.insertProjectPermissionOnUser(user2, ADMIN, privateProject2); - - // group1 has USER access on privateProject1 only - userDbTester.insertProjectPermissionOnGroup(group, USER, privateProject1); - userDbTester.insertProjectPermissionOnGroup(group, ADMIN, privateProject1); - userDbTester.insertProjectPermissionOnGroup(group, ADMIN, view1); - userDbTester.insertProjectPermissionOnGroup(group, ADMIN, application); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java deleted file mode 100644 index 1dda0b4e89a..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java +++ /dev/null @@ -1,431 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import java.util.Collection; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.utils.System2; -import org.sonar.db.DbSession; -import org.sonar.db.DbTester; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.es.EsQueueDto; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.UserDto; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.IndexType; -import org.sonar.server.es.IndexType.IndexMainType; -import org.sonar.server.es.IndexingResult; -import org.sonar.server.es.ProjectIndexer; -import org.sonar.server.tester.UserSessionRule; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.api.web.UserRole.ADMIN; -import static org.sonar.api.web.UserRole.USER; -import static org.sonar.server.es.ProjectIndexer.Cause.PERMISSION_CHANGE; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; - -public class PermissionIndexerTest { - - private static final IndexMainType INDEX_TYPE_FOO_AUTH = IndexType.main(FooIndexDefinition.DESCRIPTOR, TYPE_AUTHORIZATION); - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Rule - public DbTester db = DbTester.create(System2.INSTANCE); - @Rule - public EsTester es = EsTester.createCustom(new FooIndexDefinition()); - @Rule - public UserSessionRule userSession = UserSessionRule.standalone(); - - private FooIndex fooIndex = new FooIndex(es.client(), new WebAuthorizationTypeSupport(userSession)); - private FooIndexer fooIndexer = new FooIndexer(es.client()); - private PermissionIndexer underTest = new PermissionIndexer(db.getDbClient(), es.client(), fooIndexer); - - @Test - public void indexOnStartup_grants_access_to_any_user_and_to_group_Anyone_on_public_projects() { - ComponentDto project = createAndIndexPublicProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - - indexOnStartup(); - - verifyAnyoneAuthorized(project); - verifyAuthorized(project, user1); - verifyAuthorized(project, user2); - } - - @Test - public void deletion_resilience_will_deindex_projects() { - ComponentDto project1 = createUnindexedPublicProject(); - ComponentDto project2 = createUnindexedPublicProject(); - // UserDto user1 = db.users().insertUser(); - indexOnStartup(); - assertThat(es.countDocuments(INDEX_TYPE_FOO_AUTH)).isEqualTo(2); - - // Simulate a indexation issue - db.getDbClient().componentDao().delete(db.getSession(), project1.getId()); - underTest.prepareForRecovery(db.getSession(), asList(project1.uuid()), ProjectIndexer.Cause.PROJECT_DELETION); - assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isEqualTo(1); - Collection<EsQueueDto> esQueueDtos = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), Long.MAX_VALUE, 2); - - underTest.index(db.getSession(), esQueueDtos); - - assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isEqualTo(0); - assertThat(es.countDocuments(INDEX_TYPE_FOO_AUTH)).isEqualTo(1); - } - - @Test - public void indexOnStartup_grants_access_to_user() { - ComponentDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user1, USER, project); - db.users().insertProjectPermissionOnUser(user2, ADMIN, project); - - indexOnStartup(); - - // anonymous - verifyAnyoneNotAuthorized(project); - - // user1 has access - verifyAuthorized(project, user1); - - // user2 has not access (only USER permission is accepted) - verifyNotAuthorized(project, user2); - } - - @Test - public void indexOnStartup_grants_access_to_group_on_private_project() { - ComponentDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - UserDto user3 = db.users().insertUser(); - GroupDto group1 = db.users().insertGroup(); - GroupDto group2 = db.users().insertGroup(); - db.users().insertProjectPermissionOnGroup(group1, USER, project); - db.users().insertProjectPermissionOnGroup(group2, ADMIN, project); - - indexOnStartup(); - - // anonymous - verifyAnyoneNotAuthorized(project); - - // group1 has access - verifyAuthorized(project, user1, group1); - - // group2 has not access (only USER permission is accepted) - verifyNotAuthorized(project, user2, group2); - - // user3 is not in any group - verifyNotAuthorized(project, user3); - } - - @Test - public void indexOnStartup_grants_access_to_user_and_group() { - ComponentDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - db.users().insertMember(group, user2); - db.users().insertProjectPermissionOnUser(user1, USER, project); - db.users().insertProjectPermissionOnGroup(group, USER, project); - - indexOnStartup(); - - // anonymous - verifyAnyoneNotAuthorized(project); - - // has direct access - verifyAuthorized(project, user1); - - // has access through group - verifyAuthorized(project, user1, group); - - // no access - verifyNotAuthorized(project, user2); - } - - @Test - public void indexOnStartup_does_not_grant_access_to_anybody_on_private_project() { - ComponentDto project = createAndIndexPrivateProject(); - UserDto user = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - - indexOnStartup(); - - verifyAnyoneNotAuthorized(project); - verifyNotAuthorized(project, user); - verifyNotAuthorized(project, user, group); - } - - @Test - public void indexOnStartup_grants_access_to_anybody_on_public_project() { - ComponentDto project = createAndIndexPublicProject(); - UserDto user = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - - indexOnStartup(); - - verifyAnyoneAuthorized(project); - verifyAuthorized(project, user); - verifyAuthorized(project, user, group); - } - - @Test - public void indexOnStartup_grants_access_to_anybody_on_view() { - ComponentDto view = createAndIndexView(); - UserDto user = db.users().insertUser(); - GroupDto group = db.users().insertGroup(); - - indexOnStartup(); - - verifyAnyoneAuthorized(view); - verifyAuthorized(view, user); - verifyAuthorized(view, user, group); - } - - @Test - public void indexOnStartup_grants_access_on_many_projects() { - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - ComponentDto project = null; - for (int i = 0; i < 10; i++) { - project = createAndIndexPrivateProject(); - db.users().insertProjectPermissionOnUser(user1, USER, project); - } - - indexOnStartup(); - - verifyAnyoneNotAuthorized(project); - verifyAuthorized(project, user1); - verifyNotAuthorized(project, user2); - } - - @Test - public void public_projects_are_visible_to_anybody_whatever_the_organization() { - ComponentDto projectOnOrg1 = createAndIndexPublicProject(db.organizations().insert()); - ComponentDto projectOnOrg2 = createAndIndexPublicProject(db.organizations().insert()); - UserDto user = db.users().insertUser(); - - indexOnStartup(); - - verifyAnyoneAuthorized(projectOnOrg1); - verifyAnyoneAuthorized(projectOnOrg2); - verifyAuthorized(projectOnOrg1, user); - verifyAuthorized(projectOnOrg2, user); - } - - @Test - public void indexOnAnalysis_does_nothing_because_CE_does_not_touch_permissions() { - ComponentDto project = createAndIndexPublicProject(); - - underTest.indexOnAnalysis(project.uuid()); - - assertThatAuthIndexHasSize(0); - verifyAnyoneNotAuthorized(project); - } - - @Test - public void permissions_are_not_updated_on_project_tags_update() { - ComponentDto project = createAndIndexPublicProject(); - - indexPermissions(project, ProjectIndexer.Cause.PROJECT_TAGS_UPDATE); - - assertThatAuthIndexHasSize(0); - verifyAnyoneNotAuthorized(project); - } - - @Test - public void permissions_are_not_updated_on_project_key_update() { - ComponentDto project = createAndIndexPublicProject(); - - indexPermissions(project, ProjectIndexer.Cause.PROJECT_TAGS_UPDATE); - - assertThatAuthIndexHasSize(0); - verifyAnyoneNotAuthorized(project); - } - - @Test - public void index_permissions_on_project_creation() { - ComponentDto project = createAndIndexPrivateProject(); - UserDto user = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user, USER, project); - - indexPermissions(project, ProjectIndexer.Cause.PROJECT_CREATION); - - assertThatAuthIndexHasSize(1); - verifyAuthorized(project, user); - } - - @Test - public void index_permissions_on_permission_change() { - ComponentDto project = createAndIndexPrivateProject(); - UserDto user1 = db.users().insertUser(); - UserDto user2 = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user1, USER, project); - indexPermissions(project, ProjectIndexer.Cause.PROJECT_CREATION); - verifyAuthorized(project, user1); - verifyNotAuthorized(project, user2); - - db.users().insertProjectPermissionOnUser(user2, USER, project); - indexPermissions(project, PERMISSION_CHANGE); - - verifyAuthorized(project, user1); - verifyAuthorized(project, user1); - } - - @Test - public void delete_permissions_on_project_deletion() { - ComponentDto project = createAndIndexPrivateProject(); - UserDto user = db.users().insertUser(); - db.users().insertProjectPermissionOnUser(user, USER, project); - indexPermissions(project, ProjectIndexer.Cause.PROJECT_CREATION); - verifyAuthorized(project, user); - - db.getDbClient().componentDao().delete(db.getSession(), project.getId()); - indexPermissions(project, ProjectIndexer.Cause.PROJECT_DELETION); - - verifyNotAuthorized(project, user); - assertThatAuthIndexHasSize(0); - } - - @Test - public void errors_during_indexing_are_recovered() { - ComponentDto project = createAndIndexPublicProject(); - es.lockWrites(INDEX_TYPE_FOO_AUTH); - - IndexingResult result = indexPermissions(project, PERMISSION_CHANGE); - assertThat(result.getTotal()).isEqualTo(1L); - assertThat(result.getFailures()).isEqualTo(1L); - - // index is still read-only, fail to recover - result = recover(); - assertThat(result.getTotal()).isEqualTo(1L); - assertThat(result.getFailures()).isEqualTo(1L); - assertThatAuthIndexHasSize(0); - assertThatEsQueueTableHasSize(1); - - es.unlockWrites(INDEX_TYPE_FOO_AUTH); - - result = recover(); - assertThat(result.getTotal()).isEqualTo(1L); - assertThat(result.getFailures()).isEqualTo(0L); - verifyAnyoneAuthorized(project); - assertThatEsQueueTableHasSize(0); - } - - private void assertThatAuthIndexHasSize(int expectedSize) { - assertThat(es.countDocuments(FooIndexDefinition.TYPE_AUTHORIZATION)).isEqualTo(expectedSize); - } - - private void indexOnStartup() { - underTest.indexOnStartup(underTest.getIndexTypes()); - } - - private void verifyAuthorized(ComponentDto project, UserDto user) { - logIn(user); - verifyAuthorized(project, true); - } - - private void verifyAuthorized(ComponentDto project, UserDto user, GroupDto group) { - logIn(user).setGroups(group); - verifyAuthorized(project, true); - } - - private void verifyNotAuthorized(ComponentDto project, UserDto user) { - logIn(user); - verifyAuthorized(project, false); - } - - private void verifyNotAuthorized(ComponentDto project, UserDto user, GroupDto group) { - logIn(user).setGroups(group); - verifyAuthorized(project, false); - } - - private void verifyAnyoneAuthorized(ComponentDto project) { - userSession.anonymous(); - verifyAuthorized(project, true); - } - - private void verifyAnyoneNotAuthorized(ComponentDto project) { - userSession.anonymous(); - verifyAuthorized(project, false); - } - - private void verifyAuthorized(ComponentDto project, boolean expectedAccess) { - assertThat(fooIndex.hasAccessToProject(project.uuid())).isEqualTo(expectedAccess); - } - - private UserSessionRule logIn(UserDto u) { - userSession.logIn(u.getLogin()).setUserId(u.getId()); - return userSession; - } - - private IndexingResult indexPermissions(ComponentDto project, ProjectIndexer.Cause cause) { - DbSession dbSession = db.getSession(); - Collection<EsQueueDto> items = underTest.prepareForRecovery(dbSession, singletonList(project.uuid()), cause); - dbSession.commit(); - return underTest.index(dbSession, items); - } - - private ComponentDto createUnindexedPublicProject() { - ComponentDto project = db.components().insertPublicProject(); - return project; - } - - private ComponentDto createAndIndexPrivateProject() { - ComponentDto project = db.components().insertPrivateProject(); - fooIndexer.indexOnAnalysis(project.uuid()); - return project; - } - - private ComponentDto createAndIndexPublicProject() { - ComponentDto project = db.components().insertPublicProject(); - fooIndexer.indexOnAnalysis(project.uuid()); - return project; - } - - private ComponentDto createAndIndexView() { - ComponentDto view = db.components().insertView(); - fooIndexer.indexOnAnalysis(view.uuid()); - return view; - } - - private ComponentDto createAndIndexPublicProject(OrganizationDto org) { - ComponentDto project = db.components().insertPublicProject(org); - fooIndexer.indexOnAnalysis(project.uuid()); - return project; - } - - private IndexingResult recover() { - Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), System.currentTimeMillis() + 1_000L, 10); - return underTest.index(db.getSession(), items); - } - - private void assertThatEsQueueTableHasSize(int expectedSize) { - assertThat(db.countRowsOfTable("es_queue")).isEqualTo(expectedSize); - } - -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTester.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTester.java deleted file mode 100644 index 9fb5f837187..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTester.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import java.util.List; -import java.util.stream.Stream; -import org.sonar.db.component.ComponentDto; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.UserDto; -import org.sonar.server.es.EsTester; - -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toList; - -public class PermissionIndexerTester { - - private final PermissionIndexer permissionIndexer; - - public PermissionIndexerTester(EsTester esTester, NeedAuthorizationIndexer indexer, NeedAuthorizationIndexer... others) { - NeedAuthorizationIndexer[] indexers = Stream.concat(Stream.of(indexer), stream(others)).toArray(NeedAuthorizationIndexer[]::new); - this.permissionIndexer = new PermissionIndexer(null, esTester.client(), indexers); - } - - public PermissionIndexerTester allowOnlyAnyone(ComponentDto... projects) { - return allow(stream(projects).map(project -> new IndexPermissions(project.uuid(), project.qualifier()).allowAnyone()).collect(toList())); - } - - public PermissionIndexerTester allowOnlyUser(ComponentDto project, UserDto user) { - IndexPermissions dto = new IndexPermissions(project.uuid(), project.qualifier()) - .addUserId(user.getId()); - return allow(dto); - } - - public PermissionIndexerTester allowOnlyGroup(ComponentDto project, GroupDto group) { - IndexPermissions dto = new IndexPermissions(project.uuid(), project.qualifier()) - .addGroupId(group.getId()); - return allow(dto); - } - - public PermissionIndexerTester allow(IndexPermissions... indexPermissions) { - return allow(stream(indexPermissions).collect(toList())); - } - - public PermissionIndexerTester allow(List<IndexPermissions> indexPermissions) { - permissionIndexer.index(indexPermissions); - return this; - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/WebAuthorizationTypeSupportTest.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/WebAuthorizationTypeSupportTest.java deleted file mode 100644 index 9a94ba36f93..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/WebAuthorizationTypeSupportTest.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.permission.index; - -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.join.query.HasParentQueryBuilder; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.db.user.GroupDto; -import org.sonar.db.user.GroupTesting; -import org.sonar.server.tester.UserSessionRule; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.test.JsonAssert.assertJson; - -public class WebAuthorizationTypeSupportTest { - - @Rule - public UserSessionRule userSession = UserSessionRule.standalone(); - - private WebAuthorizationTypeSupport underTest = new WebAuthorizationTypeSupport(userSession); - - @Test - public void createQueryFilter_does_not_include_permission_filters_if_user_is_flagged_as_root() { - userSession.logIn().setRoot(); - - QueryBuilder filter = underTest.createQueryFilter(); - - assertThat(filter).isInstanceOf(MatchAllQueryBuilder.class); - } - - @Test - public void createQueryFilter_sets_filter_on_anyone_group_if_user_is_anonymous() { - userSession.anonymous(); - - HasParentQueryBuilder filter = (HasParentQueryBuilder) underTest.createQueryFilter(); - - assertJson(filter.toString()).isSimilarTo("{" + - " \"has_parent\" : {" + - " \"query\" : {" + - " \"bool\" : {" + - " \"filter\" : [{" + - " \"bool\" : {" + - " \"should\" : [{" + - " \"term\" : {" + - " \"auth_allowAnyone\" : {\"value\": true}" + - " }" + - " }]" + - " }" + - " }]" + - " }" + - " }," + - " \"parent_type\" : \"auth\"" + - " }" + - "}"); - } - - @Test - public void createQueryFilter_sets_filter_on_anyone_and_user_id_if_user_is_logged_in_but_has_no_groups() { - userSession.logIn().setUserId(1234); - - HasParentQueryBuilder filter = (HasParentQueryBuilder) underTest.createQueryFilter(); - - assertJson(filter.toString()).isSimilarTo("{" + - " \"has_parent\": {" + - " \"query\": {" + - " \"bool\": {" + - " \"filter\": [{" + - " \"bool\": {" + - " \"should\": [" + - " {" + - " \"term\": {" + - " \"auth_allowAnyone\": {\"value\": true}" + - " }" + - " }," + - " {" + - " \"term\": {" + - " \"auth_userIds\": {\"value\": 1234}" + - " }" + - " }" + - " ]" + - " }" + - " }]" + - " }" + - " }," + - " \"parent_type\": \"auth\"" + - " }" + - "}"); - } - - @Test - public void createQueryFilter_sets_filter_on_anyone_and_user_id_and_group_ids_if_user_is_logged_in_and_has_groups() { - GroupDto group1 = GroupTesting.newGroupDto().setId(10); - GroupDto group2 = GroupTesting.newGroupDto().setId(11); - userSession.logIn().setUserId(1234).setGroups(group1, group2); - - HasParentQueryBuilder filter = (HasParentQueryBuilder) underTest.createQueryFilter(); - - assertJson(filter.toString()).isSimilarTo("{" + - " \"has_parent\": {" + - " \"query\": {" + - " \"bool\": {" + - " \"filter\": [{" + - " \"bool\": {" + - " \"should\": [" + - " {" + - " \"term\": {" + - " \"auth_allowAnyone\": {\"value\": true}" + - " }" + - " }," + - " {" + - " \"term\": {" + - " \"auth_userIds\": {\"value\": 1234}" + - " }" + - " }," + - " {" + - " \"term\": {" + - " \"auth_groupIds\": {\"value\": 10}" + - " }" + - " }," + - " {" + - " \"term\": {" + - " \"auth_groupIds\": {\"value\": 11}" + - " }" + - " }" + - " ]" + - " }" + - " }]" + - " }" + - " }," + - " \"parent_type\": \"auth\"" + - " }" + - "}"); - } -} |