From 7f47ee191a4f7de6f522dafedf36d34e931aca7f Mon Sep 17 00:00:00 2001 From: Teryk Bellahsene Date: Wed, 19 Oct 2016 21:27:16 +0200 Subject: [PATCH] SONAR-8232 WS api/components/search_projects return the ncloc facet --- .../component/es/ProjectMeasuresIndex.java | 18 +++++++++ .../component/ws/SearchProjectsAction.java | 35 ++++++++++++++++- .../main/java/org/sonar/server/es/Facets.java | 8 ++++ .../es/ProjectMeasuresIndexTest.java | 39 +++++++++++++++++++ .../ws/SearchProjectsActionTest.java | 39 ++++++++++++++++--- .../src/main/protobuf/ws-components.proto | 1 + 6 files changed, 133 insertions(+), 7 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/es/ProjectMeasuresIndex.java b/server/sonar-server/src/main/java/org/sonar/server/component/es/ProjectMeasuresIndex.java index 6c2724d3fca..c8160b08959 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/es/ProjectMeasuresIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/es/ProjectMeasuresIndex.java @@ -23,6 +23,8 @@ import java.util.Set; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.sort.SortOrder; import org.sonar.server.component.es.ProjectMeasuresQuery.MetricCriterion; @@ -37,6 +39,8 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.sonar.api.measures.CoreMetrics.NCLOC_KEY; import static org.sonar.server.component.es.ProjectMeasuresIndexDefinition.FIELD_MEASURES; import static org.sonar.server.component.es.ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY; import static org.sonar.server.component.es.ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE; @@ -60,6 +64,19 @@ public class ProjectMeasuresIndex extends BaseIndex { public SearchIdResult search(ProjectMeasuresQuery query, SearchOptions searchOptions) { QueryBuilder esQuery = createEsQuery(query); + AggregationBuilder locAggregation = AggregationBuilders.nested("nested_" + NCLOC_KEY) + .path("measures") + .subAggregation( + AggregationBuilders.filter("filter_" + NCLOC_KEY) + .filter(termsQuery("measures.key", NCLOC_KEY)) + .subAggregation(AggregationBuilders.range(NCLOC_KEY) + .field("measures.value") + .addUnboundedTo(1_000d) + .addRange(1_000d, 10_000d) + .addRange(10_000d, 100_000d) + .addRange(100_000d, 500_000d) + .addUnboundedFrom(500_000))); + SearchRequestBuilder request = getClient() .prepareSearch(INDEX_PROJECT_MEASURES) .setTypes(TYPE_PROJECT_MEASURES) @@ -67,6 +84,7 @@ public class ProjectMeasuresIndex extends BaseIndex { .setQuery(esQuery) .setFrom(searchOptions.getOffset()) .setSize(searchOptions.getLimit()) + .addAggregation(locAggregation) .addSort(FIELD_NAME + "." + SORT_SUFFIX, SortOrder.ASC); return new SearchIdResult<>(request.get(), id -> id); diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchProjectsAction.java b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchProjectsAction.java index d4566f8a200..af877f99169 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchProjectsAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SearchProjectsAction.java @@ -21,7 +21,10 @@ package org.sonar.server.component.ws; import com.google.common.collect.Ordering; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Stream; import org.sonar.api.server.ws.Request; @@ -33,6 +36,7 @@ import org.sonar.db.DbSession; import org.sonar.db.component.ComponentDto; import org.sonar.server.component.es.ProjectMeasuresIndex; import org.sonar.server.component.es.ProjectMeasuresQuery; +import org.sonar.server.es.Facets; import org.sonar.server.es.SearchIdResult; import org.sonar.server.es.SearchOptions; import org.sonarqube.ws.Common; @@ -98,7 +102,7 @@ public class SearchProjectsAction implements ComponentsWsAction { Ordering ordering = Ordering.explicit(searchResult.getIds()).onResultOf(ComponentDto::uuid); List projects = ordering.immutableSortedCopy(dbClient.componentDao().selectByUuids(dbSession, searchResult.getIds())); - return new SearchResults(projects, searchResult.getTotal()); + return new SearchResults(projects, searchResult.getFacets(), searchResult.getTotal()); } private static SearchProjectsRequest toRequest(Request httpRequest) { @@ -123,11 +127,36 @@ public class SearchProjectsAction implements ComponentsWsAction { .forEach(response::addComponents); return response; }) + .map(response -> addFacets(searchResults.facets, response)) .map(SearchProjectsWsResponse.Builder::build) .findFirst() .orElseThrow(() -> new IllegalStateException("SearchProjectsWsResponse not built")); } + private static SearchProjectsWsResponse.Builder addFacets(Facets facets, SearchProjectsWsResponse.Builder wsResponse) { + Common.Facets.Builder wsFacets = Common.Facets.newBuilder(); + Common.Facet.Builder wsFacet = Common.Facet.newBuilder(); + for (Map.Entry> facet : facets.getAll().entrySet()) { + wsFacet.clear(); + wsFacet.setProperty(facet.getKey()); + LinkedHashMap buckets = facet.getValue(); + if (buckets != null) { + for (Map.Entry bucket : buckets.entrySet()) { + Common.FacetValue.Builder valueBuilder = wsFacet.addValuesBuilder(); + valueBuilder.setVal(bucket.getKey()); + valueBuilder.setCount(bucket.getValue()); + valueBuilder.build(); + } + } else { + wsFacet.addAllValues(Collections.emptyList()); + } + wsFacets.addFacets(wsFacet); + } + wsResponse.setFacets(wsFacets); + + return wsResponse; + } + private static class DbToWsComponent implements Function { private final Component.Builder wsComponent; @@ -148,11 +177,13 @@ public class SearchProjectsAction implements ComponentsWsAction { private static class SearchResults { private final List projects; + private final Facets facets; private final int total; - private SearchResults(List projects, long total) { + private SearchResults(List projects, Facets facets, long total) { this.projects = projects; this.total = (int) total; + this.facets = facets; } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java b/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java index 3a87bd5f387..e82d8eeadf7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/Facets.java @@ -32,6 +32,7 @@ import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.HasAggregations; import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; import org.elasticsearch.search.aggregations.bucket.missing.Missing; +import org.elasticsearch.search.aggregations.bucket.range.Range; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.sum.Sum; @@ -68,6 +69,8 @@ public class Facets { processDateHistogram((Histogram) aggregation); } else if (Sum.class.isAssignableFrom(aggregation.getClass())) { processSum((Sum) aggregation); + } else if (Range.class.isAssignableFrom(aggregation.getClass())) { + processRange((Range) aggregation); } else { throw new IllegalArgumentException("Aggregation type not supported yet: " + aggregation.getClass()); } @@ -123,6 +126,11 @@ public class Facets { getOrCreateFacet(aggregation.getName()).put(TOTAL, Math.round(aggregation.getValue())); } + private void processRange(Range aggregation) { + LinkedHashMap facet = getOrCreateFacet(aggregation.getName()); + aggregation.getBuckets().forEach(bucket -> facet.put(bucket.getKeyAsString(), bucket.getDocCount())); + } + public boolean contains(String facetName) { return facetsByName.containsKey(facetName); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/es/ProjectMeasuresIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/es/ProjectMeasuresIndexTest.java index 6ee2e8ebed3..c38a1d5135c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/es/ProjectMeasuresIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/es/ProjectMeasuresIndexTest.java @@ -32,6 +32,7 @@ import org.sonar.api.config.MapSettings; import org.sonar.server.component.es.ProjectMeasuresQuery.MetricCriterion; import org.sonar.server.component.es.ProjectMeasuresQuery.Operator; import org.sonar.server.es.EsTester; +import org.sonar.server.es.Facets; import org.sonar.server.es.SearchIdResult; import org.sonar.server.es.SearchOptions; import org.sonar.server.permission.index.PermissionIndexerTester; @@ -40,6 +41,7 @@ import org.sonar.server.tester.UserSessionRule; import static com.google.common.collect.Lists.newArrayList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; import static org.sonar.api.measures.Metric.Level.OK; import static org.sonar.api.security.DefaultGroups.ANYONE; import static org.sonar.server.component.es.ProjectMeasuresIndexDefinition.INDEX_PROJECT_MEASURES; @@ -210,6 +212,43 @@ public class ProjectMeasuresIndexTest { assertThat(result).containsOnly("P1"); } + @Test + public void facet_ncloc() { + addDocs( + // 3 docs with ncloc<1K + newDoc("P11", "K1", "N1").setMeasures(newArrayList(newMeasure(NCLOC, 0d))), + newDoc("P12", "K1", "N1").setMeasures(newArrayList(newMeasure(NCLOC, 0d))), + newDoc("P13", "K1", "N1").setMeasures(newArrayList(newMeasure(NCLOC, 999d))), + // 2 docs with ncloc>=1K and ncloc<10K + newDoc("P21", "K2", "N2").setMeasures(newArrayList(newMeasure(NCLOC, 1_000d))), + newDoc("P22", "K2", "N2").setMeasures(newArrayList(newMeasure(NCLOC, 9_999d))), + // 4 docs with ncloc>=10K and ncloc<100K + newDoc("P31", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 10_000d))), + newDoc("P32", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 10_000d))), + newDoc("P33", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 11_000d))), + newDoc("P34", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 99_000d))), + // 2 docs with ncloc>=100K and ncloc<500K + newDoc("P41", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 100_000d))), + newDoc("P42", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 499_000d))), + // 5 docs with ncloc>= 500K + newDoc("P51", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 500_000d))), + newDoc("P52", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 100_000_000d))), + newDoc("P53", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 500_000d))), + newDoc("P54", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 1_000_000d))), + newDoc("P55", "K3", "N3").setMeasures(newArrayList(newMeasure(NCLOC, 100_000_000_000d))) + ); + + Facets facets = underTest.search(new ProjectMeasuresQuery(), new SearchOptions()).getFacets(); + + assertThat(facets.get(NCLOC)).containsExactly( + entry("*-1000.0", 3L), + entry("1000.0-10000.0", 2L), + entry("10000.0-100000.0", 4L), + entry("100000.0-500000.0", 2L), + entry("500000.0-*", 5L) + ); + } + private void addDocs(ProjectMeasuresDoc... docs) { addDocs(null, ANYONE, docs); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java index 2bfef03e97f..09b6832ceca 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java @@ -61,6 +61,7 @@ import static com.google.common.collect.Lists.newArrayList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; import static org.sonar.db.component.ComponentTesting.newDeveloper; import static org.sonar.db.component.ComponentTesting.newDirectory; import static org.sonar.db.component.ComponentTesting.newFileDto; @@ -74,6 +75,8 @@ import static org.sonar.test.JsonAssert.assertJson; import static org.sonarqube.ws.client.component.ComponentsWsParameters.PARAM_FILTER; public class SearchProjectsActionTest { + private static final String NCLOC = "ncloc"; + private static final String COVERAGE = "coverage"; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -170,12 +173,12 @@ public class SearchProjectsActionTest { @Test public void filter_projects_with_query() { - insertProjectInDbAndEs(newProjectDto().setName("Sonar Java"), newArrayList(newMeasure("coverage", 81), newMeasure("ncloc", 10_000d))); - insertProjectInDbAndEs(newProjectDto().setName("Sonar Markdown"), newArrayList(newMeasure("coverage", 80d), newMeasure("ncloc", 10_000d))); - insertProjectInDbAndEs(newProjectDto().setName("Sonar Qube"), newArrayList(newMeasure("coverage", 80d), newMeasure("ncloc", 10_001d))); + insertProjectInDbAndEs(newProjectDto().setName("Sonar Java"), newArrayList(newMeasure(COVERAGE, 81), newMeasure(NCLOC, 10_000d))); + insertProjectInDbAndEs(newProjectDto().setName("Sonar Markdown"), newArrayList(newMeasure(COVERAGE, 80d), newMeasure(NCLOC, 10_000d))); + insertProjectInDbAndEs(newProjectDto().setName("Sonar Qube"), newArrayList(newMeasure(COVERAGE, 80d), newMeasure(NCLOC, 10_001d))); request.setFilter("coverage <= 80 and ncloc <= 10000"); - dbClient.metricDao().insert(dbSession, newMetricDto().setKey("coverage").setValueType(Metric.ValueType.FLOAT.name())); - dbClient.metricDao().insert(dbSession, newMetricDto().setKey("ncloc").setValueType(Metric.ValueType.FLOAT.name())); + dbClient.metricDao().insert(dbSession, newMetricDto().setKey(COVERAGE).setValueType(Metric.ValueType.FLOAT.name())); + dbClient.metricDao().insert(dbSession, newMetricDto().setKey(NCLOC).setValueType(Metric.ValueType.FLOAT.name())); db.commit(); SearchProjectsWsResponse result = call(request); @@ -184,6 +187,32 @@ public class SearchProjectsActionTest { assertThat(result.getComponents(0).getName()).isEqualTo("Sonar Markdown"); } + @Test + public void return_loc_facet() { + insertProjectInDbAndEs(newProjectDto().setName("Sonar Java"), newArrayList(newMeasure(COVERAGE, 81), newMeasure(NCLOC, 5d))); + insertProjectInDbAndEs(newProjectDto().setName("Sonar Groovy"), newArrayList(newMeasure(COVERAGE, 81), newMeasure(NCLOC, 5d))); + insertProjectInDbAndEs(newProjectDto().setName("Sonar Markdown"), newArrayList(newMeasure(COVERAGE, 80d), newMeasure(NCLOC, 10_000d))); + insertProjectInDbAndEs(newProjectDto().setName("Sonar Qube"), newArrayList(newMeasure(COVERAGE, 80d), newMeasure(NCLOC, 500_001d))); + dbClient.metricDao().insert(dbSession, newMetricDto().setKey(COVERAGE).setValueType(Metric.ValueType.FLOAT.name())); + dbClient.metricDao().insert(dbSession, newMetricDto().setKey(NCLOC).setValueType(Metric.ValueType.FLOAT.name())); + db.commit(); + + SearchProjectsWsResponse result = call(request); + + assertThat(result.getFacets().getFacetsCount()).isEqualTo(1); + Common.Facet facet = result.getFacets().getFacets(0); + assertThat(facet.getProperty()).isEqualTo(NCLOC); + assertThat(facet.getValuesCount()).isEqualTo(5); + assertThat(facet.getValuesList()) + .extracting(Common.FacetValue::getVal, Common.FacetValue::getCount) + .containsExactly( + tuple("*-1000.0", 2L), + tuple("1000.0-10000.0", 0L), + tuple("10000.0-100000.0", 1L), + tuple("100000.0-500000.0", 0L), + tuple("500000.0-*", 1L)); + } + @Test public void fail_if_metric_is_unknown() { expectedException.expect(IllegalArgumentException.class); diff --git a/sonar-ws/src/main/protobuf/ws-components.proto b/sonar-ws/src/main/protobuf/ws-components.proto index 4feaf85aa0a..004df58988c 100644 --- a/sonar-ws/src/main/protobuf/ws-components.proto +++ b/sonar-ws/src/main/protobuf/ws-components.proto @@ -61,6 +61,7 @@ message BulkUpdateKeyWsResponse { message SearchProjectsWsResponse { optional sonarqube.ws.commons.Paging paging = 1; repeated Component components = 2; + optional sonarqube.ws.commons.Facets facets = 3; } message Component { -- 2.39.5