From e8975d8452cf916cf907d46124d4e6c600bf122e Mon Sep 17 00:00:00 2001 From: Julien Lancelot Date: Mon, 6 Mar 2017 13:55:01 +0100 Subject: [PATCH] SONAR-8231 Return distribution of projects per language --- .../ProjectMeasuresIndexerIterator.java | 14 ++-- .../ProjectMeasuresIndexerIteratorTest.java | 2 +- .../measure/index/ProjectMeasuresDoc.java | 11 +-- .../measure/index/ProjectMeasuresIndex.java | 14 +--- .../index/ProjectMeasuresIndexDefinition.java | 9 +- .../measure/index/ProjectMeasuresIndexer.java | 2 +- .../ws/SearchProjectsActionTest.java | 32 +++---- .../index/ProjectMeasuresIndexTest.java | 83 ++++++++++++++----- 8 files changed, 91 insertions(+), 76 deletions(-) diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java index 85c72f4102b..8a81d3cfdf2 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/measure/ProjectMeasuresIndexerIterator.java @@ -20,7 +20,7 @@ package org.sonar.db.measure; import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -219,7 +219,7 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator numericMeasures = new HashMap<>(); private String qualityGateStatus; - private Map languageDistribution = new HashMap<>(); + private List languages = new ArrayList<>(); Measures addNumericMeasure(String metricKey, double value) { numericMeasures.put(metricKey, value); @@ -324,13 +324,13 @@ public class ProjectMeasuresIndexerIterator extends CloseableIterator getLanguageDistribution() { - return languageDistribution; + public List getLanguages() { + return languages; } } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java index 39bc94ec5f0..1adb5696bca 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/measure/ProjectMeasuresIndexerIteratorTest.java @@ -129,7 +129,7 @@ public class ProjectMeasuresIndexerIteratorTest { Map docsById = createResultSetAndReturnDocsById(); - assertThat(docsById.get(project.uuid()).getMeasures().getLanguageDistribution()).containsOnly(entry("", 2), entry("java", 6), entry("xoo", 18)); + assertThat(docsById.get(project.uuid()).getMeasures().getLanguages()).containsOnly("", "java", "xoo"); } @Test diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java index 4af9c866781..01f9ddbf741 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java @@ -36,8 +36,6 @@ import static org.sonar.api.measures.Metric.Level.WARN; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_LANGUAGES; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_LANGUAGES_KEY; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_LANGUAGES_VALUE; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE; @@ -130,13 +128,8 @@ public class ProjectMeasuresDoc extends BaseDoc { return this; } - public ProjectMeasuresDoc setLanguages(Map languageDistribution) { - setField(FIELD_LANGUAGES, - languageDistribution.entrySet().stream() - .map(entry -> ImmutableMap.of( - FIELD_LANGUAGES_KEY, entry.getKey(), - FIELD_LANGUAGES_VALUE, entry.getValue())) - .collect(Collectors.toList())); + public ProjectMeasuresDoc setLanguages(List languages) { + setField(FIELD_LANGUAGES, languages); return this; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java index 3efd2707796..8cc01948f18 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java @@ -99,9 +99,6 @@ public class ProjectMeasuresIndex extends BaseIndex { private static final String FIELD_MEASURES_KEY = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_KEY; private static final String FIELD_MEASURES_VALUE = FIELD_MEASURES + "." + ProjectMeasuresIndexDefinition.FIELD_MEASURES_VALUE; - private static final String FIELD_LANGUAGES_KEY = FIELD_LANGUAGES + "." + ProjectMeasuresIndexDefinition.FIELD_LANGUAGES_KEY; - private static final String FIELD_LANGUAGES_VALUE = FIELD_LANGUAGES + "." + ProjectMeasuresIndexDefinition.FIELD_LANGUAGES_VALUE; - private static final Map FACET_FACTORIES = ImmutableMap.builder() .put(NCLOC_KEY, (esSearch, filters) -> addRangeFacet(esSearch, NCLOC_KEY, ImmutableList.of(1_000d, 10_000d, 100_000d, 500_000d), filters)) .put(DUPLICATED_LINES_DENSITY_KEY, (esSearch, filters) -> addRangeFacet(esSearch, DUPLICATED_LINES_DENSITY_KEY, ImmutableList.of(3d, 5d, 10d, 20d), filters)) @@ -231,13 +228,7 @@ public class ProjectMeasuresIndex extends BaseIndex { } private static AbstractAggregationBuilder createLanguagesFacet() { - return AggregationBuilders.nested("nested_" + FILTER_LANGUAGE) - .path(FIELD_LANGUAGES) - .subAggregation( - AggregationBuilders.terms(FILTER_LANGUAGE) - .field(FIELD_LANGUAGES_KEY) - .subAggregation(AggregationBuilders.sum("size_" + FILTER_LANGUAGE) - .field(FIELD_LANGUAGES_VALUE))); + return AggregationBuilders.terms(FILTER_LANGUAGE).field(FIELD_LANGUAGES); } private static AbstractAggregationBuilder createTagsFacet() { @@ -267,8 +258,7 @@ public class ProjectMeasuresIndex extends BaseIndex { .ifPresent(projectUuids -> filters.put("ids", termsQuery("_id", projectUuids))); query.getLanguages() - .ifPresent(languages -> filters.put(FILTER_LANGUAGE, - nestedQuery(FIELD_LANGUAGES, boolQuery().filter(termsQuery(FIELD_LANGUAGES_KEY, languages))))); + .ifPresent(languages -> filters.put(FILTER_LANGUAGE, termsQuery(FIELD_LANGUAGES, languages))); query.getOrganizationUuid() .ifPresent(organizationUuid -> filters.put(FIELD_ORGANIZATION_UUID, termQuery(FIELD_ORGANIZATION_UUID, organizationUuid))); diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java index bfe3c41c249..ea28cf08a27 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java @@ -40,8 +40,6 @@ public class ProjectMeasuresIndexDefinition implements IndexDefinition { public static final String FIELD_MEASURES_KEY = "key"; public static final String FIELD_MEASURES_VALUE = "value"; public static final String FIELD_LANGUAGES = "languages"; - public static final String FIELD_LANGUAGES_KEY = "key"; - public static final String FIELD_LANGUAGES_VALUE = "value"; private final Settings settings; @@ -63,15 +61,12 @@ public class ProjectMeasuresIndexDefinition implements IndexDefinition { mapping.stringFieldBuilder(FIELD_NAME).addSubFields(SORTABLE_ANALYZER, SEARCH_GRAMS_ANALYZER).build(); mapping.stringFieldBuilder(FIELD_QUALITY_GATE_STATUS).build(); mapping.stringFieldBuilder(FIELD_TAGS).build(); - mapping.createDateTimeField(FIELD_ANALYSED_AT); + mapping.stringFieldBuilder(FIELD_LANGUAGES).build(); mapping.nestedFieldBuilder(FIELD_MEASURES) .addStringField(FIELD_MEASURES_KEY) .addDoubleField(FIELD_MEASURES_VALUE) .build(); - mapping.nestedFieldBuilder(FIELD_LANGUAGES) - .addStringField(FIELD_LANGUAGES_KEY) - .addIntegerField(FIELD_LANGUAGES_VALUE) - .build(); + mapping.createDateTimeField(FIELD_ANALYSED_AT); mapping.setEnableSource(false); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index b643b8316ff..5268ade753b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -135,6 +135,6 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization .setTags(project.getTags()) .setAnalysedAt(analysisDate == null ? null : new Date(analysisDate)) .setMeasuresFromMap(projectMeasures.getMeasures().getNumericMeasures()) - .setLanguages(projectMeasures.getMeasures().getLanguageDistribution()); + .setLanguages(projectMeasures.getMeasures().getLanguages()); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java index 605442cd4a4..034c9e97bdf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/component/ws/SearchProjectsActionTest.java @@ -62,8 +62,8 @@ import org.sonarqube.ws.WsComponents.SearchProjectsWsResponse; import org.sonarqube.ws.client.component.SearchProjectsRequest; import static com.google.common.collect.Lists.newArrayList; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Optional.ofNullable; import static org.assertj.core.api.Assertions.assertThat; @@ -283,10 +283,10 @@ public class SearchProjectsActionTest { @Test public void filter_projects_by_languages() { OrganizationDto organizationDto = db.organizations().insertForKey("my-org-key-1"); - insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Java"), newArrayList(newMeasure(COVERAGE, 81d)), null, ImmutableMap.of("", 2, "java", 6, "xoo", 18)); - insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Groovy"), newArrayList(newMeasure(COVERAGE, 81)), null, ImmutableMap.of("java", 4, "xoo", 5)); - insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Markdown"), newArrayList(newMeasure(COVERAGE, 80d)), null, ImmutableMap.of("xoo", 3)); - insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Qube"), newArrayList(newMeasure(COVERAGE, 80d)), null, ImmutableMap.of("", 5, "java", 16, "xoo", 9)); + insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Java"), newArrayList(newMeasure(COVERAGE, 81d)), null, asList("", "java", "xoo")); + insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Groovy"), newArrayList(newMeasure(COVERAGE, 81)), null,asList("java", "xoo")); + insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Markdown"), newArrayList(newMeasure(COVERAGE, 80d)), null, asList("xoo")); + insertProjectInDbAndEs(newProjectDto(organizationDto).setName("Sonar Qube"), newArrayList(newMeasure(COVERAGE, 80d)), null, asList("", "java", "xoo")); insertMetrics(COVERAGE, NCLOC_LANGUAGE_DISTRIBUTION_KEY); request.setFilter("language IN (java, js, )"); @@ -459,10 +459,10 @@ public class SearchProjectsActionTest { @Test public void return_languages_facet() { OrganizationDto organization = db.getDefaultOrganization(); - insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Java"), newArrayList(newMeasure(COVERAGE, 81d)), null, ImmutableMap.of("", 2, "java", 6, "xoo", 18)); - insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Groovy"), newArrayList(newMeasure(COVERAGE, 81)), null, ImmutableMap.of("java", 4, "xoo", 5)); - insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Markdown"), newArrayList(newMeasure(COVERAGE, 80d)), null, ImmutableMap.of("xoo", 3)); - insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Qube"), newArrayList(newMeasure(COVERAGE, 80d)), null, ImmutableMap.of("", 5, "java", 16, "xoo", 9)); + insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Java"), newArrayList(newMeasure(COVERAGE, 81d)), null, asList("", "java", "xoo")); + insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Groovy"), newArrayList(newMeasure(COVERAGE, 81)), null, asList("java", "xoo")); + insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Markdown"), newArrayList(newMeasure(COVERAGE, 80d)), null, asList("xoo")); + insertProjectInDbAndEs(newProjectDto(organization).setName("Sonar Qube"), newArrayList(newMeasure(COVERAGE, 80d)), null, asList("", "java", "xoo")); insertMetrics(COVERAGE, NCLOC_LANGUAGE_DISTRIBUTION_KEY); SearchProjectsWsResponse result = call(request.setFacets(singletonList(FILTER_LANGUAGE))); @@ -474,9 +474,9 @@ public class SearchProjectsActionTest { assertThat(facet.getValuesList()) .extracting(Common.FacetValue::getVal, Common.FacetValue::getCount) .containsExactly( - tuple("xoo", 35L), - tuple("java", 26L), - tuple("", 7L)); + tuple("xoo", 4L), + tuple("java", 3L), + tuple("", 2L)); } @Test @@ -595,15 +595,15 @@ public class SearchProjectsActionTest { } private ComponentDto insertProjectInDbAndEs(ComponentDto project, List> measures) { - return insertProjectInDbAndEs(project, measures, null, emptyMap()); + return insertProjectInDbAndEs(project, measures, null, emptyList()); } private ComponentDto insertProjectInDbAndEs(ComponentDto project, String qualityGateStatus) { - return insertProjectInDbAndEs(project, emptyList(), qualityGateStatus, emptyMap()); + return insertProjectInDbAndEs(project, emptyList(), qualityGateStatus, emptyList()); } private ComponentDto insertProjectInDbAndEs(ComponentDto project, List> measures, @Nullable String qualityGateStatus, - Map languagesDistribution) { + List languages) { ComponentDto res = componentDb.insertComponent(project); try { es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, @@ -614,7 +614,7 @@ public class SearchProjectsActionTest { .setName(project.name()) .setMeasures(measures) .setQualityGateStatus(qualityGateStatus) - .setLanguages(languagesDistribution) + .setLanguages(languages) .setTags(project.getTags())); authorizationIndexerTester.allowOnlyAnyone(project); } catch (Exception e) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java index 3d5a28e324d..bbee61446f2 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java @@ -20,6 +20,7 @@ package org.sonar.server.measure.index; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; @@ -48,6 +49,7 @@ import org.sonar.server.tester.UserSessionRule; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Sets.newHashSet; +import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; @@ -284,10 +286,10 @@ public class ProjectMeasuresIndexTest { public void filter_on_languages() { ComponentDto project4 = newProjectDto(ORG).setUuid("Project-4").setName("Project 4").setKey("key-4"); index( - newDoc(PROJECT1).setLanguages(ImmutableMap.of("java", 6)), - newDoc(PROJECT2).setLanguages(ImmutableMap.of("xoo", 8)), - newDoc(PROJECT3).setLanguages(ImmutableMap.of("xoo", 18)), - newDoc(project4).setLanguages(ImmutableMap.of("", 10, "java", 2, "xoo", 12))); + newDoc(PROJECT1).setLanguages(singletonList("java")), + newDoc(PROJECT2).setLanguages(singletonList("xoo")), + newDoc(PROJECT3).setLanguages(singletonList("xoo")), + newDoc(project4).setLanguages(asList("", "java", "xoo"))); assertResults(new ProjectMeasuresQuery().setLanguages(newHashSet("java", "xoo")), PROJECT1, PROJECT2, PROJECT3, project4); assertResults(new ProjectMeasuresQuery().setLanguages(newHashSet("java")), PROJECT1, project4); @@ -991,48 +993,83 @@ public class ProjectMeasuresIndexTest { } @Test - public void facet_languages() { + public void facet_language() { index( - newDoc().setLanguages(ImmutableMap.of("java", 6)), - newDoc().setLanguages(ImmutableMap.of("java", 8)), - newDoc().setLanguages(ImmutableMap.of("xoo", 18)), - newDoc().setLanguages(ImmutableMap.of("xml", 4)), - newDoc().setLanguages(ImmutableMap.of("", 2, "java", 5)), - newDoc().setLanguages(ImmutableMap.of("", 10, "java", 2, "xoo", 12))); + newDoc().setLanguages(singletonList("java")), + newDoc().setLanguages(singletonList("java")), + newDoc().setLanguages(singletonList("xoo")), + newDoc().setLanguages(singletonList("xml")), + newDoc().setLanguages(asList("", "java")), + newDoc().setLanguages(asList("", "java", "xoo"))); Facets facets = underTest.search(new ProjectMeasuresQuery(), new SearchOptions().addFacets(LANGUAGE)).getFacets(); assertThat(facets.get(LANGUAGE)).containsOnly( - entry("", 12L), - entry("java", 21L), - entry("xoo", 30L), - entry("xml", 4L)); + entry("", 2L), + entry("java", 4L), + entry("xoo", 2L), + entry("xml", 1L)); } - // TODO @Test - public void facet_languages_is_sticky() { + public void facet_language_is_limited_to_10_languages() { + index( + newDoc().setLanguages(asList("", "java", "xoo", "css", "cpp")), + newDoc().setLanguages(asList("xml", "php", "python", "perl", "ruby")), + newDoc().setLanguages(asList("js", "scala"))); + + Facets facets = underTest.search(new ProjectMeasuresQuery(), new SearchOptions().addFacets(LANGUAGE)).getFacets(); + + assertThat(facets.get(LANGUAGE)).hasSize(10); + } + @Test + public void facet_language_is_sticky() { + index( + newDoc(NCLOC, 10d).setLanguages(singletonList("java")), + newDoc(NCLOC, 10d).setLanguages(singletonList("java")), + newDoc(NCLOC, 10d).setLanguages(singletonList("xoo")), + newDoc(NCLOC, 100d).setLanguages(singletonList("xml")), + newDoc(NCLOC, 100d).setLanguages(asList("", "java")), + newDoc(NCLOC, 5000d).setLanguages(asList("", "java", "xoo"))); + + Facets facets = underTest.search( + new ProjectMeasuresQuery().setLanguages(ImmutableSet.of("java")), + new SearchOptions().addFacets(LANGUAGE, NCLOC)).getFacets(); + + // Sticky facet on language does not take into account language filter + assertThat(facets.get(LANGUAGE)).containsOnly( + entry("", 2L), + entry("java", 4L), + entry("xoo", 2L), + entry("xml", 1L)); + // But facet on ncloc does well take account into filters + assertThat(facets.get(NCLOC)).containsExactly( + entry("*-1000.0", 3L), + entry("1000.0-10000.0", 1L), + entry("10000.0-100000.0", 0L), + entry("100000.0-500000.0", 0L), + entry("500000.0-*", 0L)); } @Test public void facet_languages_contains_only_projects_authorized_for_user() throws Exception { // User can see these projects indexForUser(USER1, - newDoc().setLanguages(ImmutableMap.of("java", 6)), - newDoc().setLanguages(ImmutableMap.of("java", 1, "xoo", 5))); + newDoc().setLanguages(singletonList("java")), + newDoc().setLanguages(asList("java", "xoo"))); // User cannot see these projects indexForUser(USER2, - newDoc().setLanguages(ImmutableMap.of("java", 5)), - newDoc().setLanguages(ImmutableMap.of("java", 2, "xoo", 12))); + newDoc().setLanguages(singletonList("java")), + newDoc().setLanguages(asList("java", "xoo"))); userSession.logIn(USER1); LinkedHashMap result = underTest.search(new ProjectMeasuresQuery(), new SearchOptions().addFacets(LANGUAGE)).getFacets().get(LANGUAGE); assertThat(result).containsOnly( - entry("java", 7L), - entry("xoo", 5L)); + entry("java", 2L), + entry("xoo", 1L)); } @Test -- 2.39.5