From aab783fde3f151d600b204767888198711faa108 Mon Sep 17 00:00:00 2001 From: Daniel Schwarz Date: Thu, 13 Jul 2017 10:03:44 +0200 Subject: [PATCH] Search project statistics in Issue ES index --- .../sonar/server/issue/index/IssueIndex.java | 44 +++ .../server/issue/index/ProjectStatistics.java | 45 +++ .../IssueIndexProjectStatisticsTest.java | 256 ++++++++++++++++++ 3 files changed, 345 insertions(+) create mode 100644 server/sonar-server/src/main/java/org/sonar/server/issue/index/ProjectStatistics.java create mode 100644 server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index ef2ec71a48c..a19ce5bb2be 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -32,6 +32,8 @@ import java.util.Objects; import java.util.Optional; import java.util.TimeZone; import java.util.regex.Pattern; +import java.util.stream.IntStream; +import java.util.stream.Stream; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.BooleanUtils; @@ -48,8 +50,10 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInter import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.Terms.Order; import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder; +import org.elasticsearch.search.aggregations.metrics.max.InternalMax; import org.elasticsearch.search.aggregations.metrics.min.Min; import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder; +import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount; import org.joda.time.Duration; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; @@ -65,10 +69,12 @@ import org.sonar.server.permission.index.AuthorizationTypeSupport; import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; +import static com.google.common.base.Preconditions.checkState; import static java.lang.String.format; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.existsQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; @@ -636,4 +642,42 @@ public class IssueIndex { } return boolQuery; } + + public List searchProjectStatistics(List projectUuids, List froms, String assignee) { + checkState(projectUuids.size() == froms.size(), + "Expected same size for projectUuids (had size %s) and froms (had size %s)", projectUuids.size(), froms.size()); + if (projectUuids.isEmpty()) { + return Collections.emptyList(); + } + SearchRequestBuilder request = client.prepareSearch(IssueIndexDefinition.INDEX_TYPE_ISSUE) + .setQuery( + boolQuery() + .mustNot(existsQuery(IssueIndexDefinition.FIELD_ISSUE_RESOLUTION)) + .filter(termQuery(IssueIndexDefinition.FIELD_ISSUE_ASSIGNEE, assignee)) + ) + .setSize(0); + IntStream.range(0, projectUuids.size()).forEach(i -> { + String projectUuid = projectUuids.get(i); + long from = froms.get(i); + request + .addAggregation(AggregationBuilders + .filter(projectUuid) + .filter(boolQuery() + .filter(termQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, projectUuid)) + .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(new Date(from))) + ) + .subAggregation(AggregationBuilders.count(projectUuid + "_count").field(IssueIndexDefinition.FIELD_ISSUE_KEY)) + .subAggregation(AggregationBuilders.max(projectUuid + "_maxFuncCreatedAt").field(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT)) + ); + }); + SearchResponse response = request.get(); + return response.getAggregations().asList().stream().flatMap(projectBucket -> { + long count = ((InternalValueCount) projectBucket.getProperty(projectBucket.getName() + "_count")).getValue(); + if (count < 1L) { + return Stream.empty(); + } + long lastIssueDate = (long) ((InternalMax) projectBucket.getProperty(projectBucket.getName() + "_maxFuncCreatedAt")).getValue(); + return Stream.of(new ProjectStatistics(projectBucket.getName(), count, lastIssueDate)); + }).collect(MoreCollectors.toList(projectUuids.size())); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/ProjectStatistics.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/ProjectStatistics.java new file mode 100644 index 00000000000..646daa3b9fa --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/ProjectStatistics.java @@ -0,0 +1,45 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +public class ProjectStatistics { + + private final String projectUuid; + private final long issueCount; + private final long lastIssueDate; + + public ProjectStatistics(String projectUuid, long issueCount, long lastIssueDate) { + this.projectUuid = projectUuid; + this.issueCount = issueCount; + this.lastIssueDate = lastIssueDate; + } + + public String getProjectUuid() { + return projectUuid; + } + + public long getIssueCount() { + return issueCount; + } + + public long getLastIssueDate() { + return lastIssueDate; + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java new file mode 100644 index 00000000000..481c62ff55a --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java @@ -0,0 +1,256 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.issue.index; + +import java.util.Date; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.api.issue.Issue; +import org.sonar.api.utils.System2; +import org.sonar.db.component.ComponentDto; +import org.sonar.db.component.ComponentTesting; +import org.sonar.db.organization.OrganizationDto; +import org.sonar.server.es.EsTester; +import org.sonar.server.permission.index.AuthorizationTypeSupport; +import org.sonar.server.permission.index.PermissionIndexerDao; +import org.sonar.server.permission.index.PermissionIndexerTester; +import org.sonar.server.tester.UserSessionRule; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.mockito.Mockito.mock; +import static org.sonar.db.organization.OrganizationTesting.newOrganizationDto; +import static org.sonar.server.issue.IssueDocTesting.newDoc; + +public class IssueIndexProjectStatisticsTest { + + private System2 system2 = mock(System2.class); + private MapSettings settings = new MapSettings(); + @Rule + public EsTester esTester = new EsTester(new IssueIndexDefinition(settings.asConfig())); + @Rule + public UserSessionRule userSessionRule = UserSessionRule.standalone(); + + private IssueIndexer issueIndexer = new IssueIndexer(esTester.client(), new IssueIteratorFactory(null)); + private PermissionIndexerTester authorizationIndexerTester = new PermissionIndexerTester(esTester, issueIndexer); + + private IssueIndex underTest = new IssueIndex(esTester.client(), system2, userSessionRule, new AuthorizationTypeSupport(userSessionRule)); + + @Test + public void searchProjectStatistics_returns_empty_list_if_no_input() throws Exception { + List result = underTest.searchProjectStatistics(emptyList(), emptyList(), "unknownUser"); + assertThat(result).isEmpty(); + } + + @Test + public void searchProjectStatistics_returns_empty_list_if_the_input_does_not_match_anything() throws Exception { + List result = underTest.searchProjectStatistics(singletonList("unknownProjectUuid"), singletonList(1_111_234_567_890L), "unknownUser"); + assertThat(result).isEmpty(); + } + + @Test + public void searchProjectStatistics_returns_something() throws Exception { + OrganizationDto org = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org); + String userLogin = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues(newDoc("issue1", project).setAssignee(userLogin).setFuncCreationDate(new Date(from+1L))); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin); + + assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); + } + + @Test + public void searchProjectStatistics_does_not_return_results_if_assignee_does_not_match() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + String userLogin2 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L))); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin2); + + assertThat(result).isEmpty(); + } + + @Test + public void searchProjectStatistics_returns_results_if_assignee_matches() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L))); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1); + + assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); + } + + @Test + public void searchProjectStatistics_returns_results_if_functional_date_is_strictly_after_from_date() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L))); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1); + + assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); + } + + @Test + public void searchProjectStatistics_does_not_return_results_if_functional_date_is_same_as_from_date() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from))); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1); + + assertThat(result).extracting(ProjectStatistics::getProjectUuid).containsExactly(project.uuid()); + } + + @Test + public void searchProjectStatistics_does_not_return_resolved_issues() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues( + newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)).setResolution(Issue.RESOLUTION_FALSE_POSITIVE), + newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)).setResolution(Issue.RESOLUTION_FIXED), + newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)).setResolution(Issue.RESOLUTION_REMOVED), + newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)).setResolution(Issue.RESOLUTION_WONT_FIX) + ); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1); + + assertThat(result).isEmpty(); + } + + @Test + public void searchProjectStatistics_does_not_return_results_if_functional_date_is_before_from_date() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from-1L))); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1); + + assertThat(result).isEmpty(); + } + + @Test + public void searchProjectStatistics_returns_issue_count() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues( + newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), + newDoc("issue2", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), + newDoc("issue3", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)) + ); + + List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1); + + assertThat(result).extracting(ProjectStatistics::getIssueCount).containsExactly(3L); + } + + @Test + public void searchProjectStatistics_returns_issue_count_for_multiple_projects() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project1 = ComponentTesting.newPrivateProjectDto(org1); + ComponentDto project2 = ComponentTesting.newPrivateProjectDto(org1); + ComponentDto project3 = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues( + newDoc("issue1", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), + newDoc("issue2", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), + newDoc("issue3", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), + + newDoc("issue4", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), + newDoc("issue5", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)) + ); + + List result = underTest.searchProjectStatistics( + asList(project1.uuid(),project2.uuid(), project3.uuid()), + asList(from, from, from), + userLogin1); + + assertThat(result) + .extracting(ProjectStatistics::getProjectUuid, ProjectStatistics::getIssueCount) + .containsExactlyInAnyOrder( + tuple(project1.uuid(), 3L), + tuple(project3.uuid(), 2L) + ); + } + + @Test + public void searchProjectStatistics_returns_max_date_for_multiple_projects() throws Exception { + OrganizationDto org1 = newOrganizationDto(); + ComponentDto project1 = ComponentTesting.newPrivateProjectDto(org1); + ComponentDto project2 = ComponentTesting.newPrivateProjectDto(org1); + ComponentDto project3 = ComponentTesting.newPrivateProjectDto(org1); + String userLogin1 = randomAlphanumeric(20); + long from = 1_111_234_567_890L; + indexIssues( + newDoc("issue1", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), + newDoc("issue2", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+2L)), + newDoc("issue3", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+3L)), + + newDoc("issue4", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+4L)), + newDoc("issue5", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+5L)) + ); + + List result = underTest.searchProjectStatistics( + asList(project1.uuid(),project2.uuid(), project3.uuid()), + asList(from, from, from), + userLogin1); + + assertThat(result) + .extracting(ProjectStatistics::getProjectUuid, ProjectStatistics::getLastIssueDate) + .containsExactlyInAnyOrder( + tuple(project1.uuid(), from+3L), + tuple(project3.uuid(), from+5L) + ); + } + + private void indexIssues(IssueDoc... issues) { + issueIndexer.index(asList(issues).iterator()); + for (IssueDoc issue : issues) { + PermissionIndexerDao.Dto access = new PermissionIndexerDao.Dto(issue.projectUuid(), system2.now(), "TRK"); + access.allowAnyone(); + authorizationIndexerTester.allow(access); + } + } +} -- 2.39.5