From 9c8422953b98dcb673427f9324a767b25af1b607 Mon Sep 17 00:00:00 2001 From: Belen Pruvost Date: Thu, 31 Mar 2022 16:50:35 +0200 Subject: [PATCH] SONAR-16039 - Hybrid New Code Search for Issues Not re-analyzed in 9.4 --- .../issue/index/IssueIndexDefinition.java | 3 +- .../sonar/server/issue/index/IssueIndex.java | 13 ++++--- .../sonar/server/issue/index/IssueQuery.java | 1 + .../server/issue/index/IssueQueryFactory.java | 33 ++++++++++------- .../issue/index/IssueIndexFiltersTest.java | 1 + .../issue/index/IssueQueryFactoryTest.java | 36 +++++++++++++------ .../server/issue/index/IssueQueryTest.java | 4 +++ 7 files changed, 60 insertions(+), 31 deletions(-) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index 0d3bd0f3320..abb86e6b075 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -19,6 +19,7 @@ */ package org.sonar.server.issue.index; +import javax.inject.Inject; import org.sonar.api.config.Configuration; import org.sonar.api.config.internal.MapSettings; import org.sonar.server.es.Index; @@ -32,8 +33,6 @@ import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_ import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; -import javax.inject.Inject; - /** * Definition of ES index "issues", including settings and fields. */ diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java index dd06323c3ac..6b6fe478b98 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -669,13 +669,16 @@ public class IssueIndex { Collection newCodeOnReferenceByProjectUuids = query.newCodeOnReferenceByProjectUuids(); BoolQueryBuilder boolQueryBuilder = boolQuery(); - newCodeOnReferenceByProjectUuids.forEach(projectOrProjectBranchUuid -> boolQueryBuilder.should(boolQuery() - .filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectOrProjectBranchUuid)) - .filter(termQuery(FIELD_ISSUE_NEW_CODE_REFERENCE, true)))); + if (!newCodeOnReferenceByProjectUuids.isEmpty()) { - allFilters.addFilter("__is_new_code_reference_by_project_uuids", new SimpleFieldFilterScope("newCodeReferenceByProjectUuids"), boolQueryBuilder); - } + newCodeOnReferenceByProjectUuids.forEach(projectOrProjectBranchUuid -> boolQueryBuilder.should(boolQuery() + .filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectOrProjectBranchUuid)) + .filter(termQuery(FIELD_ISSUE_NEW_CODE_REFERENCE, true)))); + allFilters.addFilter("__is_new_code_reference_by_project_uuids", + new SimpleFieldFilterScope("newCodeReferenceByProjectUuids"), boolQueryBuilder); + } + } private static void addCreatedAfterByProjectsFilter(AllFilters allFilters, IssueQuery query) { Map createdAfterByProjectUuids = query.createdAfterByProjectUuids(); diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java index 24b1ecb69c1..eef7b1db1b4 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java @@ -319,6 +319,7 @@ public class IssueQuery { return newCodeOnReferenceByProjectUuids; } + public static class Builder { private Collection issueKeys; private Collection severities; diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java index e54f7026ae1..72520964fa6 100644 --- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java +++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java @@ -64,6 +64,7 @@ import static java.util.Collections.singletonList; import static org.sonar.api.issue.Issue.STATUSES; import static org.sonar.api.issue.Issue.STATUS_REVIEWED; import static org.sonar.api.issue.Issue.STATUS_TO_REVIEW; +import static org.sonar.api.measures.CoreMetrics.ANALYSIS_FROM_SONARQUBE_9_4_KEY; import static org.sonar.api.utils.DateUtils.longToDate; import static org.sonar.api.utils.DateUtils.parseEndingDateOrDateTime; import static org.sonar.api.utils.DateUtils.parseStartingDateOrDateTime; @@ -197,14 +198,13 @@ public class IssueQueryFactory { if (!QUALIFIERS_WITHOUT_LEAK_PERIOD.contains(component.qualifier()) && request.getPullRequest() == null) { Optional snapshot = getLastAnalysis(dbSession, component); - boolean isLastAnalysisUsingReferenceBranch = isLastAnalysisUsingReferenceBranch(snapshot); - if (isLastAnalysisUsingReferenceBranch) { - builder.newCodeOnReference(true); - } else { - // if last analysis has no period date, then no issue should be considered new. - Date createdAfterFromSnapshot = findCreatedAfterFromComponentUuid(snapshot); - setCreatedAfterFromDates(builder, createdAfterFromSnapshot, null, false); + if (!snapshot.isEmpty() && isLastAnalysisFromReAnalyzedReferenceBranch(dbSession, snapshot.get())) { + builder.newCodeOnReference(true); + return; } + // if last analysis has no period date, then no issue should be considered new. + Date createdAfterFromSnapshot = findCreatedAfterFromComponentUuid(snapshot); + setCreatedAfterFromDates(builder, createdAfterFromSnapshot, null, false); } } } @@ -230,9 +230,12 @@ public class IssueQueryFactory { return snapshot.map(s -> longToDate(s.getPeriodDate())).orElseGet(() -> new Date(clock.millis())); } - private static boolean isLastAnalysisUsingReferenceBranch(Optional snapshot) { - String periodMode = snapshot.map(SnapshotDto::getPeriodMode).orElse(""); - return periodMode.equals(REFERENCE_BRANCH.name()); + private static boolean isLastAnalysisUsingReferenceBranch(SnapshotDto snapshot) { + return !isNullOrEmpty(snapshot.getPeriodMode()) && snapshot.getPeriodMode().equals(REFERENCE_BRANCH.name()); + } + + private boolean isLastAnalysisFromSonarQube94Onwards(DbSession dbSession, String componentUuid) { + return dbClient.liveMeasureDao().selectMeasure(dbSession, componentUuid, ANALYSIS_FROM_SONARQUBE_9_4_KEY).isPresent(); } private Optional getLastAnalysis(DbSession dbSession, ComponentDto component) { @@ -376,20 +379,24 @@ public class IssueQueryFactory { Set newCodeReferenceByProjects = snapshots .stream() - .filter(s -> !isNullOrEmpty(s.getPeriodMode()) && s.getPeriodMode().equals(REFERENCE_BRANCH.name())) + .filter(s -> isLastAnalysisFromReAnalyzedReferenceBranch(dbSession, s)) .map(SnapshotDto::getComponentUuid) .collect(toSet()); Map leakByProjects = snapshots .stream() - .filter(s -> s.getPeriodDate() != null && - (isNullOrEmpty(s.getPeriodMode()) || !s.getPeriodMode().equals(REFERENCE_BRANCH.name()))) + .filter(s -> s.getPeriodDate() != null && !isLastAnalysisFromReAnalyzedReferenceBranch(dbSession, s)) .collect(uniqueIndex(SnapshotDto::getComponentUuid, s -> new PeriodStart(longToDate(s.getPeriodDate()), false))); builder.createdAfterByProjectUuids(leakByProjects); builder.newCodeOnReferenceByProjectUuids(newCodeReferenceByProjects); } + private boolean isLastAnalysisFromReAnalyzedReferenceBranch(DbSession dbSession, SnapshotDto snapshot) { + return isLastAnalysisUsingReferenceBranch(snapshot) && + isLastAnalysisFromSonarQube94Onwards(dbSession, snapshot.getComponentUuid()); + } + private static void addDirectories(IssueQuery.Builder builder, List directories) { Collection directoryModuleUuids = new HashSet<>(); Collection directoryPaths = new HashSet<>(); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java index ac08cdadb50..29ceca25501 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java @@ -518,6 +518,7 @@ public class IssueIndexFiltersTest { project1Issue1.key(), project2Issue2.key()); } + @Test public void filter_by_new_code_reference_branches() { ComponentDto project1 = newPrivateProjectDto(); diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java index 62b0ea13546..bb8dc87c153 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java @@ -33,6 +33,7 @@ import org.sonar.api.rule.RuleKey; import org.sonar.db.DbTester; import org.sonar.db.component.ComponentDto; import org.sonar.db.component.SnapshotDto; +import org.sonar.db.metric.MetricDto; import org.sonar.db.rule.RuleDbTester; import org.sonar.db.rule.RuleDefinitionDto; import org.sonar.db.user.UserDto; @@ -47,6 +48,7 @@ import static org.assertj.core.api.Assertions.tuple; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.sonar.api.measures.CoreMetrics.ANALYSIS_FROM_SONARQUBE_9_4_KEY; import static org.sonar.api.resources.Qualifiers.APP; import static org.sonar.api.utils.DateUtils.addDays; import static org.sonar.api.utils.DateUtils.parseDateTime; @@ -190,7 +192,7 @@ public class IssueQueryFactoryTest { } @Test - public void leak_period_does_not_rely_on_date_for_reference_branch() { + public void leak_period_relies_on_date_for_reference_branch_with_analysis_after_sonarqube_94() { long leakPeriodStart = addDays(new Date(), -14).getTime(); ComponentDto project = db.components().insertPublicProject(); @@ -199,6 +201,9 @@ public class IssueQueryFactoryTest { SnapshotDto analysis = db.components().insertSnapshot(project, s -> s.setPeriodMode(REFERENCE_BRANCH.name()) .setPeriodParam("master")); + MetricDto analysisMetric = db.measures().insertMetric(m -> m.setKey(ANALYSIS_FROM_SONARQUBE_9_4_KEY)); + db.measures().insertLiveMeasure(project, analysisMetric, measure -> measure.setData("true")); + SearchRequest request = new SearchRequest() .setComponentUuids(Collections.singletonList(file.uuid())) .setOnComponentOnly(true) @@ -210,16 +215,17 @@ public class IssueQueryFactoryTest { assertThat(query.newCodeOnReference()).isTrue(); assertThat(query.createdAfter()).isNull(); } - @Test - public void new_code_period_does_not_rely_on_date_for_reference_branch() { - + public void new_code_period_does_not_rely_on_date_for_reference_branch_with_analysis_after_sonarqube_94() { ComponentDto project = db.components().insertPublicProject(); ComponentDto file = db.components().insertComponent(newFileDto(project)); - SnapshotDto analysis = db.components().insertSnapshot(project, s -> s.setPeriodMode(REFERENCE_BRANCH.name()) + db.components().insertSnapshot(project, s -> s.setPeriodMode(REFERENCE_BRANCH.name()) .setPeriodParam("master")); + MetricDto analysisMetric = db.measures().insertMetric(m -> m.setKey(ANALYSIS_FROM_SONARQUBE_9_4_KEY)); + db.measures().insertLiveMeasure(project, analysisMetric, measure -> measure.setData("true")); + SearchRequest request = new SearchRequest() .setComponentUuids(Collections.singletonList(file.uuid())) .setOnComponentOnly(true) @@ -377,7 +383,7 @@ public class IssueQueryFactoryTest { } @Test - public void application_search_project_issues_on_leak() { + public void application_search_project_issues_on_leak_with_and_without_analysis_after_sonarqube_94() { Date now = new Date(); when(clock.millis()).thenReturn(now.getTime()); ComponentDto project1 = db.components().insertPublicProject(); @@ -386,9 +392,14 @@ public class IssueQueryFactoryTest { db.components().insertSnapshot(project2, s -> s.setPeriodDate(null)); ComponentDto project3 = db.components().insertPublicProject(); ComponentDto project4 = db.components().insertPublicProject(); - SnapshotDto analysis2 = db.components().insertSnapshot(project4, + SnapshotDto analysis3 =db.components().insertSnapshot(project3, + s -> s.setPeriodMode(REFERENCE_BRANCH.name()).setPeriodParam("master") + .setPeriodDate(addDays(now, -14).getTime())); + db.components().insertSnapshot(project4, s -> s.setPeriodMode(REFERENCE_BRANCH.name()).setPeriodParam("master")); ComponentDto application = db.components().insertPublicApplication(); + MetricDto analysisMetric = db.measures().insertMetric(m -> m.setKey(ANALYSIS_FROM_SONARQUBE_9_4_KEY)); + db.measures().insertLiveMeasure(project4, analysisMetric, measure -> measure.setData("true")); db.components().insertComponents(newProjectCopy("PC1", project1, application)); db.components().insertComponents(newProjectCopy("PC2", project2, application)); db.components().insertComponents(newProjectCopy("PC3", project3, application)); @@ -399,16 +410,17 @@ public class IssueQueryFactoryTest { .setComponentUuids(singletonList(application.uuid())) .setSinceLeakPeriod(true)); - assertThat(result.createdAfterByProjectUuids()).hasSize(1); + assertThat(result.createdAfterByProjectUuids()).hasSize(2); assertThat(result.createdAfterByProjectUuids().entrySet()).extracting(Map.Entry::getKey, e -> e.getValue().date(), e -> e.getValue().inclusive()).containsOnly( - tuple(project1.uuid(), new Date(analysis1.getPeriodDate()), false)); + tuple(project1.uuid(), new Date(analysis1.getPeriodDate()), false), + tuple(project3.uuid(), new Date(analysis3.getPeriodDate()), false)); assertThat(result.newCodeOnReferenceByProjectUuids()).hasSize(1); assertThat(result.newCodeOnReferenceByProjectUuids()).containsOnly(project4.uuid()); assertThat(result.viewUuids()).containsExactlyInAnyOrder(application.uuid()); } @Test - public void application_search_project_issues_in_new_code() { + public void application_search_project_issues_in_new_code_with_and_without_analysis_after_sonarqube_94() { Date now = new Date(); when(clock.millis()).thenReturn(now.getTime()); ComponentDto project1 = db.components().insertPublicProject(); @@ -420,6 +432,8 @@ public class IssueQueryFactoryTest { SnapshotDto analysis2 = db.components().insertSnapshot(project4, s -> s.setPeriodMode(REFERENCE_BRANCH.name()).setPeriodParam("master")); ComponentDto application = db.components().insertPublicApplication(); + MetricDto analysisMetric = db.measures().insertMetric(m -> m.setKey(ANALYSIS_FROM_SONARQUBE_9_4_KEY)); + db.measures().insertLiveMeasure(project4, analysisMetric, measure -> measure.setData("true")); db.components().insertComponents(newProjectCopy("PC1", project1, application)); db.components().insertComponents(newProjectCopy("PC2", project2, application)); db.components().insertComponents(newProjectCopy("PC3", project3, application)); @@ -718,7 +732,7 @@ public class IssueQueryFactoryTest { @Test public void fail_if_no_component_provided_with_since_leak_period() { - assertThatThrownBy(() -> underTest.create(new SearchRequest().setSinceLeakPeriod(true))) + assertThatThrownBy(() -> underTest.create(new SearchRequest().setInNewCodePeriod(true))) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("One and only one component must be provided when searching in new code period"); } diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryTest.java index b99c33d67b2..1c8781ef6ca 100644 --- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryTest.java +++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryTest.java @@ -62,6 +62,8 @@ public class IssueQueryTest { .createdBefore(new Date()) .createdAt(new Date()) .resolved(true) + .newCodeOnReference(true) + .newCodeOnReferenceByProjectUuids(List.of("PROJECT")) .sort(IssueQuery.SORT_BY_CREATION_DATE) .asc(true) .build(); @@ -88,6 +90,8 @@ public class IssueQueryTest { assertThat(query.createdBefore()).isNotNull(); assertThat(query.createdAt()).isNotNull(); assertThat(query.resolved()).isTrue(); + assertThat(query.newCodeOnReference()).isTrue(); + assertThat(query.newCodeOnReferenceByProjectUuids()).containsOnly("PROJECT"); assertThat(query.sort()).isEqualTo(IssueQuery.SORT_BY_CREATION_DATE); assertThat(query.asc()).isTrue(); } -- 2.39.5