SONAR-15904 - Adapt Issue Search to work with new code reference

5 files changed, 182 insertions, 16 deletions

diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java
index 09313ee6909..5b44483753a 100644
--- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java
+++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java
@@ -149,6 +149,7 @@ import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_LANG
 import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_LINE;
 import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_MODULE_PATH;
 import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_MODULE_UUID;
+import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_NEW_CODE_REFERENCE;
 import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_OWASP_TOP_10;
 import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID;
 import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_RESOLUTION;
@@ -452,6 +453,8 @@ public class IssueIndex {
     addComponentRelatedFilters(query, filters);
     addDatesFilter(filters, query);
     addCreatedAfterByProjectsFilter(filters, query);
+    addNewCodeReferenceFilter(filters, query);
+    addNewCodeReferenceFilterByProjectsFilter(filters, query);
     return filters;
   }
 
@@ -562,11 +565,11 @@ public class IssueIndex {
   private static RequestFiltersComputer newFilterComputer(SearchOptions options, AllFilters allFilters) {
     Collection<String> facetNames = options.getFacets();
     Set<TopAggregationDefinition<?>> facets = Stream.concat(
-      Stream.of(EFFORT_TOP_AGGREGATION),
-      facetNames.stream()
-        .map(FACETS_BY_NAME::get)
-        .filter(Objects::nonNull)
-        .map(Facet::getTopAggregationDef))
+        Stream.of(EFFORT_TOP_AGGREGATION),
+        facetNames.stream()
+          .map(FACETS_BY_NAME::get)
+          .filter(Objects::nonNull)
+          .map(Facet::getTopAggregationDef))
       .collect(MoreCollectors.toSet(facetNames.size()));
 
     return new RequestFiltersComputer(allFilters, facets);
@@ -645,6 +648,28 @@ public class IssueIndex {
     }
   }
 
+  private static void addNewCodeReferenceFilter(AllFilters filters, IssueQuery query) {
+    Boolean newCodeOnReference = query.newCodeOnReference();
+
+    if (newCodeOnReference != null) {
+      filters.addFilter(
+        FIELD_ISSUE_NEW_CODE_REFERENCE, new SimpleFieldFilterScope(FIELD_ISSUE_NEW_CODE_REFERENCE),
+        termQuery(FIELD_ISSUE_NEW_CODE_REFERENCE, true));
+    }
+  }
+
+  private static void addNewCodeReferenceFilterByProjectsFilter(AllFilters allFilters, IssueQuery query) {
+    Collection<String> newCodeOnReferenceByProjectUuids = query.newCodeOnReferenceByProjectUuids();
+    BoolQueryBuilder boolQueryBuilder = boolQuery();
+
+    newCodeOnReferenceByProjectUuids.forEach(projectOrProjectBranchUuid -> boolQueryBuilder.should(boolQuery()
+      .filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectOrProjectBranchUuid))
+      .filter(termQuery(FIELD_ISSUE_NEW_CODE_REFERENCE, true))));
+
+    allFilters.addFilter("__is_new_code_reference_by_project_uuids", new SimpleFieldFilterScope("newCodeReferenceByProjectUuids"), boolQueryBuilder);
+  }
+
+
   private static void addCreatedAfterByProjectsFilter(AllFilters allFilters, IssueQuery query) {
     Map<String, PeriodStart> createdAfterByProjectUuids = query.createdAfterByProjectUuids();
     BoolQueryBuilder boolQueryBuilder = boolQuery();
@@ -865,7 +890,7 @@ public class IssueIndex {
         t ->
           // add sub-aggregation to return issue count for current user
           aggregationHelper.getSubAggregationHelper()
-            .buildSelectedItemsAggregation(ASSIGNED_TO_ME.getName(), ASSIGNED_TO_ME.getTopAggregationDef(), new String[] {uuid})
+            .buildSelectedItemsAggregation(ASSIGNED_TO_ME.getName(), ASSIGNED_TO_ME.getTopAggregationDef(), new String[]{uuid})
             .ifPresent(t::subAggregation));
       esRequest.aggregation(aggregation);
     }
diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java
index d3be5a82a4f..c18b04fa4a1 100644
--- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java
+++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQuery.java
@@ -97,6 +97,8 @@ public class IssueQuery {
   private final String branchUuid;
   private final Boolean mainBranch;
   private final ZoneId timeZone;
+  private final Boolean newCodeOnReference;
+  private final Collection<String> newCodeOnReferenceByProjectUuids;
 
   private IssueQuery(Builder builder) {
     this.issueKeys = defaultCollection(builder.issueKeys);
@@ -135,6 +137,8 @@ public class IssueQuery {
     this.branchUuid = builder.branchUuid;
     this.mainBranch = builder.mainBranch;
     this.timeZone = builder.timeZone;
+    this.newCodeOnReference = builder.newCodeOnReference;
+    this.newCodeOnReferenceByProjectUuids = defaultCollection(builder.newCodeOnReferenceByProjectUuids);
   }
 
   public Collection<String> issueKeys() {
@@ -300,6 +304,15 @@ public class IssueQuery {
     return timeZone;
   }
 
+  @CheckForNull
+  public Boolean newCodeOnReference() {
+    return newCodeOnReference;
+  }
+
+  public Collection<String> newCodeOnReferenceByProjectUuids() {
+    return newCodeOnReferenceByProjectUuids;
+  }
+
   public static class Builder {
     private Collection<String> issueKeys;
     private Collection<String> severities;
@@ -337,6 +350,8 @@ public class IssueQuery {
     private String branchUuid;
     private Boolean mainBranch = true;
     private ZoneId timeZone;
+    private Boolean newCodeOnReference = null;
+    private Collection<String> newCodeOnReferenceByProjectUuids;
 
     private Builder() {
 
@@ -548,6 +563,16 @@ public class IssueQuery {
       this.timeZone = timeZone;
       return this;
     }
+
+    public Builder newCodeOnReference(@Nullable Boolean newCodeOnReference) {
+      this.newCodeOnReference = newCodeOnReference;
+      return this;
+    }
+
+    public Builder newCodeOnReferenceByProjectUuids(@Nullable Collection<String> newCodeOnReferenceByProjectUuids) {
+      this.newCodeOnReferenceByProjectUuids = newCodeOnReferenceByProjectUuids;
+      return this;
+    }
   }
 
   private static <T> Collection<T> defaultCollection(@Nullable Collection<T> c) {
diff --git a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java
index 5647bab8e1f..fd7a17885ca 100644
--- a/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java
+++ b/server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueQueryFactory.java
@@ -20,7 +20,6 @@
 package org.sonar.server.issue.index;
 
 import com.google.common.base.Joiner;
-import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import java.time.Clock;
 import java.time.DateTimeException;
@@ -57,6 +56,7 @@ import org.sonar.server.issue.index.IssueQuery.PeriodStart;
 import org.sonar.server.user.UserSession;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.Collections2.transform;
 import static java.lang.String.format;
 import static java.util.Collections.singleton;
@@ -72,6 +72,7 @@ import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
 import static org.sonar.core.util.stream.MoreCollectors.toList;
 import static org.sonar.core.util.stream.MoreCollectors.toSet;
 import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex;
+import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_COMPONENT_KEYS;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_COMPONENT_UUIDS;
 import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_CREATED_AFTER;
@@ -146,7 +147,7 @@ public class IssueQueryFactory {
 
       setCreatedAfterFromRequest(dbSession, builder, request, allComponents, timeZone);
       String sort = request.getSort();
-      if (!Strings.isNullOrEmpty(sort)) {
+      if (!isNullOrEmpty(sort)) {
         builder.sort(sort);
         builder.asc(request.getAsc());
       }
@@ -184,6 +185,7 @@ public class IssueQueryFactory {
       checkArgument(createdAfter == null || createdInLast == null, format("Parameters %s and %s cannot be set simultaneously", PARAM_CREATED_AFTER, PARAM_CREATED_IN_LAST));
       setCreatedAfterFromDates(builder, createdAfter, createdInLast, true);
     } else {
+      // If the filter is on leak period
       checkArgument(createdAfter == null, "Parameters '%s' and '%s' cannot be set simultaneously", PARAM_CREATED_AFTER, PARAM_SINCE_LEAK_PERIOD);
       checkArgument(createdInLast == null, format("Parameters %s and %s cannot be set simultaneously", PARAM_CREATED_IN_LAST, PARAM_SINCE_LEAK_PERIOD));
 
@@ -191,18 +193,36 @@ public class IssueQueryFactory {
       ComponentDto component = componentUuids.iterator().next();
 
       if (!QUALIFIERS_WITHOUT_LEAK_PERIOD.contains(component.qualifier()) && request.getPullRequest() == null) {
-        Date createdAfterFromSnapshot = findCreatedAfterFromComponentUuid(dbSession, component);
-        setCreatedAfterFromDates(builder, createdAfterFromSnapshot, null, false);
+        Optional<SnapshotDto> snapshot = getLastAnalysis(dbSession, component);
+        boolean isLastAnalysisUsingReferenceBranch = isLastAnalysisUsingReferenceBranch(snapshot);
+        if (isLastAnalysisUsingReferenceBranch) {
+          builder.newCodeOnReference(true);
+        } else {
+          // if last analysis has no period date, then no issue should be considered new.
+          Date createdAfterFromSnapshot = findCreatedAfterFromComponentUuid(snapshot);
+          setCreatedAfterFromDates(builder, createdAfterFromSnapshot, null, false);
+        }
       }
     }
   }
 
-  private Date findCreatedAfterFromComponentUuid(DbSession dbSession, ComponentDto component) {
-    Optional<SnapshotDto> snapshot = dbClient.snapshotDao().selectLastAnalysisByComponentUuid(dbSession, component.uuid());
-    // if last analysis has no period date, then no issue should be considered new.
+  private Date findCreatedAfterFromComponentUuid(Optional<SnapshotDto> snapshot) {
     return snapshot.map(s -> longToDate(s.getPeriodDate())).orElseGet(() -> new Date(clock.millis()));
   }
 
+  private static boolean isLastAnalysisUsingReferenceBranch(Optional<SnapshotDto> snapshot) {
+    String periodMode = snapshot.map(SnapshotDto::getPeriodMode).orElse("");
+    return periodMode.equals(REFERENCE_BRANCH.name());
+  }
+
+  private Optional<SnapshotDto> getLastAnalysis(DbSession dbSession, ComponentDto component) {
+    return dbClient.snapshotDao().selectLastAnalysisByComponentUuid(dbSession, component.uuid());
+  }
+
+  private List<SnapshotDto> getLastAnalysis(DbSession dbSession, Set<String> projectUuids) {
+    return dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, projectUuids);
+  }
+
   private boolean mergeDeprecatedComponentParameters(DbSession session, SearchRequest request, List<ComponentDto> allComponents) {
     Boolean onComponentOnly = request.getOnComponentOnly();
     Collection<String> components = request.getComponents();
@@ -332,12 +352,22 @@ public class IssueQueryFactory {
       .flatMap(app -> dbClient.componentDao().selectProjectsFromView(dbSession, app, app).stream())
       .collect(toSet());
 
-    Map<String, PeriodStart> leakByProjects = dbClient.snapshotDao().selectLastAnalysesByRootComponentUuids(dbSession, projectUuids)
+    List<SnapshotDto> snapshots = getLastAnalysis(dbSession, projectUuids);
+
+    Set<String> newCodeReferenceByProjects = snapshots
+      .stream()
+      .filter(s -> !isNullOrEmpty(s.getPeriodMode()) && s.getPeriodMode().equals(REFERENCE_BRANCH.name()))
+      .map(SnapshotDto::getComponentUuid)
+      .collect(toSet());
+
+    Map<String, PeriodStart> leakByProjects = snapshots
       .stream()
-      .filter(s -> s.getPeriodDate() != null)
+      .filter(s -> s.getPeriodDate() != null &&
+        (isNullOrEmpty(s.getPeriodMode()) || !s.getPeriodMode().equals(REFERENCE_BRANCH.name())))
       .collect(uniqueIndex(SnapshotDto::getComponentUuid, s -> new PeriodStart(longToDate(s.getPeriodDate()), false)));
 
     builder.createdAfterByProjectUuids(leakByProjects);
+    builder.newCodeOnReferenceByProjectUuids(newCodeReferenceByProjects);
   }
 
   private static void addDirectories(IssueQuery.Builder builder, List<ComponentDto> directories) {
diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java
index d03c3b9fe3e..ab02c967806 100644
--- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java
+++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexFiltersTest.java
@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 import org.assertj.core.api.Fail;
 import org.elasticsearch.search.SearchHit;
@@ -502,6 +503,51 @@ public class IssueIndexFiltersTest {
   }
 
   @Test
+  public void filter_by_new_code_reference_by_projects() {
+    ComponentDto project1 = newPrivateProjectDto();
+    IssueDoc project1Issue1 = newDoc(project1).setIsNewCodeReference(true);
+    IssueDoc project1Issue2 = newDoc(project1).setIsNewCodeReference(false);
+    ComponentDto project2 = newPrivateProjectDto();
+    IssueDoc project2Issue1 = newDoc(project2).setIsNewCodeReference(false);
+    IssueDoc project2Issue2 = newDoc(project2).setIsNewCodeReference(true);
+    indexIssues(project1Issue1, project1Issue2, project2Issue1, project2Issue2);
+
+    // Search for issues of project 1 and project 2 that are new code on a branch using reference for new code
+    assertThatSearchReturnsOnly(IssueQuery.builder()
+        .newCodeOnReferenceByProjectUuids(Set.of(project1.uuid(), project2.uuid())),
+      project1Issue1.key(), project2Issue2.key());
+  }
+
+  @Test
+  public void filter_by_new_code_reference_branches() {
+    ComponentDto project1 = newPrivateProjectDto();
+    IssueDoc project1Issue1 = newDoc(project1).setIsNewCodeReference(true);
+    IssueDoc project1Issue2 = newDoc(project1).setIsNewCodeReference(false);
+
+    ComponentDto project1Branch1 = db.components().insertProjectBranch(project1);
+    IssueDoc project1Branch1Issue1 = newDoc(project1Branch1).setIsNewCodeReference(false);
+    IssueDoc project1Branch1Issue2 = newDoc(project1Branch1).setIsNewCodeReference(true);
+
+    ComponentDto project2 = newPrivateProjectDto();
+
+    IssueDoc project2Issue1 = newDoc(project2).setIsNewCodeReference(true);
+    IssueDoc project2Issue2 = newDoc(project2).setIsNewCodeReference(false);
+
+    ComponentDto project2Branch1 = db.components().insertProjectBranch(project2);
+    IssueDoc project2Branch1Issue1 = newDoc(project2Branch1).setIsNewCodeReference(false);
+    IssueDoc project2Branch1Issue2 = newDoc(project2Branch1).setIsNewCodeReference(true);
+
+    indexIssues(project1Issue1, project1Issue2, project2Issue1, project2Issue2,
+      project1Branch1Issue1, project1Branch1Issue2, project2Branch1Issue1, project2Branch1Issue2);
+
+    // Search for issues of project 1 branch 1 and project 2 branch 1 that are new code on a branch using reference for new code
+    assertThatSearchReturnsOnly(IssueQuery.builder()
+        .mainBranch(false)
+        .newCodeOnReferenceByProjectUuids(Set.of(project1Branch1.uuid(), project2Branch1.uuid())),
+      project1Branch1Issue2.key(), project2Branch1Issue2.key());
+  }
+
+  @Test
   public void filter_by_severities() {
     ComponentDto project = newPrivateProjectDto();
     ComponentDto file = newFileDto(project, null);
@@ -757,6 +803,17 @@ public class IssueIndexFiltersTest {
   }
 
   @Test
+  public void filter_by_new_code_reference() {
+    ComponentDto project = newPrivateProjectDto();
+    ComponentDto file = newFileDto(project, null);
+
+    indexIssues(newDoc("I1", file).setIsNewCodeReference(true),
+      newDoc("I2", file).setIsNewCodeReference(false));
+
+    assertThatSearchReturnsOnly(IssueQuery.builder().newCodeOnReference(true), "I1");
+  }
+
+  @Test
   public void filter_by_cwe() {
     ComponentDto project = newPrivateProjectDto();
     ComponentDto file = newFileDto(project, null);
diff --git a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java
index a04621b98dd..1de6246a60e 100644
--- a/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java
+++ b/server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueQueryFactoryTest.java
@@ -56,6 +56,7 @@ import static org.sonar.db.component.ComponentTesting.newFileDto;
 import static org.sonar.db.component.ComponentTesting.newModuleDto;
 import static org.sonar.db.component.ComponentTesting.newProjectCopy;
 import static org.sonar.db.component.ComponentTesting.newSubPortfolio;
+import static org.sonar.db.newcodeperiod.NewCodePeriodType.REFERENCE_BRANCH;
 import static org.sonar.db.rule.RuleTesting.newRule;
 
 public class IssueQueryFactoryTest {
@@ -163,7 +164,29 @@ public class IssueQueryFactoryTest {
     assertThat(query.componentUuids()).containsOnly(file.uuid());
     assertThat(query.createdAfter().date()).isEqualTo(new Date(leakPeriodStart));
     assertThat(query.createdAfter().inclusive()).isFalse();
+    assertThat(query.newCodeOnReference()).isNull();
+  }
+
+  @Test
+  public void leak_period_does_not_rely_on_date_for_reference_branch() {
+    long leakPeriodStart = addDays(new Date(), -14).getTime();
+
+    ComponentDto project = db.components().insertPublicProject();
+    ComponentDto file = db.components().insertComponent(newFileDto(project));
+
+    SnapshotDto analysis = db.components().insertSnapshot(project, s -> s.setPeriodMode(REFERENCE_BRANCH.name())
+      .setPeriodParam("master"));
 
+    SearchRequest request = new SearchRequest()
+      .setComponentUuids(Collections.singletonList(file.uuid()))
+      .setOnComponentOnly(true)
+      .setSinceLeakPeriod(true);
+
+    IssueQuery query = underTest.create(request);
+
+    assertThat(query.componentUuids()).containsOnly(file.uuid());
+    assertThat(query.newCodeOnReference()).isTrue();
+    assertThat(query.createdAfter()).isNull();
   }
 
   @Test
@@ -319,11 +342,15 @@ public class IssueQueryFactoryTest {
     ComponentDto project2 = db.components().insertPublicProject();
     db.components().insertSnapshot(project2, s -> s.setPeriodDate(null));
     ComponentDto project3 = db.components().insertPublicProject();
+    ComponentDto project4 = db.components().insertPublicProject();
+    SnapshotDto analysis2 = db.components().insertSnapshot(project4,
+      s -> s.setPeriodMode(REFERENCE_BRANCH.name()).setPeriodParam("master"));
     ComponentDto application = db.components().insertPublicApplication();
     db.components().insertComponents(newProjectCopy("PC1", project1, application));
     db.components().insertComponents(newProjectCopy("PC2", project2, application));
     db.components().insertComponents(newProjectCopy("PC3", project3, application));
-    userSession.registerApplication(application, project1, project2, project3);
+    db.components().insertComponents(newProjectCopy("PC4", project4, application));
+    userSession.registerApplication(application, project1, project2, project3, project4);
 
     IssueQuery result = underTest.create(new SearchRequest()
       .setComponentUuids(singletonList(application.uuid()))
@@ -332,6 +359,8 @@ public class IssueQueryFactoryTest {
     assertThat(result.createdAfterByProjectUuids()).hasSize(1);
     assertThat(result.createdAfterByProjectUuids().entrySet()).extracting(Map.Entry::getKey, e -> e.getValue().date(), e -> e.getValue().inclusive()).containsOnly(
       tuple(project1.uuid(), new Date(analysis1.getPeriodDate()), false));
+    assertThat(result.newCodeOnReferenceByProjectUuids()).hasSize(1);
+    assertThat(result.newCodeOnReferenceByProjectUuids()).containsOnly(project4.uuid());
     assertThat(result.viewUuids()).containsExactlyInAnyOrder(application.uuid());
   }