SONAR-8798 allow "epoch_second" as alternative ES date format

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java
index 0c63beedfcd711c56836b93bcdf24cd00228b741..2ace22b946966eec0e101024ade2a95009c26441 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java
@@ -66,6 +66,9 @@ public abstract class BaseDoc {
       if (val instanceof Date) {
         return (Date)val;
       }
+      if (val instanceof Number) {
+        return epochSecondsToDate((Number) val);
+      }
       return EsUtils.parseDateTime((String) val);
     }
     return null;
@@ -93,6 +96,9 @@ public abstract class BaseDoc {
     if (value instanceof Date) {
       return (Date)value;
     }
+    if (value instanceof Number) {
+      return epochSecondsToDate((Number) value);
+    }
     return EsUtils.parseDateTime((String)value);
   }
 
@@ -100,7 +106,23 @@ public abstract class BaseDoc {
     fields.put(key, value);
   }
 
+  public void setField(String key, @Nullable Date value) {
+    fields.put(key, value == null ? null : dateToEpochSeconds(value));
+  }
+
   public Map<String, Object> getFields() {
     return fields;
   }
+
+  public static long epochMillisToEpochSeconds(long epochMillis) {
+    return epochMillis / 1000L;
+  }
+
+  private static Date epochSecondsToDate(Number value) {
+    return new Date(value.longValue() * 1000L);
+  }
+
+  public static long dateToEpochSeconds(Date date) {
+    return epochMillisToEpochSeconds(date.getTime());
+  }
 }

diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
index 5cea7f3a1c7778521cca7125fb2dd4ee5fc21af5..306c05752f627f31ba2bf2ed58973e7dadc866f1 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java
@@ -232,7 +232,7 @@ public class NewIndex {
     }
 
     public NewIndexType createDateTimeField(String fieldName) {
-      return setProperty(fieldName, ImmutableMap.of("type", "date", "format", "date_time"));
+      return setProperty(fieldName, ImmutableMap.of("type", "date", "format", "date_time||epoch_second"));
     }
 
     public NewIndexType createDoubleField(String fieldName) {

diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
index 4a955534caa24061f24f48697ced91ceebd9cd36..28494d67c94f79f36f76fe6115fe1a8a9eb24f99 100644 (file)
--- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
+++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java
@@ -65,6 +65,7 @@ import org.sonar.api.utils.DateUtils;
 import org.sonar.api.utils.System2;
 import org.sonar.core.util.stream.MoreCollectors;
 import org.sonar.db.organization.OrganizationDto;
+import org.sonar.server.es.BaseDoc;
 import org.sonar.server.es.EsClient;
 import org.sonar.server.es.EsUtils;
 import org.sonar.server.es.SearchOptions;
@@ -85,6 +86,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
+import static org.sonar.server.es.BaseDoc.epochMillisToEpochSeconds;
 import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars;
 import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_ORGANIZATION_UUID;
 import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
@@ -427,16 +429,16 @@ public class IssueIndex {
     if (createdAfter != null) {
       filters.put("__createdAfter", QueryBuilders
         .rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT)
-        .gte(createdAfter));
+        .gte(BaseDoc.dateToEpochSeconds(createdAfter)));
     }
     if (createdBefore != null) {
       filters.put("__createdBefore", QueryBuilders
         .rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT)
-        .lt(createdBefore));
+        .lt(BaseDoc.dateToEpochSeconds(createdBefore)));
     }
     Date createdAt = query.createdAt();
     if (createdAt != null) {
-      filters.put("__createdAt", termQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, createdAt));
+      filters.put("__createdAt", termQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, BaseDoc.dateToEpochSeconds(createdAt)));
     }
   }
 
@@ -445,7 +447,7 @@ public class IssueIndex {
     BoolQueryBuilder boolQueryBuilder = boolQuery();
     createdAfterByProjectUuids.forEach((projectUuid, createdAfterDate) -> boolQueryBuilder.should(boolQuery()
       .filter(termQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, projectUuid))
-      .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(createdAfterDate))));
+      .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(BaseDoc.dateToEpochSeconds(createdAfterDate)))));
     filters.put("createdAfterByProjectUuids", boolQueryBuilder);
   }
 
@@ -682,7 +684,7 @@ public class IssueIndex {
         .addAggregation(AggregationBuilders
           .filter(projectUuid, boolQuery()
             .filter(termQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, projectUuid))
-            .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(new Date(from)))
+            .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(epochMillisToEpochSeconds(from)))
           )
           .subAggregation(AggregationBuilders.count(projectUuid + "_count").field(IssueIndexDefinition.FIELD_ISSUE_KEY))
           .subAggregation(AggregationBuilders.max(projectUuid + "_maxFuncCreatedAt").field(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT))

diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java
index 6208388397443a927e8632d3985cabc5325a7104..85e67b543df0e88f827ec478cadae950a5a1c29c 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java
@@ -161,7 +161,7 @@ public class IssueIndexProjectStatisticsTest {
     ComponentDto project = ComponentTesting.newPrivateProjectDto(org1);
     String userLogin1 = randomAlphanumeric(20);
     long from = 1_111_234_567_890L;
-    indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from-1L)));
+    indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from-1000L)));
 
     List<ProjectStatistics> result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1);
 
@@ -222,14 +222,14 @@ public class IssueIndexProjectStatisticsTest {
     ComponentDto project2 = ComponentTesting.newPrivateProjectDto(org1);
     ComponentDto project3 = ComponentTesting.newPrivateProjectDto(org1);
     String userLogin1 = randomAlphanumeric(20);
-    long from = 1_111_234_567_890L;
+    long from = 1_111_234_567_000L;
     indexIssues(
-      newDoc("issue1", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)),
-      newDoc("issue2", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+2L)),
-      newDoc("issue3", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+3L)),
+      newDoc("issue1", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1_000L)),
+      newDoc("issue2", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+2_000L)),
+      newDoc("issue3", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+3_000L)),
       
-      newDoc("issue4", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+4L)),
-      newDoc("issue5", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+5L))
+      newDoc("issue4", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+4_000L)),
+      newDoc("issue5", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+5_000L))
     );
 
     List<ProjectStatistics> result = underTest.searchProjectStatistics(
@@ -240,8 +240,8 @@ public class IssueIndexProjectStatisticsTest {
     assertThat(result)
       .extracting(ProjectStatistics::getProjectUuid, ProjectStatistics::getLastIssueDate)
       .containsExactlyInAnyOrder(
-        tuple(project1.uuid(), from+3L),
-        tuple(project3.uuid(), from+5L)
+        tuple(project1.uuid(), from+3_000L),
+        tuple(project3.uuid(), from+5_000L)
       );
   }
 

diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java
index d24a40a3dbd6af91916b32f491011a146d5281ed..67f3e749b3fb3d34933ab77216a1b8892b184f25 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java
@@ -721,12 +721,12 @@ public class IssueIndexTest {
 
   @Test
   public void facet_on_created_at_with_less_than_20_weeks() {
-    SearchOptions SearchOptions = fixtureForCreatedAtFacet();
+    SearchOptions options = fixtureForCreatedAtFacet();
 
     SearchResponse result = underTest.search(IssueQuery.builder()
       .createdAfter(parseDateTime("2014-09-01T00:00:00+0100"))
       .createdBefore(parseDateTime("2014-09-21T00:00:00+0100")).build(),
-      SearchOptions);
+      options);
     Map<String, Long> createdAt = new Facets(result).get("createdAt");
     assertThat(createdAt).containsOnly(
       entry("2014-08-25T01:00:00+0000", 0L),
@@ -737,12 +737,12 @@ public class IssueIndexTest {
 
   @Test
   public void facet_on_created_at_with_less_than_20_months() {
-    SearchOptions SearchOptions = fixtureForCreatedAtFacet();
+    SearchOptions options = fixtureForCreatedAtFacet();
 
     SearchResponse result = underTest.search(IssueQuery.builder()
       .createdAfter(parseDateTime("2014-09-01T00:00:00+0100"))
       .createdBefore(parseDateTime("2015-01-19T00:00:00+0100")).build(),
-      SearchOptions);
+      options);
     Map<String, Long> createdAt = new Facets(result).get("createdAt");
     assertThat(createdAt).containsOnly(
       entry("2014-08-01T01:00:00+0000", 0L),
@@ -755,12 +755,12 @@ public class IssueIndexTest {
 
   @Test
   public void facet_on_created_at_with_more_than_20_months() {
-    SearchOptions SearchOptions = fixtureForCreatedAtFacet();
+    SearchOptions options = fixtureForCreatedAtFacet();
 
     SearchResponse result = underTest.search(IssueQuery.builder()
       .createdAfter(parseDateTime("2011-01-01T00:00:00+0100"))
       .createdBefore(parseDateTime("2016-01-01T00:00:00+0100")).build(),
-      SearchOptions);
+      options);
     Map<String, Long> createdAt = new Facets(result).get("createdAt");
     assertThat(createdAt).containsOnly(
       entry("2010-01-01T01:00:00+0000", 0L),
@@ -774,12 +774,12 @@ public class IssueIndexTest {
 
   @Test
   public void facet_on_created_at_with_one_day() {
-    SearchOptions SearchOptions = fixtureForCreatedAtFacet();
+    SearchOptions options = fixtureForCreatedAtFacet();
 
     SearchResponse result = underTest.search(IssueQuery.builder()
       .createdAfter(parseDateTime("2014-09-01T00:00:00-0100"))
       .createdBefore(parseDateTime("2014-09-02T00:00:00-0100")).build(),
-      SearchOptions);
+      options);
     Map<String, Long> createdAt = new Facets(result).get("createdAt");
     assertThat(createdAt).containsOnly(
       entry("2014-09-01T01:00:00+0000", 2L));

diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
index 5de19c6ad50d664cd66fff98fb94003abc9fbbae..0954b7dba439fd5a1a75d68b80d387c58eda898d 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java
@@ -21,6 +21,7 @@ package org.sonar.server.issue.index;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -121,7 +122,7 @@ public class IssueIndexerTest {
     assertThat(doc.authorLogin()).isEqualTo(issue.getAuthorLogin());
     assertThat(doc.componentUuid()).isEqualTo(issue.getComponentUuid());
     assertThat(doc.closeDate()).isEqualTo(issue.getIssueCloseDate());
-    assertThat(doc.creationDate()).isEqualTo(issue.getIssueCreationDate());
+    assertThat(doc.creationDate()).isEqualToIgnoringMillis(issue.getIssueCreationDate());
     assertThat(doc.directoryPath()).isEqualTo(dir.path());
     assertThat(doc.filePath()).isEqualTo(file.path());
     assertThat(doc.getParent()).isEqualTo(project.uuid());
@@ -129,7 +130,7 @@ public class IssueIndexerTest {
     assertThat(doc.language()).isEqualTo(issue.getLanguage());
     assertThat(doc.line()).isEqualTo(issue.getLine());
     // functional date
-    assertThat(doc.updateDate().getTime()).isEqualTo(issue.getIssueUpdateTime());
+    assertThat(doc.updateDate()).isEqualToIgnoringMillis(new Date(issue.getIssueUpdateTime()));
   }
 
   @Test

diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java
index 4c8c567282cb4c1e9c79c58dd917cd9685431717..44480c02db0fea847640ef9bec42c16f47850d62 100644 (file)
--- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java
+++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java
@@ -106,12 +106,12 @@ public class BaseDocTest {
         return null;
       }
     };
-    long now = System.currentTimeMillis();
-    doc.setField("javaDate", new Date(now));
-    assertThat(doc.getFieldAsDate("javaDate").getTime()).isEqualTo(now);
+    Date now = new Date();
+    doc.setField("javaDate", now);
+    assertThat(doc.getFieldAsDate("javaDate")).isEqualToIgnoringMillis(now);
 
-    doc.setField("stringDate", EsUtils.formatDateTime(new Date(now)));
-    assertThat(doc.getFieldAsDate("stringDate").getTime()).isEqualTo(now);
+    doc.setField("stringDate", EsUtils.formatDateTime(now));
+    assertThat(doc.getFieldAsDate("stringDate")).isEqualToIgnoringMillis(now);
   }
 
   @Test
@@ -132,12 +132,12 @@ public class BaseDocTest {
         return null;
       }
     };
-    long now = System.currentTimeMillis();
-    doc.setField("javaDate", new Date(now));
-    assertThat(doc.getNullableFieldAsDate("javaDate").getTime()).isEqualTo(now);
+    Date now = new Date();
+    doc.setField("javaDate", now);
+    assertThat(doc.getNullableFieldAsDate("javaDate")).isEqualToIgnoringMillis(now);
 
-    doc.setField("stringDate", EsUtils.formatDateTime(new Date(now)));
-    assertThat(doc.getNullableFieldAsDate("stringDate").getTime()).isEqualTo(now);
+    doc.setField("stringDate", EsUtils.formatDateTime(now));
+    assertThat(doc.getNullableFieldAsDate("stringDate")).isEqualToIgnoringMillis(now);
 
     doc.setField("noValue", null);
     assertThat(doc.getNullableFieldAsDate("noValue")).isNull();