From: Daniel Schwarz Date: Thu, 27 Jul 2017 12:50:33 +0000 (+0200) Subject: SONAR-8798 allow "epoch_second" as alternative ES date format X-Git-Tag: 6.6-RC1~689 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=01472db102cb0d3128354011802dd4b1648ff9b5;p=sonarqube.git SONAR-8798 allow "epoch_second" as alternative ES date format --- diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java index 0c63beedfcd..2ace22b9469 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BaseDoc.java @@ -66,6 +66,9 @@ public abstract class BaseDoc { if (val instanceof Date) { return (Date)val; } + if (val instanceof Number) { + return epochSecondsToDate((Number) val); + } return EsUtils.parseDateTime((String) val); } return null; @@ -93,6 +96,9 @@ public abstract class BaseDoc { if (value instanceof Date) { return (Date)value; } + if (value instanceof Number) { + return epochSecondsToDate((Number) value); + } return EsUtils.parseDateTime((String)value); } @@ -100,7 +106,23 @@ public abstract class BaseDoc { fields.put(key, value); } + public void setField(String key, @Nullable Date value) { + fields.put(key, value == null ? null : dateToEpochSeconds(value)); + } + public Map getFields() { return fields; } + + public static long epochMillisToEpochSeconds(long epochMillis) { + return epochMillis / 1000L; + } + + private static Date epochSecondsToDate(Number value) { + return new Date(value.longValue() * 1000L); + } + + public static long dateToEpochSeconds(Date date) { + return epochMillisToEpochSeconds(date.getTime()); + } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java index 5cea7f3a1c7..306c05752f6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/NewIndex.java @@ -232,7 +232,7 @@ public class NewIndex { } public NewIndexType createDateTimeField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "date", "format", "date_time")); + return setProperty(fieldName, ImmutableMap.of("type", "date", "format", "date_time||epoch_second")); } public NewIndexType createDoubleField(String fieldName) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 4a955534caa..28494d67c94 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -65,6 +65,7 @@ import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.organization.OrganizationDto; +import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; import org.sonar.server.es.SearchOptions; @@ -85,6 +86,7 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; +import static org.sonar.server.es.BaseDoc.epochMillisToEpochSeconds; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_ORGANIZATION_UUID; import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; @@ -427,16 +429,16 @@ public class IssueIndex { if (createdAfter != null) { filters.put("__createdAfter", QueryBuilders .rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT) - .gte(createdAfter)); + .gte(BaseDoc.dateToEpochSeconds(createdAfter))); } if (createdBefore != null) { filters.put("__createdBefore", QueryBuilders .rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT) - .lt(createdBefore)); + .lt(BaseDoc.dateToEpochSeconds(createdBefore))); } Date createdAt = query.createdAt(); if (createdAt != null) { - filters.put("__createdAt", termQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, createdAt)); + filters.put("__createdAt", termQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT, BaseDoc.dateToEpochSeconds(createdAt))); } } @@ -445,7 +447,7 @@ public class IssueIndex { BoolQueryBuilder boolQueryBuilder = boolQuery(); createdAfterByProjectUuids.forEach((projectUuid, createdAfterDate) -> boolQueryBuilder.should(boolQuery() .filter(termQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, projectUuid)) - .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(createdAfterDate)))); + .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(BaseDoc.dateToEpochSeconds(createdAfterDate))))); filters.put("createdAfterByProjectUuids", boolQueryBuilder); } @@ -682,7 +684,7 @@ public class IssueIndex { .addAggregation(AggregationBuilders .filter(projectUuid, boolQuery() .filter(termQuery(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, projectUuid)) - .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(new Date(from))) + .filter(rangeQuery(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT).gte(epochMillisToEpochSeconds(from))) ) .subAggregation(AggregationBuilders.count(projectUuid + "_count").field(IssueIndexDefinition.FIELD_ISSUE_KEY)) .subAggregation(AggregationBuilders.max(projectUuid + "_maxFuncCreatedAt").field(IssueIndexDefinition.FIELD_ISSUE_FUNC_CREATED_AT)) diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java index 62083883974..85e67b543df 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexProjectStatisticsTest.java @@ -161,7 +161,7 @@ public class IssueIndexProjectStatisticsTest { ComponentDto project = ComponentTesting.newPrivateProjectDto(org1); String userLogin1 = randomAlphanumeric(20); long from = 1_111_234_567_890L; - indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from-1L))); + indexIssues(newDoc("issue1", project).setAssignee(userLogin1).setFuncCreationDate(new Date(from-1000L))); List result = underTest.searchProjectStatistics(singletonList(project.uuid()), singletonList(from), userLogin1); @@ -222,14 +222,14 @@ public class IssueIndexProjectStatisticsTest { ComponentDto project2 = ComponentTesting.newPrivateProjectDto(org1); ComponentDto project3 = ComponentTesting.newPrivateProjectDto(org1); String userLogin1 = randomAlphanumeric(20); - long from = 1_111_234_567_890L; + long from = 1_111_234_567_000L; indexIssues( - newDoc("issue1", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1L)), - newDoc("issue2", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+2L)), - newDoc("issue3", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+3L)), + newDoc("issue1", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+1_000L)), + newDoc("issue2", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+2_000L)), + newDoc("issue3", project1).setAssignee(userLogin1).setFuncCreationDate(new Date(from+3_000L)), - newDoc("issue4", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+4L)), - newDoc("issue5", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+5L)) + newDoc("issue4", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+4_000L)), + newDoc("issue5", project3).setAssignee(userLogin1).setFuncCreationDate(new Date(from+5_000L)) ); List result = underTest.searchProjectStatistics( @@ -240,8 +240,8 @@ public class IssueIndexProjectStatisticsTest { assertThat(result) .extracting(ProjectStatistics::getProjectUuid, ProjectStatistics::getLastIssueDate) .containsExactlyInAnyOrder( - tuple(project1.uuid(), from+3L), - tuple(project3.uuid(), from+5L) + tuple(project1.uuid(), from+3_000L), + tuple(project3.uuid(), from+5_000L) ); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index d24a40a3dbd..67f3e749b3f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -721,12 +721,12 @@ public class IssueIndexTest { @Test public void facet_on_created_at_with_less_than_20_weeks() { - SearchOptions SearchOptions = fixtureForCreatedAtFacet(); + SearchOptions options = fixtureForCreatedAtFacet(); SearchResponse result = underTest.search(IssueQuery.builder() .createdAfter(parseDateTime("2014-09-01T00:00:00+0100")) .createdBefore(parseDateTime("2014-09-21T00:00:00+0100")).build(), - SearchOptions); + options); Map createdAt = new Facets(result).get("createdAt"); assertThat(createdAt).containsOnly( entry("2014-08-25T01:00:00+0000", 0L), @@ -737,12 +737,12 @@ public class IssueIndexTest { @Test public void facet_on_created_at_with_less_than_20_months() { - SearchOptions SearchOptions = fixtureForCreatedAtFacet(); + SearchOptions options = fixtureForCreatedAtFacet(); SearchResponse result = underTest.search(IssueQuery.builder() .createdAfter(parseDateTime("2014-09-01T00:00:00+0100")) .createdBefore(parseDateTime("2015-01-19T00:00:00+0100")).build(), - SearchOptions); + options); Map createdAt = new Facets(result).get("createdAt"); assertThat(createdAt).containsOnly( entry("2014-08-01T01:00:00+0000", 0L), @@ -755,12 +755,12 @@ public class IssueIndexTest { @Test public void facet_on_created_at_with_more_than_20_months() { - SearchOptions SearchOptions = fixtureForCreatedAtFacet(); + SearchOptions options = fixtureForCreatedAtFacet(); SearchResponse result = underTest.search(IssueQuery.builder() .createdAfter(parseDateTime("2011-01-01T00:00:00+0100")) .createdBefore(parseDateTime("2016-01-01T00:00:00+0100")).build(), - SearchOptions); + options); Map createdAt = new Facets(result).get("createdAt"); assertThat(createdAt).containsOnly( entry("2010-01-01T01:00:00+0000", 0L), @@ -774,12 +774,12 @@ public class IssueIndexTest { @Test public void facet_on_created_at_with_one_day() { - SearchOptions SearchOptions = fixtureForCreatedAtFacet(); + SearchOptions options = fixtureForCreatedAtFacet(); SearchResponse result = underTest.search(IssueQuery.builder() .createdAfter(parseDateTime("2014-09-01T00:00:00-0100")) .createdBefore(parseDateTime("2014-09-02T00:00:00-0100")).build(), - SearchOptions); + options); Map createdAt = new Facets(result).get("createdAt"); assertThat(createdAt).containsOnly( entry("2014-09-01T01:00:00+0000", 2L)); diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java index 5de19c6ad50..0954b7dba43 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java @@ -21,6 +21,7 @@ package org.sonar.server.issue.index; import java.util.Arrays; import java.util.Collection; +import java.util.Date; import java.util.List; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -121,7 +122,7 @@ public class IssueIndexerTest { assertThat(doc.authorLogin()).isEqualTo(issue.getAuthorLogin()); assertThat(doc.componentUuid()).isEqualTo(issue.getComponentUuid()); assertThat(doc.closeDate()).isEqualTo(issue.getIssueCloseDate()); - assertThat(doc.creationDate()).isEqualTo(issue.getIssueCreationDate()); + assertThat(doc.creationDate()).isEqualToIgnoringMillis(issue.getIssueCreationDate()); assertThat(doc.directoryPath()).isEqualTo(dir.path()); assertThat(doc.filePath()).isEqualTo(file.path()); assertThat(doc.getParent()).isEqualTo(project.uuid()); @@ -129,7 +130,7 @@ public class IssueIndexerTest { assertThat(doc.language()).isEqualTo(issue.getLanguage()); assertThat(doc.line()).isEqualTo(issue.getLine()); // functional date - assertThat(doc.updateDate().getTime()).isEqualTo(issue.getIssueUpdateTime()); + assertThat(doc.updateDate()).isEqualToIgnoringMillis(new Date(issue.getIssueUpdateTime())); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java index 4c8c567282c..44480c02db0 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java @@ -106,12 +106,12 @@ public class BaseDocTest { return null; } }; - long now = System.currentTimeMillis(); - doc.setField("javaDate", new Date(now)); - assertThat(doc.getFieldAsDate("javaDate").getTime()).isEqualTo(now); + Date now = new Date(); + doc.setField("javaDate", now); + assertThat(doc.getFieldAsDate("javaDate")).isEqualToIgnoringMillis(now); - doc.setField("stringDate", EsUtils.formatDateTime(new Date(now))); - assertThat(doc.getFieldAsDate("stringDate").getTime()).isEqualTo(now); + doc.setField("stringDate", EsUtils.formatDateTime(now)); + assertThat(doc.getFieldAsDate("stringDate")).isEqualToIgnoringMillis(now); } @Test @@ -132,12 +132,12 @@ public class BaseDocTest { return null; } }; - long now = System.currentTimeMillis(); - doc.setField("javaDate", new Date(now)); - assertThat(doc.getNullableFieldAsDate("javaDate").getTime()).isEqualTo(now); + Date now = new Date(); + doc.setField("javaDate", now); + assertThat(doc.getNullableFieldAsDate("javaDate")).isEqualToIgnoringMillis(now); - doc.setField("stringDate", EsUtils.formatDateTime(new Date(now))); - assertThat(doc.getNullableFieldAsDate("stringDate").getTime()).isEqualTo(now); + doc.setField("stringDate", EsUtils.formatDateTime(now)); + assertThat(doc.getNullableFieldAsDate("stringDate")).isEqualToIgnoringMillis(now); doc.setField("noValue", null); assertThat(doc.getNullableFieldAsDate("noValue")).isNull();