From b7bd2a6f2692452f4c9b126fdcd5fef8f45e1a9e Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Sun, 31 Aug 2014 17:53:17 +0200 Subject: [PATCH] SONAR-5529 - Initial materialization of Issues in ES --- .../org/sonar/server/issue/db/IssueDao.java | 3 +- .../sonar/server/issue/index/IssueDoc.java | 2 +- .../sonar/server/issue/index/IssueIndex.java | 28 +++++++--- .../server/issue/index/IssueNormalizer.java | 51 ++++++++++++++++++- .../issue/db/IssueBackendMediumTest.java | 46 ++++++++++++++++- 5 files changed, 118 insertions(+), 12 deletions(-) diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java b/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java index 3cd0b1af7fc..390e39d49b8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/db/IssueDao.java @@ -27,6 +27,7 @@ import org.sonar.core.issue.db.IssueMapper; import org.sonar.core.persistence.DaoComponent; import org.sonar.core.persistence.DbSession; import org.sonar.server.db.BaseDao; +import org.sonar.server.search.IndexDefinition; import java.sql.Timestamp; import java.util.Date; @@ -39,7 +40,7 @@ public class IssueDao extends BaseDao implements @VisibleForTesting public IssueDao(System2 system) { - super(null, IssueMapper.class, system); + super(IndexDefinition.ISSUES, IssueMapper.class, system); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java index 344a482e78d..991fba0dd9e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueDoc.java @@ -36,7 +36,7 @@ public class IssueDoc extends BaseDoc implements Issue { @Override public String key() { - return null; + return getField(IssueNormalizer.IssueField.KEY.field()); } @Override diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index ef8cb163962..92fd7aef6c7 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -19,43 +19,57 @@ */ package org.sonar.server.issue.index; +import com.google.common.base.Preconditions; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.sonar.core.issue.db.IssueDto; import org.sonar.server.search.BaseIndex; import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.IndexField; import org.sonar.server.search.SearchClient; import java.io.IOException; +import java.util.HashMap; import java.util.Map; public class IssueIndex extends BaseIndex { - protected IssueIndex(IssueNormalizer normalizer, SearchClient client) { + public IssueIndex(IssueNormalizer normalizer, SearchClient client) { super(IndexDefinition.ISSUES, normalizer, client); } @Override - protected String getKeyValue(String s) { - return s; + protected String getKeyValue(String keyString) { + return keyString; } @Override protected Settings getIndexSettings() throws IOException { - return null; + return ImmutableSettings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 1) + .build(); } @Override protected Map mapProperties() { - return null; + Map mapping = new HashMap(); + for (IndexField field : IssueNormalizer.IssueField.ALL_FIELDS) { + mapping.put(field.field(), mapField(field)); + } + return mapping; } @Override protected Map mapKey() { - return null; + Map mapping = new HashMap(); + mapping.put("path", IssueNormalizer.IssueField.KEY.field()); + return mapping; } @Override protected IssueDoc toDoc(Map fields) { - return null; + Preconditions.checkNotNull(fields, "Cannot construct Issue with null response"); + return new IssueDoc(fields); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java index 27f7116bbda..6932cb9f40e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueNormalizer.java @@ -19,22 +19,69 @@ */ package org.sonar.server.issue.index; +import com.google.common.collect.ImmutableList; import org.elasticsearch.action.update.UpdateRequest; import org.sonar.core.issue.db.IssueDto; import org.sonar.server.db.DbClient; import org.sonar.server.search.BaseNormalizer; import org.sonar.server.search.IndexDefinition; +import org.sonar.server.search.IndexField; +import org.sonar.server.search.Indexable; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; public class IssueNormalizer extends BaseNormalizer { - protected IssueNormalizer(DbClient db) { + public IssueNormalizer(DbClient db) { super(IndexDefinition.ISSUES, db); } + public static final class IssueField extends Indexable { + + public static final IndexField KEY = addSortable(IndexField.Type.STRING, "key"); + + public static final Set ALL_FIELDS = getAllFields(); + + private static final Set getAllFields() { + Set fields = new HashSet(); + for (Field classField : IssueField.class.getDeclaredFields()) { + if (classField.getType().isAssignableFrom(IndexField.class)) { + try { + fields.add(IndexField.class.cast(classField.get(null))); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Could not access Field '" + classField.getName() + "'", e); + } + } + } + return fields; + } + + public static final IndexField of(String fieldName) { + for (IndexField field : ALL_FIELDS) { + if (field.field().equals(fieldName)) { + return field; + } + } + throw new IllegalStateException("Could not find an IndexField for '" + fieldName + "'"); + } + } + @Override public List normalize(IssueDto dto) { - return null; + Map update = new HashMap(); + Map upsert = new HashMap(); + + update.put(IssueField.KEY.field(), dto.getKey()); + + return ImmutableList.of( + new UpdateRequest() + .id(dto.getKey().toString()) + .doc(update) + .upsert(update)); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueBackendMediumTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueBackendMediumTest.java index 4416feab29d..9b7e9d8829b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueBackendMediumTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/db/IssueBackendMediumTest.java @@ -29,6 +29,8 @@ import org.sonar.core.persistence.DbSession; import org.sonar.core.rule.RuleDto; import org.sonar.server.component.persistence.ComponentDao; import org.sonar.server.db.DbClient; +import org.sonar.server.issue.index.IssueDoc; +import org.sonar.server.issue.index.IssueIndex; import org.sonar.server.platform.Platform; import org.sonar.server.rule.RuleTesting; import org.sonar.server.rule.db.RuleDao; @@ -67,7 +69,42 @@ public class IssueBackendMediumTest { } @Test - public void insert_and_find_after_date() throws Exception { + public void insert_and_find_by_key() throws Exception { + RuleDto rule = RuleTesting.newXooX1(); + tester.get(RuleDao.class).insert(dbSession, rule); + + ComponentDto project = new ComponentDto() + .setId(1L) + .setProjectId(1L); + tester.get(ComponentDao.class).insert(dbSession, project); + + ComponentDto resource = new ComponentDto() + .setProjectId(1L) + .setId(2L); + tester.get(ComponentDao.class).insert(dbSession, resource); + + IssueDto issue = new IssueDto().setId(1L).setRuleId(50).setComponentId(123l).setRootComponentId(100l) + .setRuleId(rule.getId()) + .setRootComponentId(project.getId()) + .setComponentId(resource.getId()) + .setStatus("OPEN").setResolution("OPEN") + .setKee(UUID.randomUUID().toString()); + dbClient.issueDao().insert(dbSession, issue); + + dbSession.commit(); + assertThat(issue.getId()).isNotNull(); + + // Check that Issue is in Index + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + + // should find by key + IssueDoc issueDoc = indexClient.get(IssueIndex.class).getByKey(issue.getKey()); + assertThat(issueDoc).isNotNull(); + assertThat(issueDoc.key()).isEqualTo(issue.getKey()); + } + + @Test + public void insert_and_find_after_date() throws Exception { RuleDto rule = RuleTesting.newXooX1(); tester.get(RuleDao.class).insert(dbSession, rule); @@ -100,5 +137,12 @@ public class IssueBackendMediumTest { // Should not find any new issues Date t1 = new Date(); assertThat(dbClient.issueDao().findAfterDate(dbSession, t1)).hasSize(0); + + // Should synchronise + tester.clearIndexes(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(0); + tester.get(Platform.class).executeStartupTasks(); + assertThat(indexClient.get(IssueIndex.class).countAll()).isEqualTo(1); + } } -- 2.39.5