From: Stephane Gamard Date: Fri, 23 May 2014 18:27:07 +0000 (+0200) Subject: DAOv.2 - Fixed IndexDefinition for Rule and ActiveRule X-Git-Tag: 4.4-RC1~842 X-Git-Url: https://source.dussan.org/?a=commitdiff_plain;h=72ac551184fa41b77477d3d5396d2422acd0a2d9;p=sonarqube.git DAOv.2 - Fixed IndexDefinition for Rule and ActiveRule --- diff --git a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index 3ff839eedaf..9db06d73859 100644 --- a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -50,8 +50,8 @@ import org.sonar.core.qualityprofile.db.ActiveRuleKey; import org.sonar.core.qualityprofile.db.QualityProfileKey; import org.sonar.server.es.ESNode; import org.sonar.server.qualityprofile.ActiveRule; -import org.sonar.server.rule2.index.RuleIndexDefinition; import org.sonar.server.search.BaseIndex; +import org.sonar.server.search.IndexDefinition; import java.io.IOException; import java.util.ArrayList; @@ -63,7 +63,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; public class ActiveRuleIndex extends BaseIndex { public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { - super(new ActiveRuleIndexDefinition(), normalizer, workQueue, node); + super(IndexDefinition.ACTIVE_RULE, normalizer, workQueue, node); } @Override @@ -82,7 +82,7 @@ public class ActiveRuleIndex extends BaseIndex { @@ -81,20 +84,25 @@ public class ActiveRuleNormalizer extends BaseNormalizer normalize(ActiveRuleKey key) { + DbSession dbSession = db.openSession(false); + List requests = new ArrayList(); try { - return normalize(db().activeRuleDao().getByKey(key, dbSession)); + requests.addAll(normalize(db.activeRuleDao().getByKey(key, dbSession))); + for (ActiveRuleParamDto param : db.activeRuleDao().findParamsByKey(key, dbSession)) { + requests.addAll(this.normalize(param, key)); + } } finally { dbSession.close(); } + return requests; } - public UpdateRequest normalize(ActiveRuleParamDto param, ActiveRuleKey key) { + public List normalize(ActiveRuleParamDto param, ActiveRuleKey key) { Preconditions.checkArgument(key != null, "Cannot normalize ActiveRuleParamDto for null key of ActiveRule"); Map newParam = new HashMap(); @@ -102,12 +110,12 @@ public class ActiveRuleNormalizer extends BaseNormalizer normalize(ActiveRuleDto activeRuleDto) { ActiveRuleKey key = activeRuleDto.getKey(); Preconditions.checkArgument(key != null, "Cannot normalize ActiveRuleDto with null key"); @@ -143,11 +151,11 @@ public class ActiveRuleNormalizer extends BaseNormalizer(RuleIndexDefinition.INDEX_TYPE, IndexAction.Method.UPSERT, rule.getKey())); + session.enqueue(new KeyIndexAction(IndexDefinition.RULE.getIndexType(), + IndexAction.Method.UPSERT, rule.getKey())); } mergeParams(ruleDef, rule, session); @@ -282,7 +283,8 @@ public class RegisterRules implements Startable { dbClient.ruleDao().updateRuleParam(rule, paramDto, session); } else { // TODO to be replaced by synchronizer - session.enqueue(new EmbeddedIndexAction(RuleIndexDefinition.INDEX_TYPE, IndexAction.Method.UPSERT, paramDto, rule.getKey())); + session.enqueue(new EmbeddedIndexAction(IndexDefinition.RULE.getIndexType(), + IndexAction.Method.UPSERT, paramDto, rule.getKey())); } existingParamDtoNames.add(paramDto.getName()); } diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java index 0b1db75a65a..119c4c3d531 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java @@ -38,10 +38,10 @@ import org.sonar.api.rule.RuleStatus; import org.sonar.core.cluster.WorkQueue; import org.sonar.core.rule.RuleDto; import org.sonar.server.es.ESNode; -import org.sonar.server.qualityprofile.index.ActiveRuleIndexDefinition; import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; import org.sonar.server.rule2.Rule; import org.sonar.server.search.BaseIndex; +import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.QueryOptions; import java.io.IOException; @@ -56,7 +56,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; public class RuleIndex extends BaseIndex { public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { - super(new RuleIndexDefinition(), normalizer, workQueue, node); + super(IndexDefinition.RULE, normalizer, workQueue, node); } protected String getKeyValue(RuleKey key) { @@ -297,11 +297,11 @@ public class RuleIndex extends BaseIndex { if (query.getActivation() == Boolean.TRUE) { if (query.getQProfileKey() == null) { // the rules that are activated at least once - fb.must(FilterBuilders.hasChildFilter(new ActiveRuleIndexDefinition().getIndexType(), + fb.must(FilterBuilders.hasChildFilter(IndexDefinition.ACTIVE_RULE.getIndexType(), QueryBuilders.matchAllQuery())); } else { // the rules that are activated on this profile - fb.must(FilterBuilders.hasChildFilter(new ActiveRuleIndexDefinition().getIndexType(), + fb.must(FilterBuilders.hasChildFilter(IndexDefinition.ACTIVE_RULE.getIndexType(), QueryBuilders.termQuery(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.key(), query.getQProfileKey()) )); @@ -309,12 +309,12 @@ public class RuleIndex extends BaseIndex { } else if (query.getActivation() == Boolean.FALSE) { if (query.getQProfileKey() == null) { // the rules that are never activated, on any profile - fb.mustNot(FilterBuilders.hasChildFilter(new ActiveRuleIndexDefinition().getIndexType(), + fb.mustNot(FilterBuilders.hasChildFilter(IndexDefinition.ACTIVE_RULE.getIndexType(), QueryBuilders.matchAllQuery())); } else { // the rules that are not activated on this profile - fb.mustNot(FilterBuilders.hasChildFilter(new ActiveRuleIndexDefinition().getIndexType(), + fb.mustNot(FilterBuilders.hasChildFilter(IndexDefinition.ACTIVE_RULE.getIndexType(), QueryBuilders.termQuery(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.key(), query.getQProfileKey()) )); diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndexDefinition.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndexDefinition.java deleted file mode 100644 index 9519da264c8..00000000000 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndexDefinition.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SonarQube, open source software quality management tool. - * Copyright (C) 2008-2014 SonarSource - * mailto:contact AT sonarsource DOT com - * - * SonarQube is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * SonarQube is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.rule2.index; - -import org.sonar.server.search.IndexDefinition; - -public final class RuleIndexDefinition implements IndexDefinition { - - public static final String INDEX_NAME = "rules2"; - public static final String INDEX_TYPE = "rule2"; - - @Override - public String getIndexName() { - return RuleIndexDefinition.INDEX_NAME; - } - - @Override - public String getIndexType() { - return RuleIndexDefinition.INDEX_TYPE; - } -} diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java index be170087a88..bf0d08fd825 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java @@ -21,6 +21,7 @@ package org.sonar.server.rule2.index; import com.google.common.base.Function; import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -33,10 +34,12 @@ import org.sonar.core.rule.RuleParamDto; import org.sonar.core.technicaldebt.db.CharacteristicDto; import org.sonar.server.db.DbClient; import org.sonar.server.search.BaseNormalizer; +import org.sonar.server.search.IndexDefinition; import javax.annotation.Nullable; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -123,21 +126,26 @@ public class RuleNormalizer extends BaseNormalizer { } public RuleNormalizer(DbClient db) { - super(db); + super(IndexDefinition.RULE, db); } @Override - public UpdateRequest normalize(RuleKey key) { - DbSession dbSession = db().openSession(false); + public List normalize(RuleKey key) { + DbSession dbSession = db.openSession(false); + List requests = new ArrayList(); try { - return normalize(db().ruleDao().getByKey(key, dbSession)); + requests.addAll(normalize(db.ruleDao().getByKey(key, dbSession))); + for(RuleParamDto param : db.ruleDao().findRuleParamsByRuleKey(key, dbSession)){ + requests.addAll(normalize(param, key)); + } } finally { dbSession.close(); } + return requests; } @Override - public UpdateRequest normalize(RuleDto rule) { + public List normalize(RuleDto rule) { /** Update Fields */ Map update = new HashMap(); @@ -203,13 +211,12 @@ public class RuleNormalizer extends BaseNormalizer { /** Creating updateRequest */ - return new UpdateRequest() + return ImmutableList.of(new UpdateRequest() .doc(update) - .upsert(upsert); - + .upsert(upsert)); } - public UpdateRequest normalize(RuleParamDto param, RuleKey key) { + public List normalize(RuleParamDto param, RuleKey key) { Map newParam = new HashMap(); newParam.put("_id", param.getName()); newParam.put(RuleParamField.NAME.key(), param.getName()); @@ -217,7 +224,7 @@ public class RuleNormalizer extends BaseNormalizer { newParam.put(RuleParamField.DESCRIPTION.key(), param.getDescription()); newParam.put(RuleParamField.DEFAULT_VALUE.key(), param.getDefaultValue()); - return this.nestedUpsert(RuleField.PARAMS.key(), - param.getName(), newParam).id(key.toString()); + return ImmutableList.of(this.nestedUpsert(RuleField.PARAMS.key(), + param.getName(), newParam).id(key.toString())); } } diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/persistence/RuleDao.java b/sonar-server/src/main/java/org/sonar/server/rule2/persistence/RuleDao.java index b4cef739556..0cf2d034f4e 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/persistence/RuleDao.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/persistence/RuleDao.java @@ -30,7 +30,7 @@ import org.sonar.core.rule.RuleDto; import org.sonar.core.rule.RuleMapper; import org.sonar.core.rule.RuleParamDto; import org.sonar.server.db.BaseDao; -import org.sonar.server.rule2.index.RuleIndexDefinition; +import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.action.IndexAction; import org.sonar.server.search.action.KeyIndexAction; @@ -48,7 +48,7 @@ public class RuleDao extends BaseDao { @VisibleForTesting public RuleDao(System2 system) { - super(new RuleIndexDefinition(), RuleMapper.class, system); + super(IndexDefinition.RULE, RuleMapper.class, system); } @CheckForNull diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index 36133dd3d27..a0343ae1410 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -20,6 +20,7 @@ package org.sonar.server.search; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -166,12 +167,16 @@ public abstract class BaseIndex, K extends Serializable> return null; } - protected void updateDocument(UpdateRequest request, K key) throws Exception { + protected void updateDocument(Collection requests, K key) throws Exception { LOG.debug("UPDATE _id:{} in index {}", key, this.getIndexName()); - getClient().update(request + BulkRequestBuilder bulkRequest = getClient().prepareBulk(); + for(UpdateRequest request : requests){ + bulkRequest.add(request .index(this.getIndexName()) .id(this.getKeyValue(key)) - .type(this.getIndexType())).get(); + .type(this.getIndexType())); + } + bulkRequest.get(); } @@ -188,8 +193,7 @@ public abstract class BaseIndex, K extends Serializable> @Override public void upsertByDto(E item) { try { - UpdateRequest doc = normalizer.normalize(item); - this.updateDocument(doc, item.getKey()); + this.updateDocument(normalizer.normalize(item), item.getKey()); } catch (Exception e) { LOG.error("Could not update document for index {}: {}", this.getIndexName(), e.getMessage()); @@ -199,8 +203,7 @@ public abstract class BaseIndex, K extends Serializable> @Override public void upsertByKey(K key) { try { - UpdateRequest doc = normalizer.normalize(key); - this.updateDocument(doc, key); + this.updateDocument(normalizer.normalize(key), key); } catch (Exception e) { LOG.error("Could not update document for index {}: {}", this.getIndexName(), e.getMessage()); diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java b/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java index d1ea10c97fb..0c384f92802 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java @@ -20,14 +20,13 @@ package org.sonar.server.search; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.core.persistence.Dto; import org.sonar.server.db.DbClient; -import java.io.IOException; import java.io.Serializable; +import java.util.List; import java.util.Map; public abstract class BaseNormalizer, K extends Serializable> { @@ -35,13 +34,11 @@ public abstract class BaseNormalizer, K extends Serializable> { private static final Logger LOG = LoggerFactory.getLogger(BaseNormalizer.class); protected final DbClient db; + protected final IndexDefinition definition; - protected BaseNormalizer(DbClient db) { + protected BaseNormalizer(IndexDefinition definition, DbClient db) { this.db = db; - } - - protected DbClient db() { - return db; + this.definition = definition; } public boolean canNormalize(Class objectClass, Class keyClass) { @@ -52,9 +49,9 @@ public abstract class BaseNormalizer, K extends Serializable> { } } - public UpdateRequest normalizeOther(Object object, Object key) { + public List normalizeOther(Object object, Object key) { try { - return (UpdateRequest) this.getClass() + return (List) this.getClass() .getMethod("normalize", object.getClass(), key.getClass()) .invoke(this, object, key); } catch (Exception e) { @@ -62,18 +59,9 @@ public abstract class BaseNormalizer, K extends Serializable> { } } - public abstract UpdateRequest normalize(K key); - - public abstract UpdateRequest normalize(E dto); - - protected void indexField(String field, Object value, XContentBuilder document) { - try { - document.field(field, value); - } catch (IOException e) { - LOG.error("Could not set {} to {} in ESDocument", field, value); - } - } + public abstract java.util.List normalize(K key); + public abstract java.util.List normalize(E dto); protected UpdateRequest nestedUpsert(String field, String key, Map item) { return new UpdateRequest() diff --git a/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java b/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java index c7932911a7b..64f128fcea2 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java +++ b/sonar-server/src/main/java/org/sonar/server/search/IndexDefinition.java @@ -19,9 +19,24 @@ */ package org.sonar.server.search; -public interface IndexDefinition { +public class IndexDefinition { - String getIndexName(); + private final String indexName; + private final String indexType; - String getIndexType(); + private IndexDefinition(String indexName, String indexType) { + this.indexName = indexName; + this.indexType = indexType; + } + + public String getIndexName() { + return indexName; + } + + public String getIndexType() { + return indexType; + } + + public static IndexDefinition RULE = new IndexDefinition("rules","rules"); + public static IndexDefinition ACTIVE_RULE = new IndexDefinition("rules","activeRules"); } diff --git a/sonar-server/src/test/java/org/sonar/server/qualityprofile/ActiveRuleServiceMediumTest.java b/sonar-server/src/test/java/org/sonar/server/qualityprofile/ActiveRuleServiceMediumTest.java index b21bf687982..0fafd373a67 100644 --- a/sonar-server/src/test/java/org/sonar/server/qualityprofile/ActiveRuleServiceMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/qualityprofile/ActiveRuleServiceMediumTest.java @@ -20,7 +20,6 @@ package org.sonar.server.qualityprofile; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.sonar.api.rule.RuleKey; @@ -355,6 +354,32 @@ public class ActiveRuleServiceMediumTest { } } + @Test + public void synchronize_index() throws Exception { + QualityProfileDto profile1 = QualityProfileDto.createFor("p1", "java"); + dbClient.qualityProfileDao().insert(dbSession, profile1); + + RuleDto rule1 = RuleDto.createFor(RuleKey.of("java", "r1")).setSeverity(Severity.MAJOR); + dbClient.ruleDao().insert(rule1, dbSession); + + ActiveRuleDto activeRule = ActiveRuleDto.createFor(profile1, rule1).setSeverity("BLOCKER"); + dbClient.activeRuleDao().insert(activeRule, dbSession); + dbSession.commit(); + + tester.clearIndexes(); + + // 0. Assert that we have no rules in Is. + assertThat(index.getByKey(activeRule.getKey())).isNull(); + + + // 1. Synchronize since 0 + dbClient.activeRuleDao().synchronizeAfter(0,dbSession); + + // 2. Assert that we have the rule in Index + assertThat(index.getByKey(activeRule.getKey())).isNotNull(); + + } + @Test public void find_active_rules() throws Exception { QualityProfileDto profile1 = QualityProfileDto.createFor("p1", "java"); diff --git a/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesMediumTest.java b/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesMediumTest.java index f2dc4e8cf23..88033365147 100644 --- a/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/qualityprofile/RegisterQualityProfilesMediumTest.java @@ -44,6 +44,7 @@ import org.sonar.core.qualityprofile.db.QualityProfileKey; import org.sonar.core.template.LoadedTemplateDto; import org.sonar.server.db.DbClient; import org.sonar.server.platform.Platform; +import org.sonar.server.qualityprofile.index.ActiveRuleIndex; import org.sonar.server.qualityprofile.persistence.ActiveRuleDao; import org.sonar.server.tester.ServerTester; @@ -67,6 +68,48 @@ public class RegisterQualityProfilesMediumTest { } } + @Test + public void register_existing_profile_definitions() throws Exception { + tester = new ServerTester().addComponents(XooRulesDefinition.class, XooProfileDefinition.class); + tester.start(); + dbSession = dbClient().openSession(false); + QualityProfileKey qualityProfileKey = QualityProfileKey.of("Basic", "xoo"); + + // Check Profile in DB + QualityProfileDao qualityProfileDao = dbClient().qualityProfileDao(); + assertThat(qualityProfileDao.findAll(dbSession)).hasSize(1); + assertThat(qualityProfileDao.getByKey(qualityProfileKey, dbSession)).isNotNull(); + + // Check ActiveRules in DB + ActiveRuleDao activeRuleDao = dbClient().activeRuleDao(); + assertThat(activeRuleDao.findByProfileKey(qualityProfileKey, dbSession)).hasSize(2); + RuleKey ruleKey = RuleKey.of("xoo", "x1"); + + ActiveRuleKey activeRuleKey = ActiveRuleKey.of(qualityProfileKey, ruleKey); + + // 0. Check and clear ES + assertThat(tester.get(ActiveRuleIndex.class).getByKey(activeRuleKey)).isNotNull(); + tester.clearIndexes(); + assertThat(tester.get(ActiveRuleIndex.class).getByKey(activeRuleKey)).isNull(); + tester.get(Platform.class).restart(); + assertThat(tester.get(ActiveRuleIndex.class).getByKey(activeRuleKey)).isNotNull(); + + + // Check ActiveRules in ES + org.sonar.server.qualityprofile.ActiveRule activeRule = tester.get(ActiveRuleIndex.class).getByKey(activeRuleKey); + assertThat(activeRule.key().qProfile()).isEqualTo(qualityProfileKey); + assertThat(activeRule.key().ruleKey()).isEqualTo(ruleKey); + assertThat(activeRule.severity()).isEqualTo(Severity.CRITICAL); + +// // Check ActiveRuleParameters in DB +// Map params = ActiveRuleParamDto.groupByKey(activeRuleDao.findParamsByActiveRule(activeRule, dbSession)); +// assertThat(params).hasSize(2); +// // set by profile +// assertThat(params.get("acceptWhitespace").getValue()).isEqualTo("true"); +// // default value +// assertThat(params.get("max").getValue()).isEqualTo("10"); + } + @Test public void register_profile_definitions() throws Exception { tester = new ServerTester().addComponents(XooRulesDefinition.class, XooProfileDefinition.class);