From 6c2a976a24240cc219dc0ed5f7dc91f1cce7ff4e Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Fri, 16 May 2014 10:43:38 +0200 Subject: [PATCH] SONAR-5007 - ES 1.1.1 update for normalizer and indexes --- .../qualityprofile/index/ActiveRuleIndex.java | 54 +++-- .../index/ActiveRuleNormalizer.java | 97 +++----- .../org/sonar/server/rule2/index/RuleDoc.java | 4 +- .../sonar/server/rule2/index/RuleIndex.java | 23 +- .../server/rule2/index/RuleNormalizer.java | 142 +++++------ .../sonar/server/rule2/index/RuleResult.java | 20 +- .../sonar/server/rule2/ws/SearchAction.java | 1 - .../org/sonar/server/search/BaseIndex.java | 22 +- .../sonar/server/search/BaseNormalizer.java | 13 + .../org/sonar/server/search/NestedIndex.java | 11 +- .../server/rule2/RuleServiceMediumTest.java | 228 ++++++++++++++---- .../rule2/index/RuleIndexMediumTest.java | 10 +- .../server/rule2/ws/RulesWebServiceTest.java | 17 +- 13 files changed, 398 insertions(+), 244 deletions(-) diff --git a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index 5a82f7ff49d..94841e32396 100644 --- a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -44,31 +44,23 @@ import org.sonar.api.rules.ActiveRule; import org.sonar.core.cluster.WorkQueue; import org.sonar.core.qualityprofile.db.ActiveRuleDto; import org.sonar.core.qualityprofile.db.ActiveRuleKey; -import org.sonar.server.rule2.index.RuleNormalizer; +import org.sonar.server.es.ESNode; +import org.sonar.server.rule2.index.RuleIndexDefinition; import org.sonar.server.search.BaseIndex; -import org.sonar.server.search.NestedIndex; import java.io.IOException; -public class ActiveRuleIndex extends NestedIndex { +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; - public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, BaseIndex index) { - super(new ActiveRuleIndexDefinition(), normalizer, workQueue, index); - } +public class ActiveRuleIndex extends BaseIndex { - @Override - protected String getParentKeyValue(ActiveRuleKey key) { - return key.ruleKey().toString(); - } - - @Override - protected String getParentIndexType() { - return "rule2"; + public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { + super(new ActiveRuleIndexDefinition(), normalizer, workQueue, node); } @Override - protected String getIndexField() { - return RuleNormalizer.RuleField.ACTIVE.key(); + protected String getKeyValue(ActiveRuleKey key) { + return key.toString(); } @Override @@ -78,12 +70,38 @@ public class ActiveRuleIndex extends NestedIndex { @@ -39,7 +39,8 @@ public class ActiveRuleNormalizer extends BaseNormalizer newParam = new HashMap(); + newParam.put("_id", param.getKey()); + newParam.put(ActiveRuleParamField.NAME.key(), param.getKey()); + newParam.put(ActiveRuleParamField.VALUE.key(), param.getValue()); + + return this.nestedUpsert(ActiveRuleField.PARAMS.key(), param.getKey(), newParam); } @Override public UpdateRequest normalize(ActiveRuleDto rule) { - try { - - XContentBuilder document = jsonBuilder().startObject(); - - ActiveRuleKey key = rule.getKey(); - if (key == null) { - throw new IllegalStateException("Cannot normalize ActiveRuleDto with null key"); - } + ActiveRuleKey key = rule.getKey(); + if (key == null) { + throw new IllegalStateException("Cannot normalize ActiveRuleDto with null key"); + } + Map newRule = new HashMap(); + newRule.put("_parent", key.ruleKey().toString()); + newRule.put("ruleKey", key.ruleKey().toString()); + newRule.put(ActiveRuleField.KEY.key(), key.toString()); + newRule.put(ActiveRuleField.INHERITANCE.key(), rule.getInheritance()); + newRule.put(ActiveRuleField.PROFILE_ID.key(), rule.getProfileId()); + newRule.put(ActiveRuleField.SEVERITY.key(), rule.getSeverityString()); + + //TODO this should be generated by RegisterRule and modified in DTO. + if (rule.getParentId() != null) { + DbSession session = db.openSession(false); + ActiveRuleDto dto = this.db.activeRuleDao().getById(rule.getParentId(), session); + session.close(); + newRule.put(ActiveRuleField.PARENT_KEY.key(), dto.getKey().toString()); + } - document.startObject(RuleNormalizer.RuleField.ACTIVE.key()); - document.startObject(key.toString()); - indexField(ActiveRuleField.KEY.key(), rule.getKey(), document); - indexField(ActiveRuleField.INHERITANCE.key(), rule.getInheritance(), document); - indexField(ActiveRuleField.PROFILE_ID.key(), rule.getProfileId(), document); - indexField(ActiveRuleField.SEVERITY.key(), rule.getSeverityString(), document); - - //TODO this should be generated by RegisterRule and modified in DTO. - if(rule.getParentId() != null){ - DbSession session = db.openSession(false); - ActiveRuleDto dto = this.db.activeRuleDao().getById(rule.getParentId(), session); - session.close(); - indexField(ActiveRuleField.PARENT_KEY.key(), dto.getKey().toString(), document); - } - - - /* Done normalizing for Rule */ - document.endObject(); - document.endObject(); + Map upsert = new HashMap(newRule); + upsert.put(ActiveRuleField.PARAMS.key(), new ArrayList()); /* Creating updateRequest */ - UpdateRequest request = new UpdateRequest() - .doc(document); - request.docAsUpsert(true); - return request; - } catch (Exception e) { - throw new IllegalStateException(String.format("Could not normalize Object with key %s", rule.getKey().toString()), e); - } + return new UpdateRequest() + .id(rule.getKey().toString()) + .parent(rule.getKey().ruleKey().toString()) + .doc(newRule) + .upsert(upsert); } } diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleDoc.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleDoc.java index c750cdb2eb9..7f2ba41279f 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleDoc.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleDoc.java @@ -115,8 +115,8 @@ class RuleDoc implements Rule { public List params() { List params = new ArrayList(); if (this.fields.get(RuleField.PARAMS.key()) != null) { - Map> esParams = (Map>) this.fields.get(RuleField.PARAMS.key()); - for (final Map param : esParams.values()) { + List> esParams = (List>) this.fields.get(RuleField.PARAMS.key()); + for (final Map param : esParams) { params.add(new RuleParam() { { this.fields = param; diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java index 1c6b4858326..cb059014710 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleIndex.java @@ -66,7 +66,6 @@ public class RuleIndex extends BaseIndex { RuleNormalizer.RuleField.CREATED_AT.key(), RuleNormalizer.RuleField.REPOSITORY.key(), RuleNormalizer.RuleField.PARAMS.key(), - RuleNormalizer.RuleField.ACTIVE.key(), RuleNormalizer.RuleField.TEMPLATE.key(), RuleNormalizer.RuleField.INTERNAL_KEY.key(), RuleNormalizer.RuleField.UPDATED_AT.key(), @@ -89,7 +88,7 @@ public class RuleIndex extends BaseIndex { return jsonBuilder().startObject() .startObject("index") .field("number_of_replicas", 0) - .field("number_of_shards", 3) + .field("number_of_shards", 1) .startObject("mapper") .field("dynamic", true) .endObject() @@ -179,11 +178,6 @@ public class RuleIndex extends BaseIndex { .endObject() .endObject(); - mapping.startObject(RuleNormalizer.RuleField.ACTIVE.key()) - .field("type", "nested") - .field("dynamic", true) - .endObject(); - mapping.startObject(RuleNormalizer.RuleField.PARAMS.key()) .field("type", "nested") .field("dynamic", true) @@ -223,19 +217,24 @@ public class RuleIndex extends BaseIndex { esSearch.setSize(options.getLimit()); /* integrate Option's Fields */ + Set fields = new HashSet(); if (options.getFieldsToReturn() != null && !options.getFieldsToReturn().isEmpty()) { for (String field : options.getFieldsToReturn()) { - esSearch.addField(field); + fields.add(field); } } else { for (RuleNormalizer.RuleField field : RuleNormalizer.RuleField.values()) { - esSearch.addField(field.key()); + fields.add(field.key()); } } //Add required fields: - esSearch.addField(RuleNormalizer.RuleField.KEY.key()); - esSearch.addField(RuleNormalizer.RuleField.REPOSITORY.key()); + fields.add(RuleNormalizer.RuleField.KEY.key()); + fields.add(RuleNormalizer.RuleField.REPOSITORY.key()); + + //TODO limit source for available fields. + //esSearch.addFields(fields.toArray(new String[fields.size()])); + //esSearch.setSource(StringUtils.join(fields,',')); return esSearch; } @@ -320,6 +319,8 @@ public class RuleIndex extends BaseIndex { esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); + System.out.println("esSearch = " + esSearch); + SearchResponse esResult = esSearch.get(); return new RuleResult(esResult); diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java index 00d54321baf..b0d1639ae01 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleNormalizer.java @@ -20,7 +20,6 @@ package org.sonar.server.rule2.index; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.sonar.api.rule.RuleKey; import org.sonar.check.Cardinality; import org.sonar.core.persistence.DbSession; @@ -29,7 +28,9 @@ import org.sonar.core.rule.RuleParamDto; import org.sonar.server.db.DbClient; import org.sonar.server.search.BaseNormalizer; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; public class RuleNormalizer extends BaseNormalizer { @@ -48,7 +49,6 @@ public class RuleNormalizer extends BaseNormalizer { TEMPLATE("template"), UPDATED_AT("updatedAt"), PARAMS("params"), - ACTIVE("active"), DEBT_FUNCTION_TYPE("debtFunction"), DEBT_FUNCTION_COEFFICIENT("debtCoefficient"), DEBT_FUNCTION_OFFSET("debtOffset"), @@ -108,86 +108,74 @@ public class RuleNormalizer extends BaseNormalizer { @Override public UpdateRequest normalize(RuleDto rule) { - try { - XContentBuilder document = jsonBuilder().startObject(); - indexField(RuleField.KEY.key(), rule.getKey(), document); - indexField(RuleField.REPOSITORY.key(), rule.getRepositoryKey(), document); - indexField(RuleField.NAME.key(), rule.getName(), document); - indexField(RuleField.CREATED_AT.key(), rule.getCreatedAt(), document); - indexField(RuleField.UPDATED_AT.key(), rule.getUpdatedAt(), document); - indexField(RuleField.HTML_DESCRIPTION.key(), rule.getDescription(), document); - indexField(RuleField.SEVERITY.key(), rule.getSeverityString(), document); - indexField(RuleField.STATUS.key(), rule.getStatus(), document); - indexField(RuleField.LANGUAGE.key(), rule.getLanguage(), document); - indexField(RuleField.INTERNAL_KEY.key(), rule.getConfigKey(), document); - indexField(RuleField.TEMPLATE.key(), rule.getCardinality() == Cardinality.MULTIPLE, document); - - //TODO Change on key when available - String subChar = null; - if(rule.getDefaultSubCharacteristicId() != null){ - subChar = rule.getDefaultSubCharacteristicId().toString(); - } - if(rule.getSubCharacteristicId() != null){ - subChar = rule.getSubCharacteristicId().toString(); - } - indexField(RuleField.SUB_CHARACTERISTIC.key(),subChar,document); - - String dType = null, dCoefficient = null, dOffset = null; - if(rule.getDefaultRemediationFunction() != null){ - dType = rule.getDefaultRemediationFunction(); - dCoefficient = rule.getDefaultRemediationCoefficient(); - dOffset= rule.getDefaultRemediationOffset(); - } - if(rule.getRemediationFunction() != null){ - dType = rule.getRemediationFunction(); - dCoefficient = rule.getRemediationCoefficient(); - dOffset= rule.getRemediationOffset(); - } - indexField(RuleField.DEBT_FUNCTION_TYPE.key(), dType, document); - indexField(RuleField.DEBT_FUNCTION_COEFFICIENT.key(), dCoefficient, document); - indexField(RuleField.DEBT_FUNCTION_OFFSET.key(), dOffset, document); - - - - - document.array(RuleField.TAGS.key(), rule.getTags().toArray(new String[rule.getTags().size()])); - document.array(RuleField.SYSTEM_TAGS.key(), rule.getSystemTags().toArray(new String[rule.getSystemTags().size()])); - document.startObject(RuleField.PARAMS.key()).endObject(); - document.startObject(RuleField.ACTIVE.key()).endObject(); - - /* Done normalizing for Rule */ - document.endObject(); - /* Creating updateRequest */ - UpdateRequest request = new UpdateRequest().doc(document); - request.docAsUpsert(true); - return request; - } catch (Exception e) { - throw new IllegalStateException(String.format("Could not normalize RuleDto with key %s", rule.getKey().toString()), e); + Map update = new HashMap(); + update.put(RuleField.KEY.key(), rule.getKey().toString()); + update.put(RuleField.REPOSITORY.key(), rule.getRepositoryKey()); + update.put(RuleField.NAME.key(), rule.getName()); + update.put(RuleField.CREATED_AT.key(), rule.getCreatedAt()); + update.put(RuleField.UPDATED_AT.key(), rule.getUpdatedAt()); + update.put(RuleField.HTML_DESCRIPTION.key(), rule.getDescription()); + update.put(RuleField.SEVERITY.key(), rule.getSeverityString()); + update.put(RuleField.STATUS.key(), rule.getStatus()); + update.put(RuleField.LANGUAGE.key(), rule.getLanguage()); + update.put(RuleField.INTERNAL_KEY.key(), rule.getConfigKey()); + update.put(RuleField.TEMPLATE.key(), rule.getCardinality() == Cardinality.MULTIPLE); + + //TODO Change on key when available + String subChar = null; + if (rule.getDefaultSubCharacteristicId() != null) { + subChar = rule.getDefaultSubCharacteristicId().toString(); + } + if (rule.getSubCharacteristicId() != null) { + subChar = rule.getSubCharacteristicId().toString(); + } + update.put(RuleField.SUB_CHARACTERISTIC.key(), subChar); + + String dType = null, dCoefficient = null, dOffset = null; + if (rule.getDefaultRemediationFunction() != null) { + dType = rule.getDefaultRemediationFunction(); + dCoefficient = rule.getDefaultRemediationCoefficient(); + dOffset = rule.getDefaultRemediationOffset(); } + if (rule.getRemediationFunction() != null) { + dType = rule.getRemediationFunction(); + dCoefficient = rule.getRemediationCoefficient(); + dOffset = rule.getRemediationOffset(); + } + update.put(RuleField.DEBT_FUNCTION_TYPE.key(), dType); + update.put(RuleField.DEBT_FUNCTION_COEFFICIENT.key(), dCoefficient); + update.put(RuleField.DEBT_FUNCTION_OFFSET.key(), dOffset); + + update.put(RuleField.TAGS.key(), rule.getTags()); + update.put(RuleField.SYSTEM_TAGS.key(), rule.getSystemTags()); + + + + /* Upsert elements */ + Map upsert = new HashMap(update); + upsert.put(RuleField.KEY.key(), rule.getKey().toString()); + upsert.put(RuleField.PARAMS.key(), new ArrayList()); + + + /* Creating updateRequest */ + return new UpdateRequest() + .doc(update) + .upsert(upsert); + } public UpdateRequest normalize(RuleParamDto param, RuleKey key) { - try { - /* Normalize the params */ - XContentBuilder document = jsonBuilder().startObject(); - document.startObject(RuleField.PARAMS.key()); - document.startObject(param.getName()); - indexField(RuleParamField.NAME.key(), param.getName(), document); - indexField(RuleParamField.TYPE.key(), param.getType(), document); - indexField(RuleParamField.DESCRIPTION.key(), param.getDescription(), document); - indexField(RuleParamField.DEFAULT_VALUE.key(), param.getDefaultValue(), document); - document.endObject(); - document.endObject(); - /* Creating updateRequest */ - UpdateRequest request = new UpdateRequest().doc(document); - request.docAsUpsert(true); - return request; - } catch (Exception e) { - throw new IllegalStateException(String.format("Could not normalize Object (%s) for key %s", - param.getClass().getSimpleName(), key.toString()), e); - } + Map newParam = new HashMap(); + newParam.put("_id", param.getName()); + newParam.put(RuleParamField.NAME.key(), param.getName()); + newParam.put(RuleParamField.TYPE.key(), param.getType()); + newParam.put(RuleParamField.DESCRIPTION.key(), param.getDescription()); + newParam.put(RuleParamField.DEFAULT_VALUE.key(), param.getDefaultValue()); + + return this.nestedUpsert(RuleField.PARAMS.key(), + param.getName(), newParam).id(key.toString()); } } diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleResult.java b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleResult.java index baf1ffbab7c..35dddc9f6bf 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleResult.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/index/RuleResult.java @@ -23,7 +23,6 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHit; -import org.sonar.core.qualityprofile.db.ActiveRuleKey; import org.sonar.server.qualityprofile.index.ActiveRuleDoc; import org.sonar.server.rule2.Rule; import org.sonar.server.search.Result; @@ -39,15 +38,16 @@ public class RuleResult extends Result { super(response); for (SearchHit hit : response.getHits()) { - String ruleKey = hit.getFields().get(RuleNormalizer.RuleField.KEY.key()).getValue(); - if (hit.getFields().containsKey(RuleNormalizer.RuleField.ACTIVE.key())) { - Map> activeRulesForHit = - hit.getFields().get(RuleNormalizer.RuleField.ACTIVE.key()).getValue(); - for (Map.Entry> activeRule : activeRulesForHit.entrySet()) { - activeRules.put(ruleKey, - new ActiveRuleDoc(ActiveRuleKey.parse(activeRule.getKey()), activeRule.getValue())); - } - } + + String ruleKey = (String) hit.getSource().get(RuleNormalizer.RuleField.KEY.key()); +// if (hit.getFields().containsKey(RuleNormalizer.RuleField.ACTIVE.key())) { +// Map> activeRulesForHit = +// hit.getFields().get(RuleNormalizer.RuleField.ACTIVE.key()).getValue(); +// for (Map.Entry> activeRule : activeRulesForHit.entrySet()) { +// activeRules.put(ruleKey, +// new ActiveRuleDoc(ActiveRuleKey.parse(activeRule.getKey()), activeRule.getValue())); +// } +// } } } diff --git a/sonar-server/src/main/java/org/sonar/server/rule2/ws/SearchAction.java b/sonar-server/src/main/java/org/sonar/server/rule2/ws/SearchAction.java index d1709494a13..9374bfce3ba 100644 --- a/sonar-server/src/main/java/org/sonar/server/rule2/ws/SearchAction.java +++ b/sonar-server/src/main/java/org/sonar/server/rule2/ws/SearchAction.java @@ -256,7 +256,6 @@ public class SearchAction implements RequestHandler { json.endArray(); /** ActiveRules */ - System.out.println("rule.key() = " + rule.key()); json.name("actives").beginArray(); for (ActiveRule activeRule : result.getActiveRules().get(rule.key().toString())) { json diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index 15176dbf1a9..ed28ad4a5f8 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -98,18 +98,25 @@ public abstract class BaseIndex, K extends Serializable> IndicesExistsResponse indexExistsResponse = getClient().admin().indices() .prepareExists(index).execute().actionGet(); + try { - if (!indexExistsResponse.isExists()) { - - try { - LOG.info("Setup of index {}", this.getIndexName()); + if (!indexExistsResponse.isExists()) { + LOG.info("Setup of {} for type {}", this.getIndexName(), this.getIndexType()); getClient().admin().indices().prepareCreate(index) .setSettings(getIndexSettings()) - .addMapping(this.indexDefinition.getIndexType(), getMapping()) + .addMapping(getIndexType(), getMapping()) + .execute().actionGet(); + + } else { + LOG.info("Update of index {} for type {}", this.getIndexName(), this.getIndexType()); + getClient().admin().indices().preparePutMapping(index) + .setType(getIndexType()) + .setIgnoreConflicts(true) + .setSource(getMapping()) .execute().actionGet(); - } catch (Exception e) { - throw new RuntimeException("Invalid configuration for index " + this.getIndexName(), e); } + } catch (Exception e) { + throw new RuntimeException("Invalid configuration for index " + this.getIndexName(), e); } } @@ -196,6 +203,7 @@ public abstract class BaseIndex, K extends Serializable> .index(this.getIndexName()) .id(this.getKeyValue(key)) .type(this.getIndexType())).get(); + } diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java b/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java index fa30e98922d..7a2bf15e904 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java @@ -28,6 +28,7 @@ import org.sonar.server.db.DbClient; import java.io.IOException; import java.io.Serializable; +import java.util.Map; public abstract class BaseNormalizer, K extends Serializable> { @@ -72,4 +73,16 @@ public abstract class BaseNormalizer, K extends Serializable> { LOG.error("Could not set {} to {} in ESDocument", field, value); } } + + + protected UpdateRequest nestedUpsert(String field, String key, Map item) { + return new UpdateRequest() + .script("if(ctx._source.containsKey(\""+field+"\")){for (int i = 0; i < ctx._source." + field + ".size(); i++){" + + "if(ctx._source." + field + "[i]._id == update_id){ ctx._source." + field + "[i] = update_doc; update_done = true;}}" + + "if(!update_done){ ctx._source." + field + " += update_doc; }\n} else {ctx._source." + field + " = [update_doc];}") + .addScriptParam("update_id", key) + .addScriptParam("update_doc", item) + .addScriptParam("update_done", false); + + } } diff --git a/sonar-server/src/main/java/org/sonar/server/search/NestedIndex.java b/sonar-server/src/main/java/org/sonar/server/search/NestedIndex.java index 42c5f2a6979..47c20d8c005 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/NestedIndex.java +++ b/sonar-server/src/main/java/org/sonar/server/search/NestedIndex.java @@ -53,11 +53,10 @@ public abstract class NestedIndex, K extends Serializable> protected abstract String getIndexField(); - protected String getKeyValue(K key) { - return this.getParentKeyValue(key); - } - protected void initializeIndex() { + @Override + protected String getKeyValue(K key) { + return key.toString(); } @Override @@ -71,8 +70,8 @@ public abstract class NestedIndex, K extends Serializable> LOG.debug("UPDATE _id:{} in index {}", key, this.getIndexName()); getClient().update(request .index(this.getIndexName()) - .id(this.getKeyValue(key)) - .type(this.getParentIndexType())).get(); + .id(this.getParentKeyValue(key)) + .type("rules2")).get(); } } diff --git a/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java index 7990aa17b05..d44ee21738a 100644 --- a/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/rule2/RuleServiceMediumTest.java @@ -20,6 +20,7 @@ package org.sonar.server.rule2; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import org.junit.After; import org.junit.Before; @@ -28,11 +29,13 @@ import org.junit.Test; import org.sonar.api.rule.RuleKey; import org.sonar.api.rule.RuleStatus; import org.sonar.api.rule.Severity; +import org.sonar.api.server.rule.RuleParamType; import org.sonar.check.Cardinality; import org.sonar.core.permission.GlobalPermissions; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.MyBatis; import org.sonar.core.rule.RuleDto; +import org.sonar.core.rule.RuleParamDto; import org.sonar.server.exceptions.ForbiddenException; import org.sonar.server.exceptions.NotFoundException; import org.sonar.server.rule2.index.RuleIndex; @@ -41,6 +44,7 @@ import org.sonar.server.tester.ServerTester; import org.sonar.server.user.MockUserSession; import java.util.Collections; +import java.util.List; import java.util.Set; import static org.fest.assertions.Assertions.assertThat; @@ -49,8 +53,10 @@ import static org.fest.assertions.Fail.fail; public class RuleServiceMediumTest { @ClassRule - public static ServerTester tester = new ServerTester(); + public static ServerTester tester = new ServerTester() + .setProperty("sonar.es.http.port","9200"); + MyBatis myBatis = tester.get(MyBatis.class); RuleDao dao = tester.get(RuleDao.class); RuleIndex index = tester.get(RuleIndex.class); RuleService service = tester.get(RuleService.class); @@ -59,7 +65,7 @@ public class RuleServiceMediumTest { @Before public void before() { tester.clearDataStores(); - dbSession = tester.get(MyBatis.class).openSession(false); + dbSession = myBatis.openSession(false); } @After @@ -67,6 +73,175 @@ public class RuleServiceMediumTest { dbSession.close(); } + @Test + public void insert_in_db_and_index_in_es() throws InterruptedException { + // insert db + RuleKey ruleKey = RuleKey.of("javascript", "S001"); + dao.insert(newRuleDto(ruleKey), dbSession); + dbSession.commit(); + + // verify that rule is persisted in db + RuleDto persistedDto = dao.getByKey(ruleKey, dbSession); + assertThat(persistedDto).isNotNull(); + assertThat(persistedDto.getId()).isGreaterThanOrEqualTo(0); + assertThat(persistedDto.getRuleKey()).isEqualTo(ruleKey.rule()); + assertThat(persistedDto.getLanguage()).isEqualTo("js"); + assertThat(persistedDto.getTags()).containsOnly("tag1", "tag2"); + assertThat(persistedDto.getSystemTags()).containsOnly("systag1", "systag2"); + assertThat(persistedDto.getCreatedAt()).isNotNull(); + assertThat(persistedDto.getUpdatedAt()).isNotNull(); + + // verify that rule is indexed in es + index.refresh(); + Rule hit = index.getByKey(ruleKey); + assertThat(hit).isNotNull(); + assertThat(hit.key().repository()).isEqualTo(ruleKey.repository()); + assertThat(hit.key().rule()).isEqualTo(ruleKey.rule()); + assertThat(hit.language()).isEqualTo("js"); + assertThat(hit.name()).isEqualTo("Rule S001"); + assertThat(hit.htmlDescription()).isEqualTo("Description S001"); + assertThat(hit.status()).isEqualTo(RuleStatus.READY); + assertThat(hit.createdAt()).isNotNull(); + assertThat(hit.updatedAt()).isNotNull(); + assertThat(hit.internalKey()).isEqualTo("InternalKeyS001"); + assertThat(hit.severity()).isEqualTo("INFO"); + assertThat(hit.template()).isFalse(); + assertThat(hit.tags()).containsOnly("tag1", "tag2"); + assertThat(hit.systemTags()).containsOnly("systag1", "systag2"); + } + + @Test + public void insert_and_index_rule_parameters() throws InterruptedException { + // insert db + RuleKey ruleKey = RuleKey.of("javascript", "S001"); + RuleDto ruleDto = newRuleDto(ruleKey); + dao.insert(ruleDto, dbSession); + dbSession.commit(); + + RuleParamDto minParamDto = new RuleParamDto() + .setName("min") + .setType(RuleParamType.INTEGER.type()) + .setDefaultValue("2") + .setDescription("Minimum"); + dao.addRuleParam(ruleDto, minParamDto, dbSession); + RuleParamDto maxParamDto = new RuleParamDto() + .setName("max") + .setType(RuleParamType.INTEGER.type()) + .setDefaultValue("10") + .setDescription("Maximum"); + dao.addRuleParam(ruleDto, maxParamDto, dbSession); + dbSession.commit(); + + //Verify that RuleDto has date from insertion + RuleDto theRule = dao.getByKey(ruleKey, dbSession); + assertThat(theRule.getCreatedAt()).isNotNull(); + assertThat(theRule.getUpdatedAt()).isNotNull(); + + // verify that parameters are persisted in db + List persistedDtos = dao.findRuleParamsByRuleKey(theRule.getKey(), dbSession); + assertThat(persistedDtos).hasSize(2); + + // verify that parameters are indexed in es + index.refresh(); + + Rule hit = index.getByKey(ruleKey); + assertThat(hit).isNotNull(); + assertThat(hit.key()).isNotNull(); + + RuleService service = tester.get(RuleService.class); + Rule rule = service.getByKey(ruleKey); + + assertThat(rule.params()).hasSize(2); + assertThat(Iterables.getLast(rule.params(), null).key()).isEqualTo("max"); + } + + @Test + public void insert_and_update_rule() { + + // insert db + RuleKey ruleKey = RuleKey.of("javascript", "S001"); + RuleDto ruleDto = newRuleDto(ruleKey) + .setTags(ImmutableSet.of("hello")) + .setName("first name"); + dao.insert(ruleDto, dbSession); + dbSession.commit(); + + // verify that parameters are indexed in es + index.refresh(); + Rule hit = index.getByKey(ruleKey); + assertThat(hit.tags()).containsExactly("hello"); + assertThat(hit.name()).isEqualTo("first name"); + + //Update in DB + ruleDto.setTags(ImmutableSet.of("world")) + .setName("second name"); + dao.update(ruleDto, dbSession); + dbSession.commit(); + + // verify that parameters are updated in es + index.refresh(); + hit = index.getByKey(ruleKey); + assertThat(hit.tags()).containsExactly("world"); + assertThat(hit.name()).isEqualTo("second name"); + } + + @Test + public void insert_and_update_rule_param() throws InterruptedException { + + // insert db + RuleKey ruleKey = RuleKey.of("javascript", "S001"); + RuleDto ruleDto = newRuleDto(ruleKey) + .setTags(ImmutableSet.of("hello")) + .setName("first name"); + dao.insert(ruleDto, dbSession); + dbSession.commit(); + + RuleParamDto minParamDto = new RuleParamDto() + .setName("min") + .setType(RuleParamType.INTEGER.type()) + .setDefaultValue("2") + .setDescription("Minimum"); + dao.addRuleParam(ruleDto, minParamDto, dbSession); + + RuleParamDto maxParamDto = new RuleParamDto() + .setName("max") + .setType(RuleParamType.INTEGER.type()) + .setDefaultValue("10") + .setDescription("Maximum"); + dao.addRuleParam(ruleDto, maxParamDto, dbSession); + dbSession.commit(); + + // verify that parameters are indexed in es + index.refresh(); + + Rule hit = index.getByKey(ruleKey); + assertThat(hit.params()).hasSize(2); + + RuleParam param = hit.params().get(0); + assertThat(param.key()).isEqualTo("min"); + assertThat(param.defaultValue()).isEqualTo("2"); + assertThat(param.description()).isEqualTo("Minimum"); + + + //Update in DB + minParamDto + .setDefaultValue("0.5") + .setDescription("new description"); + dao.updateRuleParam(ruleDto, minParamDto, dbSession); + dbSession.commit(); + + // verify that parameters are updated in es + index.refresh(); + + hit = index.getByKey(ruleKey); + assertThat(hit.params()).hasSize(2); + + param = hit.params().get(0); + assertThat(param.key()).isEqualTo("min"); + assertThat(param.defaultValue()).isEqualTo("0.5"); + assertThat(param.description()).isEqualTo("new description"); + } + @Test public void setTags() throws InterruptedException { MockUserSession.set().setGlobalPermissions(GlobalPermissions.QUALITY_PROFILE_ADMIN); @@ -76,7 +251,8 @@ public class RuleServiceMediumTest { dao.insert(newRuleDto(rule1) .setTags(Sets.newHashSet("security")) .setSystemTags(Collections.emptySet()), - dbSession); + dbSession + ); RuleKey rule2 = RuleKey.of("java", "S001"); dao.insert(newRuleDto(rule2) @@ -101,7 +277,7 @@ public class RuleServiceMediumTest { service.setTags(RuleKey.of("java", "S001"), Sets.newHashSet("bug", "security")); fail(); } catch (NotFoundException e) { - assertThat(e).hasMessage("Key 'java:S001' not found"); + assertThat(e).hasMessage("Rule java:S001 not found"); } } @@ -116,50 +292,6 @@ public class RuleServiceMediumTest { } } - @Test - public void setNote() throws Exception { - MockUserSession.set().setGlobalPermissions(GlobalPermissions.QUALITY_PROFILE_ADMIN).setLogin("marius"); - RuleKey ruleKey = RuleKey.of("javascript", "S001"); - dao.insert(newRuleDto(ruleKey), dbSession); - dbSession.commit(); - - // 1. CREATE NOTE - service.setNote(ruleKey, "my *note*"); - - // verify db - RuleDto dto = dao.getNonNullByKey(ruleKey, dbSession); - assertThat(dto.getNoteData()).isEqualTo("my *note*"); - assertThat(dto.getNoteCreatedAt()).isNotNull(); - assertThat(dto.getNoteUpdatedAt()).isNotNull(); - assertThat(dto.getNoteUserLogin()).isEqualTo("marius"); - - // verify es - index.refresh(); - Rule rule = index.getByKey(ruleKey); - // TODO -// assertThat(rule.getNote()).isEqualTo("my *note*"); -// assertThat(rule.getNoteCreatedAt()).isNotNull(); -// assertThat(rule.getNoteUpdatedAt()).isNotNull(); -// assertThat(rule.getNoteUserLogin()).isEqualTo("marius"); - - // 2. DELETE NOTE - service.setNote(ruleKey, null); - dbSession.clearCache(); - dto = dao.getNonNullByKey(ruleKey, dbSession); - assertThat(dto.getNoteData()).isNull(); - assertThat(dto.getNoteCreatedAt()).isNull(); - assertThat(dto.getNoteUpdatedAt()).isNull(); - assertThat(dto.getNoteUserLogin()).isNull(); - index.refresh(); - rule = index.getByKey(ruleKey); - // TODO - // assertThat(rule.getNote()).isNull(); -// assertThat(rule.getNoteCreatedAt()).isNull(); -// assertThat(rule.getNoteUpdatedAt()).isNull(); -// assertThat(rule.getNoteUserLogin()).isNull(); - - } - private RuleDto newRuleDto(RuleKey ruleKey) { return new RuleDto() .setRuleKey(ruleKey.rule()) diff --git a/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java index ae7174fe0d5..f1752111069 100644 --- a/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/rule2/index/RuleIndexMediumTest.java @@ -46,7 +46,8 @@ import static org.fest.assertions.Assertions.assertThat; public class RuleIndexMediumTest { @ClassRule - public static ServerTester tester = new ServerTester(); + public static ServerTester tester = new ServerTester() + .setProperty("sonar.es.http.port","9200"); MyBatis myBatis = tester.get(MyBatis.class); RuleDao dao = tester.get(RuleDao.class); @@ -198,7 +199,7 @@ public class RuleIndexMediumTest { } @Test - public void search_all_rules() { + public void search_all_rules() throws InterruptedException { dao.insert(newRuleDto(RuleKey.of("javascript", "S001")), dbSession); dao.insert(newRuleDto(RuleKey.of("java", "S002")), dbSession); dbSession.commit(); @@ -331,12 +332,13 @@ public class RuleIndexMediumTest { } @Test - public void sort_by_language() { + public void sort_by_language() throws InterruptedException { dao.insert(newRuleDto(RuleKey.of("java", "S001")).setLanguage("java"), dbSession); dao.insert(newRuleDto(RuleKey.of("java", "S002")).setLanguage("php"), dbSession); dbSession.commit(); index.refresh(); + Thread.sleep(100000000); // ascending RuleQuery query = new RuleQuery().setSortField(RuleQuery.SortField.LANGUAGE); Result results = index.search(query, new QueryOptions()); @@ -351,7 +353,7 @@ public class RuleIndexMediumTest { } @Test - public void search_by_tag() { + public void search_by_tag() throws InterruptedException { dao.insert(newRuleDto(RuleKey.of("java", "S001")).setTags(ImmutableSet.of("tag1")), dbSession); dao.insert(newRuleDto(RuleKey.of("java", "S002")).setTags(ImmutableSet.of("tag2")), dbSession); dbSession.commit(); diff --git a/sonar-server/src/test/java/org/sonar/server/rule2/ws/RulesWebServiceTest.java b/sonar-server/src/test/java/org/sonar/server/rule2/ws/RulesWebServiceTest.java index 81991b3f4ce..cf99501bb04 100644 --- a/sonar-server/src/test/java/org/sonar/server/rule2/ws/RulesWebServiceTest.java +++ b/sonar-server/src/test/java/org/sonar/server/rule2/ws/RulesWebServiceTest.java @@ -49,7 +49,8 @@ import static org.fest.assertions.Assertions.assertThat; public class RulesWebServiceTest { @ClassRule - public static ServerTester tester = new ServerTester(); + public static ServerTester tester = new ServerTester() + .setProperty("sonar.es.http.port","9200"); private RulesWebService ws; @@ -158,6 +159,7 @@ public class RulesWebServiceTest { request.setParam("q","S001"); WsTester.Result result = request.execute(); + Thread.sleep(1000000); result.assertJson(this.getClass(),"search_active_rules.json"); } @@ -169,6 +171,9 @@ public class RulesWebServiceTest { RuleDto rule = newRuleDto(RuleKey.of(profile.getLanguage(), "S001")); ruleDao.insert(rule, session); + session.commit(); + + RuleParamDto param = RuleParamDto.createFor(rule) .setDefaultValue("some value") .setType("string") @@ -176,17 +181,23 @@ public class RulesWebServiceTest { .setName("my_var"); ruleDao.addRuleParam(rule,param, session); + RuleParamDto param2 = RuleParamDto.createFor(rule) + .setDefaultValue("other value") + .setType("integer") + .setDescription("My small description") + .setName("the_var"); + ruleDao.addRuleParam(rule,param2, session); + ActiveRuleDto activeRule = newActiveRule(profile, rule); tester.get(ActiveRuleDao.class).insert(activeRule, session); + ActiveRuleParamDto activeRuleParam = ActiveRuleParamDto.createFor(param) .setValue("The VALUE"); tester.get(ActiveRuleDao.class).addParam(activeRule, activeRuleParam, session); - session.commit(); tester.get(RuleService.class).refresh(); - MockUserSession.set(); WsTester.TestRequest request = wsTester.newGetRequest("api/rules2", "search"); request.setParam("q", "S001"); -- 2.39.5