From 66a65885e799cf83216d2112ab56c654791a5d81 Mon Sep 17 00:00:00 2001 From: Stephane Gamard Date: Wed, 4 Jun 2014 11:14:42 +0200 Subject: [PATCH] SONAR-5007 - Switched to native scripts for ListUpdate --- .../index/ActiveRuleNormalizer.java | 10 +- .../server/rule/index/RuleNormalizer.java | 18 +++- .../sonar/server/search/BaseNormalizer.java | 13 --- .../java/org/sonar/server/search/ESNode.java | 6 +- .../sonar/server/search/es/ListUpdate.java | 99 +++++++++++++++++++ .../rule/index/RuleIndexMediumTest.java | 21 ++-- 6 files changed, 138 insertions(+), 29 deletions(-) create mode 100644 sonar-server/src/main/java/org/sonar/server/search/es/ListUpdate.java diff --git a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleNormalizer.java b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleNormalizer.java index e24a4f49e09..8058e8813ea 100644 --- a/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleNormalizer.java @@ -32,6 +32,7 @@ import org.sonar.server.search.BaseNormalizer; import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.IndexField; import org.sonar.server.search.Indexable; +import org.sonar.server.search.es.ListUpdate; import java.lang.reflect.Field; import java.util.ArrayList; @@ -121,8 +122,13 @@ public class ActiveRuleNormalizer extends BaseNormalizer { @@ -230,6 +236,7 @@ public class RuleNormalizer extends BaseNormalizer { } public List normalize(RuleParamDto param, RuleKey key) { + Map newParam = new HashMap(); newParam.put("_id", param.getName()); newParam.put(RuleParamField.NAME.field(), param.getName()); @@ -237,7 +244,12 @@ public class RuleNormalizer extends BaseNormalizer { newParam.put(RuleParamField.DESCRIPTION.field(), param.getDescription()); newParam.put(RuleParamField.DEFAULT_VALUE.field(), param.getDefaultValue()); - return ImmutableList.of(this.nestedUpsert(RuleField.PARAMS.field(), - param.getName(), newParam).id(key.toString())); + return ImmutableList.of(new UpdateRequest() + .id(key.toString()) + .script(ListUpdate.NAME) + .addScriptParam(ListUpdate.FIELD, RuleField.PARAMS.field()) + .addScriptParam(ListUpdate.VALUE, newParam) + .addScriptParam(ListUpdate.ID, param.getName()) + ); } } diff --git a/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java b/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java index 7981e9f3563..3cbbf6716d3 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java +++ b/sonar-server/src/main/java/org/sonar/server/search/BaseNormalizer.java @@ -25,7 +25,6 @@ import org.sonar.server.db.DbClient; import java.io.Serializable; import java.util.List; -import java.util.Map; public abstract class BaseNormalizer, K extends Serializable> { @@ -58,16 +57,4 @@ public abstract class BaseNormalizer, K extends Serializable> { public abstract java.util.List normalize(K key); public abstract java.util.List normalize(E dto); - - protected UpdateRequest nestedUpsert(String field, String key, Map item) { - return new UpdateRequest() - .script("for (int i = 0; i < ctx._source." + field + ".size(); i++){" + - "if(ctx._source." + field + "[i]._id == update_id){" - + " ctx._source." + field + "[i] = update_doc; " - + " update_done = true;}} " - + "if(!update_done){ ctx._source." + field + " += update_doc; }") - .addScriptParam("update_id", key) - .addScriptParam("update_doc", item) - .addScriptParam("update_done", false); - } } diff --git a/sonar-server/src/main/java/org/sonar/server/search/ESNode.java b/sonar-server/src/main/java/org/sonar/server/search/ESNode.java index 157737511d9..cd7c4fb3fdd 100644 --- a/sonar-server/src/main/java/org/sonar/server/search/ESNode.java +++ b/sonar-server/src/main/java/org/sonar/server/search/ESNode.java @@ -36,6 +36,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonar.api.config.Settings; import org.sonar.api.platform.ServerFileSystem; +import org.sonar.server.search.es.ListUpdate; +import org.sonar.server.search.es.ListUpdate.UpdateListScriptFactory; import java.io.File; @@ -77,7 +79,9 @@ public class ESNode implements Startable { IndexProperties.ES_TYPE.valueOf(settings.getString(IndexProperties.TYPE)) : IndexProperties.ES_TYPE.DATA; - ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder(); + ImmutableSettings.Builder esSettings = ImmutableSettings.settingsBuilder() + .put("script.default_lang", "native") + .put("script.native." + ListUpdate.NAME + ".type", UpdateListScriptFactory.class.getName()); switch (type) { case MEMORY: diff --git a/sonar-server/src/main/java/org/sonar/server/search/es/ListUpdate.java b/sonar-server/src/main/java/org/sonar/server/search/es/ListUpdate.java new file mode 100644 index 00000000000..ddfa4983318 --- /dev/null +++ b/sonar-server/src/main/java/org/sonar/server/search/es/ListUpdate.java @@ -0,0 +1,99 @@ +package org.sonar.server.search.es; + +import com.google.common.collect.ImmutableSet; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.script.AbstractExecutableScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Collection; +import java.util.Map; + +/** + * @since 4.4 + */ +public class ListUpdate extends AbstractExecutableScript { + + public static final String NAME = "listUpdate"; + + public static final String ID = "id"; + public static final String FIELD = "field"; + public static final String VALUE = "value"; + + public static class UpdateListScriptFactory implements NativeScriptFactory { + @Override + public ExecutableScript newScript(@Nullable Map params) { + String id = XContentMapValues.nodeStringValue(params.get(ID), null); + String field = XContentMapValues.nodeStringValue(params.get(FIELD), null); + Map value = XContentMapValues.nodeMapValue(params.get(VALUE), "Update item"); + + if (id == null) { + throw new IllegalStateException("Missing '" + ID + "' parameter"); + } + if (field == null) { + throw new IllegalStateException("Missing '" + FIELD + "' parameter"); + } + if (value == null) { + throw new IllegalStateException("Missing '" + VALUE + "' parameter"); + } + + return new ListUpdate(id, field, value); + } + } + + + private final String id; + private final String field; + private final Map value; + + private Map ctx; + + public ListUpdate(String id, String field, Map value) { + this.id = id; + this.field = field; + this.value = value; + } + + @Override + public void setNextVar(String name, Object value) { + if (name.equals("ctx")) { + ctx = (Map) value; + } + } + + @Override + public Object unwrap(Object value) { + return value; + } + + @Override + public Object run() { + + //Get the Document's source from ctx + Map source = XContentMapValues.nodeMapValue(ctx.get("_source"), "source from context"); + + //Get the Object for list update + Object fieldValue = source.get(field); + + if (fieldValue == null) { + // 0. The field does not exist (this is a upsert then) + source.put(field, value); + } else if (!XContentMapValues.isArray(fieldValue)) { + // 1. The field is not yet a list + source.put(field, ImmutableSet.of(fieldValue, value)); + } else { + // 3. field is a list + Collection items = ((Collection) fieldValue); + for (Object item : items) { + String idValue = XContentMapValues.nodeStringValue(item, null); + if (idValue != null && idValue.equals(id)) { + items.remove(item); + } + } + items.add(value); + source.put(field, items); + } + return null; + } +} diff --git a/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java b/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java index 77940ab5bfd..4752c0fcd9b 100644 --- a/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java +++ b/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexMediumTest.java @@ -33,7 +33,6 @@ import org.sonar.api.server.debt.DebtRemediationFunction; import org.sonar.check.Cardinality; import org.sonar.core.persistence.DbSession; import org.sonar.core.qualityprofile.db.ActiveRuleDto; -import org.sonar.core.qualityprofile.db.ActiveRuleParamDto; import org.sonar.core.qualityprofile.db.QualityProfileDto; import org.sonar.core.rule.RuleDto; import org.sonar.core.rule.RuleParamDto; @@ -44,11 +43,17 @@ import org.sonar.server.qualityprofile.ActiveRule; import org.sonar.server.rule.Rule; import org.sonar.server.rule.db.RuleDao; import org.sonar.server.search.FacetValue; +import org.sonar.server.search.IndexProperties; import org.sonar.server.search.QueryOptions; import org.sonar.server.search.Result; import org.sonar.server.tester.ServerTester; -import java.util.*; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; import static org.fest.assertions.Assertions.assertThat; import static org.fest.assertions.Fail.fail; @@ -56,7 +61,8 @@ import static org.fest.assertions.Fail.fail; public class RuleIndexMediumTest { @ClassRule - public static ServerTester tester = new ServerTester(); + public static ServerTester tester = new ServerTester() + .setProperty(IndexProperties.HTTP_PORT, "9200"); RuleDao dao = tester.get(RuleDao.class); RuleIndex index = tester.get(RuleIndex.class); @@ -576,7 +582,7 @@ public class RuleIndexMediumTest { } @Test - public void complex_param_value() { + public void complex_param_value() throws InterruptedException { String value = "//expression[primary/qualifiedIdentifier[count(IDENTIFIER) = 2]/IDENTIFIER[2]/@tokenValue = 'firstOf' and primary/identifierSuffix/arguments/expression[not(primary) or primary[not(qualifiedIdentifier) or identifierSuffix]]]"; QualityProfileDto profile = QualityProfileDto.createFor("name", "Language"); @@ -591,14 +597,9 @@ public class RuleIndexMediumTest { .setDefaultValue(value); dao.addRuleParam(dbSession, rule, param); - ActiveRuleDto activeRule = ActiveRuleDto.createFor(profile, rule) - .setSeverity("BLOCKER"); - - ActiveRuleParamDto activeRuleParam = ActiveRuleParamDto.createFor(param); - db.activeRuleDao().insert(dbSession, activeRule); - db.activeRuleDao().addParam(dbSession, activeRule, activeRuleParam); dbSession.commit(); + assertThat(index.getByKey(rule.getKey()).params()).hasSize(1); assertThat(index.getByKey(rule.getKey()).params().get(0).defaultValue()).isEqualTo(value); } -- 2.39.5