import org.sonar.core.persistence.MyBatis;
import javax.annotation.CheckForNull;
+
import java.sql.Timestamp;
import java.util.Collection;
import java.util.List;
*/
package org.sonar.server.cluster;
+import org.jfree.util.Log;
+import org.picocontainer.Startable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.jfree.util.Log;
+import org.sonar.api.ServerComponent;
import org.sonar.core.cluster.IndexAction;
+import org.sonar.core.cluster.WorkQueue;
import org.sonar.server.search.Index;
import java.util.HashMap;
import java.util.Map;
-import org.picocontainer.Startable;
-import org.sonar.api.ServerComponent;
-import org.sonar.core.cluster.WorkQueue;
-
public class LocalQueueWorker implements ServerComponent, Startable {
private static final Logger LOG = LoggerFactory.getLogger(LocalNonBlockingWorkQueue.class);
Thread thisThread = Thread.currentThread();
while (worker == thisThread) {
try {
- Thread.sleep(200);
+ Thread.sleep(20);
} catch (InterruptedException e) {
Log.error("Oops");
}
IndexAction action = queue.dequeue();
if (action != null && indexes.containsKey(action.getIndexName())) {
+ LOG.info("Dequeued action {}",action);
indexes.get(action.getIndexName()).executeAction(action);
}
}
}
private void initRestConsole(ImmutableSettings.Builder esSettings) {
- int httpPort = settings.getInt("sonar.es.http.port");
+ int httpPort = 8888;//settings.getInt("sonar.es.http.port");
if (httpPort > 0) {
LOG.warn("Elasticsearch HTTP console enabled on port {}. Only for debugging purpose.", httpPort);
esSettings.put(HTTP_ENABLED, true);
*/
package org.sonar.server.rule2;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
+import org.apache.commons.beanutils.BeanUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonar.api.rule.RuleKey;
import org.sonar.core.profiling.Profiling;
import org.sonar.core.rule.RuleConstants;
import org.sonar.core.rule.RuleDao;
+import org.sonar.core.rule.RuleDto;
import org.sonar.server.es.ESNode;
import org.sonar.server.search.BaseIndex;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.util.Map;
+import java.util.Map.Entry;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-
-public class RuleIndex extends BaseIndex<RuleKey> {
+public class RuleIndex extends BaseIndex<RuleKey, RuleDto> {
private static final Logger LOG = LoggerFactory.getLogger(RuleIndex.class);
return RuleConstants.ES_TYPE;
}
- protected QueryBuilder getKeyQuery(RuleKey key){
- return QueryBuilders.boolQuery()
- .must(QueryBuilders.termQuery("repositoryKey", key.repository()))
- .must(QueryBuilders.termQuery("ruleKey", key.rule()));
+ protected String getKeyValue(RuleKey key) {
+ return key.toString();
}
@Override
- protected Settings getIndexSettings() {
+ protected XContentBuilder getIndexSettings() {
try {
- return ImmutableSettings.builder()
- .loadFromSource(
- jsonBuilder().startObject()
- .startObject("index")
- .field("number_of_replicas", 0)
- .field("number_of_shards",4)
- .startObject("mapper")
- .field("dynamic", true)
- .endObject()
- .startObject("analysis")
- .startObject("analyzer")
- .startObject("path_analyzer")
- .field("type", "custom")
- .field("tokenizer", "path_hierarchy")
- .endObject()
- .endObject()
- .endObject()
- .endObject().toString())
- .build();
+ return jsonBuilder().startObject()
+ .startObject("index")
+ .field("number_of_replicas", 0)
+ .field("number_of_shards", 3)
+ .startObject("mapper")
+ .field("dynamic", true)
+ .endObject()
+ .startObject("analysis")
+ .startObject("analyzer")
+ .startObject("path_analyzer")
+ .field("type", "custom")
+ .field("tokenizer", "path_hierarchy")
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject().endObject();
} catch (IOException e) {
- LOG.error("Could not create index settings for {}",this.getIndexName());
- return ImmutableSettings.builder().build();
+ LOG.error("Could not create index settings for {}", this.getIndexName());
+ return null;
}
}
protected XContentBuilder getMapping() {
try {
return jsonBuilder().startObject()
- .startObject(this.getType()).endObject()
- .endObject();
-
-// return jsonBuilder().startObject()
-// .startObject("issue")
-// .startObject("properties")
-// .startObject("component.path")
-// .field("type", "string")
-// .field("index_analyzer", "path_analyzer")
-// .field("search_analyzer", "keyword")
-// .endObject()
-// .startObject("rule.name")
-// .field("type", "string")
-// .field("analyzer", "keyword")
-// .endObject()
-// .startObject("root.id")
-// .field("type", "multi_field")
-// .startObject("fields")
-// .startObject("str")
-// .field("type", "string")
-// .field("index","analyzed")
-// .field("analyzer", "default")
-// .endObject()
-// .startObject("num")
-// .field("type", "long")
-// .field("index","analyzed")
-// .endObject()
-// .endObject()
-// .endObject()
-// .endObject().endObject();
+ .startObject(this.getType())
+ .field("dynamic",true)
+ .startObject("properties")
+ .startObject("id")
+ .field("type", "string")
+ .field("index", "not_analyzed")
+ .endObject()
+ .startObject("key")
+ .field("type", "string")
+ .field("index", "not_analyzed")
+ .endObject()
+ .startObject("repositoryKey")
+ .field("type", "string")
+ .field("index", "not_analyzed")
+ .endObject()
+ .startObject("severity")
+ .field("type", "string")
+ .field("index", "not_analyzed")
+ .endObject()
+ .endObject()
+ .endObject().endObject();
+
+ // return jsonBuilder().startObject()
+ // .startObject("issue")
+ // .startObject("properties")
+ // .startObject("component.path")
+ // .field("type", "string")
+ // .field("index_analyzer", "path_analyzer")
+ // .field("search_analyzer", "keyword")
+ // .endObject()
+ // .startObject("rule.name")
+ // .field("type", "string")
+ // .field("analyzer", "keyword")
+ // .endObject()
+ // .startObject("root.id")
+ // .field("type", "multi_field")
+ // .startObject("fields")
+ // .startObject("str")
+ // .field("type", "string")
+ // .field("index","analyzed")
+ // .field("analyzer", "default")
+ // .endObject()
+ // .startObject("num")
+ // .field("type", "long")
+ // .field("index","analyzed")
+ // .endObject()
+ // .endObject()
+ // .endObject()
+ // .endObject().endObject();
} catch (IOException e) {
- LOG.error("Could not create mapping for {}",this.getIndexName());
+ LOG.error("Could not create mapping for {}", this.getIndexName());
return null;
}
}
@Override
- public Map<String, Object> normalize(RuleKey key) {
- //Use a MyBatis to normalize the Rule form multiple Table
+ public XContentBuilder normalize(RuleKey key) {
+
+ RuleDto rule = dao.getByKey(key);
+
+ try {
+
+ XContentBuilder document = jsonBuilder().startObject();
+
+ Map<String, Object> properties = BeanUtils.describe(rule);
+
+ for (Entry<String, Object> property : properties.entrySet()) {
+ LOG.trace("NORMALIZING: {} -> {}",property.getKey(), property.getValue());
+ document.field(property.getKey(), property.getValue());
+ }
+
+ return document.endObject();
+ } catch (IOException e) {
+ LOG.error("Could not normalize {} in {}", key, this.getClass().getSimpleName());
+ e.printStackTrace();
+ } catch (IllegalAccessException e) {
+ LOG.error("Could not normalize {} in {}", key, this.getClass().getSimpleName());
+ e.printStackTrace();
+ } catch (InvocationTargetException e) {
+ LOG.error("Could not normalize {} in {}", key, this.getClass().getSimpleName());
+ e.printStackTrace();
+ } catch (NoSuchMethodException e) {
+ LOG.error("Could not normalize {} in {}", key, this.getClass().getSimpleName());
+ e.printStackTrace();
+ }
return null;
}
}
*/
package org.sonar.server.search;
-import org.sonar.server.es.ESNode;
-
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonar.core.cluster.IndexAction;
import org.sonar.core.cluster.IndexAction.Method;
import org.sonar.core.cluster.WorkQueue;
import org.sonar.core.db.Dao;
+import org.sonar.core.db.Dto;
import org.sonar.core.profiling.Profiling;
import org.sonar.core.profiling.Profiling.Level;
import org.sonar.core.profiling.StopWatch;
+import org.sonar.server.es.ESNode;
+import java.io.IOException;
import java.io.Serializable;
-public abstract class BaseIndex<K extends Serializable> implements Index<K> {
+public abstract class BaseIndex<K extends Serializable, E extends Dto<K>> implements Index<K> {
private static final String PROFILE_DOMAIN = "es";
+
private static final Logger LOG = LoggerFactory.getLogger(BaseIndex.class);
public static final String ES_CLUSTER_NAME = "sonarqube";
private final ESNode node;
private WorkQueue workQueue;
private IndexSynchronizer<K> synchronizer;
- protected Dao<?, K> dao;
+ protected Dao<E, K> dao;
- public BaseIndex(WorkQueue workQueue, Dao<?, K> dao, Profiling profiling, ESNode node) {
+ public BaseIndex(WorkQueue workQueue, Dao<E, K> dao, Profiling profiling, ESNode node) {
this.profiling = profiling;
this.workQueue = workQueue;
this.synchronizer = new IndexSynchronizer<K>(this, dao, this.workQueue);
if (!indexExistsResponse.isExists()) {
+ LOG.info("Setup of index {}",this.getIndexName());
+
+ try {
+ LOG.info("Settings: {}",getIndexSettings().string());
+ LOG.info("Mapping: {}", getMapping().string());
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
client.admin().indices().prepareCreate(index)
.setSettings(getIndexSettings())
.addMapping(getType(), getMapping())
/* Index management methods */
- protected abstract Settings getIndexSettings();
+ protected abstract XContentBuilder getIndexSettings();
protected abstract String getType();
/* Base CRUD methods */
- protected abstract QueryBuilder getKeyQuery(K key);
+ protected abstract String getKeyValue(K key);
@Override
public Hit getByKey(K key) {
- getClient().prepareSearch(this.getIndexName())
- .setQuery(getKeyQuery(key))
+ GetResponse result = getClient().prepareGet(this.getIndexName(), this.getType(), this.getKeyValue(key))
.get();
- return null;
+ return Hit.fromMap(0, result.getSourceAsMap());
}
@Override
@Override
public void update(K key) {
+ LOG.info("Update document with key {}", key);
IndexResponse result = getClient().index(new IndexRequest()
.type(this.getType())
.index(this.getIndexName())
+ .id(this.getKeyValue(key))
.source(this.normalize(key))).actionGet();
}
@Override
public void delete(K key) {
- DeleteByQueryResponse result = getClient().prepareDeleteByQuery(this.getIndexName())
- .setQuery(getKeyQuery(key)).get();
+ LOG.info("Deleting document with key {}", key);
+ DeleteResponse result = getClient()
+ .prepareDelete(this.getIndexName(), this.getType(), this.getKeyValue(key))
+ .get();
}
/* Synchronization methods */
*/
package org.sonar.server.search;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
public class Hit implements Comparable<Hit> {
- private Map<String, Collection<Serializable>> fields;
+ private Map<String, Object> fields;
private Integer rank;
public Hit(Integer rank){
- this.fields = new HashMap<String, Collection<Serializable>>();
+ this.fields = new HashMap<String, Object>();
this.rank = rank;
}
- public Collection<Serializable> getFieldValues(String field){
- return this.fields.get(field);
- }
-
- public Serializable getFieldValue(String field){
- if(this.hasField(field)){
- return fields.get(field).iterator().next();
- } else {
- return null;
- }
- }
-
- public Hit addFieldValue(String field, Serializable value){
- if(!this.hasField(field)){
- this.fields.put(field, new ArrayList<Serializable>());
- }
- return this;
- }
-
- public boolean hasField(String field){
- return this.fields.containsKey(field) &&
- !this.fields.get(field).isEmpty();
+ public Map<String, Object> getFields(){
+ return this.fields;
}
public Integer getRank(){
public int compareTo(Hit hit) {
return this.getRank().compareTo(hit.getRank());
}
+
+ public static Hit fromMap(Integer rank, Map<String, Object> fieldMap) {
+ Hit hit = new Hit(0);
+ hit.fields = fieldMap;
+ return hit;
+ }
}
*/
package org.sonar.server.search;
+import org.elasticsearch.common.xcontent.XContentBuilder;
import org.picocontainer.Startable;
import org.sonar.core.cluster.IndexAction;
import java.io.Serializable;
-import java.util.Map;
public interface Index<K extends Serializable> extends Startable {
void delete(K key);
- Map<String, Object> normalize(K key);
+ XContentBuilder normalize(K key);
Long getLastSynchronization();
package org.sonar.server.search;
import com.google.common.collect.ImmutableList;
+
import org.sonar.server.rule2.RuleIndex;
+import org.sonar.server.cluster.LocalQueueWorker;
import java.util.List;
@SuppressWarnings("unchecked")
public static List<Class> getIndexClasses() {
return ImmutableList.<Class>of(
- RuleIndex.class
+ RuleIndex.class,
+ LocalQueueWorker.class
);
}
}
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
-package org.sonar.server.rule2;
-
-import com.github.tlrx.elasticsearch.test.annotations.ElasticsearchNode;
-import com.github.tlrx.elasticsearch.test.support.junit.runners.ElasticsearchRunner;
-import org.elasticsearch.node.Node;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.sonar.api.config.Settings;
-import org.sonar.core.profiling.Profiling;
-import org.sonar.server.cluster.LocalNonBlockingWorkQueue;
-import org.sonar.server.search.BaseIndex;
-
-import static org.fest.assertions.Assertions.assertThat;
-
-@RunWith(ElasticsearchRunner.class)
-@Ignore("Same problem as with BaseIndex test")
-public class RuleIndexTest {
-
+//package org.sonar.server.rule2;
+//
+//import com.github.tlrx.elasticsearch.test.annotations.ElasticsearchNode;
+//import com.github.tlrx.elasticsearch.test.support.junit.runners.ElasticsearchRunner;
+//import org.elasticsearch.node.Node;
+//import org.junit.Before;
+//import org.junit.Ignore;
+//import org.junit.runner.RunWith;
+//import org.sonar.server.es.ESNode;
+//import org.sonar.server.search.BaseIndex;
+//
+//@RunWith(ElasticsearchRunner.class)
+//@Ignore("Same problem as with BaseIndex test")
+//public class RuleIndexTest {
+//
// private static final String TEST_NODE_NAME = "es_node_for_tests";
//
// @ElasticsearchNode(name = TEST_NODE_NAME,
// clusterName = BaseIndex.ES_CLUSTER_NAME,
-// local = false, data = true)
+// local = true, data = true)
// private Node node;
//
+// private ESNode esNode;
+//
// @Before
// public void setUp() throws Exception {
-//
-// }
-//
-// private RuleIndex getRuleIndex(){
-// LocalNonBlockingWorkQueue queue = new LocalNonBlockingWorkQueue();
-// Settings settings = new Settings();
-// settings.setProperty("sonar.log.profilingLevel", "BASIC");
-// RuleIndex rindex = new RuleIndex(queue, null, new Profiling(settings));
-// return rindex;
-// }
-//
-// @After
-// public void tearDown() {
-// if (node != null && !node.isClosed()) {
-// node.close();
-// }
-// }
-//
-// @Test
-// public void test_ruleIndex_conencts_to_es() {
-//
-// RuleIndex ruleIndex = getRuleIndex();
-// ruleIndex.connect();
-//
-// assertThat(node.client().admin().cluster().prepareClusterStats().get().getNodesStats().getCounts().getTotal())
-// .isEqualTo(ruleIndex.getNodesStats().getCounts().getTotal());
-//
-// ruleIndex.stop();
-//
+// esNode = new ESNode(fileSystem, settings)
// }
-}
+////
+//// private RuleIndex getRuleIndex(){
+//// LocalNonBlockingWorkQueue queue = new LocalNonBlockingWorkQueue();
+//// Settings settings = new Settings();
+//// settings.setProperty("sonar.log.profilingLevel", "BASIC");
+//// RuleIndex rindex = new RuleIndex(queue, null, new Profiling(settings));
+//// return rindex;
+//// }
+////
+//// @After
+//// public void tearDown() {
+//// if (node != null && !node.isClosed()) {
+//// node.close();
+//// }
+//// }
+////
+//// @Test
+//// public void test_ruleIndex_conencts_to_es() {
+////
+//// RuleIndex ruleIndex = getRuleIndex();
+//// ruleIndex.connect();
+////
+//// assertThat(node.client().admin().cluster().prepareClusterStats().get().getNodesStats().getCounts().getTotal())
+//// .isEqualTo(ruleIndex.getNodesStats().getCounts().getTotal());
+////
+//// ruleIndex.stop();
+////
+//// }
+//}
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.rule2;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.rule.Severity;
+import org.sonar.api.utils.DateUtils;
+import org.sonar.check.Cardinality;
+import org.sonar.core.rule.RuleDao;
+import org.sonar.core.rule.RuleDto;
+import org.sonar.server.search.Hit;
+import org.sonar.server.tester.ServerTester;
+
+import static org.fest.assertions.Assertions.assertThat;
+
+public class RuleMediumTest {
+
+ @Rule
+ public ServerTester tester = new ServerTester();
+
+
+ private RuleDto getRuleDto(){
+ return new RuleDto()
+ .setId(1)
+ .setRuleKey("NewRuleKey")
+ .setRepositoryKey("plugin")
+ .setName("new name")
+ .setDescription("new description")
+ .setStatus(org.sonar.api.rules.Rule.STATUS_DEPRECATED)
+ .setConfigKey("NewConfigKey")
+ .setSeverity(Severity.INFO)
+ .setCardinality(Cardinality.MULTIPLE)
+ .setLanguage("dart")
+ .setParentId(3)
+ .setSubCharacteristicId(100)
+ .setDefaultSubCharacteristicId(101)
+ .setRemediationFunction("linear")
+ .setDefaultRemediationFunction("linear_offset")
+ .setRemediationCoefficient("1h")
+ .setDefaultRemediationCoefficient("5d")
+ .setRemediationOffset("5min")
+ .setDefaultRemediationOffset("10h")
+ .setEffortToFixDescription("squid.S115.effortToFix")
+ .setCreatedAt(DateUtils.parseDate("2013-12-16"))
+ .setUpdatedAt(DateUtils.parseDate("2013-12-17"));
+ }
+
+ @Test
+ public void test_dao_queue_es_search_loop(){
+ RuleDao dao = tester.get(RuleDao.class);
+ RuleIndex index = tester.get(RuleIndex.class);
+
+ RuleDto dto = getRuleDto();
+ dao.insert(dto);
+
+ try {
+ Thread.sleep(1000000);
+ } catch (InterruptedException e) {
+ ;
+ }
+
+ Hit hit = index.getByKey(dto.getKey());
+ assertThat(hit.getFields().get("key")).isEqualTo(dto.getRuleKey());
+ }
+}
projects / sonarqube.git / commitdiff