private ScheduledFuture<?> pingTask = null;
private ScheduledExecutorService monitor;
+ private final boolean isMonitored;
+
protected MonitoredProcess(Props props) throws Exception {
+ this(props, false);
+ }
+
+ protected MonitoredProcess(Props props, boolean monitor) throws Exception {
+ this.isMonitored = monitor;
this.props = props;
this.name = props.of(NAME_PROPERTY);
Logger logger = LoggerFactory.getLogger(getClass());
logger.debug("Process[{}] starting", name);
- scheduleAutokill();
+ if (this.isMonitored) {
+ scheduleAutokill();
+ }
doStart();
logger.debug("Process[{}] started", name);
}
import javax.annotation.Nullable;
import javax.management.JMX;
import javax.management.MBeanServerConnection;
+import javax.management.Notification;
+import javax.management.NotificationListener;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
Thread.sleep(1000L);
LOGGER.debug("Try #{} to connect to JMX server for process '{}'", i, processName);
JMXConnector jmxConnector = JMXConnectorFactory.connect(jmxUrl, null);
+ jmxConnector.addConnectionNotificationListener(new NotificationListener() {
+ @Override
+ public void handleNotification(Notification notification, Object handback) {
+ LOGGER.info("JMX Connection Notification:{}",notification.getMessage());
+ }
+ }, null, null);
MBeanServerConnection mBeanServer = jmxConnector.getMBeanServerConnection();
ProcessMXBean bean = JMX.newMBeanProxy(mBeanServer, JmxUtils.objectName(processName), ProcessMXBean.class);
return bean;
private static final Integer MINIMUM_INDEX_REPLICATION = 1;
private final Set<String> nodes = new HashSet<String>();
+ private final boolean isBlocking;
private Node node;
- SearchServer(Props props) throws Exception {
- super(props);
+ public SearchServer(Props props, boolean monitored, boolean blocking) throws Exception {
+ super(props, monitored);
+ this.isBlocking = blocking;
new MinimumViableSystem().check();
if (StringUtils.isNotEmpty(props.of(ES_CLUSTER_INET, null))) {
}
}
+ public SearchServer(Props props) throws Exception {
+ this(props, true, true);
+ }
+
@Override
protected boolean doIsReady() {
return (node.client().admin().cluster().prepareHealth()
// Disable MCast
.put("discovery.zen.ping.multicast.enabled", "false")
- // Index storage policies
+ // Index storage policies
.put("index.merge.policy.max_merge_at_once", "200")
.put("index.merge.policy.segments_per_tier", "200")
.put("index.number_of_shards", "1")
.put("indices.store.throttle.type", "merge")
.put("indices.store.throttle.max_bytes_per_sec", "200mb")
- // Install our own listUpdate scripts
+ // Install our own listUpdate scripts
.put("script.default_lang", "native")
.put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName())
- // Node is pure transport
+ // Node is pure transport
.put("transport.tcp.port", port)
.put("http.enabled", false)
- // Setting up ES paths
+ // Setting up ES paths
.put("path.data", new File(dataDir, "es").getAbsolutePath())
.put("path.work", new File(tempDir).getAbsolutePath())
.put("path.logs", new File(logDir).getAbsolutePath());
// Set cluster coordinates
esSettings.put("cluster.name", clusterName);
esSettings.put("node.rack_id", StringUtils.defaultIfEmpty(props.of(SONAR_NODE_NAME), "unknown"));
- esSettings.put("cluster.routing.allocation.awareness.attributes","rack_id");
+ esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id");
if (props.contains(SONAR_NODE_NAME)) {
esSettings.put("node.name", props.of(SONAR_NODE_NAME));
} else {
.settings(esSettings)
.build().start();
- while (node != null && !node.isClosed()) {
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- // Ignore
+ if (this.isBlocking) {
+ while (node != null && !node.isClosed()) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
}
}
}
// Disallow dynamic mapping (too expensive)
.put("index.mapper.dynamic", false)
- // Sortable text analyzer
+ // Sortable text analyzer
.put("index.analysis.analyzer.sortable.type", "custom")
.put("index.analysis.analyzer.sortable.tokenizer", "keyword")
.putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate")
- // Edge NGram index-analyzer
+ // Edge NGram index-analyzer
.put("index.analysis.analyzer.index_grams.type", "custom")
.put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
.putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
- // Edge NGram search-analyzer
+ // Edge NGram search-analyzer
.put("index.analysis.analyzer.search_grams.type", "custom")
.put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
.putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
- // Word index-analyzer
+ // Word index-analyzer
.put("index.analysis.analyzer.index_words.type", "custom")
.put("index.analysis.analyzer.index_words.tokenizer", "standard")
.putArray("index.analysis.analyzer.index_words.filter",
"standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
- // Word search-analyzer
+ // Word search-analyzer
.put("index.analysis.analyzer.search_words.type", "custom")
.put("index.analysis.analyzer.search_words.tokenizer", "standard")
.putArray("index.analysis.analyzer.search_words.filter",
"standard", "lowercase", "stop", "asciifolding", "porter_stem")
- // Edge NGram filter
+ // Edge NGram filter
.put("index.analysis.filter.gram_filter.type", "edgeNGram")
.put("index.analysis.filter.gram_filter.min_gram", 2)
.put("index.analysis.filter.gram_filter.max_gram", 15)
.putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
- // Word filter
+ // Word filter
.put("index.analysis.filter.word_filter.type", "word_delimiter")
.put("index.analysis.filter.word_filter.generate_word_parts", true)
.put("index.analysis.filter.word_filter.catenate_words", true)
.put("index.analysis.filter.word_filter.split_on_numerics", true)
.put("index.analysis.filter.word_filter.stem_english_possessive", true)
- // Path Analyzer
+ // Path Analyzer
.put("index.analysis.analyzer.path_analyzer.type", "custom")
.put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy");
<artifactId>http-request</artifactId>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>org.codehaus.sonar</groupId>
+ <artifactId>sonar-search</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
import org.sonar.core.activity.db.ActivityDto;
import org.sonar.core.cluster.WorkQueue;
import org.sonar.server.search.BaseIndex;
-import org.sonar.server.search.ESNode;
+import org.sonar.server.search.SearchClient;
import org.sonar.server.search.IndexDefinition;
import org.sonar.server.search.IndexField;
import org.sonar.server.search.QueryOptions;
*/
public class ActivityIndex extends BaseIndex<Activity, ActivityDto, String> {
- public ActivityIndex(ActivityNormalizer normalizer, WorkQueue workQueue, ESNode node) {
+ public ActivityIndex(ActivityNormalizer normalizer, WorkQueue workQueue, SearchClient node) {
super(IndexDefinition.LOG, normalizer, workQueue, node);
}
.setQuery(QueryBuilders.matchAllQuery())
.setTypes(this.getIndexType())
.setSize(Integer.MAX_VALUE);
- SearchResponse response = node.execute(request);
+ SearchResponse response = getClient().execute(request);
return new Result<Activity>(this, response);
}
esSearch.setScroll(TimeValue.timeValueMinutes(3));
}
- SearchResponse response = node.execute(esSearch);
+ SearchResponse response = getClient().execute(esSearch);
return response;
}
ActivityDao.class,
// Elasticsearch
- ESNode.class,
+ SearchClient.class,
RuleNormalizer.class,
ActiveRuleNormalizer.class,
RuleIndex.class,
import org.sonar.server.qualityprofile.ActiveRule;
import org.sonar.server.rule.index.RuleNormalizer;
import org.sonar.server.search.BaseIndex;
-import org.sonar.server.search.ESNode;
+import org.sonar.server.search.SearchClient;
import org.sonar.server.search.FacetValue;
import org.sonar.server.search.IndexDefinition;
import org.sonar.server.search.IndexField;
public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, ActiveRuleKey> {
- public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, ESNode node) {
+ public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, SearchClient node) {
super(IndexDefinition.ACTIVE_RULE, normalizer, workQueue, node);
}
// TODO replace by scrolling
.setSize(Integer.MAX_VALUE);
- SearchResponse response = node.execute(request);
+ SearchResponse response = getClient().execute(request);
List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
for (SearchHit hit : response.getHits()) {
.setRouting(key)
// TODO replace by scrolling
.setSize(Integer.MAX_VALUE);
- SearchResponse response = node.execute(request);
+ SearchResponse response = getClient().execute(request);
List<ActiveRule> activeRules = new ArrayList<ActiveRule>();
for (SearchHit hit : response.getHits()) {
activeRules.add(toDoc(hit.getSource()));
.subAggregation(AggregationBuilders.count("countActiveRules")))
.setSize(0)
.setTypes(this.getIndexType());
- SearchResponse response = node.execute(request);
+ SearchResponse response = getClient().execute(request);
Map<String, Multimap<String, FacetValue>> stats = new HashMap<String, Multimap<String, FacetValue>>();
Aggregation aggregation = response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field());
for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) {
import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer;
import org.sonar.server.rule.Rule;
import org.sonar.server.search.BaseIndex;
-import org.sonar.server.search.ESNode;
+import org.sonar.server.search.SearchClient;
import org.sonar.server.search.IndexDefinition;
import org.sonar.server.search.IndexField;
import org.sonar.server.search.QueryOptions;
public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> {
- public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, ESNode node) {
+ public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, SearchClient node) {
super(IndexDefinition.RULE, normalizer, workQueue, node);
}
QueryBuilder qb = this.getQuery(query, options);
esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb));
- SearchResponse esResult = node.execute(esSearch);
+ SearchResponse esResult = getClient().execute(esSearch);
return new Result<Rule>(this, esResult);
}
.size(Integer.MAX_VALUE)
.minDocCount(1));
- SearchResponse esResponse = node.execute(request);
+ SearchResponse esResponse = getClient().execute(request);
Terms aggregation = esResponse.getAggregations().get(key);
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.termQuery(RuleNormalizer.RuleField.ID.field(), id))
.setSize(1);
- SearchResponse response = node.execute(request);
+ SearchResponse response = getClient().execute(request);
SearchHit hit = response.getHits().getAt(0);
if (hit == null) {
.setScroll(TimeValue.timeValueSeconds(3L))
.setSize(100)
.setQuery(QueryBuilders.termsQuery(RuleNormalizer.RuleField.ID.field(), ids));
- SearchResponse scrollResp = node.execute(request);
+ SearchResponse scrollResp = getClient().execute(request);
List<Rule> rules = newArrayList();
while (true) {
.prepareSearchScroll(scrollResp.getScrollId())
.setScroll(TimeValue.timeValueSeconds(3L));
- scrollResp = node.execute(scrollRequest);
+ scrollResp = getClient().execute(scrollRequest);
for (SearchHit hit : scrollResp.getHits()) {
rules.add(toDoc(hit.getSource()));
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * Inclient., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.search;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolFilterBuilder;
private static final Logger LOG = LoggerFactory.getLogger(BaseIndex.class);
- protected final ESNode node;
+ private final SearchClient client;
private final BaseNormalizer<DTO, KEY> normalizer;
private final IndexDefinition indexDefinition;
protected BaseIndex(IndexDefinition indexDefinition, BaseNormalizer<DTO, KEY> normalizer,
- WorkQueue workQueue, ESNode node) {
+ WorkQueue workQueue, SearchClient client) {
this.normalizer = normalizer;
- this.node = node;
+ this.client = client;
this.indexDefinition = indexDefinition;
}
return this.indexDefinition.getIndexType();
}
- protected final Client getClient() {
- return node.client();
- }
-
/* Component Methods */
@Override
public void start() {
-
/* Setup the index if necessary */
initializeIndex();
}
// nothing to do
}
+ public SearchClient getClient(){
+ return client;
+ }
+
// Scrolling within the index
public Iterator<DOMAIN> scroll(final String scrollId) {
return new Iterator<DOMAIN>() {
private void fillQueue() {
try {
- SearchScrollRequestBuilder esRequest = getClient().prepareSearchScroll(scrollId)
+ SearchScrollRequestBuilder esRequest = client.prepareSearchScroll(scrollId)
.setScroll(TimeValue.timeValueMinutes(3));
- Collections.addAll(hits, ((SearchResponse) node.execute(esRequest)).getHits().getHits());
+ Collections.addAll(hits, ((SearchResponse) client.execute(esRequest)).getHits().getHits());
} catch (Exception e) {
throw new IllegalStateException("Error while filling in the scroll buffer", e);
}
protected void initializeIndex() {
String index = this.getIndexName();
- IndicesExistsResponse indexExistsResponse = getClient().admin().indices()
+ IndicesExistsResponse indexExistsResponse = client.admin().indices()
.prepareExists(index).execute().actionGet();
try {
if (!indexExistsResponse.isExists()) {
LOG.debug("Setup of {} for type {}", this.getIndexName(), this.getIndexType());
- getClient().admin().indices().prepareCreate(index)
+ client.admin().indices().prepareCreate(index)
.setSettings(getIndexSettings())
.execute().actionGet();
}
LOG.debug("Update of index {} for type {}", this.getIndexName(), this.getIndexType());
- getClient().admin().indices().preparePutMapping(index)
+ client.admin().indices().preparePutMapping(index)
.setType(getIndexType())
.setIgnoreConflicts(true)
.setSource(mapDomain())
IndexStat stat = new IndexStat();
/** get total document count */
- CountRequestBuilder countRequest = getClient().prepareCount(this.getIndexName())
+ CountRequestBuilder countRequest = client.prepareCount(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.matchAllQuery());
- CountResponse response = node.execute(countRequest);
+ CountResponse response = client.execute(countRequest);
stat.setDocumentCount(response.getCount());
/** get Management information */
public Date getLastSynchronization() {
Date date;
- SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName())
+ SearchRequestBuilder request = client.prepareSearch(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.matchAllQuery())
.setSize(0)
.addAggregation(AggregationBuilders.max("latest")
.field(BaseNormalizer.UPDATED_AT_FIELD));
- SearchResponse response = node.execute(request);
+ SearchResponse response = client.execute(request);
Max max = (Max) response.getAggregations().get("latest");
@Override
public void refresh() {
- node.execute(getClient()
+ client.execute(client
.admin()
.indices()
.prepareRefresh(this.getIndexName())
protected abstract DOMAIN toDoc(Map<String, Object> fields);
public DOMAIN getByKey(KEY key) {
- GetRequestBuilder request = getClient().prepareGet()
+ GetRequestBuilder request = client.prepareGet()
.setType(this.getIndexType())
.setIndex(this.getIndexName())
.setId(this.getKeyValue(key))
.setRouting(this.getKeyValue(key));
- GetResponse response = node.execute(request);
+ GetResponse response = client.execute(request);
if (response.isExists()) {
return toDoc(response.getSource());
protected void updateDocument(Collection<UpdateRequest> requests, KEY key) {
LOG.debug("UPDATE _id:{} in index {}", key, this.getIndexName());
- BulkRequestBuilder bulkRequest = getClient().prepareBulk();
+ BulkRequestBuilder bulkRequest = client.prepareBulk();
for (UpdateRequest request : requests) {
// if request has no ID then no upsert possible!
if (request.id() == null || request.id().isEmpty()) {
.type(this.getIndexType()));
}
}
- BulkResponse response = node.execute(bulkRequest);
+ BulkResponse response = client.execute(bulkRequest);
}
@Override
private void deleteDocument(KEY key) throws ExecutionException, InterruptedException {
LOG.debug("DELETE _id:{} in index {}", key, this.getIndexName());
- DeleteRequestBuilder request = getClient()
+ DeleteRequestBuilder request = client
.prepareDelete()
.setIndex(this.getIndexName())
.setType(this.getIndexType())
.setId(this.getKeyValue(key));
- DeleteResponse response = node.execute(request);
+ DeleteResponse response = client.execute(request);
}
@Override
}
public Long countAll() {
- return getClient().prepareCount(this.getIndexName())
+ return client.prepareCount(this.getIndexName())
.setTypes(this.getIndexType())
.get().getCount();
}
public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) {
Map<String, Long> counts = new HashMap<String, Long>();
- SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName())
+ SearchRequestBuilder request = client.prepareSearch(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.filteredQuery(
QueryBuilders.matchAllQuery(),
.size(Integer.MAX_VALUE)
.minDocCount(0));
- SearchResponse response = node.execute(request);
+ SearchResponse response = client.execute(request);
Terms values =
response.getAggregations().get(indexField.field());
}
public static final String TYPE = "sonar.search.type";
- public static final String HTTP_PORT = "sonar.search.http.port";
public static final String NODE_PORT = "sonar.search.port";
public static final String CLUSTER_NAME = "sonar.cluster.name";
public static final String NODE_NAME = "sonar.node.name";
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.search;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodeResponse;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonar.api.config.Settings;
+import org.sonar.core.profiling.Profiling;
+import org.sonar.core.profiling.StopWatch;
+
+import javax.annotation.CheckForNull;
+import java.io.File;
+import java.net.InetAddress;
+
+/**
+ * ElasticSearch Node used to connect to index.
+ */
+public class SearchClient extends TransportClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SearchClient.class);
+ private static final String DEFAULT_HEALTH_TIMEOUT = "30s";
+
+ private final Settings settings;
+ private final String healthTimeout;
+
+ protected final Profiling profiling;
+
+ public SearchClient(Settings settings) {
+ this(settings, DEFAULT_HEALTH_TIMEOUT);
+ }
+
+ @VisibleForTesting
+ SearchClient(Settings settings, String healthTimeout) {
+ super(ImmutableSettings.settingsBuilder()
+ .put("network.bind_host", "localhost")
+ .put("node.rack_id", StringUtils.defaultIfEmpty(settings.getString(IndexProperties.NODE_NAME), "unknown"))
+ .put("cluster.name", StringUtils.defaultIfBlank(settings.getString(IndexProperties.CLUSTER_NAME), "sonarqube"))
+ .build()
+ );
+ initLogging();
+ this.addTransportAddress(new InetSocketTransportAddress("localhost",
+ settings.getInt(IndexProperties.NODE_PORT)));
+ this.settings = settings;
+ this.healthTimeout = healthTimeout;
+ this.profiling = new Profiling(settings);
+ }
+
+ public NodeHealth getNodeHealth() {
+ NodeHealth health = new NodeHealth();
+ ClusterStatsResponse clusterStatsResponse = this.admin().cluster().prepareClusterStats().get();
+
+ // Cluster health
+ health.setClusterAvailable(clusterStatsResponse.getStatus() != ClusterHealthStatus.RED);
+
+ ClusterStatsNodes nodesStats = clusterStatsResponse.getNodesStats();
+
+ // JVM Heap Usage
+ health.setJvmHeapMax(nodesStats.getJvm().getHeapMax().bytes());
+ health.setJvmHeapUsed(nodesStats.getJvm().getHeapUsed().bytes());
+
+ // OS Memory Usage ?
+
+ // Disk Usage
+ health.setFsTotal(nodesStats.getFs().getTotal().bytes());
+ health.setFsAvailable(nodesStats.getFs().getAvailable().bytes());
+
+ // Ping ?
+
+ // Threads
+ health.setJvmThreads(nodesStats.getJvm().getThreads());
+
+ // CPU
+ health.setProcessCpuPercent(nodesStats.getProcess().getCpuPercent());
+
+ // Open Files
+ health.setOpenFiles(nodesStats.getProcess().getAvgOpenFileDescriptors());
+
+ // Uptime
+ health.setJvmUptimeMillis(nodesStats.getJvm().getMaxUpTime().getMillis());
+
+ return health;
+ }
+
+ private ClusterHealthStatus getClusterHealthStatus() {
+ return this.admin().cluster().prepareHealth()
+ .setWaitForYellowStatus()
+ .setTimeout(healthTimeout)
+ .get()
+ .getStatus();
+ }
+
+ @CheckForNull
+ private NodeInfo getLocalNodeInfoByHostName(String hostname) {
+ for (ClusterStatsNodeResponse nodeResp : this.admin().cluster().prepareClusterStats().get().getNodes()) {
+ if (hostname.equals(nodeResp.nodeInfo().getHostname())) {
+ return nodeResp.nodeInfo();
+ }
+ }
+ return null;
+ }
+
+ @CheckForNull
+ private NodeInfo getLocalNodeInfoByNodeName(String nodeName) {
+ return this.admin().cluster().prepareClusterStats().get().getNodesMap().get(nodeName).nodeInfo();
+ }
+
+ @CheckForNull
+ private NodeInfo getLocalNodeInfo() {
+ if (settings.hasKey(IndexProperties.NODE_NAME)) {
+ return getLocalNodeInfoByNodeName(settings.getString(IndexProperties.NODE_NAME));
+ } else {
+ try {
+ String LocalhostName = InetAddress.getLocalHost().getHostName();
+ return getLocalNodeInfoByHostName(LocalhostName);
+ } catch (Exception exception) {
+ throw new IllegalStateException("Could not get localhost hostname", exception);
+ }
+ }
+ }
+
+ // TODO that has nothing to do here!!!
+ private void addIndexTemplates() {
+ this.admin().indices()
+ .preparePutTemplate("default")
+ .setTemplate("*")
+ .addMapping("_default_", "{\"dynamic\": \"strict\"}")
+ .get();
+ }
+
+ private void initLogging() {
+ ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory());
+ }
+
+ private File esHomeDir() {
+ if (!settings.hasKey("sonar.path.home")) {
+ throw new IllegalStateException("property 'sonar.path.home' is required");
+ }
+ return new File(settings.getString("sonar.path.home"));
+ }
+
+ private File esDataDir() {
+ if (settings.hasKey("sonar.path.data")) {
+ return new File(settings.getString("sonar.path.data"), "es");
+ } else {
+ return new File(settings.getString("sonar.path.home"), "data/es");
+ }
+ }
+
+ private File esLogDir() {
+ if (settings.hasKey("sonar.path.log")) {
+ return new File(settings.getString("sonar.path.log"));
+ } else {
+ return new File(settings.getString("sonar.path.home"), "log");
+ }
+ }
+
+ public <K extends ActionResponse> K execute(ActionRequestBuilder request) {
+ StopWatch basicProfile = profiling.start("search", Profiling.Level.BASIC);
+ StopWatch fullProfile = profiling.start("search", Profiling.Level.FULL);
+ ListenableActionFuture acc = request.execute();
+ try {
+
+ K response = (K) acc.get();
+
+ if (profiling.isProfilingEnabled(Profiling.Level.BASIC)) {
+ if (ToXContent.class.isAssignableFrom(request.getClass())) {
+ XContentBuilder debugResponse = XContentFactory.jsonBuilder();
+ debugResponse.startObject();
+ ((ToXContent) request).toXContent(debugResponse, ToXContent.EMPTY_PARAMS);
+ debugResponse.endObject();
+ fullProfile.stop("ES Request: %s", debugResponse.string());
+ } else {
+ fullProfile.stop("ES Request: %s", request.toString().replaceAll("\n", ""));
+ }
+ }
+
+ if (profiling.isProfilingEnabled(Profiling.Level.FULL)) {
+ if (ToXContent.class.isAssignableFrom(response.getClass())) {
+ XContentBuilder debugResponse = XContentFactory.jsonBuilder();
+ debugResponse.startObject();
+ ((ToXContent) response).toXContent(debugResponse, ToXContent.EMPTY_PARAMS);
+ debugResponse.endObject();
+ fullProfile.stop("ES Response: %s", debugResponse.string());
+ } else {
+ fullProfile.stop("ES Response: %s", response.toString());
+ }
+ }
+
+ return response;
+ } catch (Exception e) {
+ throw new IllegalStateException("ES error: ", e);
+ }
+ }
+
+
+
+ static public ImmutableSettings.Builder getGlobalIndexSettings(ImmutableSettings.Builder builder) {
+ return builder
+
+ // Disable MCast
+ .put("discovery.zen.ping.multicast.enabled", "false")
+
+ // Disable dynamic mapping
+ .put("index.mapper.dynamic", false)
+
+ // Sortable text analyzer
+ .put("index.analysis.analyzer.sortable.type", "custom")
+ .put("index.analysis.analyzer.sortable.tokenizer", "keyword")
+ .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate")
+
+ // Edge NGram index-analyzer
+ .put("index.analysis.analyzer.index_grams.type", "custom")
+ .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace")
+ .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter")
+
+ // Edge NGram search-analyzer
+ .put("index.analysis.analyzer.search_grams.type", "custom")
+ .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace")
+ .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase")
+
+ // Word index-analyzer
+ .put("index.analysis.analyzer.index_words.type", "custom")
+ .put("index.analysis.analyzer.index_words.tokenizer", "standard")
+ .putArray("index.analysis.analyzer.index_words.filter",
+ "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem")
+
+ // Word search-analyzer
+ .put("index.analysis.analyzer.search_words.type", "custom")
+ .put("index.analysis.analyzer.search_words.tokenizer", "standard")
+ .putArray("index.analysis.analyzer.search_words.filter",
+ "standard", "lowercase", "stop", "asciifolding", "porter_stem")
+
+ // Edge NGram filter
+ .put("index.analysis.filter.gram_filter.type", "edgeNGram")
+ .put("index.analysis.filter.gram_filter.min_gram", 2)
+ .put("index.analysis.filter.gram_filter.max_gram", 15)
+ .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol")
+
+ // Word filter
+ .put("index.analysis.filter.word_filter.type", "word_delimiter")
+ .put("index.analysis.filter.word_filter.generate_word_parts", true)
+ .put("index.analysis.filter.word_filter.catenate_words", true)
+ .put("index.analysis.filter.word_filter.catenate_numbers", true)
+ .put("index.analysis.filter.word_filter.catenate_all", true)
+ .put("index.analysis.filter.word_filter.split_on_case_change", true)
+ .put("index.analysis.filter.word_filter.preserve_original", true)
+ .put("index.analysis.filter.word_filter.split_on_numerics", true)
+ .put("index.analysis.filter.word_filter.stem_english_possessive", true)
+
+ // Path Analyzer
+ .put("index.analysis.analyzer.path_analyzer.type", "custom")
+ .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy");
+ }
+
+}
public class SearchHealth {
- private ESNode node;
+ private SearchClient searchClient;
private IndexClient indexClient;
- public SearchHealth(ESNode node, IndexClient indexClient) {
- this.node = node;
+ public SearchHealth(SearchClient searchClient, IndexClient indexClient) {
+ this.searchClient = searchClient;
this.indexClient = indexClient;
}
public NodeHealth getNodeHealth() {
- return node.getNodeHealth();
+ return searchClient.getNodeHealth();
}
public Map<String, IndexHealth> getIndexHealth() {
newIndexHealth.documentCount = indexStat.getDocumentCount();
newIndexHealth.lastSync = indexStat.getLastUpdate();
- IndicesStatsRequestBuilder statRequest = node.client().admin().indices().prepareStats(index.getIndexName())
+ IndicesStatsRequestBuilder statRequest = searchClient.admin().indices().prepareStats(index.getIndexName())
.setTypes(index.getIndexType());
- IndicesStatsResponse indicesStatsResponse = node.execute(statRequest);
+ IndicesStatsResponse indicesStatsResponse = searchClient.execute(statRequest);
newIndexHealth.segmentCount = indicesStatsResponse.getTotal().getSegments().getCount();
newIndexHealth.pendingDeletion = indicesStatsResponse.getTotal().getDocs().getDeleted();
@Rule
public TemporaryFolder temp = new TemporaryFolder();
- ESNode node;
+ SearchClient node;
@Before
public void setup() throws IOException {
Settings settings = new Settings();
settings.setProperty("sonar.path.home", dataDir.getAbsolutePath());
settings.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name());
- node = new ESNode(settings);
- node.start();
+ node = new SearchClient(settings);
+ //node.start();
}
@After
public void tearDown() {
- node.stop();
+ //node.stop();
}
@Test
}
- private BaseIndex getIndex(final ESNode esNode) {
+ private BaseIndex getIndex(final SearchClient searchClient) {
BaseIndex index = new BaseIndex(
IndexDefinition.TEST,
- null, new NullQueue(), esNode) {
+ null, new NullQueue(), searchClient) {
@Override
protected String getKeyValue(Serializable key) {
return null;
--- /dev/null
+/*
+ * SonarQube, open source software quality management tool.
+ * Copyright (C) 2008-2014 SonarSource
+ * mailto:contact AT sonarsource DOT com
+ *
+ * SonarQube is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * SonarQube is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.search;
+
+public class SearchClientTest {
+
+// File dataDir;
+// Settings settings;
+//
+// @Rule
+// public TemporaryFolder temp = new TemporaryFolder();
+//
+// @Before
+// public void createMocks() throws IOException {
+// dataDir = temp.newFolder();
+// settings = new Settings();
+// settings.setProperty("sonar.path.home", dataDir.getAbsolutePath());
+// settings.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name());
+// }
+//
+// @Test(expected = StrictDynamicMappingException.class)
+// public void should_use_default_settings_for_index() throws Exception {
+// SearchClient node = new SearchClient(settings);
+// node.start();
+//
+// node.client().admin().indices().prepareCreate("strict")
+// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}")
+// .execute().actionGet();
+// node.client().admin().cluster().prepareHealth("strict").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000));
+//
+// // strict mapping is enforced
+// try {
+// node.client().prepareIndex("strict", "type1", "666").setSource(
+// XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject()
+// ).get();
+// } finally {
+// node.stop();
+// }
+// }
+//
+// @Test
+// public void check_path_analyzer() throws Exception {
+// SearchClient node = new SearchClient(settings);
+// node.start();
+//
+// node.client().admin().indices().prepareCreate("path")
+// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}")
+// .execute().actionGet();
+// node.client().admin().cluster().prepareHealth("path").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000));
+//
+// // default "path_analyzer" analyzer is defined for all indices
+// AnalyzeResponse response = node.client().admin().indices()
+// .prepareAnalyze("path", "/temp/65236/test path/MyFile.java").setAnalyzer("path_analyzer").get();
+// // default "path_analyzer" analyzer is defined for all indices
+// assertThat(response.getTokens()).hasSize(4);
+// assertThat(response.getTokens().get(0).getTerm()).isEqualTo("/temp");
+// assertThat(response.getTokens().get(1).getTerm()).isEqualTo("/temp/65236");
+// assertThat(response.getTokens().get(2).getTerm()).isEqualTo("/temp/65236/test path");
+// assertThat(response.getTokens().get(3).getTerm()).isEqualTo("/temp/65236/test path/MyFile.java");
+//
+// node.stop();
+// }
+//
+// @Test
+// public void check_sortable_analyzer() throws Exception {
+// SearchClient node = new SearchClient(settings);
+// node.start();
+//
+// node.client().admin().indices().prepareCreate("sort")
+// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}")
+// .execute().actionGet();
+// node.client().admin().cluster().prepareHealth("sort").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000));
+//
+// // default "sortable" analyzer is defined for all indices
+// assertThat(node.client().admin().indices()
+// .prepareAnalyze("sort", "This Is A Wonderful Text").setAnalyzer("sortable").get()
+// .getTokens().get(0).getTerm()).isEqualTo("this is a ");
+//
+// node.stop();
+// }
+//
+// @Test
+// public void check_gram_analyzer() throws Exception {
+// SearchClient node = new SearchClient(settings);
+// node.start();
+//
+// node.client().admin().indices().prepareCreate("gram")
+// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}")
+// .execute().actionGet();
+// node.client().admin().cluster().prepareHealth("gram").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000));
+//
+// // default "string_gram" analyzer is defined for all indices
+// AnalyzeResponse response = node.client().admin().indices()
+// .prepareAnalyze("gram", "he.llo w@rl#d").setAnalyzer("index_grams").get();
+// assertThat(response.getTokens()).hasSize(10);
+// assertThat(response.getTokens().get(0).getTerm()).isEqualTo("he");
+// assertThat(response.getTokens().get(7).getTerm()).isEqualTo("w@rl");
+//
+// node.stop();
+// }
+//
+// @Test
+// public void should_fail_to_get_client_if_not_started() {
+// SearchClient node = new SearchClient(settings);
+// try {
+// node.client();
+// fail();
+// } catch (IllegalStateException e) {
+// assertThat(e).hasMessage("Elasticsearch is not started");
+// }
+// }
+}
*/
package org.sonar.server.tester;
-import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.QueryBuilders;
import org.sonar.api.ServerComponent;
import org.sonar.core.persistence.DatabaseVersion;
import org.sonar.core.persistence.DbSession;
import org.sonar.core.persistence.MyBatis;
-import org.sonar.server.search.ESNode;
+import org.sonar.server.search.SearchClient;
import java.sql.Connection;
public class BackendCleanup implements ServerComponent {
- private final ESNode esNode;
+ private final SearchClient searchClient;
private final MyBatis myBatis;
- public BackendCleanup(ESNode esNode, MyBatis myBatis) {
- this.esNode = esNode;
+ public BackendCleanup(SearchClient searchClient, MyBatis myBatis) {
+ this.searchClient = searchClient;
this.myBatis = myBatis;
}
}
public void clearIndexes() {
- Client client = esNode.client();
- client.prepareDeleteByQuery(client.admin().cluster().prepareState().get()
+ searchClient.prepareDeleteByQuery(searchClient.admin().cluster().prepareState().get()
.getState().getMetaData().concreteAllIndices())
.setQuery(QueryBuilders.matchAllQuery())
.get();
import org.apache.commons.lang.StringUtils;
import org.junit.rules.ExternalResource;
import org.sonar.api.database.DatabaseProperties;
+import org.sonar.process.MonitoredProcess;
+import org.sonar.process.NetworkUtils;
+import org.sonar.process.Props;
+import org.sonar.search.SearchServer;
import org.sonar.server.platform.Platform;
import org.sonar.server.search.IndexProperties;
import org.sonar.server.ws.WsTester;
private static final String PROP_PREFIX = "mediumTests.";
+ private final String clusterName;
+ private final Integer clusterPort;
+
+ private SearchServer searchServer;
private final Platform platform;
private final File homeDir;
private final List components = Lists.newArrayList(BackendCleanup.class, WsTester.class);
public ServerTester() {
homeDir = createTempDir();
platform = new Platform();
+
+ clusterName = "cluster-mem-" + System.currentTimeMillis();
+ clusterPort = NetworkUtils.freePort();
+ Properties properties = new Properties();
+ properties.setProperty(IndexProperties.CLUSTER_NAME, clusterName);
+ properties.setProperty(IndexProperties.NODE_PORT, clusterPort.toString());
+ properties.setProperty(MonitoredProcess.NAME_PROPERTY, "ES");
+ properties.setProperty("sonar.path.data", new File(homeDir, "data/es").getAbsolutePath());
+ properties.setProperty("sonar.path.logs", new File(homeDir, "logs").getAbsolutePath());
+ properties.setProperty("sonar.path.temp", new File(homeDir, "temp").getAbsolutePath());
+ try {
+ searchServer = new SearchServer(new Props(properties), false, false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
/**
Properties properties = new Properties();
properties.putAll(initialProps);
- properties.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name());
+
+ try {
+ searchServer.start();
+ System.out.println("searchServer = " + searchServer.isReady());
+ while (!searchServer.isReady()) {
+ Thread.sleep(100);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ properties.setProperty(IndexProperties.CLUSTER_NAME, clusterName);
+ properties.setProperty(IndexProperties.NODE_PORT, clusterPort.toString());
+ properties.setProperty(MonitoredProcess.NAME_PROPERTY, "ES");
+
properties.setProperty("sonar.path.home", homeDir.getAbsolutePath());
properties.setProperty(DatabaseProperties.PROP_URL, "jdbc:h2:" + homeDir.getAbsolutePath() + "/h2");
for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
*/
public void stop() {
platform.doStop();
+ searchServer.terminate();
FileUtils.deleteQuietly(homeDir);
}
projects / sonarqube.git / commitdiff