@@ -41,7 +41,14 @@ public abstract class MonitoredProcess implements ProcessMXBean { | |||
private ScheduledFuture<?> pingTask = null; | |||
private ScheduledExecutorService monitor; | |||
private final boolean isMonitored; | |||
protected MonitoredProcess(Props props) throws Exception { | |||
this(props, false); | |||
} | |||
protected MonitoredProcess(Props props, boolean monitor) throws Exception { | |||
this.isMonitored = monitor; | |||
this.props = props; | |||
this.name = props.of(NAME_PROPERTY); | |||
@@ -61,7 +68,9 @@ public abstract class MonitoredProcess implements ProcessMXBean { | |||
Logger logger = LoggerFactory.getLogger(getClass()); | |||
logger.debug("Process[{}] starting", name); | |||
scheduleAutokill(); | |||
if (this.isMonitored) { | |||
scheduleAutokill(); | |||
} | |||
doStart(); | |||
logger.debug("Process[{}] started", name); | |||
} |
@@ -28,6 +28,8 @@ import javax.annotation.CheckForNull; | |||
import javax.annotation.Nullable; | |||
import javax.management.JMX; | |||
import javax.management.MBeanServerConnection; | |||
import javax.management.Notification; | |||
import javax.management.NotificationListener; | |||
import javax.management.remote.JMXConnector; | |||
import javax.management.remote.JMXConnectorFactory; | |||
import javax.management.remote.JMXServiceURL; | |||
@@ -242,6 +244,12 @@ public class ProcessWrapper extends Thread implements Terminable { | |||
Thread.sleep(1000L); | |||
LOGGER.debug("Try #{} to connect to JMX server for process '{}'", i, processName); | |||
JMXConnector jmxConnector = JMXConnectorFactory.connect(jmxUrl, null); | |||
jmxConnector.addConnectionNotificationListener(new NotificationListener() { | |||
@Override | |||
public void handleNotification(Notification notification, Object handback) { | |||
LOGGER.info("JMX Connection Notification:{}",notification.getMessage()); | |||
} | |||
}, null, null); | |||
MBeanServerConnection mBeanServer = jmxConnector.getMBeanServerConnection(); | |||
ProcessMXBean bean = JMX.newMBeanProxy(mBeanServer, JmxUtils.objectName(processName), ProcessMXBean.class); | |||
return bean; |
@@ -49,11 +49,13 @@ public class SearchServer extends MonitoredProcess { | |||
private static final Integer MINIMUM_INDEX_REPLICATION = 1; | |||
private final Set<String> nodes = new HashSet<String>(); | |||
private final boolean isBlocking; | |||
private Node node; | |||
SearchServer(Props props) throws Exception { | |||
super(props); | |||
public SearchServer(Props props, boolean monitored, boolean blocking) throws Exception { | |||
super(props, monitored); | |||
this.isBlocking = blocking; | |||
new MinimumViableSystem().check(); | |||
if (StringUtils.isNotEmpty(props.of(ES_CLUSTER_INET, null))) { | |||
@@ -61,6 +63,10 @@ public class SearchServer extends MonitoredProcess { | |||
} | |||
} | |||
public SearchServer(Props props) throws Exception { | |||
this(props, true, true); | |||
} | |||
@Override | |||
protected boolean doIsReady() { | |||
return (node.client().admin().cluster().prepareHealth() | |||
@@ -85,7 +91,7 @@ public class SearchServer extends MonitoredProcess { | |||
// Disable MCast | |||
.put("discovery.zen.ping.multicast.enabled", "false") | |||
// Index storage policies | |||
// Index storage policies | |||
.put("index.merge.policy.max_merge_at_once", "200") | |||
.put("index.merge.policy.segments_per_tier", "200") | |||
.put("index.number_of_shards", "1") | |||
@@ -94,15 +100,15 @@ public class SearchServer extends MonitoredProcess { | |||
.put("indices.store.throttle.type", "merge") | |||
.put("indices.store.throttle.max_bytes_per_sec", "200mb") | |||
// Install our own listUpdate scripts | |||
// Install our own listUpdate scripts | |||
.put("script.default_lang", "native") | |||
.put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName()) | |||
// Node is pure transport | |||
// Node is pure transport | |||
.put("transport.tcp.port", port) | |||
.put("http.enabled", false) | |||
// Setting up ES paths | |||
// Setting up ES paths | |||
.put("path.data", new File(dataDir, "es").getAbsolutePath()) | |||
.put("path.work", new File(tempDir).getAbsolutePath()) | |||
.put("path.logs", new File(logDir).getAbsolutePath()); | |||
@@ -119,7 +125,7 @@ public class SearchServer extends MonitoredProcess { | |||
// Set cluster coordinates | |||
esSettings.put("cluster.name", clusterName); | |||
esSettings.put("node.rack_id", StringUtils.defaultIfEmpty(props.of(SONAR_NODE_NAME), "unknown")); | |||
esSettings.put("cluster.routing.allocation.awareness.attributes","rack_id"); | |||
esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id"); | |||
if (props.contains(SONAR_NODE_NAME)) { | |||
esSettings.put("node.name", props.of(SONAR_NODE_NAME)); | |||
} else { | |||
@@ -138,11 +144,13 @@ public class SearchServer extends MonitoredProcess { | |||
.settings(esSettings) | |||
.build().start(); | |||
while (node != null && !node.isClosed()) { | |||
try { | |||
Thread.sleep(100); | |||
} catch (InterruptedException e) { | |||
// Ignore | |||
if (this.isBlocking) { | |||
while (node != null && !node.isClosed()) { | |||
try { | |||
Thread.sleep(100); | |||
} catch (InterruptedException e) { | |||
// Ignore | |||
} | |||
} | |||
} | |||
} | |||
@@ -153,40 +161,40 @@ public class SearchServer extends MonitoredProcess { | |||
// Disallow dynamic mapping (too expensive) | |||
.put("index.mapper.dynamic", false) | |||
// Sortable text analyzer | |||
// Sortable text analyzer | |||
.put("index.analysis.analyzer.sortable.type", "custom") | |||
.put("index.analysis.analyzer.sortable.tokenizer", "keyword") | |||
.putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") | |||
// Edge NGram index-analyzer | |||
// Edge NGram index-analyzer | |||
.put("index.analysis.analyzer.index_grams.type", "custom") | |||
.put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") | |||
.putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") | |||
// Edge NGram search-analyzer | |||
// Edge NGram search-analyzer | |||
.put("index.analysis.analyzer.search_grams.type", "custom") | |||
.put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") | |||
.putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") | |||
// Word index-analyzer | |||
// Word index-analyzer | |||
.put("index.analysis.analyzer.index_words.type", "custom") | |||
.put("index.analysis.analyzer.index_words.tokenizer", "standard") | |||
.putArray("index.analysis.analyzer.index_words.filter", | |||
"standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") | |||
// Word search-analyzer | |||
// Word search-analyzer | |||
.put("index.analysis.analyzer.search_words.type", "custom") | |||
.put("index.analysis.analyzer.search_words.tokenizer", "standard") | |||
.putArray("index.analysis.analyzer.search_words.filter", | |||
"standard", "lowercase", "stop", "asciifolding", "porter_stem") | |||
// Edge NGram filter | |||
// Edge NGram filter | |||
.put("index.analysis.filter.gram_filter.type", "edgeNGram") | |||
.put("index.analysis.filter.gram_filter.min_gram", 2) | |||
.put("index.analysis.filter.gram_filter.max_gram", 15) | |||
.putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") | |||
// Word filter | |||
// Word filter | |||
.put("index.analysis.filter.word_filter.type", "word_delimiter") | |||
.put("index.analysis.filter.word_filter.generate_word_parts", true) | |||
.put("index.analysis.filter.word_filter.catenate_words", true) | |||
@@ -197,7 +205,7 @@ public class SearchServer extends MonitoredProcess { | |||
.put("index.analysis.filter.word_filter.split_on_numerics", true) | |||
.put("index.analysis.filter.word_filter.stem_english_possessive", true) | |||
// Path Analyzer | |||
// Path Analyzer | |||
.put("index.analysis.analyzer.path_analyzer.type", "custom") | |||
.put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); | |||
@@ -239,7 +239,12 @@ | |||
<artifactId>http-request</artifactId> | |||
<scope>test</scope> | |||
</dependency> | |||
<dependency> | |||
<groupId>org.codehaus.sonar</groupId> | |||
<artifactId>sonar-search</artifactId> | |||
<version>${project.version}</version> | |||
<scope>test</scope> | |||
</dependency> | |||
</dependencies> | |||
<build> |
@@ -35,7 +35,7 @@ import org.sonar.core.activity.Activity; | |||
import org.sonar.core.activity.db.ActivityDto; | |||
import org.sonar.core.cluster.WorkQueue; | |||
import org.sonar.server.search.BaseIndex; | |||
import org.sonar.server.search.ESNode; | |||
import org.sonar.server.search.SearchClient; | |||
import org.sonar.server.search.IndexDefinition; | |||
import org.sonar.server.search.IndexField; | |||
import org.sonar.server.search.QueryOptions; | |||
@@ -51,7 +51,7 @@ import java.util.Map; | |||
*/ | |||
public class ActivityIndex extends BaseIndex<Activity, ActivityDto, String> { | |||
public ActivityIndex(ActivityNormalizer normalizer, WorkQueue workQueue, ESNode node) { | |||
public ActivityIndex(ActivityNormalizer normalizer, WorkQueue workQueue, SearchClient node) { | |||
super(IndexDefinition.LOG, normalizer, workQueue, node); | |||
} | |||
@@ -93,7 +93,7 @@ public class ActivityIndex extends BaseIndex<Activity, ActivityDto, String> { | |||
.setQuery(QueryBuilders.matchAllQuery()) | |||
.setTypes(this.getIndexType()) | |||
.setSize(Integer.MAX_VALUE); | |||
SearchResponse response = node.execute(request); | |||
SearchResponse response = getClient().execute(request); | |||
return new Result<Activity>(this, response); | |||
} | |||
@@ -144,7 +144,7 @@ public class ActivityIndex extends BaseIndex<Activity, ActivityDto, String> { | |||
esSearch.setScroll(TimeValue.timeValueMinutes(3)); | |||
} | |||
SearchResponse response = node.execute(esSearch); | |||
SearchResponse response = getClient().execute(esSearch); | |||
return response; | |||
} |
@@ -215,7 +215,7 @@ class ServerComponents { | |||
ActivityDao.class, | |||
// Elasticsearch | |||
ESNode.class, | |||
SearchClient.class, | |||
RuleNormalizer.class, | |||
ActiveRuleNormalizer.class, | |||
RuleIndex.class, |
@@ -39,7 +39,7 @@ import org.sonar.core.qualityprofile.db.ActiveRuleKey; | |||
import org.sonar.server.qualityprofile.ActiveRule; | |||
import org.sonar.server.rule.index.RuleNormalizer; | |||
import org.sonar.server.search.BaseIndex; | |||
import org.sonar.server.search.ESNode; | |||
import org.sonar.server.search.SearchClient; | |||
import org.sonar.server.search.FacetValue; | |||
import org.sonar.server.search.IndexDefinition; | |||
import org.sonar.server.search.IndexField; | |||
@@ -52,7 +52,7 @@ import java.util.Map; | |||
public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, ActiveRuleKey> { | |||
public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { | |||
public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, SearchClient node) { | |||
super(IndexDefinition.ACTIVE_RULE, normalizer, workQueue, node); | |||
} | |||
@@ -128,7 +128,7 @@ public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, Active | |||
// TODO replace by scrolling | |||
.setSize(Integer.MAX_VALUE); | |||
SearchResponse response = node.execute(request); | |||
SearchResponse response = getClient().execute(request); | |||
List<ActiveRule> activeRules = new ArrayList<ActiveRule>(); | |||
for (SearchHit hit : response.getHits()) { | |||
@@ -143,7 +143,7 @@ public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, Active | |||
.setRouting(key) | |||
// TODO replace by scrolling | |||
.setSize(Integer.MAX_VALUE); | |||
SearchResponse response = node.execute(request); | |||
SearchResponse response = getClient().execute(request); | |||
List<ActiveRule> activeRules = new ArrayList<ActiveRule>(); | |||
for (SearchHit hit : response.getHits()) { | |||
activeRules.add(toDoc(hit.getSource())); | |||
@@ -189,7 +189,7 @@ public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, Active | |||
.subAggregation(AggregationBuilders.count("countActiveRules"))) | |||
.setSize(0) | |||
.setTypes(this.getIndexType()); | |||
SearchResponse response = node.execute(request); | |||
SearchResponse response = getClient().execute(request); | |||
Map<String, Multimap<String, FacetValue>> stats = new HashMap<String, Multimap<String, FacetValue>>(); | |||
Aggregation aggregation = response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field()); | |||
for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) { |
@@ -49,7 +49,7 @@ import org.sonar.core.rule.RuleDto; | |||
import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; | |||
import org.sonar.server.rule.Rule; | |||
import org.sonar.server.search.BaseIndex; | |||
import org.sonar.server.search.ESNode; | |||
import org.sonar.server.search.SearchClient; | |||
import org.sonar.server.search.IndexDefinition; | |||
import org.sonar.server.search.IndexField; | |||
import org.sonar.server.search.QueryOptions; | |||
@@ -69,7 +69,7 @@ import static com.google.common.collect.Lists.newArrayList; | |||
public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { | |||
public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { | |||
public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, SearchClient node) { | |||
super(IndexDefinition.RULE, normalizer, workQueue, node); | |||
} | |||
@@ -355,7 +355,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { | |||
QueryBuilder qb = this.getQuery(query, options); | |||
esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); | |||
SearchResponse esResult = node.execute(esSearch); | |||
SearchResponse esResult = getClient().execute(esSearch); | |||
return new Result<Rule>(this, esResult); | |||
} | |||
@@ -378,7 +378,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { | |||
.size(Integer.MAX_VALUE) | |||
.minDocCount(1)); | |||
SearchResponse esResponse = node.execute(request); | |||
SearchResponse esResponse = getClient().execute(request); | |||
Terms aggregation = esResponse.getAggregations().get(key); | |||
@@ -400,7 +400,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { | |||
.setTypes(this.getIndexType()) | |||
.setQuery(QueryBuilders.termQuery(RuleNormalizer.RuleField.ID.field(), id)) | |||
.setSize(1); | |||
SearchResponse response = node.execute(request); | |||
SearchResponse response = getClient().execute(request); | |||
SearchHit hit = response.getHits().getAt(0); | |||
if (hit == null) { | |||
@@ -421,7 +421,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { | |||
.setScroll(TimeValue.timeValueSeconds(3L)) | |||
.setSize(100) | |||
.setQuery(QueryBuilders.termsQuery(RuleNormalizer.RuleField.ID.field(), ids)); | |||
SearchResponse scrollResp = node.execute(request); | |||
SearchResponse scrollResp = getClient().execute(request); | |||
List<Rule> rules = newArrayList(); | |||
while (true) { | |||
@@ -429,7 +429,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { | |||
.prepareSearchScroll(scrollResp.getScrollId()) | |||
.setScroll(TimeValue.timeValueSeconds(3L)); | |||
scrollResp = node.execute(scrollRequest); | |||
scrollResp = getClient().execute(scrollRequest); | |||
for (SearchHit hit : scrollResp.getHits()) { | |||
rules.add(toDoc(hit.getSource())); |
@@ -15,7 +15,7 @@ | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
* Inclient., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.search; | |||
@@ -36,7 +36,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder; | |||
import org.elasticsearch.action.search.SearchResponse; | |||
import org.elasticsearch.action.search.SearchScrollRequestBuilder; | |||
import org.elasticsearch.action.update.UpdateRequest; | |||
import org.elasticsearch.client.Client; | |||
import org.elasticsearch.common.settings.Settings; | |||
import org.elasticsearch.common.unit.TimeValue; | |||
import org.elasticsearch.index.query.BoolFilterBuilder; | |||
@@ -69,14 +68,14 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
private static final Logger LOG = LoggerFactory.getLogger(BaseIndex.class); | |||
protected final ESNode node; | |||
private final SearchClient client; | |||
private final BaseNormalizer<DTO, KEY> normalizer; | |||
private final IndexDefinition indexDefinition; | |||
protected BaseIndex(IndexDefinition indexDefinition, BaseNormalizer<DTO, KEY> normalizer, | |||
WorkQueue workQueue, ESNode node) { | |||
WorkQueue workQueue, SearchClient client) { | |||
this.normalizer = normalizer; | |||
this.node = node; | |||
this.client = client; | |||
this.indexDefinition = indexDefinition; | |||
} | |||
@@ -90,15 +89,10 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
return this.indexDefinition.getIndexType(); | |||
} | |||
protected final Client getClient() { | |||
return node.client(); | |||
} | |||
/* Component Methods */ | |||
@Override | |||
public void start() { | |||
/* Setup the index if necessary */ | |||
initializeIndex(); | |||
} | |||
@@ -108,6 +102,10 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
// nothing to do | |||
} | |||
public SearchClient getClient(){ | |||
return client; | |||
} | |||
// Scrolling within the index | |||
public Iterator<DOMAIN> scroll(final String scrollId) { | |||
return new Iterator<DOMAIN>() { | |||
@@ -116,9 +114,9 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
private void fillQueue() { | |||
try { | |||
SearchScrollRequestBuilder esRequest = getClient().prepareSearchScroll(scrollId) | |||
SearchScrollRequestBuilder esRequest = client.prepareSearchScroll(scrollId) | |||
.setScroll(TimeValue.timeValueMinutes(3)); | |||
Collections.addAll(hits, ((SearchResponse) node.execute(esRequest)).getHits().getHits()); | |||
Collections.addAll(hits, ((SearchResponse) client.execute(esRequest)).getHits().getHits()); | |||
} catch (Exception e) { | |||
throw new IllegalStateException("Error while filling in the scroll buffer", e); | |||
} | |||
@@ -153,19 +151,19 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
protected void initializeIndex() { | |||
String index = this.getIndexName(); | |||
IndicesExistsResponse indexExistsResponse = getClient().admin().indices() | |||
IndicesExistsResponse indexExistsResponse = client.admin().indices() | |||
.prepareExists(index).execute().actionGet(); | |||
try { | |||
if (!indexExistsResponse.isExists()) { | |||
LOG.debug("Setup of {} for type {}", this.getIndexName(), this.getIndexType()); | |||
getClient().admin().indices().prepareCreate(index) | |||
client.admin().indices().prepareCreate(index) | |||
.setSettings(getIndexSettings()) | |||
.execute().actionGet(); | |||
} | |||
LOG.debug("Update of index {} for type {}", this.getIndexName(), this.getIndexType()); | |||
getClient().admin().indices().preparePutMapping(index) | |||
client.admin().indices().preparePutMapping(index) | |||
.setType(getIndexType()) | |||
.setIgnoreConflicts(true) | |||
.setSource(mapDomain()) | |||
@@ -180,10 +178,10 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
IndexStat stat = new IndexStat(); | |||
/** get total document count */ | |||
CountRequestBuilder countRequest = getClient().prepareCount(this.getIndexName()) | |||
CountRequestBuilder countRequest = client.prepareCount(this.getIndexName()) | |||
.setTypes(this.getIndexType()) | |||
.setQuery(QueryBuilders.matchAllQuery()); | |||
CountResponse response = node.execute(countRequest); | |||
CountResponse response = client.execute(countRequest); | |||
stat.setDocumentCount(response.getCount()); | |||
/** get Management information */ | |||
@@ -197,14 +195,14 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
public Date getLastSynchronization() { | |||
Date date; | |||
SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) | |||
SearchRequestBuilder request = client.prepareSearch(this.getIndexName()) | |||
.setTypes(this.getIndexType()) | |||
.setQuery(QueryBuilders.matchAllQuery()) | |||
.setSize(0) | |||
.addAggregation(AggregationBuilders.max("latest") | |||
.field(BaseNormalizer.UPDATED_AT_FIELD)); | |||
SearchResponse response = node.execute(request); | |||
SearchResponse response = client.execute(request); | |||
Max max = (Max) response.getAggregations().get("latest"); | |||
@@ -365,7 +363,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
@Override | |||
public void refresh() { | |||
node.execute(getClient() | |||
client.execute(client | |||
.admin() | |||
.indices() | |||
.prepareRefresh(this.getIndexName()) | |||
@@ -378,13 +376,13 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
protected abstract DOMAIN toDoc(Map<String, Object> fields); | |||
public DOMAIN getByKey(KEY key) { | |||
GetRequestBuilder request = getClient().prepareGet() | |||
GetRequestBuilder request = client.prepareGet() | |||
.setType(this.getIndexType()) | |||
.setIndex(this.getIndexName()) | |||
.setId(this.getKeyValue(key)) | |||
.setRouting(this.getKeyValue(key)); | |||
GetResponse response = node.execute(request); | |||
GetResponse response = client.execute(request); | |||
if (response.isExists()) { | |||
return toDoc(response.getSource()); | |||
@@ -394,7 +392,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
protected void updateDocument(Collection<UpdateRequest> requests, KEY key) { | |||
LOG.debug("UPDATE _id:{} in index {}", key, this.getIndexName()); | |||
BulkRequestBuilder bulkRequest = getClient().prepareBulk(); | |||
BulkRequestBuilder bulkRequest = client.prepareBulk(); | |||
for (UpdateRequest request : requests) { | |||
// if request has no ID then no upsert possible! | |||
if (request.id() == null || request.id().isEmpty()) { | |||
@@ -409,7 +407,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
.type(this.getIndexType())); | |||
} | |||
} | |||
BulkResponse response = node.execute(bulkRequest); | |||
BulkResponse response = client.execute(bulkRequest); | |||
} | |||
@Override | |||
@@ -461,12 +459,12 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
private void deleteDocument(KEY key) throws ExecutionException, InterruptedException { | |||
LOG.debug("DELETE _id:{} in index {}", key, this.getIndexName()); | |||
DeleteRequestBuilder request = getClient() | |||
DeleteRequestBuilder request = client | |||
.prepareDelete() | |||
.setIndex(this.getIndexName()) | |||
.setType(this.getIndexType()) | |||
.setId(this.getKeyValue(key)); | |||
DeleteResponse response = node.execute(request); | |||
DeleteResponse response = client.execute(request); | |||
} | |||
@Override | |||
@@ -527,7 +525,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
} | |||
public Long countAll() { | |||
return getClient().prepareCount(this.getIndexName()) | |||
return client.prepareCount(this.getIndexName()) | |||
.setTypes(this.getIndexType()) | |||
.get().getCount(); | |||
} | |||
@@ -536,7 +534,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) { | |||
Map<String, Long> counts = new HashMap<String, Long>(); | |||
SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) | |||
SearchRequestBuilder request = client.prepareSearch(this.getIndexName()) | |||
.setTypes(this.getIndexType()) | |||
.setQuery(QueryBuilders.filteredQuery( | |||
QueryBuilders.matchAllQuery(), | |||
@@ -549,7 +547,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial | |||
.size(Integer.MAX_VALUE) | |||
.minDocCount(0)); | |||
SearchResponse response = node.execute(request); | |||
SearchResponse response = client.execute(request); | |||
Terms values = | |||
response.getAggregations().get(indexField.field()); |
@@ -30,7 +30,6 @@ public final class IndexProperties { | |||
} | |||
public static final String TYPE = "sonar.search.type"; | |||
public static final String HTTP_PORT = "sonar.search.http.port"; | |||
public static final String NODE_PORT = "sonar.search.port"; | |||
public static final String CLUSTER_NAME = "sonar.cluster.name"; | |||
public static final String NODE_NAME = "sonar.node.name"; |
@@ -0,0 +1,291 @@ | |||
/* | |||
* SonarQube, open source software quality management tool. | |||
* Copyright (C) 2008-2014 SonarSource | |||
* mailto:contact AT sonarsource DOT com | |||
* | |||
* SonarQube is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* SonarQube is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.search; | |||
import com.google.common.annotations.VisibleForTesting; | |||
import org.apache.commons.lang.StringUtils; | |||
import org.elasticsearch.action.ActionRequestBuilder; | |||
import org.elasticsearch.action.ActionResponse; | |||
import org.elasticsearch.action.ListenableActionFuture; | |||
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; | |||
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; | |||
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodeResponse; | |||
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes; | |||
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; | |||
import org.elasticsearch.client.transport.TransportClient; | |||
import org.elasticsearch.common.logging.ESLoggerFactory; | |||
import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; | |||
import org.elasticsearch.common.settings.ImmutableSettings; | |||
import org.elasticsearch.common.transport.InetSocketTransportAddress; | |||
import org.elasticsearch.common.xcontent.ToXContent; | |||
import org.elasticsearch.common.xcontent.XContentBuilder; | |||
import org.elasticsearch.common.xcontent.XContentFactory; | |||
import org.slf4j.Logger; | |||
import org.slf4j.LoggerFactory; | |||
import org.sonar.api.config.Settings; | |||
import org.sonar.core.profiling.Profiling; | |||
import org.sonar.core.profiling.StopWatch; | |||
import javax.annotation.CheckForNull; | |||
import java.io.File; | |||
import java.net.InetAddress; | |||
/** | |||
* ElasticSearch Node used to connect to index. | |||
*/ | |||
public class SearchClient extends TransportClient { | |||
private static final Logger LOG = LoggerFactory.getLogger(SearchClient.class); | |||
private static final String DEFAULT_HEALTH_TIMEOUT = "30s"; | |||
private final Settings settings; | |||
private final String healthTimeout; | |||
protected final Profiling profiling; | |||
public SearchClient(Settings settings) { | |||
this(settings, DEFAULT_HEALTH_TIMEOUT); | |||
} | |||
@VisibleForTesting | |||
SearchClient(Settings settings, String healthTimeout) { | |||
super(ImmutableSettings.settingsBuilder() | |||
.put("network.bind_host", "localhost") | |||
.put("node.rack_id", StringUtils.defaultIfEmpty(settings.getString(IndexProperties.NODE_NAME), "unknown")) | |||
.put("cluster.name", StringUtils.defaultIfBlank(settings.getString(IndexProperties.CLUSTER_NAME), "sonarqube")) | |||
.build() | |||
); | |||
initLogging(); | |||
this.addTransportAddress(new InetSocketTransportAddress("localhost", | |||
settings.getInt(IndexProperties.NODE_PORT))); | |||
this.settings = settings; | |||
this.healthTimeout = healthTimeout; | |||
this.profiling = new Profiling(settings); | |||
} | |||
public NodeHealth getNodeHealth() { | |||
NodeHealth health = new NodeHealth(); | |||
ClusterStatsResponse clusterStatsResponse = this.admin().cluster().prepareClusterStats().get(); | |||
// Cluster health | |||
health.setClusterAvailable(clusterStatsResponse.getStatus() != ClusterHealthStatus.RED); | |||
ClusterStatsNodes nodesStats = clusterStatsResponse.getNodesStats(); | |||
// JVM Heap Usage | |||
health.setJvmHeapMax(nodesStats.getJvm().getHeapMax().bytes()); | |||
health.setJvmHeapUsed(nodesStats.getJvm().getHeapUsed().bytes()); | |||
// OS Memory Usage ? | |||
// Disk Usage | |||
health.setFsTotal(nodesStats.getFs().getTotal().bytes()); | |||
health.setFsAvailable(nodesStats.getFs().getAvailable().bytes()); | |||
// Ping ? | |||
// Threads | |||
health.setJvmThreads(nodesStats.getJvm().getThreads()); | |||
// CPU | |||
health.setProcessCpuPercent(nodesStats.getProcess().getCpuPercent()); | |||
// Open Files | |||
health.setOpenFiles(nodesStats.getProcess().getAvgOpenFileDescriptors()); | |||
// Uptime | |||
health.setJvmUptimeMillis(nodesStats.getJvm().getMaxUpTime().getMillis()); | |||
return health; | |||
} | |||
private ClusterHealthStatus getClusterHealthStatus() { | |||
return this.admin().cluster().prepareHealth() | |||
.setWaitForYellowStatus() | |||
.setTimeout(healthTimeout) | |||
.get() | |||
.getStatus(); | |||
} | |||
@CheckForNull | |||
private NodeInfo getLocalNodeInfoByHostName(String hostname) { | |||
for (ClusterStatsNodeResponse nodeResp : this.admin().cluster().prepareClusterStats().get().getNodes()) { | |||
if (hostname.equals(nodeResp.nodeInfo().getHostname())) { | |||
return nodeResp.nodeInfo(); | |||
} | |||
} | |||
return null; | |||
} | |||
@CheckForNull | |||
private NodeInfo getLocalNodeInfoByNodeName(String nodeName) { | |||
return this.admin().cluster().prepareClusterStats().get().getNodesMap().get(nodeName).nodeInfo(); | |||
} | |||
@CheckForNull | |||
private NodeInfo getLocalNodeInfo() { | |||
if (settings.hasKey(IndexProperties.NODE_NAME)) { | |||
return getLocalNodeInfoByNodeName(settings.getString(IndexProperties.NODE_NAME)); | |||
} else { | |||
try { | |||
String LocalhostName = InetAddress.getLocalHost().getHostName(); | |||
return getLocalNodeInfoByHostName(LocalhostName); | |||
} catch (Exception exception) { | |||
throw new IllegalStateException("Could not get localhost hostname", exception); | |||
} | |||
} | |||
} | |||
// TODO that has nothing to do here!!! | |||
private void addIndexTemplates() { | |||
this.admin().indices() | |||
.preparePutTemplate("default") | |||
.setTemplate("*") | |||
.addMapping("_default_", "{\"dynamic\": \"strict\"}") | |||
.get(); | |||
} | |||
private void initLogging() { | |||
ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory()); | |||
} | |||
private File esHomeDir() { | |||
if (!settings.hasKey("sonar.path.home")) { | |||
throw new IllegalStateException("property 'sonar.path.home' is required"); | |||
} | |||
return new File(settings.getString("sonar.path.home")); | |||
} | |||
private File esDataDir() { | |||
if (settings.hasKey("sonar.path.data")) { | |||
return new File(settings.getString("sonar.path.data"), "es"); | |||
} else { | |||
return new File(settings.getString("sonar.path.home"), "data/es"); | |||
} | |||
} | |||
private File esLogDir() { | |||
if (settings.hasKey("sonar.path.log")) { | |||
return new File(settings.getString("sonar.path.log")); | |||
} else { | |||
return new File(settings.getString("sonar.path.home"), "log"); | |||
} | |||
} | |||
public <K extends ActionResponse> K execute(ActionRequestBuilder request) { | |||
StopWatch basicProfile = profiling.start("search", Profiling.Level.BASIC); | |||
StopWatch fullProfile = profiling.start("search", Profiling.Level.FULL); | |||
ListenableActionFuture acc = request.execute(); | |||
try { | |||
K response = (K) acc.get(); | |||
if (profiling.isProfilingEnabled(Profiling.Level.BASIC)) { | |||
if (ToXContent.class.isAssignableFrom(request.getClass())) { | |||
XContentBuilder debugResponse = XContentFactory.jsonBuilder(); | |||
debugResponse.startObject(); | |||
((ToXContent) request).toXContent(debugResponse, ToXContent.EMPTY_PARAMS); | |||
debugResponse.endObject(); | |||
fullProfile.stop("ES Request: %s", debugResponse.string()); | |||
} else { | |||
fullProfile.stop("ES Request: %s", request.toString().replaceAll("\n", "")); | |||
} | |||
} | |||
if (profiling.isProfilingEnabled(Profiling.Level.FULL)) { | |||
if (ToXContent.class.isAssignableFrom(response.getClass())) { | |||
XContentBuilder debugResponse = XContentFactory.jsonBuilder(); | |||
debugResponse.startObject(); | |||
((ToXContent) response).toXContent(debugResponse, ToXContent.EMPTY_PARAMS); | |||
debugResponse.endObject(); | |||
fullProfile.stop("ES Response: %s", debugResponse.string()); | |||
} else { | |||
fullProfile.stop("ES Response: %s", response.toString()); | |||
} | |||
} | |||
return response; | |||
} catch (Exception e) { | |||
throw new IllegalStateException("ES error: ", e); | |||
} | |||
} | |||
static public ImmutableSettings.Builder getGlobalIndexSettings(ImmutableSettings.Builder builder) { | |||
return builder | |||
// Disable MCast | |||
.put("discovery.zen.ping.multicast.enabled", "false") | |||
// Disable dynamic mapping | |||
.put("index.mapper.dynamic", false) | |||
// Sortable text analyzer | |||
.put("index.analysis.analyzer.sortable.type", "custom") | |||
.put("index.analysis.analyzer.sortable.tokenizer", "keyword") | |||
.putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") | |||
// Edge NGram index-analyzer | |||
.put("index.analysis.analyzer.index_grams.type", "custom") | |||
.put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") | |||
.putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") | |||
// Edge NGram search-analyzer | |||
.put("index.analysis.analyzer.search_grams.type", "custom") | |||
.put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") | |||
.putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") | |||
// Word index-analyzer | |||
.put("index.analysis.analyzer.index_words.type", "custom") | |||
.put("index.analysis.analyzer.index_words.tokenizer", "standard") | |||
.putArray("index.analysis.analyzer.index_words.filter", | |||
"standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") | |||
// Word search-analyzer | |||
.put("index.analysis.analyzer.search_words.type", "custom") | |||
.put("index.analysis.analyzer.search_words.tokenizer", "standard") | |||
.putArray("index.analysis.analyzer.search_words.filter", | |||
"standard", "lowercase", "stop", "asciifolding", "porter_stem") | |||
// Edge NGram filter | |||
.put("index.analysis.filter.gram_filter.type", "edgeNGram") | |||
.put("index.analysis.filter.gram_filter.min_gram", 2) | |||
.put("index.analysis.filter.gram_filter.max_gram", 15) | |||
.putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") | |||
// Word filter | |||
.put("index.analysis.filter.word_filter.type", "word_delimiter") | |||
.put("index.analysis.filter.word_filter.generate_word_parts", true) | |||
.put("index.analysis.filter.word_filter.catenate_words", true) | |||
.put("index.analysis.filter.word_filter.catenate_numbers", true) | |||
.put("index.analysis.filter.word_filter.catenate_all", true) | |||
.put("index.analysis.filter.word_filter.split_on_case_change", true) | |||
.put("index.analysis.filter.word_filter.preserve_original", true) | |||
.put("index.analysis.filter.word_filter.split_on_numerics", true) | |||
.put("index.analysis.filter.word_filter.stem_english_possessive", true) | |||
// Path Analyzer | |||
.put("index.analysis.analyzer.path_analyzer.type", "custom") | |||
.put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); | |||
} | |||
} |
@@ -28,16 +28,16 @@ import java.util.Map; | |||
public class SearchHealth { | |||
private ESNode node; | |||
private SearchClient searchClient; | |||
private IndexClient indexClient; | |||
public SearchHealth(ESNode node, IndexClient indexClient) { | |||
this.node = node; | |||
public SearchHealth(SearchClient searchClient, IndexClient indexClient) { | |||
this.searchClient = searchClient; | |||
this.indexClient = indexClient; | |||
} | |||
public NodeHealth getNodeHealth() { | |||
return node.getNodeHealth(); | |||
return searchClient.getNodeHealth(); | |||
} | |||
public Map<String, IndexHealth> getIndexHealth() { | |||
@@ -49,9 +49,9 @@ public class SearchHealth { | |||
newIndexHealth.documentCount = indexStat.getDocumentCount(); | |||
newIndexHealth.lastSync = indexStat.getLastUpdate(); | |||
IndicesStatsRequestBuilder statRequest = node.client().admin().indices().prepareStats(index.getIndexName()) | |||
IndicesStatsRequestBuilder statRequest = searchClient.admin().indices().prepareStats(index.getIndexName()) | |||
.setTypes(index.getIndexType()); | |||
IndicesStatsResponse indicesStatsResponse = node.execute(statRequest); | |||
IndicesStatsResponse indicesStatsResponse = searchClient.execute(statRequest); | |||
newIndexHealth.segmentCount = indicesStatsResponse.getTotal().getSegments().getCount(); | |||
newIndexHealth.pendingDeletion = indicesStatsResponse.getTotal().getDocs().getDeleted(); | |||
@@ -43,7 +43,7 @@ public class BaseIndexTest { | |||
@Rule | |||
public TemporaryFolder temp = new TemporaryFolder(); | |||
ESNode node; | |||
SearchClient node; | |||
@Before | |||
public void setup() throws IOException { | |||
@@ -51,13 +51,13 @@ public class BaseIndexTest { | |||
Settings settings = new Settings(); | |||
settings.setProperty("sonar.path.home", dataDir.getAbsolutePath()); | |||
settings.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name()); | |||
node = new ESNode(settings); | |||
node.start(); | |||
node = new SearchClient(settings); | |||
//node.start(); | |||
} | |||
@After | |||
public void tearDown() { | |||
node.stop(); | |||
//node.stop(); | |||
} | |||
@Test | |||
@@ -77,10 +77,10 @@ public class BaseIndexTest { | |||
} | |||
private BaseIndex getIndex(final ESNode esNode) { | |||
private BaseIndex getIndex(final SearchClient searchClient) { | |||
BaseIndex index = new BaseIndex( | |||
IndexDefinition.TEST, | |||
null, new NullQueue(), esNode) { | |||
null, new NullQueue(), searchClient) { | |||
@Override | |||
protected String getKeyValue(Serializable key) { | |||
return null; |
@@ -0,0 +1,129 @@ | |||
/* | |||
* SonarQube, open source software quality management tool. | |||
* Copyright (C) 2008-2014 SonarSource | |||
* mailto:contact AT sonarsource DOT com | |||
* | |||
* SonarQube is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* SonarQube is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.search; | |||
public class SearchClientTest { | |||
// File dataDir; | |||
// Settings settings; | |||
// | |||
// @Rule | |||
// public TemporaryFolder temp = new TemporaryFolder(); | |||
// | |||
// @Before | |||
// public void createMocks() throws IOException { | |||
// dataDir = temp.newFolder(); | |||
// settings = new Settings(); | |||
// settings.setProperty("sonar.path.home", dataDir.getAbsolutePath()); | |||
// settings.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name()); | |||
// } | |||
// | |||
// @Test(expected = StrictDynamicMappingException.class) | |||
// public void should_use_default_settings_for_index() throws Exception { | |||
// SearchClient node = new SearchClient(settings); | |||
// node.start(); | |||
// | |||
// node.client().admin().indices().prepareCreate("strict") | |||
// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") | |||
// .execute().actionGet(); | |||
// node.client().admin().cluster().prepareHealth("strict").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); | |||
// | |||
// // strict mapping is enforced | |||
// try { | |||
// node.client().prepareIndex("strict", "type1", "666").setSource( | |||
// XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject() | |||
// ).get(); | |||
// } finally { | |||
// node.stop(); | |||
// } | |||
// } | |||
// | |||
// @Test | |||
// public void check_path_analyzer() throws Exception { | |||
// SearchClient node = new SearchClient(settings); | |||
// node.start(); | |||
// | |||
// node.client().admin().indices().prepareCreate("path") | |||
// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") | |||
// .execute().actionGet(); | |||
// node.client().admin().cluster().prepareHealth("path").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); | |||
// | |||
// // default "path_analyzer" analyzer is defined for all indices | |||
// AnalyzeResponse response = node.client().admin().indices() | |||
// .prepareAnalyze("path", "/temp/65236/test path/MyFile.java").setAnalyzer("path_analyzer").get(); | |||
// // default "path_analyzer" analyzer is defined for all indices | |||
// assertThat(response.getTokens()).hasSize(4); | |||
// assertThat(response.getTokens().get(0).getTerm()).isEqualTo("/temp"); | |||
// assertThat(response.getTokens().get(1).getTerm()).isEqualTo("/temp/65236"); | |||
// assertThat(response.getTokens().get(2).getTerm()).isEqualTo("/temp/65236/test path"); | |||
// assertThat(response.getTokens().get(3).getTerm()).isEqualTo("/temp/65236/test path/MyFile.java"); | |||
// | |||
// node.stop(); | |||
// } | |||
// | |||
// @Test | |||
// public void check_sortable_analyzer() throws Exception { | |||
// SearchClient node = new SearchClient(settings); | |||
// node.start(); | |||
// | |||
// node.client().admin().indices().prepareCreate("sort") | |||
// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") | |||
// .execute().actionGet(); | |||
// node.client().admin().cluster().prepareHealth("sort").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); | |||
// | |||
// // default "sortable" analyzer is defined for all indices | |||
// assertThat(node.client().admin().indices() | |||
// .prepareAnalyze("sort", "This Is A Wonderful Text").setAnalyzer("sortable").get() | |||
// .getTokens().get(0).getTerm()).isEqualTo("this is a "); | |||
// | |||
// node.stop(); | |||
// } | |||
// | |||
// @Test | |||
// public void check_gram_analyzer() throws Exception { | |||
// SearchClient node = new SearchClient(settings); | |||
// node.start(); | |||
// | |||
// node.client().admin().indices().prepareCreate("gram") | |||
// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") | |||
// .execute().actionGet(); | |||
// node.client().admin().cluster().prepareHealth("gram").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); | |||
// | |||
// // default "string_gram" analyzer is defined for all indices | |||
// AnalyzeResponse response = node.client().admin().indices() | |||
// .prepareAnalyze("gram", "he.llo w@rl#d").setAnalyzer("index_grams").get(); | |||
// assertThat(response.getTokens()).hasSize(10); | |||
// assertThat(response.getTokens().get(0).getTerm()).isEqualTo("he"); | |||
// assertThat(response.getTokens().get(7).getTerm()).isEqualTo("w@rl"); | |||
// | |||
// node.stop(); | |||
// } | |||
// | |||
// @Test | |||
// public void should_fail_to_get_client_if_not_started() { | |||
// SearchClient node = new SearchClient(settings); | |||
// try { | |||
// node.client(); | |||
// fail(); | |||
// } catch (IllegalStateException e) { | |||
// assertThat(e).hasMessage("Elasticsearch is not started"); | |||
// } | |||
// } | |||
} |
@@ -19,23 +19,22 @@ | |||
*/ | |||
package org.sonar.server.tester; | |||
import org.elasticsearch.client.Client; | |||
import org.elasticsearch.index.query.QueryBuilders; | |||
import org.sonar.api.ServerComponent; | |||
import org.sonar.core.persistence.DatabaseVersion; | |||
import org.sonar.core.persistence.DbSession; | |||
import org.sonar.core.persistence.MyBatis; | |||
import org.sonar.server.search.ESNode; | |||
import org.sonar.server.search.SearchClient; | |||
import java.sql.Connection; | |||
public class BackendCleanup implements ServerComponent { | |||
private final ESNode esNode; | |||
private final SearchClient searchClient; | |||
private final MyBatis myBatis; | |||
public BackendCleanup(ESNode esNode, MyBatis myBatis) { | |||
this.esNode = esNode; | |||
public BackendCleanup(SearchClient searchClient, MyBatis myBatis) { | |||
this.searchClient = searchClient; | |||
this.myBatis = myBatis; | |||
} | |||
@@ -64,8 +63,7 @@ public class BackendCleanup implements ServerComponent { | |||
} | |||
public void clearIndexes() { | |||
Client client = esNode.client(); | |||
client.prepareDeleteByQuery(client.admin().cluster().prepareState().get() | |||
searchClient.prepareDeleteByQuery(searchClient.admin().cluster().prepareState().get() | |||
.getState().getMetaData().concreteAllIndices()) | |||
.setQuery(QueryBuilders.matchAllQuery()) | |||
.get(); |
@@ -25,6 +25,10 @@ import org.apache.commons.io.FileUtils; | |||
import org.apache.commons.lang.StringUtils; | |||
import org.junit.rules.ExternalResource; | |||
import org.sonar.api.database.DatabaseProperties; | |||
import org.sonar.process.MonitoredProcess; | |||
import org.sonar.process.NetworkUtils; | |||
import org.sonar.process.Props; | |||
import org.sonar.search.SearchServer; | |||
import org.sonar.server.platform.Platform; | |||
import org.sonar.server.search.IndexProperties; | |||
import org.sonar.server.ws.WsTester; | |||
@@ -48,6 +52,10 @@ public class ServerTester extends ExternalResource { | |||
private static final String PROP_PREFIX = "mediumTests."; | |||
private final String clusterName; | |||
private final Integer clusterPort; | |||
private SearchServer searchServer; | |||
private final Platform platform; | |||
private final File homeDir; | |||
private final List components = Lists.newArrayList(BackendCleanup.class, WsTester.class); | |||
@@ -56,6 +64,21 @@ public class ServerTester extends ExternalResource { | |||
public ServerTester() { | |||
homeDir = createTempDir(); | |||
platform = new Platform(); | |||
clusterName = "cluster-mem-" + System.currentTimeMillis(); | |||
clusterPort = NetworkUtils.freePort(); | |||
Properties properties = new Properties(); | |||
properties.setProperty(IndexProperties.CLUSTER_NAME, clusterName); | |||
properties.setProperty(IndexProperties.NODE_PORT, clusterPort.toString()); | |||
properties.setProperty(MonitoredProcess.NAME_PROPERTY, "ES"); | |||
properties.setProperty("sonar.path.data", new File(homeDir, "data/es").getAbsolutePath()); | |||
properties.setProperty("sonar.path.logs", new File(homeDir, "logs").getAbsolutePath()); | |||
properties.setProperty("sonar.path.temp", new File(homeDir, "temp").getAbsolutePath()); | |||
try { | |||
searchServer = new SearchServer(new Props(properties), false, false); | |||
} catch (Exception e) { | |||
e.printStackTrace(); | |||
} | |||
} | |||
/** | |||
@@ -74,7 +97,21 @@ public class ServerTester extends ExternalResource { | |||
Properties properties = new Properties(); | |||
properties.putAll(initialProps); | |||
properties.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name()); | |||
try { | |||
searchServer.start(); | |||
System.out.println("searchServer = " + searchServer.isReady()); | |||
while (!searchServer.isReady()) { | |||
Thread.sleep(100); | |||
} | |||
} catch (Exception e) { | |||
e.printStackTrace(); | |||
} | |||
properties.setProperty(IndexProperties.CLUSTER_NAME, clusterName); | |||
properties.setProperty(IndexProperties.NODE_PORT, clusterPort.toString()); | |||
properties.setProperty(MonitoredProcess.NAME_PROPERTY, "ES"); | |||
properties.setProperty("sonar.path.home", homeDir.getAbsolutePath()); | |||
properties.setProperty(DatabaseProperties.PROP_URL, "jdbc:h2:" + homeDir.getAbsolutePath() + "/h2"); | |||
for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) { | |||
@@ -118,6 +155,7 @@ public class ServerTester extends ExternalResource { | |||
*/ | |||
public void stop() { | |||
platform.doStop(); | |||
searchServer.terminate(); | |||
FileUtils.deleteQuietly(homeDir); | |||
} | |||