diff options
author | Stephane Gamard <stephane.gamard@searchbox.com> | 2014-08-11 10:10:11 +0200 |
---|---|---|
committer | Stephane Gamard <stephane.gamard@searchbox.com> | 2014-08-11 10:38:39 +0200 |
commit | 7c3daba67adfda896f528d35b2f6120d685c3983 (patch) | |
tree | 12b6760e5c088669468a9becb2c7ab4bf1ad0a69 /server | |
parent | e1937fdb8b0f5e7a695a39ed5a97241601da18fe (diff) | |
download | sonarqube-7c3daba67adfda896f528d35b2f6120d685c3983.tar.gz sonarqube-7c3daba67adfda896f528d35b2f6120d685c3983.zip |
Extracted ES from sonar-server & updated ServerTester for mediumTests
Diffstat (limited to 'server')
16 files changed, 572 insertions, 89 deletions
diff --git a/server/process/sonar-process/src/main/java/org/sonar/process/MonitoredProcess.java b/server/process/sonar-process/src/main/java/org/sonar/process/MonitoredProcess.java index 670dc645555..3dde4e021c5 100644 --- a/server/process/sonar-process/src/main/java/org/sonar/process/MonitoredProcess.java +++ b/server/process/sonar-process/src/main/java/org/sonar/process/MonitoredProcess.java @@ -41,7 +41,14 @@ public abstract class MonitoredProcess implements ProcessMXBean { private ScheduledFuture<?> pingTask = null; private ScheduledExecutorService monitor; + private final boolean isMonitored; + protected MonitoredProcess(Props props) throws Exception { + this(props, false); + } + + protected MonitoredProcess(Props props, boolean monitor) throws Exception { + this.isMonitored = monitor; this.props = props; this.name = props.of(NAME_PROPERTY); @@ -61,7 +68,9 @@ public abstract class MonitoredProcess implements ProcessMXBean { Logger logger = LoggerFactory.getLogger(getClass()); logger.debug("Process[{}] starting", name); - scheduleAutokill(); + if (this.isMonitored) { + scheduleAutokill(); + } doStart(); logger.debug("Process[{}] started", name); } diff --git a/server/process/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java b/server/process/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java index 8ff2f758974..12e59cd8f06 100644 --- a/server/process/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java +++ b/server/process/sonar-process/src/main/java/org/sonar/process/ProcessWrapper.java @@ -28,6 +28,8 @@ import javax.annotation.CheckForNull; import javax.annotation.Nullable; import javax.management.JMX; import javax.management.MBeanServerConnection; +import javax.management.Notification; +import javax.management.NotificationListener; import javax.management.remote.JMXConnector; import javax.management.remote.JMXConnectorFactory; import javax.management.remote.JMXServiceURL; @@ -242,6 +244,12 @@ public class ProcessWrapper extends Thread implements Terminable { Thread.sleep(1000L); LOGGER.debug("Try #{} to connect to JMX server for process '{}'", i, processName); JMXConnector jmxConnector = JMXConnectorFactory.connect(jmxUrl, null); + jmxConnector.addConnectionNotificationListener(new NotificationListener() { + @Override + public void handleNotification(Notification notification, Object handback) { + LOGGER.info("JMX Connection Notification:{}",notification.getMessage()); + } + }, null, null); MBeanServerConnection mBeanServer = jmxConnector.getMBeanServerConnection(); ProcessMXBean bean = JMX.newMBeanProxy(mBeanServer, JmxUtils.objectName(processName), ProcessMXBean.class); return bean; diff --git a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java index f1de561955c..486e8e612ef 100644 --- a/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java +++ b/server/sonar-search/src/main/java/org/sonar/search/SearchServer.java @@ -49,11 +49,13 @@ public class SearchServer extends MonitoredProcess { private static final Integer MINIMUM_INDEX_REPLICATION = 1; private final Set<String> nodes = new HashSet<String>(); + private final boolean isBlocking; private Node node; - SearchServer(Props props) throws Exception { - super(props); + public SearchServer(Props props, boolean monitored, boolean blocking) throws Exception { + super(props, monitored); + this.isBlocking = blocking; new MinimumViableSystem().check(); if (StringUtils.isNotEmpty(props.of(ES_CLUSTER_INET, null))) { @@ -61,6 +63,10 @@ public class SearchServer extends MonitoredProcess { } } + public SearchServer(Props props) throws Exception { + this(props, true, true); + } + @Override protected boolean doIsReady() { return (node.client().admin().cluster().prepareHealth() @@ -85,7 +91,7 @@ public class SearchServer extends MonitoredProcess { // Disable MCast .put("discovery.zen.ping.multicast.enabled", "false") - // Index storage policies + // Index storage policies .put("index.merge.policy.max_merge_at_once", "200") .put("index.merge.policy.segments_per_tier", "200") .put("index.number_of_shards", "1") @@ -94,15 +100,15 @@ public class SearchServer extends MonitoredProcess { .put("indices.store.throttle.type", "merge") .put("indices.store.throttle.max_bytes_per_sec", "200mb") - // Install our own listUpdate scripts + // Install our own listUpdate scripts .put("script.default_lang", "native") .put("script.native." + ListUpdate.NAME + ".type", ListUpdate.UpdateListScriptFactory.class.getName()) - // Node is pure transport + // Node is pure transport .put("transport.tcp.port", port) .put("http.enabled", false) - // Setting up ES paths + // Setting up ES paths .put("path.data", new File(dataDir, "es").getAbsolutePath()) .put("path.work", new File(tempDir).getAbsolutePath()) .put("path.logs", new File(logDir).getAbsolutePath()); @@ -119,7 +125,7 @@ public class SearchServer extends MonitoredProcess { // Set cluster coordinates esSettings.put("cluster.name", clusterName); esSettings.put("node.rack_id", StringUtils.defaultIfEmpty(props.of(SONAR_NODE_NAME), "unknown")); - esSettings.put("cluster.routing.allocation.awareness.attributes","rack_id"); + esSettings.put("cluster.routing.allocation.awareness.attributes", "rack_id"); if (props.contains(SONAR_NODE_NAME)) { esSettings.put("node.name", props.of(SONAR_NODE_NAME)); } else { @@ -138,11 +144,13 @@ public class SearchServer extends MonitoredProcess { .settings(esSettings) .build().start(); - while (node != null && !node.isClosed()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - // Ignore + if (this.isBlocking) { + while (node != null && !node.isClosed()) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + // Ignore + } } } } @@ -153,40 +161,40 @@ public class SearchServer extends MonitoredProcess { // Disallow dynamic mapping (too expensive) .put("index.mapper.dynamic", false) - // Sortable text analyzer + // Sortable text analyzer .put("index.analysis.analyzer.sortable.type", "custom") .put("index.analysis.analyzer.sortable.tokenizer", "keyword") .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") - // Edge NGram index-analyzer + // Edge NGram index-analyzer .put("index.analysis.analyzer.index_grams.type", "custom") .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") - // Edge NGram search-analyzer + // Edge NGram search-analyzer .put("index.analysis.analyzer.search_grams.type", "custom") .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") - // Word index-analyzer + // Word index-analyzer .put("index.analysis.analyzer.index_words.type", "custom") .put("index.analysis.analyzer.index_words.tokenizer", "standard") .putArray("index.analysis.analyzer.index_words.filter", "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") - // Word search-analyzer + // Word search-analyzer .put("index.analysis.analyzer.search_words.type", "custom") .put("index.analysis.analyzer.search_words.tokenizer", "standard") .putArray("index.analysis.analyzer.search_words.filter", "standard", "lowercase", "stop", "asciifolding", "porter_stem") - // Edge NGram filter + // Edge NGram filter .put("index.analysis.filter.gram_filter.type", "edgeNGram") .put("index.analysis.filter.gram_filter.min_gram", 2) .put("index.analysis.filter.gram_filter.max_gram", 15) .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") - // Word filter + // Word filter .put("index.analysis.filter.word_filter.type", "word_delimiter") .put("index.analysis.filter.word_filter.generate_word_parts", true) .put("index.analysis.filter.word_filter.catenate_words", true) @@ -197,7 +205,7 @@ public class SearchServer extends MonitoredProcess { .put("index.analysis.filter.word_filter.split_on_numerics", true) .put("index.analysis.filter.word_filter.stem_english_possessive", true) - // Path Analyzer + // Path Analyzer .put("index.analysis.analyzer.path_analyzer.type", "custom") .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); diff --git a/server/sonar-server/pom.xml b/server/sonar-server/pom.xml index 92c5dbaf7d6..e1b8f74bb22 100644 --- a/server/sonar-server/pom.xml +++ b/server/sonar-server/pom.xml @@ -239,7 +239,12 @@ <artifactId>http-request</artifactId> <scope>test</scope> </dependency> - + <dependency> + <groupId>org.codehaus.sonar</groupId> + <artifactId>sonar-search</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java index 6d81789168f..01ffc420f82 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/activity/index/ActivityIndex.java @@ -35,7 +35,7 @@ import org.sonar.core.activity.Activity; import org.sonar.core.activity.db.ActivityDto; import org.sonar.core.cluster.WorkQueue; import org.sonar.server.search.BaseIndex; -import org.sonar.server.search.ESNode; +import org.sonar.server.search.SearchClient; import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.IndexField; import org.sonar.server.search.QueryOptions; @@ -51,7 +51,7 @@ import java.util.Map; */ public class ActivityIndex extends BaseIndex<Activity, ActivityDto, String> { - public ActivityIndex(ActivityNormalizer normalizer, WorkQueue workQueue, ESNode node) { + public ActivityIndex(ActivityNormalizer normalizer, WorkQueue workQueue, SearchClient node) { super(IndexDefinition.LOG, normalizer, workQueue, node); } @@ -93,7 +93,7 @@ public class ActivityIndex extends BaseIndex<Activity, ActivityDto, String> { .setQuery(QueryBuilders.matchAllQuery()) .setTypes(this.getIndexType()) .setSize(Integer.MAX_VALUE); - SearchResponse response = node.execute(request); + SearchResponse response = getClient().execute(request); return new Result<Activity>(this, response); } @@ -144,7 +144,7 @@ public class ActivityIndex extends BaseIndex<Activity, ActivityDto, String> { esSearch.setScroll(TimeValue.timeValueMinutes(3)); } - SearchResponse response = node.execute(esSearch); + SearchResponse response = getClient().execute(esSearch); return response; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java index caa430bedd7..4cfce8e9a81 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/ServerComponents.java @@ -215,7 +215,7 @@ class ServerComponents { ActivityDao.class, // Elasticsearch - ESNode.class, + SearchClient.class, RuleNormalizer.class, ActiveRuleNormalizer.class, RuleIndex.class, diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java index a851c038c72..e71647d3415 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndex.java @@ -39,7 +39,7 @@ import org.sonar.core.qualityprofile.db.ActiveRuleKey; import org.sonar.server.qualityprofile.ActiveRule; import org.sonar.server.rule.index.RuleNormalizer; import org.sonar.server.search.BaseIndex; -import org.sonar.server.search.ESNode; +import org.sonar.server.search.SearchClient; import org.sonar.server.search.FacetValue; import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.IndexField; @@ -52,7 +52,7 @@ import java.util.Map; public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, ActiveRuleKey> { - public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { + public ActiveRuleIndex(ActiveRuleNormalizer normalizer, WorkQueue workQueue, SearchClient node) { super(IndexDefinition.ACTIVE_RULE, normalizer, workQueue, node); } @@ -128,7 +128,7 @@ public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, Active // TODO replace by scrolling .setSize(Integer.MAX_VALUE); - SearchResponse response = node.execute(request); + SearchResponse response = getClient().execute(request); List<ActiveRule> activeRules = new ArrayList<ActiveRule>(); for (SearchHit hit : response.getHits()) { @@ -143,7 +143,7 @@ public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, Active .setRouting(key) // TODO replace by scrolling .setSize(Integer.MAX_VALUE); - SearchResponse response = node.execute(request); + SearchResponse response = getClient().execute(request); List<ActiveRule> activeRules = new ArrayList<ActiveRule>(); for (SearchHit hit : response.getHits()) { activeRules.add(toDoc(hit.getSource())); @@ -189,7 +189,7 @@ public class ActiveRuleIndex extends BaseIndex<ActiveRule, ActiveRuleDto, Active .subAggregation(AggregationBuilders.count("countActiveRules"))) .setSize(0) .setTypes(this.getIndexType()); - SearchResponse response = node.execute(request); + SearchResponse response = getClient().execute(request); Map<String, Multimap<String, FacetValue>> stats = new HashMap<String, Multimap<String, FacetValue>>(); Aggregation aggregation = response.getAggregations().get(ActiveRuleNormalizer.ActiveRuleField.PROFILE_KEY.field()); for (Terms.Bucket value : ((Terms) aggregation).getBuckets()) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java index ca4e42f7a9d..a80f4d13ef6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -49,7 +49,7 @@ import org.sonar.core.rule.RuleDto; import org.sonar.server.qualityprofile.index.ActiveRuleNormalizer; import org.sonar.server.rule.Rule; import org.sonar.server.search.BaseIndex; -import org.sonar.server.search.ESNode; +import org.sonar.server.search.SearchClient; import org.sonar.server.search.IndexDefinition; import org.sonar.server.search.IndexField; import org.sonar.server.search.QueryOptions; @@ -69,7 +69,7 @@ import static com.google.common.collect.Lists.newArrayList; public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { - public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, ESNode node) { + public RuleIndex(RuleNormalizer normalizer, WorkQueue workQueue, SearchClient node) { super(IndexDefinition.RULE, normalizer, workQueue, node); } @@ -355,7 +355,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { QueryBuilder qb = this.getQuery(query, options); esSearch.setQuery(QueryBuilders.filteredQuery(qb, fb)); - SearchResponse esResult = node.execute(esSearch); + SearchResponse esResult = getClient().execute(esSearch); return new Result<Rule>(this, esResult); } @@ -378,7 +378,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { .size(Integer.MAX_VALUE) .minDocCount(1)); - SearchResponse esResponse = node.execute(request); + SearchResponse esResponse = getClient().execute(request); Terms aggregation = esResponse.getAggregations().get(key); @@ -400,7 +400,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { .setTypes(this.getIndexType()) .setQuery(QueryBuilders.termQuery(RuleNormalizer.RuleField.ID.field(), id)) .setSize(1); - SearchResponse response = node.execute(request); + SearchResponse response = getClient().execute(request); SearchHit hit = response.getHits().getAt(0); if (hit == null) { @@ -421,7 +421,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { .setScroll(TimeValue.timeValueSeconds(3L)) .setSize(100) .setQuery(QueryBuilders.termsQuery(RuleNormalizer.RuleField.ID.field(), ids)); - SearchResponse scrollResp = node.execute(request); + SearchResponse scrollResp = getClient().execute(request); List<Rule> rules = newArrayList(); while (true) { @@ -429,7 +429,7 @@ public class RuleIndex extends BaseIndex<Rule, RuleDto, RuleKey> { .prepareSearchScroll(scrollResp.getScrollId()) .setScroll(TimeValue.timeValueSeconds(3L)); - scrollResp = node.execute(scrollRequest); + scrollResp = getClient().execute(scrollRequest); for (SearchHit hit : scrollResp.getHits()) { rules.add(toDoc(hit.getSource())); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java index d3ab18ae5cb..271d439801d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/BaseIndex.java @@ -15,7 +15,7 @@ * * You should have received a copy of the GNU Lesser General Public License * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * Inclient., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package org.sonar.server.search; @@ -36,7 +36,6 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequestBuilder; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolFilterBuilder; @@ -69,14 +68,14 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial private static final Logger LOG = LoggerFactory.getLogger(BaseIndex.class); - protected final ESNode node; + private final SearchClient client; private final BaseNormalizer<DTO, KEY> normalizer; private final IndexDefinition indexDefinition; protected BaseIndex(IndexDefinition indexDefinition, BaseNormalizer<DTO, KEY> normalizer, - WorkQueue workQueue, ESNode node) { + WorkQueue workQueue, SearchClient client) { this.normalizer = normalizer; - this.node = node; + this.client = client; this.indexDefinition = indexDefinition; } @@ -90,15 +89,10 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial return this.indexDefinition.getIndexType(); } - protected final Client getClient() { - return node.client(); - } - /* Component Methods */ @Override public void start() { - /* Setup the index if necessary */ initializeIndex(); } @@ -108,6 +102,10 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial // nothing to do } + public SearchClient getClient(){ + return client; + } + // Scrolling within the index public Iterator<DOMAIN> scroll(final String scrollId) { return new Iterator<DOMAIN>() { @@ -116,9 +114,9 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial private void fillQueue() { try { - SearchScrollRequestBuilder esRequest = getClient().prepareSearchScroll(scrollId) + SearchScrollRequestBuilder esRequest = client.prepareSearchScroll(scrollId) .setScroll(TimeValue.timeValueMinutes(3)); - Collections.addAll(hits, ((SearchResponse) node.execute(esRequest)).getHits().getHits()); + Collections.addAll(hits, ((SearchResponse) client.execute(esRequest)).getHits().getHits()); } catch (Exception e) { throw new IllegalStateException("Error while filling in the scroll buffer", e); } @@ -153,19 +151,19 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial protected void initializeIndex() { String index = this.getIndexName(); - IndicesExistsResponse indexExistsResponse = getClient().admin().indices() + IndicesExistsResponse indexExistsResponse = client.admin().indices() .prepareExists(index).execute().actionGet(); try { if (!indexExistsResponse.isExists()) { LOG.debug("Setup of {} for type {}", this.getIndexName(), this.getIndexType()); - getClient().admin().indices().prepareCreate(index) + client.admin().indices().prepareCreate(index) .setSettings(getIndexSettings()) .execute().actionGet(); } LOG.debug("Update of index {} for type {}", this.getIndexName(), this.getIndexType()); - getClient().admin().indices().preparePutMapping(index) + client.admin().indices().preparePutMapping(index) .setType(getIndexType()) .setIgnoreConflicts(true) .setSource(mapDomain()) @@ -180,10 +178,10 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial IndexStat stat = new IndexStat(); /** get total document count */ - CountRequestBuilder countRequest = getClient().prepareCount(this.getIndexName()) + CountRequestBuilder countRequest = client.prepareCount(this.getIndexName()) .setTypes(this.getIndexType()) .setQuery(QueryBuilders.matchAllQuery()); - CountResponse response = node.execute(countRequest); + CountResponse response = client.execute(countRequest); stat.setDocumentCount(response.getCount()); /** get Management information */ @@ -197,14 +195,14 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial public Date getLastSynchronization() { Date date; - SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) + SearchRequestBuilder request = client.prepareSearch(this.getIndexName()) .setTypes(this.getIndexType()) .setQuery(QueryBuilders.matchAllQuery()) .setSize(0) .addAggregation(AggregationBuilders.max("latest") .field(BaseNormalizer.UPDATED_AT_FIELD)); - SearchResponse response = node.execute(request); + SearchResponse response = client.execute(request); Max max = (Max) response.getAggregations().get("latest"); @@ -365,7 +363,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial @Override public void refresh() { - node.execute(getClient() + client.execute(client .admin() .indices() .prepareRefresh(this.getIndexName()) @@ -378,13 +376,13 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial protected abstract DOMAIN toDoc(Map<String, Object> fields); public DOMAIN getByKey(KEY key) { - GetRequestBuilder request = getClient().prepareGet() + GetRequestBuilder request = client.prepareGet() .setType(this.getIndexType()) .setIndex(this.getIndexName()) .setId(this.getKeyValue(key)) .setRouting(this.getKeyValue(key)); - GetResponse response = node.execute(request); + GetResponse response = client.execute(request); if (response.isExists()) { return toDoc(response.getSource()); @@ -394,7 +392,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial protected void updateDocument(Collection<UpdateRequest> requests, KEY key) { LOG.debug("UPDATE _id:{} in index {}", key, this.getIndexName()); - BulkRequestBuilder bulkRequest = getClient().prepareBulk(); + BulkRequestBuilder bulkRequest = client.prepareBulk(); for (UpdateRequest request : requests) { // if request has no ID then no upsert possible! if (request.id() == null || request.id().isEmpty()) { @@ -409,7 +407,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial .type(this.getIndexType())); } } - BulkResponse response = node.execute(bulkRequest); + BulkResponse response = client.execute(bulkRequest); } @Override @@ -461,12 +459,12 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial private void deleteDocument(KEY key) throws ExecutionException, InterruptedException { LOG.debug("DELETE _id:{} in index {}", key, this.getIndexName()); - DeleteRequestBuilder request = getClient() + DeleteRequestBuilder request = client .prepareDelete() .setIndex(this.getIndexName()) .setType(this.getIndexType()) .setId(this.getKeyValue(key)); - DeleteResponse response = node.execute(request); + DeleteResponse response = client.execute(request); } @Override @@ -527,7 +525,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial } public Long countAll() { - return getClient().prepareCount(this.getIndexName()) + return client.prepareCount(this.getIndexName()) .setTypes(this.getIndexType()) .get().getCount(); } @@ -536,7 +534,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial public Map<String, Long> countByField(IndexField indexField, FilterBuilder filter) { Map<String, Long> counts = new HashMap<String, Long>(); - SearchRequestBuilder request = getClient().prepareSearch(this.getIndexName()) + SearchRequestBuilder request = client.prepareSearch(this.getIndexName()) .setTypes(this.getIndexType()) .setQuery(QueryBuilders.filteredQuery( QueryBuilders.matchAllQuery(), @@ -549,7 +547,7 @@ public abstract class BaseIndex<DOMAIN, DTO extends Dto<KEY>, KEY extends Serial .size(Integer.MAX_VALUE) .minDocCount(0)); - SearchResponse response = node.execute(request); + SearchResponse response = client.execute(request); Terms values = response.getAggregations().get(indexField.field()); diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/IndexProperties.java b/server/sonar-server/src/main/java/org/sonar/server/search/IndexProperties.java index fbaad7a4834..d809845e770 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/IndexProperties.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/IndexProperties.java @@ -30,7 +30,6 @@ public final class IndexProperties { } public static final String TYPE = "sonar.search.type"; - public static final String HTTP_PORT = "sonar.search.http.port"; public static final String NODE_PORT = "sonar.search.port"; public static final String CLUSTER_NAME = "sonar.cluster.name"; public static final String NODE_NAME = "sonar.node.name"; diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/SearchClient.java b/server/sonar-server/src/main/java/org/sonar/server/search/SearchClient.java new file mode 100644 index 00000000000..1987943e848 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/search/SearchClient.java @@ -0,0 +1,291 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.search; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodeResponse; +import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes; +import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.logging.slf4j.Slf4jESLoggerFactory; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonar.api.config.Settings; +import org.sonar.core.profiling.Profiling; +import org.sonar.core.profiling.StopWatch; + +import javax.annotation.CheckForNull; +import java.io.File; +import java.net.InetAddress; + +/** + * ElasticSearch Node used to connect to index. + */ +public class SearchClient extends TransportClient { + + private static final Logger LOG = LoggerFactory.getLogger(SearchClient.class); + private static final String DEFAULT_HEALTH_TIMEOUT = "30s"; + + private final Settings settings; + private final String healthTimeout; + + protected final Profiling profiling; + + public SearchClient(Settings settings) { + this(settings, DEFAULT_HEALTH_TIMEOUT); + } + + @VisibleForTesting + SearchClient(Settings settings, String healthTimeout) { + super(ImmutableSettings.settingsBuilder() + .put("network.bind_host", "localhost") + .put("node.rack_id", StringUtils.defaultIfEmpty(settings.getString(IndexProperties.NODE_NAME), "unknown")) + .put("cluster.name", StringUtils.defaultIfBlank(settings.getString(IndexProperties.CLUSTER_NAME), "sonarqube")) + .build() + ); + initLogging(); + this.addTransportAddress(new InetSocketTransportAddress("localhost", + settings.getInt(IndexProperties.NODE_PORT))); + this.settings = settings; + this.healthTimeout = healthTimeout; + this.profiling = new Profiling(settings); + } + + public NodeHealth getNodeHealth() { + NodeHealth health = new NodeHealth(); + ClusterStatsResponse clusterStatsResponse = this.admin().cluster().prepareClusterStats().get(); + + // Cluster health + health.setClusterAvailable(clusterStatsResponse.getStatus() != ClusterHealthStatus.RED); + + ClusterStatsNodes nodesStats = clusterStatsResponse.getNodesStats(); + + // JVM Heap Usage + health.setJvmHeapMax(nodesStats.getJvm().getHeapMax().bytes()); + health.setJvmHeapUsed(nodesStats.getJvm().getHeapUsed().bytes()); + + // OS Memory Usage ? + + // Disk Usage + health.setFsTotal(nodesStats.getFs().getTotal().bytes()); + health.setFsAvailable(nodesStats.getFs().getAvailable().bytes()); + + // Ping ? + + // Threads + health.setJvmThreads(nodesStats.getJvm().getThreads()); + + // CPU + health.setProcessCpuPercent(nodesStats.getProcess().getCpuPercent()); + + // Open Files + health.setOpenFiles(nodesStats.getProcess().getAvgOpenFileDescriptors()); + + // Uptime + health.setJvmUptimeMillis(nodesStats.getJvm().getMaxUpTime().getMillis()); + + return health; + } + + private ClusterHealthStatus getClusterHealthStatus() { + return this.admin().cluster().prepareHealth() + .setWaitForYellowStatus() + .setTimeout(healthTimeout) + .get() + .getStatus(); + } + + @CheckForNull + private NodeInfo getLocalNodeInfoByHostName(String hostname) { + for (ClusterStatsNodeResponse nodeResp : this.admin().cluster().prepareClusterStats().get().getNodes()) { + if (hostname.equals(nodeResp.nodeInfo().getHostname())) { + return nodeResp.nodeInfo(); + } + } + return null; + } + + @CheckForNull + private NodeInfo getLocalNodeInfoByNodeName(String nodeName) { + return this.admin().cluster().prepareClusterStats().get().getNodesMap().get(nodeName).nodeInfo(); + } + + @CheckForNull + private NodeInfo getLocalNodeInfo() { + if (settings.hasKey(IndexProperties.NODE_NAME)) { + return getLocalNodeInfoByNodeName(settings.getString(IndexProperties.NODE_NAME)); + } else { + try { + String LocalhostName = InetAddress.getLocalHost().getHostName(); + return getLocalNodeInfoByHostName(LocalhostName); + } catch (Exception exception) { + throw new IllegalStateException("Could not get localhost hostname", exception); + } + } + } + + // TODO that has nothing to do here!!! + private void addIndexTemplates() { + this.admin().indices() + .preparePutTemplate("default") + .setTemplate("*") + .addMapping("_default_", "{\"dynamic\": \"strict\"}") + .get(); + } + + private void initLogging() { + ESLoggerFactory.setDefaultFactory(new Slf4jESLoggerFactory()); + } + + private File esHomeDir() { + if (!settings.hasKey("sonar.path.home")) { + throw new IllegalStateException("property 'sonar.path.home' is required"); + } + return new File(settings.getString("sonar.path.home")); + } + + private File esDataDir() { + if (settings.hasKey("sonar.path.data")) { + return new File(settings.getString("sonar.path.data"), "es"); + } else { + return new File(settings.getString("sonar.path.home"), "data/es"); + } + } + + private File esLogDir() { + if (settings.hasKey("sonar.path.log")) { + return new File(settings.getString("sonar.path.log")); + } else { + return new File(settings.getString("sonar.path.home"), "log"); + } + } + + public <K extends ActionResponse> K execute(ActionRequestBuilder request) { + StopWatch basicProfile = profiling.start("search", Profiling.Level.BASIC); + StopWatch fullProfile = profiling.start("search", Profiling.Level.FULL); + ListenableActionFuture acc = request.execute(); + try { + + K response = (K) acc.get(); + + if (profiling.isProfilingEnabled(Profiling.Level.BASIC)) { + if (ToXContent.class.isAssignableFrom(request.getClass())) { + XContentBuilder debugResponse = XContentFactory.jsonBuilder(); + debugResponse.startObject(); + ((ToXContent) request).toXContent(debugResponse, ToXContent.EMPTY_PARAMS); + debugResponse.endObject(); + fullProfile.stop("ES Request: %s", debugResponse.string()); + } else { + fullProfile.stop("ES Request: %s", request.toString().replaceAll("\n", "")); + } + } + + if (profiling.isProfilingEnabled(Profiling.Level.FULL)) { + if (ToXContent.class.isAssignableFrom(response.getClass())) { + XContentBuilder debugResponse = XContentFactory.jsonBuilder(); + debugResponse.startObject(); + ((ToXContent) response).toXContent(debugResponse, ToXContent.EMPTY_PARAMS); + debugResponse.endObject(); + fullProfile.stop("ES Response: %s", debugResponse.string()); + } else { + fullProfile.stop("ES Response: %s", response.toString()); + } + } + + return response; + } catch (Exception e) { + throw new IllegalStateException("ES error: ", e); + } + } + + + + static public ImmutableSettings.Builder getGlobalIndexSettings(ImmutableSettings.Builder builder) { + return builder + + // Disable MCast + .put("discovery.zen.ping.multicast.enabled", "false") + + // Disable dynamic mapping + .put("index.mapper.dynamic", false) + + // Sortable text analyzer + .put("index.analysis.analyzer.sortable.type", "custom") + .put("index.analysis.analyzer.sortable.tokenizer", "keyword") + .putArray("index.analysis.analyzer.sortable.filter", "trim", "lowercase", "truncate") + + // Edge NGram index-analyzer + .put("index.analysis.analyzer.index_grams.type", "custom") + .put("index.analysis.analyzer.index_grams.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.index_grams.filter", "trim", "lowercase", "gram_filter") + + // Edge NGram search-analyzer + .put("index.analysis.analyzer.search_grams.type", "custom") + .put("index.analysis.analyzer.search_grams.tokenizer", "whitespace") + .putArray("index.analysis.analyzer.search_grams.filter", "trim", "lowercase") + + // Word index-analyzer + .put("index.analysis.analyzer.index_words.type", "custom") + .put("index.analysis.analyzer.index_words.tokenizer", "standard") + .putArray("index.analysis.analyzer.index_words.filter", + "standard", "word_filter", "lowercase", "stop", "asciifolding", "porter_stem") + + // Word search-analyzer + .put("index.analysis.analyzer.search_words.type", "custom") + .put("index.analysis.analyzer.search_words.tokenizer", "standard") + .putArray("index.analysis.analyzer.search_words.filter", + "standard", "lowercase", "stop", "asciifolding", "porter_stem") + + // Edge NGram filter + .put("index.analysis.filter.gram_filter.type", "edgeNGram") + .put("index.analysis.filter.gram_filter.min_gram", 2) + .put("index.analysis.filter.gram_filter.max_gram", 15) + .putArray("index.analysis.filter.gram_filter.token_chars", "letter", "digit", "punctuation", "symbol") + + // Word filter + .put("index.analysis.filter.word_filter.type", "word_delimiter") + .put("index.analysis.filter.word_filter.generate_word_parts", true) + .put("index.analysis.filter.word_filter.catenate_words", true) + .put("index.analysis.filter.word_filter.catenate_numbers", true) + .put("index.analysis.filter.word_filter.catenate_all", true) + .put("index.analysis.filter.word_filter.split_on_case_change", true) + .put("index.analysis.filter.word_filter.preserve_original", true) + .put("index.analysis.filter.word_filter.split_on_numerics", true) + .put("index.analysis.filter.word_filter.stem_english_possessive", true) + + // Path Analyzer + .put("index.analysis.analyzer.path_analyzer.type", "custom") + .put("index.analysis.analyzer.path_analyzer.tokenizer", "path_hierarchy"); + } + +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/search/SearchHealth.java b/server/sonar-server/src/main/java/org/sonar/server/search/SearchHealth.java index c8b2e57e5ce..e4d077a121c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/search/SearchHealth.java +++ b/server/sonar-server/src/main/java/org/sonar/server/search/SearchHealth.java @@ -28,16 +28,16 @@ import java.util.Map; public class SearchHealth { - private ESNode node; + private SearchClient searchClient; private IndexClient indexClient; - public SearchHealth(ESNode node, IndexClient indexClient) { - this.node = node; + public SearchHealth(SearchClient searchClient, IndexClient indexClient) { + this.searchClient = searchClient; this.indexClient = indexClient; } public NodeHealth getNodeHealth() { - return node.getNodeHealth(); + return searchClient.getNodeHealth(); } public Map<String, IndexHealth> getIndexHealth() { @@ -49,9 +49,9 @@ public class SearchHealth { newIndexHealth.documentCount = indexStat.getDocumentCount(); newIndexHealth.lastSync = indexStat.getLastUpdate(); - IndicesStatsRequestBuilder statRequest = node.client().admin().indices().prepareStats(index.getIndexName()) + IndicesStatsRequestBuilder statRequest = searchClient.admin().indices().prepareStats(index.getIndexName()) .setTypes(index.getIndexType()); - IndicesStatsResponse indicesStatsResponse = node.execute(statRequest); + IndicesStatsResponse indicesStatsResponse = searchClient.execute(statRequest); newIndexHealth.segmentCount = indicesStatsResponse.getTotal().getSegments().getCount(); newIndexHealth.pendingDeletion = indicesStatsResponse.getTotal().getDocs().getDeleted(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java index c270dc6df9a..604a925524f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseIndexTest.java @@ -43,7 +43,7 @@ public class BaseIndexTest { @Rule public TemporaryFolder temp = new TemporaryFolder(); - ESNode node; + SearchClient node; @Before public void setup() throws IOException { @@ -51,13 +51,13 @@ public class BaseIndexTest { Settings settings = new Settings(); settings.setProperty("sonar.path.home", dataDir.getAbsolutePath()); settings.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name()); - node = new ESNode(settings); - node.start(); + node = new SearchClient(settings); + //node.start(); } @After public void tearDown() { - node.stop(); + //node.stop(); } @Test @@ -77,10 +77,10 @@ public class BaseIndexTest { } - private BaseIndex getIndex(final ESNode esNode) { + private BaseIndex getIndex(final SearchClient searchClient) { BaseIndex index = new BaseIndex( IndexDefinition.TEST, - null, new NullQueue(), esNode) { + null, new NullQueue(), searchClient) { @Override protected String getKeyValue(Serializable key) { return null; diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/SearchClientTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/SearchClientTest.java new file mode 100644 index 00000000000..fb92fd80106 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/search/SearchClientTest.java @@ -0,0 +1,129 @@ +/* + * SonarQube, open source software quality management tool. + * Copyright (C) 2008-2014 SonarSource + * mailto:contact AT sonarsource DOT com + * + * SonarQube is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * SonarQube is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.search; + +public class SearchClientTest { + +// File dataDir; +// Settings settings; +// +// @Rule +// public TemporaryFolder temp = new TemporaryFolder(); +// +// @Before +// public void createMocks() throws IOException { +// dataDir = temp.newFolder(); +// settings = new Settings(); +// settings.setProperty("sonar.path.home", dataDir.getAbsolutePath()); +// settings.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name()); +// } +// +// @Test(expected = StrictDynamicMappingException.class) +// public void should_use_default_settings_for_index() throws Exception { +// SearchClient node = new SearchClient(settings); +// node.start(); +// +// node.client().admin().indices().prepareCreate("strict") +// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") +// .execute().actionGet(); +// node.client().admin().cluster().prepareHealth("strict").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); +// +// // strict mapping is enforced +// try { +// node.client().prepareIndex("strict", "type1", "666").setSource( +// XContentFactory.jsonBuilder().startObject().field("unknown", "plouf").endObject() +// ).get(); +// } finally { +// node.stop(); +// } +// } +// +// @Test +// public void check_path_analyzer() throws Exception { +// SearchClient node = new SearchClient(settings); +// node.start(); +// +// node.client().admin().indices().prepareCreate("path") +// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") +// .execute().actionGet(); +// node.client().admin().cluster().prepareHealth("path").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); +// +// // default "path_analyzer" analyzer is defined for all indices +// AnalyzeResponse response = node.client().admin().indices() +// .prepareAnalyze("path", "/temp/65236/test path/MyFile.java").setAnalyzer("path_analyzer").get(); +// // default "path_analyzer" analyzer is defined for all indices +// assertThat(response.getTokens()).hasSize(4); +// assertThat(response.getTokens().get(0).getTerm()).isEqualTo("/temp"); +// assertThat(response.getTokens().get(1).getTerm()).isEqualTo("/temp/65236"); +// assertThat(response.getTokens().get(2).getTerm()).isEqualTo("/temp/65236/test path"); +// assertThat(response.getTokens().get(3).getTerm()).isEqualTo("/temp/65236/test path/MyFile.java"); +// +// node.stop(); +// } +// +// @Test +// public void check_sortable_analyzer() throws Exception { +// SearchClient node = new SearchClient(settings); +// node.start(); +// +// node.client().admin().indices().prepareCreate("sort") +// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") +// .execute().actionGet(); +// node.client().admin().cluster().prepareHealth("sort").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); +// +// // default "sortable" analyzer is defined for all indices +// assertThat(node.client().admin().indices() +// .prepareAnalyze("sort", "This Is A Wonderful Text").setAnalyzer("sortable").get() +// .getTokens().get(0).getTerm()).isEqualTo("this is a "); +// +// node.stop(); +// } +// +// @Test +// public void check_gram_analyzer() throws Exception { +// SearchClient node = new SearchClient(settings); +// node.start(); +// +// node.client().admin().indices().prepareCreate("gram") +// .addMapping("type1", "{\"type1\": {\"properties\": {\"value\": {\"type\": \"string\"}}}}") +// .execute().actionGet(); +// node.client().admin().cluster().prepareHealth("gram").setWaitForYellowStatus().get(TimeValue.timeValueMillis(1000)); +// +// // default "string_gram" analyzer is defined for all indices +// AnalyzeResponse response = node.client().admin().indices() +// .prepareAnalyze("gram", "he.llo w@rl#d").setAnalyzer("index_grams").get(); +// assertThat(response.getTokens()).hasSize(10); +// assertThat(response.getTokens().get(0).getTerm()).isEqualTo("he"); +// assertThat(response.getTokens().get(7).getTerm()).isEqualTo("w@rl"); +// +// node.stop(); +// } +// +// @Test +// public void should_fail_to_get_client_if_not_started() { +// SearchClient node = new SearchClient(settings); +// try { +// node.client(); +// fail(); +// } catch (IllegalStateException e) { +// assertThat(e).hasMessage("Elasticsearch is not started"); +// } +// } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/tester/BackendCleanup.java b/server/sonar-server/src/test/java/org/sonar/server/tester/BackendCleanup.java index 96ec59e0543..cb79ed6d569 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/tester/BackendCleanup.java +++ b/server/sonar-server/src/test/java/org/sonar/server/tester/BackendCleanup.java @@ -19,23 +19,22 @@ */ package org.sonar.server.tester; -import org.elasticsearch.client.Client; import org.elasticsearch.index.query.QueryBuilders; import org.sonar.api.ServerComponent; import org.sonar.core.persistence.DatabaseVersion; import org.sonar.core.persistence.DbSession; import org.sonar.core.persistence.MyBatis; -import org.sonar.server.search.ESNode; +import org.sonar.server.search.SearchClient; import java.sql.Connection; public class BackendCleanup implements ServerComponent { - private final ESNode esNode; + private final SearchClient searchClient; private final MyBatis myBatis; - public BackendCleanup(ESNode esNode, MyBatis myBatis) { - this.esNode = esNode; + public BackendCleanup(SearchClient searchClient, MyBatis myBatis) { + this.searchClient = searchClient; this.myBatis = myBatis; } @@ -64,8 +63,7 @@ public class BackendCleanup implements ServerComponent { } public void clearIndexes() { - Client client = esNode.client(); - client.prepareDeleteByQuery(client.admin().cluster().prepareState().get() + searchClient.prepareDeleteByQuery(searchClient.admin().cluster().prepareState().get() .getState().getMetaData().concreteAllIndices()) .setQuery(QueryBuilders.matchAllQuery()) .get(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java b/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java index 480382fae23..74faa976d6b 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java +++ b/server/sonar-server/src/test/java/org/sonar/server/tester/ServerTester.java @@ -25,6 +25,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.junit.rules.ExternalResource; import org.sonar.api.database.DatabaseProperties; +import org.sonar.process.MonitoredProcess; +import org.sonar.process.NetworkUtils; +import org.sonar.process.Props; +import org.sonar.search.SearchServer; import org.sonar.server.platform.Platform; import org.sonar.server.search.IndexProperties; import org.sonar.server.ws.WsTester; @@ -48,6 +52,10 @@ public class ServerTester extends ExternalResource { private static final String PROP_PREFIX = "mediumTests."; + private final String clusterName; + private final Integer clusterPort; + + private SearchServer searchServer; private final Platform platform; private final File homeDir; private final List components = Lists.newArrayList(BackendCleanup.class, WsTester.class); @@ -56,6 +64,21 @@ public class ServerTester extends ExternalResource { public ServerTester() { homeDir = createTempDir(); platform = new Platform(); + + clusterName = "cluster-mem-" + System.currentTimeMillis(); + clusterPort = NetworkUtils.freePort(); + Properties properties = new Properties(); + properties.setProperty(IndexProperties.CLUSTER_NAME, clusterName); + properties.setProperty(IndexProperties.NODE_PORT, clusterPort.toString()); + properties.setProperty(MonitoredProcess.NAME_PROPERTY, "ES"); + properties.setProperty("sonar.path.data", new File(homeDir, "data/es").getAbsolutePath()); + properties.setProperty("sonar.path.logs", new File(homeDir, "logs").getAbsolutePath()); + properties.setProperty("sonar.path.temp", new File(homeDir, "temp").getAbsolutePath()); + try { + searchServer = new SearchServer(new Props(properties), false, false); + } catch (Exception e) { + e.printStackTrace(); + } } /** @@ -74,7 +97,21 @@ public class ServerTester extends ExternalResource { Properties properties = new Properties(); properties.putAll(initialProps); - properties.setProperty(IndexProperties.TYPE, IndexProperties.ES_TYPE.MEMORY.name()); + + try { + searchServer.start(); + System.out.println("searchServer = " + searchServer.isReady()); + while (!searchServer.isReady()) { + Thread.sleep(100); + } + } catch (Exception e) { + e.printStackTrace(); + } + + properties.setProperty(IndexProperties.CLUSTER_NAME, clusterName); + properties.setProperty(IndexProperties.NODE_PORT, clusterPort.toString()); + properties.setProperty(MonitoredProcess.NAME_PROPERTY, "ES"); + properties.setProperty("sonar.path.home", homeDir.getAbsolutePath()); properties.setProperty(DatabaseProperties.PROP_URL, "jdbc:h2:" + homeDir.getAbsolutePath() + "/h2"); for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) { @@ -118,6 +155,7 @@ public class ServerTester extends ExternalResource { */ public void stop() { platform.doStop(); + searchServer.terminate(); FileUtils.deleteQuietly(homeDir); } |