abstract class AbstractProcessMonitor implements ProcessMonitor {
- private static final Logger LOG = LoggerFactory.getLogger(EsProcessMonitor.class);
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractProcessMonitor.class);
private static final int EXPECTED_EXIT_VALUE = 0;
protected final Process process;
settingsMap.forEach((key, value) -> res.addEsOption("-E" + key + "=" + value));
return res;
-
- // FIXME quid of proxy settings and sonar.search.javaOpts/javaAdditionalOpts
- // defaults of HTTPS are the same than HTTP defaults
- // setSystemPropertyToDefaultIfNotSet(command, HTTPS_PROXY_HOST, HTTP_PROXY_HOST);
- // setSystemPropertyToDefaultIfNotSet(command, HTTPS_PROXY_PORT, HTTP_PROXY_PORT);
- // command
- // .addJavaOptions(settings.getProps().nonNullValue(ProcessProperties.SEARCH_JAVA_OPTS))
- // .addJavaOptions(settings.getProps().nonNullValue(ProcessProperties.SEARCH_JAVA_ADDITIONAL_OPTS));
}
private Properties buildLog4j2Properties(File logDir) {
import io.netty.util.ThreadDeathWatcher;
import io.netty.util.concurrent.GlobalEventExecutor;
import java.net.InetAddress;
-import java.net.MalformedURLException;
import java.net.UnknownHostException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
private final EsCommand esCommand;
private AtomicReference<TransportClient> transportClient = new AtomicReference<>(null);
- public EsProcessMonitor(Process process, ProcessId processId, EsCommand esCommand) throws MalformedURLException {
+ public EsProcessMonitor(Process process, ProcessId processId, EsCommand esCommand) {
super(process, processId);
this.esCommand = esCommand;
}
}
private TransportClient buildTransportClient() {
- org.elasticsearch.common.settings.Settings.Builder esSettings = org.elasticsearch.common.settings.Settings.builder();
+ Settings.Builder esSettings = Settings.builder();
// mandatory property defined by bootstrap process
esSettings.put("cluster.name", esCommand.getClusterName());
}
}
- private void configureAction(Map<String, String> builder) {
+ private static void configureAction(Map<String, String> builder) {
builder.put("action.auto_create_index", String.valueOf(false));
}
}
try {
writeConfFiles(esCommand);
ProcessBuilder processBuilder = create(esCommand);
- LOG.info("Launch process[{}]: {}", processId.getKey(), String.join(" ", processBuilder.command()));
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Launch process[{}]: {}", processId.getKey(), String.join(" ", processBuilder.command()));
+ }
process = processBuilder.start();
IOUtils.copy(getClass().getResourceAsStream("jvm.options"), new FileOutputStream(new File(confDir, "jvm.options")));
esCommand.getLog4j2Properties().store(new FileOutputStream(new File(confDir, "log4j2.properties")), "log42 properties file for ES bundled in SonarQube");
} catch (IOException e) {
- e.printStackTrace();
throw new IllegalStateException("Failed to write ES configuration files", e);
}
}
ProcessCommands commands = allProcessesCommands.createAfterClean(processId.getIpcIndex());
ProcessBuilder processBuilder = create(javaCommand);
- LOG.info("Launch process[{}]: {}", processId.getKey(), String.join(" ", processBuilder.command()));
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Launch process[{}]: {}", processId.getKey(), String.join(" ", processBuilder.command()));
+ }
process = processBuilder.start();
return new ProcessCommandsProcessMonitor(process, processId, commands);
} catch (Exception e) {
return create(esCommand, commands);
}
- private void writeJvmOptions(EsCommand esCommand) {
+ private static void writeJvmOptions(EsCommand esCommand) {
Path jvmOptionsFile = esCommand.getJvmOptionsFile();
String jvmOptions = esCommand.getJvmOptions()
.stream()
- .map(s -> s.split(" (?=-)"))// FIXME this pattern does not allow escaping
+
+ // we do not expect the user to use parameters containing " -"
+ .map(s -> s.split(" (?=-)"))
.flatMap(Arrays::stream)
.collect(Collectors.joining("\n"));
String jvmOptionsContent = ELASTICSEARCH_JVM_OPTIONS_HEADER + jvmOptions;
try {
Files.write(jvmOptionsFile, jvmOptionsContent.getBytes(Charset.forName("UTF-8")));
} catch (IOException e) {
- throw new RuntimeException("Cannot write Elasticsearch jvm options file", e);
+ throw new IllegalStateException("Cannot write Elasticsearch jvm options file", e);
}
}
props.putAll(javaCommand.getArguments());
props.setProperty(PROPERTY_PROCESS_KEY, javaCommand.getProcessId().getKey());
props.setProperty(PROPERTY_PROCESS_INDEX, Integer.toString(javaCommand.getProcessId().getIpcIndex()));
- // FIXME is it the responsibility of child process to have this timeout (too) ?
props.setProperty(PROPERTY_TERMINATION_TIMEOUT, "60000");
props.setProperty(PROPERTY_SHARED_PATH, tempDir.getAbsolutePath());
try (OutputStream out = new FileOutputStream(propertiesFile)) {
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator;
import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters.InternalBucket;
SearchHits hitList = docs.getHits();
SearchHit[] hits = hitList.getHits();
- return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.totalHits());
+ return new ComponentHitsPerQualifier(bucket.getKey(), ComponentHit.fromSearchHits(hits), hitList.getTotalHits());
}
}
while (true) {
SearchHit[] hits = searchResponse.getHits().getHits();
for (SearchHit hit : hits) {
- SearchHitField routing = hit.field("_routing");
- DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.index(), hit.type(), hit.getId());
+ SearchHitField routing = hit.getField("_routing");
+ DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId());
if (routing != null) {
deleteRequestBuilder.setRouting(routing.getValue());
}
return docs;
}
- public static LinkedHashMap<String, Long> termsToMap(Terms terms) {
+ public static Map<String, Long> termsToMap(Terms terms) {
LinkedHashMap<String, Long> map = new LinkedHashMap<>();
List<? extends Terms.Bucket> buckets = terms.getBuckets();
for (Terms.Bucket bucket : buckets) {
return SPECIAL_REGEX_CHARS.matcher(str).replaceAll("\\\\$0");
}
- private static class DocScrollIterator<D extends BaseDoc> implements Iterator<D> {
-
- private final EsClient esClient;
- private final String scrollId;
- private final Function<Map<String, Object>, D> docConverter;
-
- private final Queue<SearchHit> hits = new ArrayDeque<>();
-
- private DocScrollIterator(EsClient esClient, SearchResponse scrollResponse, Function<Map<String, Object>, D> docConverter) {
- this.esClient = esClient;
- this.scrollId = scrollResponse.getScrollId();
- this.docConverter = docConverter;
- Collections.addAll(hits, scrollResponse.getHits().getHits());
- }
-
- @Override
- public boolean hasNext() {
- if (hits.isEmpty()) {
- SearchScrollRequestBuilder esRequest = esClient.prepareSearchScroll(scrollId)
- .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES));
- Collections.addAll(hits, esRequest.get().getHits().getHits());
- }
- return !hits.isEmpty();
- }
-
- @Override
- public D next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- return docConverter.apply(hits.poll().getSource());
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException("Cannot remove item when scrolling");
- }
- }
-
- public static <ID> Iterator<ID> scrollIds(EsClient esClient, SearchResponse scrollResponse, Function<String, ID> idConverter) {
+ public static <I> Iterator<I> scrollIds(EsClient esClient, SearchResponse scrollResponse, Function<String, I> idConverter) {
return new IdScrollIterator<>(esClient, scrollResponse, idConverter);
}
- private static class IdScrollIterator<ID> implements Iterator<ID> {
+ private static class IdScrollIterator<I> implements Iterator<I> {
private final EsClient esClient;
private final String scrollId;
- private final Function<String, ID> idConverter;
+ private final Function<String, I> idConverter;
private final Queue<SearchHit> hits = new ArrayDeque<>();
- private IdScrollIterator(EsClient esClient, SearchResponse scrollResponse, Function<String, ID> idConverter) {
+ private IdScrollIterator(EsClient esClient, SearchResponse scrollResponse, Function<String, I> idConverter) {
this.esClient = esClient;
this.scrollId = scrollResponse.getScrollId();
this.idConverter = idConverter;
}
@Override
- public ID next() {
+ public I next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
}
public NewIndexType build() {
- Map<String, Object> hash = new TreeMap<>();
if (subFields.isEmpty()) {
- hash.putAll(ImmutableMap.of(
- "type", getFieldType(),
- "index", disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE,
- "norms", valueOf(!disableNorms),
- "store", valueOf(store)));
- if (getFieldData()) {
- hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
- }
- } else {
- hash.put("type", getFieldType());
-
- Map<String, Object> multiFields = new TreeMap<>(subFields);
-
- if (termVectorWithPositionOffsets) {
- multiFields.entrySet().forEach(entry -> {
- Object subFieldMapping = entry.getValue();
- if (subFieldMapping instanceof Map) {
- entry.setValue(
- addFieldToMapping(
- (Map<String, String>) subFieldMapping,
- FIELD_TERM_VECTOR, "with_positions_offsets"));
- }
- });
- hash.put(FIELD_TERM_VECTOR, "with_positions_offsets");
- }
- if (getFieldData()) {
- multiFields.entrySet().forEach(entry -> {
- Object subFieldMapping = entry.getValue();
- if (subFieldMapping instanceof Map) {
- entry.setValue(
- addFieldToMapping(
- (Map<String, String>) subFieldMapping,
- FIELD_FIELDDATA, FIELDDATA_ENABLED));
- }
- });
- hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
- }
-
- multiFields.put(fieldName, ImmutableMap.of(
- "type", getFieldType(),
- "index", INDEX_SEARCHABLE,
- "norms", "false",
- "store", valueOf(store)));
-
- hash.put("fields", multiFields);
+ return buildWithoutSubfields();
+ }
+ return buildWithSubfields();
+ }
+
+ private NewIndexType buildWithoutSubfields() {
+ Map<String, Object> hash = new TreeMap<>();
+ hash.putAll(ImmutableMap.of(
+ "type", getFieldType(),
+ INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE,
+ "norms", valueOf(!disableNorms),
+ "store", valueOf(store)));
+ if (getFieldData()) {
+ hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
+ }
+ return indexType.setProperty(fieldName, hash);
+ }
+
+ private NewIndexType buildWithSubfields() {
+ Map<String, Object> hash = new TreeMap<>();
+ hash.put("type", getFieldType());
+
+ Map<String, Object> multiFields = new TreeMap<>(subFields);
+
+ if (termVectorWithPositionOffsets) {
+ multiFields.entrySet().forEach(entry -> {
+ Object subFieldMapping = entry.getValue();
+ if (subFieldMapping instanceof Map) {
+ entry.setValue(
+ addFieldToMapping(
+ (Map<String, String>) subFieldMapping,
+ FIELD_TERM_VECTOR, "with_positions_offsets"));
+ }
+ });
+ hash.put(FIELD_TERM_VECTOR, "with_positions_offsets");
+ }
+ if (getFieldData()) {
+ multiFields.entrySet().forEach(entry -> {
+ Object subFieldMapping = entry.getValue();
+ if (subFieldMapping instanceof Map) {
+ entry.setValue(
+ addFieldToMapping(
+ (Map<String, String>) subFieldMapping,
+ FIELD_FIELDDATA, FIELDDATA_ENABLED));
+ }
+ });
+ hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
}
+ multiFields.put(fieldName, ImmutableMap.of(
+ "type", getFieldType(),
+ INDEX, INDEX_SEARCHABLE,
+ "norms", "false",
+ "store", valueOf(store)));
+ hash.put("fields", multiFields);
+
return indexType.setProperty(fieldName, hash);
}
public NestedFieldBuilder addKeywordField(String fieldName) {
return setProperty(fieldName, ImmutableMap.of(
"type", FIELD_TYPE_KEYWORD,
- "index", INDEX_SEARCHABLE));
+ INDEX, INDEX_SEARCHABLE));
}
public NestedFieldBuilder addDoubleField(String fieldName) {
* @param selected the terms, that the user already has selected
* @return the (global) aggregation, that can be added on top level of the elasticsearch request
*/
- public AggregationBuilder buildStickyFacet(String fieldName, String facetName, Function<TermsAggregationBuilder, AggregationBuilder> additionalAggregationFilter, Object... selected) {
+ public AggregationBuilder buildStickyFacet(String fieldName, String facetName, Function<TermsAggregationBuilder, AggregationBuilder> additionalAggregationFilter,
+ Object... selected) {
return buildStickyFacet(fieldName, facetName, FACET_DEFAULT_SIZE, additionalAggregationFilter, selected);
}
filters.put("__authorization", authorizationTypeSupport.createQueryFilter());
Multimap<String, MetricCriterion> metricCriterionMultimap = ArrayListMultimap.create();
query.getMetricCriteria().forEach(metricCriterion -> metricCriterionMultimap.put(metricCriterion.getMetricKey(), metricCriterion));
- metricCriterionMultimap.asMap().entrySet().forEach(entry -> {
+ metricCriterionMultimap.asMap().forEach((key, value) -> {
BoolQueryBuilder metricFilters = boolQuery();
- entry.getValue()
+ value
.stream()
.map(ProjectMeasuresIndex::toQuery)
.forEach(metricFilters::must);
- filters.put(entry.getKey(), metricFilters);
+ filters.put(key, metricFilters);
});
query.getQualityGateStatus()
query.getTags()
.ifPresent(tags -> filters.put(FIELD_TAGS, termsQuery(FIELD_TAGS, tags)));
- createTextQueryFilter(query).ifPresent(queryBuilder -> filters.put("textQuery", queryBuilder));
+ query.getQueryText()
+ .map(ProjectsTextSearchQueryFactory::createQuery)
+ .ifPresent(queryBuilder -> filters.put("textQuery", queryBuilder));
return filters;
}
- private static Optional<QueryBuilder> createTextQueryFilter(ProjectMeasuresQuery query) {
- Optional<String> queryText = query.getQueryText();
- if (!queryText.isPresent()) {
- return Optional.empty();
- }
- return Optional.of(ProjectsTextSearchQueryFactory.createQuery(queryText.get()));
- }
-
private static QueryBuilder toQuery(MetricCriterion criterion) {
if (criterion.isNoData()) {
return boolQuery().mustNot(
/* Build main filter (match based) */
private static Map<String, QueryBuilder> buildFilters(RuleQuery query) {
-
Map<String, QueryBuilder> filters = new HashMap<>();
/* Add enforced filter on rules that are REMOVED */
}
if (isNotEmpty(query.getTags())) {
- BoolQueryBuilder q = boolQuery();
- query.getTags().stream()
- .map(tag -> boolQuery()
- .filter(QueryBuilders.termQuery(FIELD_RULE_EXTENSION_TAGS, tag))
- .filter(termsQuery(FIELD_RULE_EXTENSION_SCOPE, RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(query.getOrganization()).getScope())))
- .map(childQuery -> JoinQueryBuilders.hasChildQuery(INDEX_TYPE_RULE_EXTENSION.getType(), childQuery, ScoreMode.None))
- .forEach(q::should);
- filters.put(FIELD_RULE_EXTENSION_TAGS, q);
+ filters.put(FIELD_RULE_EXTENSION_TAGS,
+ buildTagsFilter(query.getTags(), query.getOrganization()));
}
Collection<RuleType> types = query.getTypes();
/* Implementation of activation query */
QProfileDto profile = query.getQProfile();
if (query.getActivation() != null && profile != null) {
-
- // ActiveRule Filter (profile and inheritance)
- BoolQueryBuilder activeRuleFilter = boolQuery();
- addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_PROFILE_UUID, profile.getRulesProfileUuid());
- addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance());
- addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_SEVERITY, query.getActiveSeverities());
-
- // ChildQuery
- QueryBuilder childQuery;
- if (activeRuleFilter.hasClauses()) {
- childQuery = activeRuleFilter;
- } else {
- childQuery = matchAllQuery();
- }
+ QueryBuilder childQuery = buildActivationFilter(query, profile);
if (TRUE.equals(query.getActivation())) {
filters.put("activation",
return filters;
}
+ private static BoolQueryBuilder buildTagsFilter(Collection<String> tags, OrganizationDto organization) {
+ BoolQueryBuilder q = boolQuery();
+ tags.stream()
+ .map(tag -> boolQuery()
+ .filter(QueryBuilders.termQuery(FIELD_RULE_EXTENSION_TAGS, tag))
+ .filter(termsQuery(FIELD_RULE_EXTENSION_SCOPE, RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(organization).getScope())))
+ .map(childQuery -> JoinQueryBuilders.hasChildQuery(INDEX_TYPE_RULE_EXTENSION.getType(), childQuery, ScoreMode.None))
+ .forEach(q::should);
+ return q;
+ }
+
+ private static QueryBuilder buildActivationFilter(RuleQuery query, QProfileDto profile) {
+ // ActiveRule Filter (profile and inheritance)
+ BoolQueryBuilder activeRuleFilter = boolQuery();
+ addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_PROFILE_UUID, profile.getRulesProfileUuid());
+ addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance());
+ addTermFilter(activeRuleFilter, FIELD_ACTIVE_RULE_SEVERITY, query.getActiveSeverities());
+
+ // ChildQuery
+ QueryBuilder childQuery;
+ if (activeRuleFilter.hasClauses()) {
+ childQuery = activeRuleFilter;
+ } else {
+ childQuery = matchAllQuery();
+ }
+ return childQuery;
+ }
+
private static BoolQueryBuilder addTermFilter(BoolQueryBuilder filter, String field, @Nullable Collection<String> values) {
if (isNotEmpty(values)) {
BoolQueryBuilder valuesFilter = boolQuery();
.setSize(1)
.setQuery(boolQuery().must(matchAllQuery()).filter(termQuery(FIELD_TEST_UUID, testUuid)))
.get().getHits().getHits()) {
- coveredFiles.addAll(new TestDoc(hit.sourceAsMap()).coveredFiles());
+ coveredFiles.addAll(new TestDoc(hit.getSourceAsMap()).coveredFiles());
}
return coveredFiles;
.setQuery(boolQuery().must(matchAllQuery()).filter(termQuery(FIELD_TEST_UUID, testUuid)))
.get().getHits().getHits();
if (hits.length > 0) {
- return Optional.of(new TestDoc(hits[0].sourceAsMap()));
+ return Optional.of(new TestDoc(hits[0].getSourceAsMap()));
}
return Optional.absent();
}
.should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount))))
.setSize(3);
for (SearchHit hit : request.get().getHits().getHits()) {
- result.add(new UserDoc(hit.sourceAsMap()));
+ result.add(new UserDoc(hit.getSourceAsMap()));
}
}
return result;
import java.util.List;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
.assertJson(this.getClass(), "no_issue.json");
}
- @Ignore
@Test
public void search_since_leak_period_on_project() throws Exception {
ComponentDto project = insertComponent(ComponentTesting.newPublicProjectDto(otherOrganization2, "P1").setDbKey("PK1"));
.assertJson(this.getClass(), "search_since_leak_period.json");
}
- @Ignore
@Test
public void search_since_leak_period_on_file_in_module_project() throws Exception {
ComponentDto project = insertComponent(ComponentTesting.newPublicProjectDto(defaultOrganization, "P1").setDbKey("PK1"));
projects / sonarqube.git / commitdiff