assertThat(reloaded.getSeverity()).isNull();
assertThat(reloaded.getName()).isEqualTo("eslint:no-cond-assign");
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1l);
- assertThat(es.getDocuments(RuleIndexDefinition.INDEX_TYPE_RULE).iterator().next().getId()).isEqualTo(Integer.toString(reloaded.getId()));
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1l);
+ assertThat(es.getDocuments(RuleIndexDefinition.TYPE_RULE).iterator().next().getId()).isEqualTo(Integer.toString(reloaded.getId()));
}
@Test
RuleDao ruleDao = dbClient.ruleDao();
assertThat(ruleDao.selectAllDefinitions(dbClient.openSession(false))).hasSize(1);
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isZero();
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isZero();
}
}
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.sonar.server.es.BaseDoc;
+import org.sonar.server.permission.index.AuthorizationDoc;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_KEY;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_LANGUAGE;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_PROJECT_UUID;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_QUALIFIER;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_UUID;
+import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT;
public class ComponentDoc extends BaseDoc {
public ComponentDoc() {
- super(new HashMap<>(6));
+ super(TYPE_COMPONENT, new HashMap<>(6));
}
public ComponentDoc(Map<String, Object> fields) {
- super(fields);
+ super(TYPE_COMPONENT, fields);
}
@Override
return getField(FIELD_UUID);
}
- @Override
- public String getRouting() {
- return getProjectUuid();
- }
-
- @Override
- public String getParent() {
- return getProjectUuid();
- }
-
public ComponentDoc setId(String s) {
setField(FIELD_UUID, s);
return this;
public ComponentDoc setProjectUuid(String s) {
setField(FIELD_PROJECT_UUID, s);
+ setParent(AuthorizationDoc.idOf(s));
return this;
}
package org.sonar.server.component.index;
import org.sonar.api.config.Configuration;
-import org.sonar.server.es.DefaultIndexSettingsElement;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.newindex.DefaultIndexSettingsElement;
+import org.sonar.server.es.newindex.NewAuthorizedIndex;
+import org.sonar.server.es.newindex.TypeMapping;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
public class ComponentIndexDefinition implements IndexDefinition {
- public static final IndexType INDEX_TYPE_COMPONENT = new IndexType("components", "component");
+ public static final Index DESCRIPTOR = Index.withRelations("components");
+ public static final IndexType.IndexRelationType TYPE_COMPONENT = IndexType.relation(IndexType.main(DESCRIPTOR, TYPE_AUTHORIZATION), "component");
public static final String FIELD_UUID = "uuid";
public static final String FIELD_PROJECT_UUID = "project_uuid";
public static final String FIELD_ORGANIZATION_UUID = "organization_uuid";
static final DefaultIndexSettingsElement[] NAME_ANALYZERS = {SORTABLE_ANALYZER, SEARCH_PREFIX_ANALYZER, SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER, SEARCH_GRAMS_ANALYZER};
private final Configuration config;
+ private final boolean enableSource;
- public ComponentIndexDefinition(Configuration config) {
+ private ComponentIndexDefinition(Configuration config, boolean enableSource) {
this.config = config;
+ this.enableSource = enableSource;
+ }
+
+ public ComponentIndexDefinition(Configuration config) {
+ this(config, false);
+ }
+
+ /**
+ * Keep the document sources in index so that indexer tests can verify content
+ * of indexed documents.
+ */
+ public static ComponentIndexDefinition createForTest() {
+ return new ComponentIndexDefinition(new MapSettings().asConfig(), true);
}
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(
- INDEX_TYPE_COMPONENT.getIndex(),
+ NewAuthorizedIndex index = context.createWithAuthorization(
+ DESCRIPTOR,
newBuilder(config)
.setRefreshInterval(MANUAL_REFRESH_INTERVAL)
.setDefaultNbOfShards(DEFAULT_NUMBER_OF_SHARDS)
- .build());
-
- NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_COMPONENT.getType())
- .requireProjectAuthorization();
+ .build())
+ .setEnableSource(enableSource);
+ TypeMapping mapping = index.createTypeMapping(TYPE_COMPONENT);
mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_PROJECT_UUID).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_KEY).addSubFields(SORTABLE_ANALYZER).build();
import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
-import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.DbSession;
import org.sonar.db.component.ComponentDto;
import org.sonar.db.es.EsQueueDto;
+import org.sonar.server.es.BaseDoc;
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.BulkIndexer.Size;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexingResult;
import org.sonar.server.es.OneToManyResilientIndexingListener;
import org.sonar.server.es.ProjectIndexer;
+import org.sonar.server.permission.index.AuthorizationDoc;
import org.sonar.server.permission.index.AuthorizationScope;
import org.sonar.server.permission.index.NeedAuthorizationIndexer;
import static java.util.Collections.emptyList;
-import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_TYPE_COMPONENT;
+import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT;
public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexer {
- private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_COMPONENT, project -> true);
- private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(INDEX_TYPE_COMPONENT);
+ private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_COMPONENT, project -> true);
+ private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(TYPE_COMPONENT);
private final DbClient dbClient;
private final EsClient esClient;
case PROJECT_DELETION:
case PROJECT_KEY_UPDATE:
List<EsQueueDto> items = projectUuids.stream()
- .map(branchUuid -> EsQueueDto.create(INDEX_TYPE_COMPONENT.format(), branchUuid, null, branchUuid))
+ .map(branchUuid -> EsQueueDto.create(TYPE_COMPONENT.format(), branchUuid, null, branchUuid))
.collect(MoreCollectors.toArrayList(projectUuids.size()));
return dbClient.esQueueDao().insert(dbSession, items);
}
OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items);
- BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR, listener);
+ BulkIndexer bulkIndexer = new BulkIndexer(esClient, TYPE_COMPONENT, Size.REGULAR, listener);
bulkIndexer.start();
Set<String> branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size()));
Set<String> remaining = new HashSet<>(branchUuids);
// TODO allow scrolling multiple projects at the same time
dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> {
ComponentDto dto = context.getResultObject();
- bulkIndexer.add(newIndexRequest(toDocument(dto)));
+ bulkIndexer.add(toDocument(dto).toIndexRequest());
remaining.remove(dto.projectUuid());
});
}
* <b>Warning:</b> only use {@code null} during startup.
*/
private void doIndexByProjectUuid(@Nullable String projectUuid, Size bulkSize) {
- BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, bulkSize);
+ BulkIndexer bulk = new BulkIndexer(esClient, TYPE_COMPONENT, bulkSize);
bulk.start();
try (DbSession dbSession = dbClient.openSession(false)) {
dbClient.componentDao()
.scrollForIndexing(dbSession, projectUuid, context -> {
ComponentDto dto = context.getResultObject();
- bulk.add(newIndexRequest(toDocument(dto)));
+ bulk.add(toDocument(dto).toIndexRequest());
});
}
bulk.stop();
}
private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) {
- SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX_TYPE_COMPONENT)
+ SearchRequestBuilder searchRequest = esClient.prepareSearch(TYPE_COMPONENT.getMainType())
.setQuery(QueryBuilders.termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid))
- .setRouting(projectUuid);
+ .setRouting(AuthorizationDoc.idOf(projectUuid));
bulkIndexer.addDeletion(searchRequest);
}
public void delete(String projectUuid, Collection<String> disabledComponentUuids) {
- BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR);
+ BulkIndexer bulk = new BulkIndexer(esClient, TYPE_COMPONENT, Size.REGULAR);
bulk.start();
- disabledComponentUuids.forEach(uuid -> bulk.addDeletion(INDEX_TYPE_COMPONENT, uuid, projectUuid));
+ disabledComponentUuids.forEach(uuid -> bulk.addDeletion(TYPE_COMPONENT, uuid, AuthorizationDoc.idOf(projectUuid)));
bulk.stop();
}
@VisibleForTesting
void index(ComponentDto... docs) {
- BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR);
+ BulkIndexer bulk = new BulkIndexer(esClient, TYPE_COMPONENT, Size.REGULAR);
bulk.start();
Arrays.stream(docs)
.map(ComponentIndexer::toDocument)
- .map(ComponentIndexer::newIndexRequest)
+ .map(BaseDoc::toIndexRequest)
.forEach(bulk::add);
bulk.stop();
}
- private static IndexRequest newIndexRequest(ComponentDoc doc) {
- return new IndexRequest(INDEX_TYPE_COMPONENT.getIndex(), INDEX_TYPE_COMPONENT.getType(), doc.getId())
- .routing(doc.getRouting())
- .parent(doc.getParent())
- .source(doc.getFields());
- }
-
public static ComponentDoc toDocument(ComponentDto component) {
return new ComponentDoc()
.setId(component.uuid())
*/
package org.sonar.server.es;
+import com.google.common.collect.ImmutableMap;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
+import org.elasticsearch.action.index.IndexRequest;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.IndexType.IndexRelationType;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
/**
* Base implementation for business objects based on elasticsearch document
*/
public abstract class BaseDoc {
+ private static final String SETPARENT_NOT_CALLED = "parent must be set on a doc associated to a IndexRelationType (see BaseDoc#setParent(String))";
+ private final IndexType indexType;
+ private String parentId = null;
protected final Map<String, Object> fields;
- protected BaseDoc() {
- this.fields = new HashMap<>();
+ protected BaseDoc(IndexType indexType) {
+ this(indexType, new HashMap<>());
}
- protected BaseDoc(Map<String, Object> fields) {
+ protected BaseDoc(IndexType indexType, Map<String, Object> fields) {
+ this.indexType = indexType;
this.fields = fields;
+ if (indexType instanceof IndexMainType) {
+ IndexMainType mainType = (IndexMainType) indexType;
+ if (mainType.getIndex().acceptsRelations()) {
+ setField(mainType.getIndex().getJoinField(), ImmutableMap.of("name", mainType.getType()));
+ setField(FIELD_INDEX_TYPE, mainType.getType());
+ }
+ }
+ }
+
+ protected void setParent(String parentId) {
+ checkState(this.indexType instanceof IndexRelationType, "Doc must be associated to a IndexRelationType to set a parent");
+ checkArgument(parentId != null && !parentId.isEmpty(), "parentId can't be null nor empty");
+ this.parentId = parentId;
+ IndexRelationType indexRelationType = (IndexRelationType) this.indexType;
+ setField(indexRelationType.getMainType().getIndex().getJoinField(), ImmutableMap.of("name", indexRelationType.getName(), "parent", parentId));
+ setField(FIELD_INDEX_TYPE, indexRelationType.getName());
}
public abstract String getId();
- @CheckForNull
- public abstract String getRouting();
+ public Optional<String> getRouting() {
+ // when using relations, routing MUST be defined and MUST be the id of the parent
+ if (this.indexType instanceof IndexRelationType) {
+ ensureSetParentCalled();
+ return Optional.of(this.parentId);
+ }
+ if (this.indexType instanceof IndexMainType && indexType.getMainType().getIndex().acceptsRelations()) {
+ return Optional.of(getId());
+ }
+ return getSimpleMainTypeRouting();
+ }
- @CheckForNull
- public abstract String getParent();
+ /**
+ * Intended to be overridden by subclass which wants to define a routing and which indexType is a {@link IndexMainType}
+ * (if not a {@link IndexMainType}, this method will never be called).
+ */
+ protected Optional<String> getSimpleMainTypeRouting() {
+ return Optional.empty();
+ }
/**
* Use this method when field value can be null. See warning in {@link #getField(String)}
Object val = getNullableField(key);
if (val != null) {
if (val instanceof Date) {
- return (Date)val;
+ return (Date) val;
}
if (val instanceof Number) {
return epochSecondsToDate((Number) val);
public Date getFieldAsDate(String key) {
Object value = getField(key);
if (value instanceof Date) {
- return (Date)value;
+ return (Date) value;
}
if (value instanceof Number) {
return epochSecondsToDate((Number) value);
}
- return EsUtils.parseDateTime((String)value);
+ return EsUtils.parseDateTime((String) value);
}
public void setField(String key, @Nullable Object value) {
fields.put(key, value);
}
- public Map<String, Object> getFields() {
+ public final Map<String, Object> getFields() {
+ if (indexType instanceof IndexRelationType) {
+ ensureSetParentCalled();
+ }
return fields;
}
+ private void ensureSetParentCalled() {
+ checkState(this.parentId != null, SETPARENT_NOT_CALLED);
+ }
+
+ public IndexRequest toIndexRequest() {
+ IndexMainType mainType = this.indexType.getMainType();
+ return new IndexRequest(mainType.getIndex().getName(), mainType.getType())
+ .id(getId())
+ .routing(getRouting().orElse(null))
+ .source(getFields());
+ }
+
public static long epochMillisToEpochSeconds(long epochMillis) {
return epochMillis / 1000L;
}
Thread.currentThread().interrupt();
throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e);
}
- client.prepareRefresh(indexType.getIndex()).get();
+ client.prepareRefresh(indexType.getMainType().getIndex()).get();
sizeHandler.afterStop(this);
indexingListener.onFinish(result);
return result;
@Override
void beforeStart(BulkIndexer bulkIndexer) {
- this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexType.getIndex()), bulkIndexer.result.total, LOGGER)
+ String index = bulkIndexer.indexType.getMainType().getIndex().getName();
+ this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", index), bulkIndexer.result.total, LOGGER)
.setPluralLabel("requests");
this.progress.start();
Map<String, Object> temporarySettings = new HashMap<>();
- GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(bulkIndexer.indexType.getIndex()).get();
+ GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(index).get();
// deactivate replicas
- int initialReplicas = Integer.parseInt(settingsResp.getSetting(bulkIndexer.indexType.getIndex(), IndexMetaData.SETTING_NUMBER_OF_REPLICAS));
+ int initialReplicas = Integer.parseInt(settingsResp.getSetting(index, IndexMetaData.SETTING_NUMBER_OF_REPLICAS));
if (initialReplicas > 0) {
initialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas);
temporarySettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
}
// deactivate periodical refresh
- String refreshInterval = settingsResp.getSetting(bulkIndexer.indexType.getIndex(), REFRESH_INTERVAL_SETTING);
+ String refreshInterval = settingsResp.getSetting(index, REFRESH_INTERVAL_SETTING);
initialSettings.put(REFRESH_INTERVAL_SETTING, refreshInterval);
temporarySettings.put(REFRESH_INTERVAL_SETTING, "-1");
// optimize lucene segments and revert index settings
// Optimization must be done before re-applying replicas:
// http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/
- bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getIndex()).get();
+ bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getMainType().getIndex().getName()).get();
updateSettings(bulkIndexer, initialSettings);
this.progress.stop();
}
private static void updateSettings(BulkIndexer bulkIndexer, Map<String, Object> settings) {
- UpdateSettingsRequestBuilder req = bulkIndexer.client.nativeClient().admin().indices().prepareUpdateSettings(bulkIndexer.indexType.getIndex());
+ UpdateSettingsRequestBuilder req = bulkIndexer.client.nativeClient().admin().indices().prepareUpdateSettings(bulkIndexer.indexType.getMainType().getIndex().getName());
req.setSettings(settings);
req.get();
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import java.util.Arrays;
-import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-
-public class DefaultIndexSettings {
-
- /** Minimum length of ngrams. */
- public static final int MINIMUM_NGRAM_LENGTH = 2;
- /** Maximum length of ngrams. */
- public static final int MAXIMUM_NGRAM_LENGTH = 15;
-
- /** Pattern, that splits the user search input **/
- public static final String SEARCH_TERM_TOKENIZER_PATTERN = "[\\s]+";
-
- public static final String ANALYSIS = "index.analysis";
- public static final String DELIMITER = ".";
-
- public static final String TOKENIZER = "tokenizer";
- public static final String FILTER = "filter";
- public static final String CHAR_FILTER = "char_filter";
- public static final String ANALYZER = "analyzer";
- public static final String SEARCH_ANALYZER = "search_analyzer";
-
- public static final String TYPE = "type";
- public static final String INDEX = "index";
- public static final String INDEX_SEARCHABLE = "true";
- public static final String INDEX_NOT_SEARCHABLE = "false";
- public static final String FIELD_TYPE_TEXT = "text";
- public static final String FIELD_TYPE_KEYWORD = "keyword";
- public static final String NORMS = "norms";
- public static final String STORE = "store";
- public static final String FIELD_FIELDDATA = "fielddata";
- public static final String FIELDDATA_ENABLED = "true";
- public static final String FIELD_TERM_VECTOR = "term_vector";
- public static final String STANDARD = "standard";
- public static final String PATTERN = "pattern";
- public static final String CUSTOM = "custom";
- public static final String KEYWORD = "keyword";
- public static final String CLASSIC = "classic";
- public static final RefreshPolicy REFRESH_IMMEDIATE = RefreshPolicy.IMMEDIATE;
- public static final RefreshPolicy REFRESH_NONE = RefreshPolicy.NONE;
-
- public static final String TRUNCATE = "truncate";
-
- public static final String SUB_FIELD_DELIMITER = ".";
- public static final String TRIM = "trim";
- public static final String LOWERCASE = "lowercase";
- public static final String WHITESPACE = "whitespace";
- public static final String STOP = "stop";
- public static final String ASCIIFOLDING = "asciifolding";
- public static final String PORTER_STEM = "porter_stem";
- public static final String MIN_GRAM = "min_gram";
- public static final String MAX_GRAM = "max_gram";
- public static final String LENGTH = "length";
- public static final String HTML_STRIP = "html_strip";
-
- private DefaultIndexSettings() {
- // only static stuff
- }
-
- static Settings.Builder defaults() {
- Settings.Builder builder = Settings.builder()
- .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
- .put("index.refresh_interval", "30s")
- .put("index.mapper.dynamic", false);
-
- Arrays.stream(DefaultIndexSettingsElement.values())
- .map(DefaultIndexSettingsElement::settings)
- .forEach(builder::put);
-
- return builder;
- }
-}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import com.google.common.collect.ImmutableSortedMap;
-import java.util.Arrays;
-import java.util.Locale;
-import java.util.SortedMap;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.Settings.Builder;
-
-import static org.sonar.server.es.DefaultIndexSettings.ANALYSIS;
-import static org.sonar.server.es.DefaultIndexSettings.ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettings.ASCIIFOLDING;
-import static org.sonar.server.es.DefaultIndexSettings.CHAR_FILTER;
-import static org.sonar.server.es.DefaultIndexSettings.DELIMITER;
-import static org.sonar.server.es.DefaultIndexSettings.FIELDDATA_ENABLED;
-import static org.sonar.server.es.DefaultIndexSettings.FIELD_FIELDDATA;
-import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT;
-import static org.sonar.server.es.DefaultIndexSettings.FILTER;
-import static org.sonar.server.es.DefaultIndexSettings.HTML_STRIP;
-import static org.sonar.server.es.DefaultIndexSettings.INDEX;
-import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE;
-import static org.sonar.server.es.DefaultIndexSettings.KEYWORD;
-import static org.sonar.server.es.DefaultIndexSettings.LOWERCASE;
-import static org.sonar.server.es.DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH;
-import static org.sonar.server.es.DefaultIndexSettings.MAX_GRAM;
-import static org.sonar.server.es.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH;
-import static org.sonar.server.es.DefaultIndexSettings.MIN_GRAM;
-import static org.sonar.server.es.DefaultIndexSettings.PATTERN;
-import static org.sonar.server.es.DefaultIndexSettings.PORTER_STEM;
-import static org.sonar.server.es.DefaultIndexSettings.SEARCH_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettings.STANDARD;
-import static org.sonar.server.es.DefaultIndexSettings.STOP;
-import static org.sonar.server.es.DefaultIndexSettings.SUB_FIELD_DELIMITER;
-import static org.sonar.server.es.DefaultIndexSettings.TOKENIZER;
-import static org.sonar.server.es.DefaultIndexSettings.TRIM;
-import static org.sonar.server.es.DefaultIndexSettings.TYPE;
-import static org.sonar.server.es.DefaultIndexSettings.WHITESPACE;
-
-public enum DefaultIndexSettingsElement {
-
- // Filters
-
- WORD_FILTER(FILTER) {
-
- @Override
- protected void setup() {
- set(TYPE, "word_delimiter");
- set("generate_word_parts", true);
- set("catenate_words", true);
- set("catenate_numbers", true);
- set("catenate_all", true);
- set("split_on_case_change", true);
- set("preserve_original", true);
- set("split_on_numerics", true);
- set("stem_english_possessive", true);
- }
- },
- NGRAM_FILTER(FILTER) {
-
- @Override
- protected void setup() {
- set(TYPE, "nGram");
- set(MIN_GRAM, MINIMUM_NGRAM_LENGTH);
- set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH);
- setArray("token_chars", "letter", "digit", "punctuation", "symbol");
- }
- },
-
- // Tokenizers
-
- GRAM_TOKENIZER(TOKENIZER) {
-
- @Override
- protected void setup() {
- set(TYPE, "nGram");
- set(MIN_GRAM, MINIMUM_NGRAM_LENGTH);
- set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH);
- setArray("token_chars", "letter", "digit", "punctuation", "symbol");
- }
- },
- PREFIX_TOKENIZER(TOKENIZER) {
-
- @Override
- protected void setup() {
- set(TYPE, "edgeNGram");
- set(MIN_GRAM, MINIMUM_NGRAM_LENGTH);
- set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH);
- }
- },
- UUID_MODULE_TOKENIZER(TOKENIZER) {
-
- @Override
- protected void setup() {
- set(TYPE, PATTERN);
- set(PATTERN, "\\.");
- }
- },
-
- // Analyzers
-
- SORTABLE_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, KEYWORD);
- setArray(FILTER, TRIM, LOWERCASE);
- }
-
- @Override
- public SortedMap<String, String> fieldMapping() {
- return ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, INDEX_SEARCHABLE,
- ANALYZER, getName(),
- FIELD_FIELDDATA, FIELDDATA_ENABLED);
- }
- },
- INDEX_GRAMS_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, GRAM_TOKENIZER);
- setArray(FILTER, TRIM, LOWERCASE);
- }
- },
- SEARCH_GRAMS_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, WHITESPACE);
- setArray(FILTER, TRIM, LOWERCASE);
- }
-
- @Override
- public SortedMap<String, String> fieldMapping() {
- return ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, INDEX_SEARCHABLE,
- ANALYZER, INDEX_GRAMS_ANALYZER.getName(),
- SEARCH_ANALYZER, getName());
- }
- },
- INDEX_PREFIX_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, PREFIX_TOKENIZER);
- setArray(FILTER, TRIM);
- }
- },
- SEARCH_PREFIX_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, WHITESPACE);
- setArray(FILTER, TRIM);
- }
-
- @Override
- public SortedMap<String, String> fieldMapping() {
- return ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, INDEX_SEARCHABLE,
- ANALYZER, INDEX_PREFIX_ANALYZER.getName(),
- SEARCH_ANALYZER, getName());
- }
- },
- INDEX_PREFIX_CASE_INSENSITIVE_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, PREFIX_TOKENIZER);
- setArray(FILTER, TRIM, LOWERCASE);
- }
- },
- SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, WHITESPACE);
- setArray(FILTER, TRIM, LOWERCASE);
- }
-
- @Override
- public SortedMap<String, String> fieldMapping() {
- return ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, INDEX_SEARCHABLE,
- ANALYZER, INDEX_PREFIX_CASE_INSENSITIVE_ANALYZER.getName(),
- SEARCH_ANALYZER, getName());
- }
- },
- USER_INDEX_GRAMS_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, WHITESPACE);
- setArray(FILTER, TRIM, LOWERCASE, NGRAM_FILTER.getName());
- }
- },
- USER_SEARCH_GRAMS_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, WHITESPACE);
- setArray(FILTER, TRIM, LOWERCASE);
- }
-
- @Override
- public SortedMap<String, String> fieldMapping() {
- return ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, INDEX_SEARCHABLE,
- ANALYZER, USER_INDEX_GRAMS_ANALYZER.getName(),
- SEARCH_ANALYZER, getName());
- }
- },
- INDEX_WORDS_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, STANDARD);
- setArray(FILTER, STANDARD, "word_filter", LOWERCASE, STOP, ASCIIFOLDING, PORTER_STEM);
- }
- },
- SEARCH_WORDS_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, STANDARD);
- setArray(FILTER, STANDARD, LOWERCASE, STOP, ASCIIFOLDING, PORTER_STEM);
- }
-
- @Override
- public SortedMap<String, String> fieldMapping() {
- return ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, INDEX_SEARCHABLE,
- ANALYZER, INDEX_WORDS_ANALYZER.getName(),
- SEARCH_ANALYZER, getName());
- }
- },
- ENGLISH_HTML_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, STANDARD);
- setArray(FILTER, STANDARD, LOWERCASE, STOP, ASCIIFOLDING, PORTER_STEM);
- setArray(CHAR_FILTER, HTML_STRIP);
- }
-
- @Override
- public SortedMap<String, String> fieldMapping() {
- return ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, INDEX_SEARCHABLE,
- ANALYZER, getName());
- }
- },
- PATH_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, "path_hierarchy");
- }
- },
- UUID_MODULE_ANALYZER(ANALYZER) {
-
- @Override
- protected void setup() {
- set(TOKENIZER, UUID_MODULE_TOKENIZER);
- setArray(FILTER, TRIM);
- }
- },
-
- ;
-
- private final String type;
- private final String name;
-
- private Builder builder = Settings.builder();
-
- DefaultIndexSettingsElement(String type) {
- this.type = type;
- this.name = name().toLowerCase(Locale.ENGLISH);
- setup();
- }
-
- protected void set(String settingSuffix, int value) {
- put(localName(settingSuffix), Integer.toString(value));
- }
-
- protected void set(String settingSuffix, boolean value) {
- put(localName(settingSuffix), Boolean.toString(value));
- }
-
- protected void set(String settingSuffix, DefaultIndexSettingsElement otherElement) {
- put(localName(settingSuffix), otherElement.name);
- }
-
- protected void set(String settingSuffix, String value) {
- put(localName(settingSuffix), value);
- }
-
- protected void setArray(String settingSuffix, String... values) {
- putArray(localName(settingSuffix), values);
- }
-
- protected void setArray(String settingSuffix, DefaultIndexSettingsElement... values) {
- putArray(localName(settingSuffix), Arrays.stream(values).map(DefaultIndexSettingsElement::getName).toArray(String[]::new));
- }
-
- private void put(String setting, String value) {
- builder = builder.put(setting, value);
- }
-
- private void putArray(String setting, String... values) {
- builder = builder.putArray(setting, values);
- }
-
- private String localName(String settingSuffix) {
- return ANALYSIS + DELIMITER + type + DELIMITER + name + DELIMITER + settingSuffix;
- }
-
- public Settings settings() {
- return builder.build();
- }
-
- protected abstract void setup();
-
- public SortedMap<String, String> fieldMapping() {
- throw new UnsupportedOperationException("The elasticsearch configuration element '" + name + "' cannot be used as field mapping.");
- }
-
- public String subField(String fieldName) {
- return fieldName + SUB_FIELD_DELIMITER + getSubFieldSuffix();
- }
-
- public String getSubFieldSuffix() {
- return getName();
- }
-
- public String getName() {
- return name;
- }
-}
import javax.annotation.concurrent.Immutable;
+import static java.util.Objects.requireNonNull;
+
@Immutable
class DocId {
private final String indexType;
private final String id;
- DocId(IndexType indexType, String id) {
- this(indexType.getIndex(), indexType.getType(), id);
- }
-
DocId(String index, String indexType, String id) {
- this.index = index;
- this.indexType = indexType;
- this.id = id;
+ this.index = requireNonNull(index, "index can't be null");
+ this.indexType = requireNonNull(indexType,"type can't be null");
+ this.id = requireNonNull(id, "id can't be null");
}
@Override
}
DocId docId = (DocId) o;
- if (!index.equals(docId.index)) {
- return false;
- }
- if (!indexType.equals(docId.indexType)) {
- return false;
- }
- return id.equals(docId.id);
+ return index.equals(docId.index) && indexType.equals(docId.indexType) && id.equals(docId.id);
}
@Override
result = 31 * result + id.hashCode();
return result;
}
+
+ @Override
+ public String toString() {
+ return "DocId{" + index + '/' + indexType + '/' + id + '}';
+ }
}
import org.elasticsearch.common.Priority;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
+import org.sonar.server.es.IndexType.IndexMainType;
import org.sonar.server.es.request.ProxyClearCacheRequestBuilder;
import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder;
import org.sonar.server.es.request.ProxyClusterStateRequestBuilder;
this.nativeClient = null;
}
- public RefreshRequestBuilder prepareRefresh(String... indices) {
- return new ProxyRefreshRequestBuilder(nativeClient()).setIndices(indices);
+ public RefreshRequestBuilder prepareRefresh(Index index) {
+ return new ProxyRefreshRequestBuilder(nativeClient()).setIndices(index.getName());
}
- public IndicesStatsRequestBuilder prepareStats(String... indices) {
- return new ProxyIndicesStatsRequestBuilder(nativeClient()).setIndices(indices);
+ public IndicesStatsRequestBuilder prepareStats() {
+ return new ProxyIndicesStatsRequestBuilder(nativeClient());
+ }
+
+ public IndicesStatsRequestBuilder prepareStats(Index index) {
+ return new ProxyIndicesStatsRequestBuilder(nativeClient()).setIndices(index.getName());
}
public NodesStatsRequestBuilder prepareNodesStats(String... nodesIds) {
return new ProxyClusterStateRequestBuilder(nativeClient());
}
- public ClusterHealthRequestBuilder prepareHealth(String... indices) {
- return new ProxyClusterHealthRequestBuilder(nativeClient()).setIndices(indices);
+ public ClusterHealthRequestBuilder prepareHealth() {
+ return new ProxyClusterHealthRequestBuilder(nativeClient());
}
public void waitForStatus(ClusterHealthStatus status) {
prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForStatus(status).get();
}
- public IndicesExistsRequestBuilder prepareIndicesExist(String... indices) {
- return new ProxyIndicesExistsRequestBuilder(nativeClient(), indices);
+ public IndicesExistsRequestBuilder prepareIndicesExist(Index index) {
+ return new ProxyIndicesExistsRequestBuilder(nativeClient(), index.getName());
}
- public CreateIndexRequestBuilder prepareCreate(String index) {
- return new ProxyCreateIndexRequestBuilder(nativeClient(), index);
+ public CreateIndexRequestBuilder prepareCreate(Index index) {
+ return new ProxyCreateIndexRequestBuilder(nativeClient(), index.getName());
}
- public PutMappingRequestBuilder preparePutMapping(String... indices) {
- return new ProxyPutMappingRequestBuilder(nativeClient()).setIndices(indices);
+ public PutMappingRequestBuilder preparePutMapping(Index index) {
+ return new ProxyPutMappingRequestBuilder(nativeClient()).setIndices(index.getName());
}
- public SearchRequestBuilder prepareSearch(String... indices) {
- return new ProxySearchRequestBuilder(nativeClient()).setIndices(indices);
+ public SearchRequestBuilder prepareSearch(Index index) {
+ return new ProxySearchRequestBuilder(nativeClient()).setIndices(index.getName());
}
- public SearchRequestBuilder prepareSearch(IndexType... indexType) {
+ public SearchRequestBuilder prepareSearch(IndexMainType indexType) {
return new ProxySearchRequestBuilder(nativeClient())
- .setIndices(IndexType.getIndices(indexType))
- .setTypes(IndexType.getTypes(indexType));
+ .setIndices(indexType.getIndex().getName())
+ .setTypes(indexType.getType());
}
public SearchScrollRequestBuilder prepareSearchScroll(String scrollId) {
}
public GetRequestBuilder prepareGet(IndexType indexType, String id) {
- return new ProxyGetRequestBuilder(nativeClient()).setIndex(indexType.getIndex()).setType(indexType.getType()).setId(id);
+ IndexMainType mainType = indexType.getMainType();
+ return new ProxyGetRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType()).setId(id);
}
public DeleteRequestBuilder prepareDelete(IndexType indexType, String id) {
- return new ProxyDeleteRequestBuilder(nativeClient(), indexType.getIndex()).setType(indexType.getType()).setId(id);
+ IndexMainType mainType = indexType.getMainType();
+ return new ProxyDeleteRequestBuilder(nativeClient(), mainType.getIndex().getName()).setType(mainType.getType()).setId(id);
}
- public DeleteRequestBuilder prepareDelete(String index, String type, String id) {
+ DeleteRequestBuilder prepareDelete(String index, String type, String id) {
return new ProxyDeleteRequestBuilder(nativeClient(), index).setType(type).setId(id);
}
public IndexRequestBuilder prepareIndex(IndexType indexType) {
- return new ProxyIndexRequestBuilder(nativeClient()).setIndex(indexType.getIndex()).setType(indexType.getType());
+ IndexMainType mainType = indexType.getMainType();
+ return new ProxyIndexRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType());
}
public ForceMergeRequestBuilder prepareForceMerge(String indexName) {
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import java.util.Objects;
+import org.apache.commons.lang.StringUtils;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+public final class Index {
+ public static final Index ALL_INDICES = Index.simple("_all");
+
+ private final String name;
+ private final boolean relations;
+
+ private Index(String name, boolean acceptsRelations) {
+ checkArgument(name != null && !name.isEmpty(), "Index name can't be null nor empty");
+ checkArgument("_all".equals(name) || StringUtils.isAllLowerCase(name), "Index name must be lower-case letters or '_all': %s", name);
+ this.name = name;
+ this.relations = acceptsRelations;
+ }
+
+ public static Index simple(String name) {
+ return new Index(name, false);
+ }
+
+ public static Index withRelations(String name) {
+ return new Index(name, true);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean acceptsRelations() {
+ return relations;
+ }
+
+ /**
+ * @return the name of the join field for this index if it accepts relations
+ * @throws IllegalStateException if index does not accept relations
+ * @see #acceptsRelations()
+ */
+ public String getJoinField() {
+ checkState(relations, "Only index accepting relations has a join field");
+ return "join_" + name;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Index index = (Index) o;
+ return relations == index.relations && name.equals(index.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, relations);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + name + (relations ? "|*" : "|") + ']';
+ }
+}
*/
package org.sonar.server.es;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
-import org.elasticsearch.common.settings.Settings;
+import java.util.Map;
import org.sonar.api.server.ServerSide;
+import org.sonar.server.es.newindex.NewAuthorizedIndex;
+import org.sonar.server.es.newindex.NewIndex;
+import org.sonar.server.es.newindex.NewRegularIndex;
+import org.sonar.server.es.newindex.SettingsConfiguration;
-import java.util.Map;
+import static com.google.common.base.Preconditions.checkArgument;
@ServerSide
public interface IndexDefinition {
class IndexDefinitionContext {
private final Map<String, NewIndex> byKey = Maps.newHashMap();
- public NewIndex create(String key, NewIndex.SettingsConfiguration settingsConfiguration) {
- Preconditions.checkArgument(!byKey.containsKey(key), String.format("Index already exists: %s", key));
- NewIndex index = new NewIndex(key, settingsConfiguration);
- byKey.put(key, index);
- return index;
- }
-
- public Map<String, NewIndex> getIndices() {
- return byKey;
- }
- }
-
- void define(IndexDefinitionContext context);
-
- /**
- * Immutable copy of {@link NewIndex}
- */
- class Index {
- private final String name;
- private final Settings settings;
- private final Map<String, Type> types;
-
- Index(NewIndex newIndex) {
- this.name = newIndex.getName();
- this.settings = newIndex.getSettings().build();
- ImmutableMap.Builder<String, Type> builder = ImmutableMap.builder();
- for (NewIndex.NewIndexType newIndexType : newIndex.getTypes().values()) {
- Type type = new Type(newIndexType);
- builder.put(type.getName(), type);
- }
- this.types = builder.build();
+ public NewRegularIndex create(Index index, SettingsConfiguration settingsConfiguration) {
+ String indexName = index.getName();
+ checkArgument(!byKey.containsKey(indexName), String.format("Index already exists: %s", indexName));
+ NewRegularIndex newIndex = new NewRegularIndex(index, settingsConfiguration);
+ byKey.put(indexName, newIndex);
+ return newIndex;
}
- public String getName() {
- return name;
- }
+ public NewAuthorizedIndex createWithAuthorization(Index index, SettingsConfiguration settingsConfiguration) {
+ checkArgument(index.acceptsRelations(), "Index with authorization must accept relations");
+ String indexName = index.getName();
+ checkArgument(!byKey.containsKey(indexName), String.format("Index already exists: %s", indexName));
- public Settings getSettings() {
- return settings;
+ NewAuthorizedIndex newIndex = new NewAuthorizedIndex(index, settingsConfiguration);
+ byKey.put(indexName, newIndex);
+ return newIndex;
}
- public Map<String, Type> getTypes() {
- return types;
+ public Map<String, NewIndex> getIndices() {
+ return byKey;
}
}
- /**
- * Immutable copy of {@link NewIndex.NewIndexType}
- */
- class Type {
- private final String name;
- private final Map<String, Object> attributes;
-
- private Type(NewIndex.NewIndexType newType) {
- this.name = newType.getName();
- this.attributes = ImmutableMap.copyOf(newType.getAttributes());
- }
-
- public String getName() {
- return name;
- }
+ void define(IndexDefinitionContext context);
- public Map<String, Object> getAttributes() {
- return attributes;
- }
- }
}
*/
package org.sonar.server.es;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Lists;
-import org.apache.commons.codec.digest.DigestUtils;
-
-import java.util.Collections;
-import java.util.List;
+import java.util.Arrays;
import java.util.Map;
import java.util.SortedMap;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.sonar.server.es.newindex.BuiltIndex;
+
+import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex;
/**
* Hash of index definition is stored in the index itself in order to detect changes of mappings
* and re-populated from scratch. There's no attempt to migrate existing data.
*/
class IndexDefinitionHash {
-
private static final char DELIMITER = ',';
private IndexDefinitionHash() {
}
- static String of(IndexDefinition.Index index) {
- return of(index.getSettings().getAsMap(), index.getTypes());
+ static String of(BuiltIndex<?> index) {
+ IndexType.IndexMainType mainType = index.getMainType();
+ return of(
+ ImmutableMap.of(mainType.getIndex(), mainType),
+ index.getRelationTypes().stream().collect(uniqueIndex(IndexType.IndexRelationType::getName, t -> t)),
+ index.getSettings().getAsMap(),
+ index.getAttributes());
}
private static String of(Map... maps) {
return DigestUtils.sha256Hex(sb.toString());
}
- private static void appendObject(StringBuilder sb, Object value) {
- if (value instanceof IndexDefinition.Type) {
- appendIndexType(sb, (IndexDefinition.Type) value);
- } else if (value instanceof Map) {
- appendMap(sb, (Map) value);
- } else if (value instanceof Iterable) {
- appendIterable(sb, (Iterable) value);
- } else {
- sb.append(String.valueOf(value));
- }
- }
-
- private static void appendIndexType(StringBuilder sb, IndexDefinition.Type type) {
- appendMap(sb, type.getAttributes());
- }
-
private static void appendMap(StringBuilder sb, Map attributes) {
for (Object entry : sort(attributes).entrySet()) {
sb.append(((Map.Entry) entry).getKey());
}
}
- private static void appendIterable(StringBuilder sb, Iterable value) {
- List sorted = Lists.newArrayList(value);
- Collections.sort(sorted);
- for (Object o : sorted) {
- appendObject(sb, o);
- sb.append(DELIMITER);
+ private static void appendObject(StringBuilder sb, Object value) {
+ if (value instanceof Object[]) {
+ sb.append(Arrays.toString((Object[]) value));
+ } else if (value instanceof Map) {
+ appendMap(sb, (Map) value);
+ } else if (value instanceof IndexType) {
+ sb.append(((IndexType) value).format());
+ } else {
+ sb.append(String.valueOf(value));
}
}
private static SortedMap sort(Map map) {
+ if (map instanceof ImmutableSortedMap) {
+ return (ImmutableSortedMap) map;
+ }
return ImmutableSortedMap.copyOf(map);
}
}
package org.sonar.server.es;
import com.google.common.base.Splitter;
-import java.util.Arrays;
import java.util.List;
-import java.util.function.Function;
-import org.sonar.core.util.stream.MoreCollectors;
+import java.util.Objects;
+import javax.annotation.concurrent.Immutable;
+import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Objects.requireNonNull;
-public class IndexType {
+public abstract class IndexType {
+
+ public static final String FIELD_INDEX_TYPE = "indexType";
private static final String SEPARATOR = "/";
private static final Splitter SEPARATOR_SPLITTER = Splitter.on(SEPARATOR);
- private final String index;
- private final String type;
- private final String key;
+ public abstract IndexMainType getMainType();
- public IndexType(String index, String type) {
- this.index = requireNonNull(index);
- this.type = requireNonNull(type);
- this.key = index + SEPARATOR + type;
- }
+ public abstract String format();
- public String getIndex() {
- return index;
- }
+ /**
+ * Parse a String generated by {@link #format()} to extract the simple details of the main type.
+ * <p>
+ * Note: neither {@link IndexMainType} nor {@link IndexRelationType} can be parsed from the string generated by
+ * {@link #format()} as the generated string does not contain the {@link Index#acceptsRelations() acceptsRelations}
+ * flag).
+ */
+ public static SimpleIndexMainType parseMainType(String s) {
+ List<String> split = SEPARATOR_SPLITTER.splitToList(s);
+ checkArgument(split.size() >= 2, "Unsupported IndexType value: %s", s);
- public String getType() {
- return type;
+ return new SimpleIndexMainType(split.get(0), split.get(1));
}
- public static String[] getIndices(IndexType... indexTypes) {
- return getDetails(IndexType::getIndex, indexTypes);
- }
+ @Immutable
+ public static final class SimpleIndexMainType {
+ private final String index;
+ private final String type;
+
+ private SimpleIndexMainType(String index, String type) {
+ this.index = index;
+ this.type = type;
+ }
+
+ public String getIndex() {
+ return index;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ SimpleIndexMainType that = (SimpleIndexMainType) o;
+ return index.equals(that.index) && type.equals(that.type);
+ }
- public static String[] getTypes(IndexType... indexTypes) {
- return getDetails(IndexType::getType, indexTypes);
+ @Override
+ public int hashCode() {
+ return Objects.hash(index, type);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + index + '/' + type + ']';
+ }
}
- private static String[] getDetails(Function<? super IndexType, ? extends String> function, IndexType... indexTypes) {
- return Arrays.stream(indexTypes).map(function).collect(MoreCollectors.toSet(indexTypes.length)).toArray(new String[0]);
+ public static IndexMainType main(Index index, String type) {
+ return new IndexMainType(index, type);
}
- public String format() {
- return key;
+ public static IndexRelationType relation(IndexMainType mainType, String name) {
+ checkArgument(mainType.getIndex().acceptsRelations(), "Index must define a join field to have relations");
+
+ return new IndexRelationType(mainType, name);
}
- /**
- * Parse a String generated by {@link #format()}
- */
- public static IndexType parse(String s) {
- List<String> split = SEPARATOR_SPLITTER.splitToList(s);
- if (split.size() != 2) {
- throw new IllegalArgumentException("Unsupported IndexType value: " + s);
+ @Immutable
+ public static final class IndexMainType extends IndexType {
+ private final Index index;
+ private final String type;
+ private final String key;
+
+ private IndexMainType(Index index, String type) {
+ this.index = requireNonNull(index);
+ checkArgument(type != null && !type.isEmpty(), "type name can't be null nor empty");
+ this.type = type;
+ this.key = index.getName() + SEPARATOR + type;
}
- return new IndexType(split.get(0), split.get(1));
- }
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
+ @Override
+ public IndexMainType getMainType() {
+ return this;
}
- if (o == null || getClass() != o.getClass()) {
- return false;
+
+ public Index getIndex() {
+ return index;
}
- IndexType indexType = (IndexType) o;
- return key.equals(indexType.key);
- }
+ public String getType() {
+ return type;
+ }
+
+ @Override
+ public String format() {
+ return key;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ IndexMainType indexType = (IndexMainType) o;
+ return index.equals(indexType.index) && type.equals(indexType.type);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(index, type);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + key + "]";
+ }
- @Override
- public int hashCode() {
- return key.hashCode();
}
- @Override
- public String toString() {
- return "[" + index + "/" + type + "]";
+ @Immutable
+ public static final class IndexRelationType extends IndexType {
+ private final IndexMainType mainType;
+ private final String name;
+ private final String key;
+
+ private IndexRelationType(IndexMainType mainType, String name) {
+ this.mainType = mainType;
+ checkArgument(name != null && !name.isEmpty(), "type name can't be null nor empty");
+ this.name = name;
+ this.key = mainType.index.getName() + "/" + mainType.type + "/" + name;
+ }
+
+ @Override
+ public IndexMainType getMainType() {
+ return mainType;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String format() {
+ return key;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ IndexRelationType indexType = (IndexRelationType) o;
+ return mainType.equals(indexType.mainType) && name.equals(indexType.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(mainType, name);
+ }
+
+ @Override
+ public String toString() {
+ return "[" + key + "]";
+ }
}
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedMap;
-import com.google.common.collect.Maps;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import javax.annotation.CheckForNull;
-import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.sonar.api.config.Configuration;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static java.lang.String.format;
-import static java.lang.String.valueOf;
-import static java.util.Objects.requireNonNull;
-import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
-import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS;
-import static org.sonar.server.es.DefaultIndexSettings.ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettings.FIELDDATA_ENABLED;
-import static org.sonar.server.es.DefaultIndexSettings.FIELD_FIELDDATA;
-import static org.sonar.server.es.DefaultIndexSettings.FIELD_TERM_VECTOR;
-import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_KEYWORD;
-import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT;
-import static org.sonar.server.es.DefaultIndexSettings.INDEX;
-import static org.sonar.server.es.DefaultIndexSettings.INDEX_NOT_SEARCHABLE;
-import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE;
-import static org.sonar.server.es.DefaultIndexSettings.NORMS;
-import static org.sonar.server.es.DefaultIndexSettings.STORE;
-import static org.sonar.server.es.DefaultIndexSettings.TYPE;
-import static org.sonar.server.es.DefaultIndexSettingsElement.UUID_MODULE_ANALYZER;
-import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_ALLOW_ANYONE;
-import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_GROUP_IDS;
-import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_USER_IDS;
-import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
-
-public class NewIndex {
-
- private final String indexName;
- private final Settings.Builder settings = DefaultIndexSettings.defaults();
- private final Map<String, NewIndexType> types = new LinkedHashMap<>();
-
- NewIndex(String indexName, SettingsConfiguration settingsConfiguration) {
- checkArgument(StringUtils.isAllLowerCase(indexName), "Index name must be lower-case: " + indexName);
- this.indexName = indexName;
- applySettingsConfiguration(settingsConfiguration);
- }
-
- private void applySettingsConfiguration(SettingsConfiguration settingsConfiguration) {
- settings.put("index.mapper.dynamic", valueOf(false));
- settings.put("index.refresh_interval", refreshInterval(settingsConfiguration));
-
- Configuration config = settingsConfiguration.getConfiguration();
- boolean clusterMode = config.getBoolean(CLUSTER_ENABLED.getKey()).orElse(false);
- int shards = config.getInt(format("sonar.search.%s.shards", indexName))
- .orElse(settingsConfiguration.getDefaultNbOfShards());
- int replicas = clusterMode ? config.getInt(SEARCH_REPLICAS.getKey()).orElse(1) : 0;
-
- settings.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shards);
- settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas);
- }
-
- private static String refreshInterval(SettingsConfiguration settingsConfiguration) {
- int refreshInterval = settingsConfiguration.getRefreshInterval();
- if (refreshInterval == -1) {
- return "-1";
- }
- return refreshInterval + "s";
- }
-
- public static class SettingsConfiguration {
- public static final int MANUAL_REFRESH_INTERVAL = -1;
-
- private final Configuration configuration;
- private final int defaultNbOfShards;
- private final int refreshInterval;
-
- private SettingsConfiguration(Builder builder) {
- this.configuration = builder.configuration;
- this.defaultNbOfShards = builder.defaultNbOfShards;
- this.refreshInterval = builder.refreshInterval;
- }
-
- public static Builder newBuilder(Configuration configuration) {
- return new Builder(configuration);
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public int getDefaultNbOfShards() {
- return defaultNbOfShards;
- }
-
- public int getRefreshInterval() {
- return refreshInterval;
- }
-
- public static class Builder {
- private final Configuration configuration;
- private int defaultNbOfShards = 1;
- private int refreshInterval = 30;
-
- public Builder(Configuration configuration) {
- this.configuration = requireNonNull(configuration, "configuration can't be null");
- }
-
- public Builder setDefaultNbOfShards(int defaultNbOfShards) {
- checkArgument(defaultNbOfShards >= 1, "defaultNbOfShards must be >= 1");
- this.defaultNbOfShards = defaultNbOfShards;
- return this;
- }
-
- public Builder setRefreshInterval(int refreshInterval) {
- checkArgument(refreshInterval == -1 || refreshInterval > 0,
- "refreshInterval must be either -1 or strictly positive");
- this.refreshInterval = refreshInterval;
- return this;
- }
-
- public SettingsConfiguration build() {
- return new SettingsConfiguration(this);
- }
- }
-
- }
-
- public String getName() {
- return indexName;
- }
-
- public Settings.Builder getSettings() {
- return settings;
- }
-
- public NewIndexType createType(String typeName) {
- NewIndexType type = new NewIndexType(this, typeName);
- types.put(typeName, type);
- return type;
- }
-
- public Map<String, NewIndexType> getTypes() {
- return types;
- }
-
- public static class NewIndexType {
- private final NewIndex index;
- private final String name;
- private final Map<String, Object> attributes = new TreeMap<>();
- private final Map<String, Object> properties = new TreeMap<>();
-
- private NewIndexType(NewIndex index, String typeName) {
- this.index = index;
- this.name = typeName;
- // defaults
- attributes.put("dynamic", false);
- attributes.put("_all", ImmutableSortedMap.of("enabled", false));
- attributes.put("_source", ImmutableSortedMap.of("enabled", true));
- attributes.put("properties", properties);
- }
-
- public NewIndexType requireProjectAuthorization() {
- enableProjectAuthorization(this);
- return this;
- }
-
- /**
- * Creates a type that requires to verify that user has the read permission
- * when searching for documents.
- *
- * Both types {@code typeName} and "authorization" are created. Documents
- * must be created with _parent and _routing having the parent uuid as values.
- *
- * @see NewIndex.NewIndexType#requireProjectAuthorization()
- */
- private static NewIndex.NewIndexType enableProjectAuthorization(NewIndex.NewIndexType type) {
- type.setAttribute("_parent", ImmutableMap.of("type", TYPE_AUTHORIZATION));
- type.setAttribute("_routing", ImmutableMap.of("required", true));
-
- NewIndex.NewIndexType authType = type.getIndex().createType(TYPE_AUTHORIZATION);
- authType.setAttribute("_routing", ImmutableMap.of("required", true));
- authType.createLongField(FIELD_GROUP_IDS);
- authType.createLongField(FIELD_USER_IDS);
- authType.createBooleanField(FIELD_ALLOW_ANYONE);
- authType.setEnableSource(false);
- return type;
- }
-
- public NewIndex getIndex() {
- return index;
- }
-
- public String getName() {
- return name;
- }
-
- /**
- * Complete the root json hash of mapping type, for example to set the attribute "_id"
- */
- public NewIndexType setAttribute(String key, Object value) {
- attributes.put(key, value);
- return this;
- }
-
- /**
- * Complete the json hash named "properties" in mapping type, usually to declare fields
- */
- public NewIndexType setProperty(String key, Object value) {
- properties.put(key, value);
- return this;
- }
-
- public NewIndexType setEnableSource(boolean enableSource) {
- attributes.put("_source", ImmutableSortedMap.of("enabled", enableSource));
- return this;
- }
-
- public KeywordFieldBuilder keywordFieldBuilder(String fieldName) {
- return new KeywordFieldBuilder(this, fieldName);
- }
-
- public TextFieldBuilder textFieldBuilder(String fieldName) {
- return new TextFieldBuilder(this, fieldName);
- }
-
- public NestedFieldBuilder nestedFieldBuilder(String fieldName) {
- return new NestedFieldBuilder(this, fieldName);
- }
-
- public NewIndexType createBooleanField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "boolean"));
- }
-
- public NewIndexType createByteField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "byte"));
- }
-
- public NewIndexType createDateTimeField(String fieldName) {
- Map<String, String> hash = new TreeMap<>();
- hash.put("type", "date");
- hash.put("format", "date_time||epoch_second");
- return setProperty(fieldName, hash);
- }
-
- public NewIndexType createDoubleField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "double"));
- }
-
- public NewIndexType createIntegerField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "integer"));
- }
-
- public NewIndexType createLongField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "long"));
- }
-
- public NewIndexType createShortField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "short"));
- }
-
- public NewIndexType createUuidPathField(String fieldName) {
- return setProperty(fieldName, ImmutableSortedMap.of(
- TYPE, FIELD_TYPE_TEXT,
- INDEX, DefaultIndexSettings.INDEX_SEARCHABLE,
- ANALYZER, UUID_MODULE_ANALYZER.getName()));
- }
-
- public Map<String, Object> getAttributes() {
- return attributes;
- }
-
- @CheckForNull
- public Object getProperty(String key) {
- return properties.get(key);
- }
- }
-
- /**
- * Helper to define a string field in mapping of index type
- */
- public abstract static class StringFieldBuilder<T extends StringFieldBuilder<T>> {
- private final NewIndexType indexType;
- private final String fieldName;
- private boolean disableSearch = false;
- private boolean disableNorms = false;
- private boolean termVectorWithPositionOffsets = false;
- private SortedMap<String, Object> subFields = Maps.newTreeMap();
- private boolean store = false;
- protected boolean disabledDocValues = false;
-
- private StringFieldBuilder(NewIndexType indexType, String fieldName) {
- this.indexType = indexType;
- this.fieldName = fieldName;
- }
-
- /**
- * Add a sub-field. A {@code SortedMap} is required for consistency of the index settings hash.
- * @see IndexDefinitionHash
- */
- private T addSubField(String fieldName, SortedMap<String, String> fieldDefinition) {
- subFields.put(fieldName, fieldDefinition);
- return castThis();
- }
-
- /**
- * Add subfields, one for each analyzer.
- */
- public T addSubFields(DefaultIndexSettingsElement... analyzers) {
- Arrays.stream(analyzers)
- .forEach(analyzer -> addSubField(analyzer.getSubFieldSuffix(), analyzer.fieldMapping()));
- return castThis();
- }
-
- /**
- * Norms consume useless memory if string field is used for filtering or aggregations.
- *
- * https://www.elastic.co/guide/en/elasticsearch/reference/2.3/norms.html
- * https://www.elastic.co/guide/en/elasticsearch/guide/current/scoring-theory.html#field-norm
- */
- public T disableNorms() {
- this.disableNorms = true;
- return castThis();
- }
-
- /**
- * Position offset term vectors are required for the fast_vector_highlighter (fvh).
- */
- public T termVectorWithPositionOffsets() {
- this.termVectorWithPositionOffsets = true;
- return castThis();
- }
-
- /**
- * "index: false" -> Make this field not searchable.
- * By default field is "true": it is searchable, but index the value exactly
- * as specified.
- */
- public T disableSearch() {
- this.disableSearch = true;
- return castThis();
- }
-
- public T store() {
- this.store = true;
- return castThis();
- }
-
- @SuppressWarnings("unchecked")
- private T castThis() {
- return (T) this;
- }
-
- public NewIndexType build() {
- if (subFields.isEmpty()) {
- return buildWithoutSubfields();
- }
- return buildWithSubfields();
- }
-
- private NewIndexType buildWithoutSubfields() {
- Map<String, Object> hash = new TreeMap<>();
- hash.put("type", getFieldType());
- hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE);
- hash.put(NORMS, valueOf(!disableNorms));
- hash.put(STORE, valueOf(store));
- if (FIELD_TYPE_KEYWORD.equals(getFieldType())) {
- hash.put("doc_values", valueOf(!disabledDocValues));
- }
- if (getFieldData()) {
- hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
- }
- return indexType.setProperty(fieldName, hash);
- }
-
- private NewIndexType buildWithSubfields() {
- Map<String, Object> hash = new TreeMap<>();
- hash.put("type", getFieldType());
- hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE);
- hash.put(NORMS, "false");
- hash.put(STORE, valueOf(store));
- if (FIELD_TYPE_KEYWORD.equals(getFieldType())) {
- hash.put("doc_values", valueOf(!disabledDocValues));
- }
- if (getFieldData()) {
- hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
- }
- if (termVectorWithPositionOffsets) {
- hash.put(FIELD_TERM_VECTOR, "with_positions_offsets");
- }
- hash.put("fields", configureSubFields());
- return indexType.setProperty(fieldName, hash);
- }
-
- private Map<String, Object> configureSubFields() {
- Map<String, Object> multiFields = new TreeMap<>(subFields);
-
- // apply this fields configuration to all subfields
- multiFields.entrySet().forEach(entry -> {
- Object subFieldMapping = entry.getValue();
- if (subFieldMapping instanceof Map) {
- entry.setValue(configureSubField((Map<String, String>) subFieldMapping));
- }
- });
- return multiFields;
- }
-
- private Map<String, String> configureSubField(Map<String, String> subFieldMapping) {
- Map<String, String> subHash = new TreeMap<>(subFieldMapping);
- subHash.put(INDEX, INDEX_SEARCHABLE);
- subHash.put(NORMS, "false");
- subHash.put(STORE, valueOf(store));
- if (termVectorWithPositionOffsets) {
- subHash.put(FIELD_TERM_VECTOR, "with_positions_offsets");
- }
- return subHash;
- }
-
- protected abstract boolean getFieldData();
-
- protected abstract String getFieldType();
- }
-
- public static class KeywordFieldBuilder extends StringFieldBuilder<KeywordFieldBuilder> {
-
- private KeywordFieldBuilder(NewIndexType indexType, String fieldName) {
- super(indexType, fieldName);
- }
-
- @Override
- protected boolean getFieldData() {
- return false;
- }
-
- protected String getFieldType() {
- return FIELD_TYPE_KEYWORD;
- }
-
- /**
- * By default, field is stored on disk in a column-stride fashion, so that it can later be used for sorting,
- * aggregations, or scripting.
- * Disabling this reduces the size of the index and drop the constraint of single term max size of
- * 32766 bytes (which, if there is no tokenizing enabled on the field, equals the size of the whole data).
- */
- public KeywordFieldBuilder disableSortingAndAggregating() {
- this.disabledDocValues = true;
- return this;
- }
- }
-
- public static class TextFieldBuilder extends StringFieldBuilder<TextFieldBuilder> {
-
- private boolean fieldData = false;
-
- private TextFieldBuilder(NewIndexType indexType, String fieldName) {
- super(indexType, fieldName);
- }
-
- protected String getFieldType() {
- return FIELD_TYPE_TEXT;
- }
-
- /**
- * Required to enable sorting, aggregation and access to field data on fields of type "text".
- * <p>Disabled by default as this can have significant memory cost</p>
- */
- public StringFieldBuilder withFieldData() {
- this.fieldData = true;
- return this;
- }
-
- @Override
- protected boolean getFieldData() {
- return fieldData;
- }
- }
-
- public static class NestedFieldBuilder {
- private final NewIndexType indexType;
- private final String fieldName;
- private final Map<String, Object> properties = new TreeMap<>();
-
- private NestedFieldBuilder(NewIndexType indexType, String fieldName) {
- this.indexType = indexType;
- this.fieldName = fieldName;
- }
-
- private NestedFieldBuilder setProperty(String fieldName, Object value) {
- properties.put(fieldName, value);
-
- return this;
- }
-
- public NestedFieldBuilder addKeywordField(String fieldName) {
- return setProperty(fieldName, ImmutableSortedMap.of(
- "type", FIELD_TYPE_KEYWORD,
- INDEX, INDEX_SEARCHABLE));
- }
-
- public NestedFieldBuilder addDoubleField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "double"));
- }
-
- public NestedFieldBuilder addIntegerField(String fieldName) {
- return setProperty(fieldName, ImmutableMap.of("type", "integer"));
- }
-
- public NewIndexType build() {
- checkArgument(!properties.isEmpty(), "At least one sub-field must be declared in nested property '%s'", fieldName);
-
- return indexType.setProperty(fieldName, ImmutableSortedMap.of(
- "type", "nested",
- "properties", properties));
- }
- }
-
-}
this.dbClient = dbClient;
this.dbSession = dbSession;
this.itemsById = items.stream()
- .collect(MoreCollectors.index(i -> new DocId(IndexType.parse(i.getDocType()), i.getDocId()), Function.identity()));
+ .collect(MoreCollectors.index(i -> {
+ IndexType.SimpleIndexMainType mainType = IndexType.parseMainType(i.getDocType());
+ return new DocId(mainType.getIndex(), mainType.getType(), i.getDocId());
+ }, Function.identity()));
}
@Override
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.index.get.GetField;
import org.sonar.server.es.EsClient;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.IndexType.IndexRelationType;
-import static org.sonar.server.es.DefaultIndexSettings.REFRESH_IMMEDIATE;
+import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.REFRESH_IMMEDIATE;
public class MetadataIndex {
this.esClient = esClient;
}
- public Optional<String> getHash(String index) {
+ public Optional<String> getHash(Index index) {
return getMetadata(hashId(index));
}
- public void setHash(String index, String hash) {
+ public void setHash(Index index, String hash) {
setMetadata(hashId(index), hash);
}
- private static String hashId(String index) {
- return index + ".indexStructure";
+ private static String hashId(Index index) {
+ return index.getName() + ".indexStructure";
}
public boolean getInitialized(IndexType indexType) {
}
private static String initializedId(IndexType indexType) {
- return indexType.getIndex() + "." + indexType.getType() + ".initialized";
+ if (indexType instanceof IndexMainType) {
+ IndexMainType mainType = (IndexMainType) indexType;
+ return mainType.getIndex().getName() + "." + mainType.getType() + ".initialized";
+ }
+ if (indexType instanceof IndexRelationType) {
+ IndexRelationType relationType = (IndexRelationType) indexType;
+ IndexMainType mainType = relationType.getMainType();
+ return mainType.getIndex().getName() + "." + mainType.getType() + "." + relationType.getName() + ".initialized";
+ }
+ throw new IllegalArgumentException("Unsupported IndexType " + indexType.getClass());
}
public Optional<String> getDbVendor() {
}
private Optional<String> getMetadata(String id) {
- GetRequestBuilder request = esClient.prepareGet(MetadataIndexDefinition.INDEX_TYPE_METADATA, id)
+ GetRequestBuilder request = esClient.prepareGet(TYPE_METADATA, id)
.setStoredFields(MetadataIndexDefinition.FIELD_VALUE);
GetResponse response = request.get();
if (response.isExists()) {
}
private void setMetadata(String id, String value) {
- esClient.prepareIndex(MetadataIndexDefinition.INDEX_TYPE_METADATA)
+ esClient.prepareIndex(TYPE_METADATA)
.setId(id)
.setSource(MetadataIndexDefinition.FIELD_VALUE, value)
.setRefreshPolicy(REFRESH_IMMEDIATE)
package org.sonar.server.es.metadata;
import org.sonar.api.config.Configuration;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition.IndexDefinitionContext;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.newindex.NewRegularIndex;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
public class MetadataIndexDefinition {
- public static final IndexType INDEX_TYPE_METADATA = new IndexType("metadatas", "metadata");
+ public static final Index DESCRIPTOR = Index.simple("metadatas");
+ public static final IndexMainType TYPE_METADATA = IndexType.main(DESCRIPTOR, "metadata");
public static final String FIELD_VALUE = "value";
private static final int DEFAULT_NUMBER_OF_SHARDS = 1;
}
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(
- INDEX_TYPE_METADATA.getIndex(),
+ NewRegularIndex index = context.create(
+ DESCRIPTOR,
newBuilder(configuration)
.setRefreshInterval(MANUAL_REFRESH_INTERVAL)
.setDefaultNbOfShards(DEFAULT_NUMBER_OF_SHARDS)
.build());
- NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_METADATA.getType());
-
- mapping.keywordFieldBuilder(FIELD_VALUE).disableSearch().store().build();
+ index.createTypeMapping(TYPE_METADATA)
+ .keywordFieldBuilder(FIELD_VALUE).disableSearch().store().build();
}
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import org.elasticsearch.common.settings.Settings;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexRelationType;
+
+import static org.sonar.core.util.stream.MoreCollectors.toSet;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.NORMS;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.STORE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE;
+
+/**
+ * Immutable copy of {@link NewIndex}
+ */
+public final class BuiltIndex<T extends NewIndex<T>> {
+ private final IndexType.IndexMainType mainType;
+ private final Set<IndexRelationType> relationTypes;
+ private final Settings settings;
+ private final Map<String, Object> attributes;
+
+ BuiltIndex(T newIndex) {
+ this.mainType = newIndex.getMainType();
+ this.settings = newIndex.getSettings().build();
+ this.relationTypes = newIndex.getRelationsStream().collect(toSet());
+ this.attributes = buildAttributes(newIndex);
+ }
+
+ private static Map<String, Object> buildAttributes(NewIndex<?> newIndex) {
+ Map<String, Object> indexAttributes = new TreeMap<>(newIndex.getAttributes());
+ setRouting(indexAttributes, newIndex);
+ indexAttributes.put("properties", buildProperties(newIndex));
+ return ImmutableSortedMap.copyOf(indexAttributes);
+ }
+
+ private static void setRouting(Map<String, Object> indexAttributes, NewIndex newIndex) {
+ if (!newIndex.getRelations().isEmpty()) {
+ indexAttributes.put("_routing", ImmutableMap.of("required", true));
+ }
+ }
+
+ private static TreeMap<String, Object> buildProperties(NewIndex<?> newIndex) {
+ TreeMap<String, Object> indexProperties = new TreeMap<>(newIndex.getProperties());
+ setTypeField(indexProperties, newIndex);
+ setJoinField(indexProperties, newIndex);
+ return indexProperties;
+ }
+
+ private static void setTypeField(TreeMap<String, Object> indexProperties, NewIndex newIndex) {
+ Collection<IndexRelationType> relations = newIndex.getRelations();
+ if (!relations.isEmpty()) {
+ indexProperties.put(
+ FIELD_INDEX_TYPE,
+ ImmutableMap.of(
+ TYPE, "keyword",
+ NORMS, false,
+ STORE, false,
+ "doc_values", false));
+ }
+ }
+
+ private static void setJoinField(TreeMap<String, Object> indexProperties, NewIndex newIndex) {
+ Collection<IndexRelationType> relations = newIndex.getRelations();
+ IndexType.IndexMainType mainType = newIndex.getMainType();
+ if (!relations.isEmpty()) {
+ indexProperties.put(mainType.getIndex().getJoinField(), ImmutableMap.of(
+ TYPE, "join",
+ "relations", ImmutableMap.of(mainType.getType(), namesToStringOrStringArray(relations))));
+ }
+ }
+
+ private static Serializable namesToStringOrStringArray(Collection<IndexRelationType> relations) {
+ if (relations.size() == 1) {
+ return relations.iterator().next().getName();
+ }
+
+ return relations.stream()
+ .map(IndexRelationType::getName)
+ .sorted()
+ .toArray(String[]::new);
+ }
+
+ public IndexType.IndexMainType getMainType() {
+ return mainType;
+ }
+
+ public Set<IndexRelationType> getRelationTypes() {
+ return relationTypes;
+ }
+
+ public Settings getSettings() {
+ return settings;
+ }
+
+ public Map<String, Object> getAttributes() {
+ return attributes;
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import java.util.Arrays;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+
+public class DefaultIndexSettings {
+
+ /** Minimum length of ngrams. */
+ public static final int MINIMUM_NGRAM_LENGTH = 2;
+ /** Maximum length of ngrams. */
+ public static final int MAXIMUM_NGRAM_LENGTH = 15;
+
+ /** Pattern, that splits the user search input **/
+ public static final String SEARCH_TERM_TOKENIZER_PATTERN = "[\\s]+";
+
+ public static final String ANALYSIS = "index.analysis";
+ public static final String DELIMITER = ".";
+
+ public static final String TOKENIZER = "tokenizer";
+ public static final String FILTER = "filter";
+ public static final String CHAR_FILTER = "char_filter";
+ public static final String ANALYZER = "analyzer";
+ public static final String SEARCH_ANALYZER = "search_analyzer";
+
+ public static final String TYPE = "type";
+ public static final String INDEX = "index";
+ public static final String INDEX_SEARCHABLE = "true";
+ public static final String INDEX_NOT_SEARCHABLE = "false";
+ public static final String FIELD_TYPE_TEXT = "text";
+ public static final String FIELD_TYPE_KEYWORD = "keyword";
+ public static final String NORMS = "norms";
+ public static final String STORE = "store";
+ public static final String FIELD_FIELDDATA = "fielddata";
+ public static final String FIELDDATA_ENABLED = "true";
+ public static final String FIELD_TERM_VECTOR = "term_vector";
+ public static final String STANDARD = "standard";
+ public static final String PATTERN = "pattern";
+ public static final String CUSTOM = "custom";
+ public static final String KEYWORD = "keyword";
+ public static final String CLASSIC = "classic";
+ public static final RefreshPolicy REFRESH_IMMEDIATE = RefreshPolicy.IMMEDIATE;
+ public static final RefreshPolicy REFRESH_NONE = RefreshPolicy.NONE;
+
+ public static final String TRUNCATE = "truncate";
+
+ public static final String SUB_FIELD_DELIMITER = ".";
+ public static final String TRIM = "trim";
+ public static final String LOWERCASE = "lowercase";
+ public static final String WHITESPACE = "whitespace";
+ public static final String STOP = "stop";
+ public static final String ASCIIFOLDING = "asciifolding";
+ public static final String PORTER_STEM = "porter_stem";
+ public static final String MIN_GRAM = "min_gram";
+ public static final String MAX_GRAM = "max_gram";
+ public static final String LENGTH = "length";
+ public static final String HTML_STRIP = "html_strip";
+
+ private DefaultIndexSettings() {
+ // only static stuff
+ }
+
+ public static Settings.Builder defaults() {
+ Settings.Builder builder = Settings.builder()
+ .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+ .put("index.refresh_interval", "30s")
+ .put("index.mapper.dynamic", false);
+
+ Arrays.stream(DefaultIndexSettingsElement.values())
+ .map(DefaultIndexSettingsElement::settings)
+ .forEach(builder::put);
+
+ return builder;
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableSortedMap;
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.SortedMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.Settings.Builder;
+
+import static org.sonar.server.es.newindex.DefaultIndexSettings.ANALYSIS;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.ASCIIFOLDING;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.CHAR_FILTER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.DELIMITER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELDDATA_ENABLED;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_FIELDDATA;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_TEXT;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FILTER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.HTML_STRIP;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_SEARCHABLE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.KEYWORD;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.LOWERCASE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.MAX_GRAM;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.MIN_GRAM;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.PATTERN;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.PORTER_STEM;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.SEARCH_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.STANDARD;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.STOP;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.SUB_FIELD_DELIMITER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.TOKENIZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.TRIM;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.WHITESPACE;
+
+public enum DefaultIndexSettingsElement {
+
+ // Filters
+
+ WORD_FILTER(FILTER) {
+
+ @Override
+ protected void setup() {
+ set(TYPE, "word_delimiter");
+ set("generate_word_parts", true);
+ set("catenate_words", true);
+ set("catenate_numbers", true);
+ set("catenate_all", true);
+ set("split_on_case_change", true);
+ set("preserve_original", true);
+ set("split_on_numerics", true);
+ set("stem_english_possessive", true);
+ }
+ },
+ NGRAM_FILTER(FILTER) {
+
+ @Override
+ protected void setup() {
+ set(TYPE, "nGram");
+ set(MIN_GRAM, MINIMUM_NGRAM_LENGTH);
+ set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH);
+ setArray("token_chars", "letter", "digit", "punctuation", "symbol");
+ }
+ },
+
+ // Tokenizers
+
+ GRAM_TOKENIZER(TOKENIZER) {
+
+ @Override
+ protected void setup() {
+ set(TYPE, "nGram");
+ set(MIN_GRAM, MINIMUM_NGRAM_LENGTH);
+ set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH);
+ setArray("token_chars", "letter", "digit", "punctuation", "symbol");
+ }
+ },
+ PREFIX_TOKENIZER(TOKENIZER) {
+
+ @Override
+ protected void setup() {
+ set(TYPE, "edgeNGram");
+ set(MIN_GRAM, MINIMUM_NGRAM_LENGTH);
+ set(MAX_GRAM, MAXIMUM_NGRAM_LENGTH);
+ }
+ },
+ UUID_MODULE_TOKENIZER(TOKENIZER) {
+
+ @Override
+ protected void setup() {
+ set(TYPE, PATTERN);
+ set(PATTERN, "\\.");
+ }
+ },
+
+ // Analyzers
+
+ SORTABLE_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, KEYWORD);
+ setArray(FILTER, TRIM, LOWERCASE);
+ }
+
+ @Override
+ public SortedMap<String, String> fieldMapping() {
+ return ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, INDEX_SEARCHABLE,
+ ANALYZER, getName(),
+ FIELD_FIELDDATA, FIELDDATA_ENABLED);
+ }
+ },
+ INDEX_GRAMS_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, GRAM_TOKENIZER);
+ setArray(FILTER, TRIM, LOWERCASE);
+ }
+ },
+ SEARCH_GRAMS_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, WHITESPACE);
+ setArray(FILTER, TRIM, LOWERCASE);
+ }
+
+ @Override
+ public SortedMap<String, String> fieldMapping() {
+ return ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, INDEX_SEARCHABLE,
+ ANALYZER, INDEX_GRAMS_ANALYZER.getName(),
+ SEARCH_ANALYZER, getName());
+ }
+ },
+ INDEX_PREFIX_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, PREFIX_TOKENIZER);
+ setArray(FILTER, TRIM);
+ }
+ },
+ SEARCH_PREFIX_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, WHITESPACE);
+ setArray(FILTER, TRIM);
+ }
+
+ @Override
+ public SortedMap<String, String> fieldMapping() {
+ return ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, INDEX_SEARCHABLE,
+ ANALYZER, INDEX_PREFIX_ANALYZER.getName(),
+ SEARCH_ANALYZER, getName());
+ }
+ },
+ INDEX_PREFIX_CASE_INSENSITIVE_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, PREFIX_TOKENIZER);
+ setArray(FILTER, TRIM, LOWERCASE);
+ }
+ },
+ SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, WHITESPACE);
+ setArray(FILTER, TRIM, LOWERCASE);
+ }
+
+ @Override
+ public SortedMap<String, String> fieldMapping() {
+ return ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, INDEX_SEARCHABLE,
+ ANALYZER, INDEX_PREFIX_CASE_INSENSITIVE_ANALYZER.getName(),
+ SEARCH_ANALYZER, getName());
+ }
+ },
+ USER_INDEX_GRAMS_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, WHITESPACE);
+ setArray(FILTER, TRIM, LOWERCASE, NGRAM_FILTER.getName());
+ }
+ },
+ USER_SEARCH_GRAMS_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, WHITESPACE);
+ setArray(FILTER, TRIM, LOWERCASE);
+ }
+
+ @Override
+ public SortedMap<String, String> fieldMapping() {
+ return ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, INDEX_SEARCHABLE,
+ ANALYZER, USER_INDEX_GRAMS_ANALYZER.getName(),
+ SEARCH_ANALYZER, getName());
+ }
+ },
+ INDEX_WORDS_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, STANDARD);
+ setArray(FILTER, STANDARD, "word_filter", LOWERCASE, STOP, ASCIIFOLDING, PORTER_STEM);
+ }
+ },
+ SEARCH_WORDS_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, STANDARD);
+ setArray(FILTER, STANDARD, LOWERCASE, STOP, ASCIIFOLDING, PORTER_STEM);
+ }
+
+ @Override
+ public SortedMap<String, String> fieldMapping() {
+ return ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, INDEX_SEARCHABLE,
+ ANALYZER, INDEX_WORDS_ANALYZER.getName(),
+ SEARCH_ANALYZER, getName());
+ }
+ },
+ ENGLISH_HTML_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, STANDARD);
+ setArray(FILTER, STANDARD, LOWERCASE, STOP, ASCIIFOLDING, PORTER_STEM);
+ setArray(CHAR_FILTER, HTML_STRIP);
+ }
+
+ @Override
+ public SortedMap<String, String> fieldMapping() {
+ return ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, INDEX_SEARCHABLE,
+ ANALYZER, getName());
+ }
+ },
+ PATH_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, "path_hierarchy");
+ }
+ },
+ UUID_MODULE_ANALYZER(ANALYZER) {
+
+ @Override
+ protected void setup() {
+ set(TOKENIZER, UUID_MODULE_TOKENIZER);
+ setArray(FILTER, TRIM);
+ }
+ },
+
+ ;
+
+ private final String type;
+ private final String name;
+
+ private Builder builder = Settings.builder();
+
+ DefaultIndexSettingsElement(String type) {
+ this.type = type;
+ this.name = name().toLowerCase(Locale.ENGLISH);
+ setup();
+ }
+
+ protected void set(String settingSuffix, int value) {
+ put(localName(settingSuffix), Integer.toString(value));
+ }
+
+ protected void set(String settingSuffix, boolean value) {
+ put(localName(settingSuffix), Boolean.toString(value));
+ }
+
+ protected void set(String settingSuffix, DefaultIndexSettingsElement otherElement) {
+ put(localName(settingSuffix), otherElement.name);
+ }
+
+ protected void set(String settingSuffix, String value) {
+ put(localName(settingSuffix), value);
+ }
+
+ protected void setArray(String settingSuffix, String... values) {
+ putArray(localName(settingSuffix), values);
+ }
+
+ protected void setArray(String settingSuffix, DefaultIndexSettingsElement... values) {
+ putArray(localName(settingSuffix), Arrays.stream(values).map(DefaultIndexSettingsElement::getName).toArray(String[]::new));
+ }
+
+ private void put(String setting, String value) {
+ builder = builder.put(setting, value);
+ }
+
+ private void putArray(String setting, String... values) {
+ builder = builder.putArray(setting, values);
+ }
+
+ private String localName(String settingSuffix) {
+ return ANALYSIS + DELIMITER + type + DELIMITER + name + DELIMITER + settingSuffix;
+ }
+
+ public Settings settings() {
+ return builder.build();
+ }
+
+ protected abstract void setup();
+
+ public SortedMap<String, String> fieldMapping() {
+ throw new UnsupportedOperationException("The elasticsearch configuration element '" + name + "' cannot be used as field mapping.");
+ }
+
+ public String subField(String fieldName) {
+ return fieldName + SUB_FIELD_DELIMITER + getSubFieldSuffix();
+ }
+
+ public String getSubFieldSuffix() {
+ return getName();
+ }
+
+ public String getName() {
+ return name;
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_TEXT;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.UUID_MODULE_ANALYZER;
+
+public abstract class FieldAware<U extends FieldAware<U>> {
+
+ abstract U setFieldImpl(String fieldName, Object attributes);
+
+ protected final U setField(String fieldName, Object attributes) {
+ checkArgument(!FIELD_INDEX_TYPE.equalsIgnoreCase(fieldName), "%s is a reserved field name", FIELD_INDEX_TYPE);
+ return setFieldImpl(fieldName, attributes);
+ }
+
+ @SuppressWarnings("unchecked")
+ public KeywordFieldBuilder<U> keywordFieldBuilder(String fieldName) {
+ return (KeywordFieldBuilder<U>) new KeywordFieldBuilder(this, fieldName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public TextFieldBuilder<U> textFieldBuilder(String fieldName) {
+ return (TextFieldBuilder<U>) new TextFieldBuilder(this, fieldName);
+ }
+
+ @SuppressWarnings("unchecked")
+ public NestedFieldBuilder<U> nestedFieldBuilder(String fieldName) {
+ return (NestedFieldBuilder<U>) new NestedFieldBuilder(this, fieldName);
+ }
+
+ public U createBooleanField(String fieldName) {
+ return setField(fieldName, ImmutableMap.of("type", "boolean"));
+ }
+
+ public U createByteField(String fieldName) {
+ return setField(fieldName, ImmutableMap.of("type", "byte"));
+ }
+
+ public U createDateTimeField(String fieldName) {
+ Map<String, String> hash = new TreeMap<>();
+ hash.put("type", "date");
+ hash.put("format", "date_time||epoch_second");
+ return setField(fieldName, hash);
+ }
+
+ public U createDoubleField(String fieldName) {
+ return setField(fieldName, ImmutableMap.of("type", "double"));
+ }
+
+ public U createIntegerField(String fieldName) {
+ return setField(fieldName, ImmutableMap.of("type", "integer"));
+ }
+
+ public U createLongField(String fieldName) {
+ return setField(fieldName, ImmutableMap.of("type", "long"));
+ }
+
+ public U createShortField(String fieldName) {
+ return setField(fieldName, ImmutableMap.of("type", "short"));
+ }
+
+ public U createUuidPathField(String fieldName) {
+ return setField(fieldName, ImmutableSortedMap.of(
+ TYPE, FIELD_TYPE_TEXT,
+ INDEX, DefaultIndexSettings.INDEX_SEARCHABLE,
+ ANALYZER, UUID_MODULE_ANALYZER.getName()));
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_KEYWORD;
+
+public class KeywordFieldBuilder<U extends FieldAware<U>> extends StringFieldBuilder<U, KeywordFieldBuilder<U>> {
+
+ protected KeywordFieldBuilder(U indexType, String fieldName) {
+ super(indexType, fieldName);
+ }
+
+ @Override
+ protected boolean getFieldData() {
+ return false;
+ }
+
+ protected String getFieldType() {
+ return FIELD_TYPE_KEYWORD;
+ }
+
+ /**
+ * By default, field is stored on disk in a column-stride fashion, so that it can later be used for sorting,
+ * aggregations, or scripting.
+ * Disabling this reduces the size of the index and drop the constraint of single term max size of
+ * 32766 bytes (which, if there is no tokenizing enabled on the field, equals the size of the whole data).
+ */
+ public KeywordFieldBuilder disableSortingAndAggregating() {
+ this.disabledDocValues = true;
+ return this;
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSortedMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_KEYWORD;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_SEARCHABLE;
+
+public class NestedFieldBuilder<U extends FieldAware<U>> {
+ private final U parent;
+ private final String fieldName;
+ private final Map<String, Object> properties = new TreeMap<>();
+
+ protected NestedFieldBuilder(U parent, String fieldName) {
+ this.parent = parent;
+ this.fieldName = fieldName;
+ }
+
+ private NestedFieldBuilder setProperty(String fieldName, Object value) {
+ properties.put(fieldName, value);
+
+ return this;
+ }
+
+ public NestedFieldBuilder addKeywordField(String fieldName) {
+ return setProperty(fieldName, ImmutableSortedMap.of(
+ "type", FIELD_TYPE_KEYWORD,
+ INDEX, INDEX_SEARCHABLE));
+ }
+
+ public NestedFieldBuilder addDoubleField(String fieldName) {
+ return setProperty(fieldName, ImmutableMap.of("type", "double"));
+ }
+
+ public NestedFieldBuilder addIntegerField(String fieldName) {
+ return setProperty(fieldName, ImmutableMap.of("type", "integer"));
+ }
+
+ public U build() {
+ checkArgument(!properties.isEmpty(), "At least one sub-field must be declared in nested property '%s'", fieldName);
+
+ return parent.setField(fieldName, ImmutableSortedMap.of(
+ "type", "nested",
+ "properties", properties));
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexRelationType;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_ALLOW_ANYONE;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_GROUP_IDS;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_USER_IDS;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
+
+public class NewAuthorizedIndex extends NewIndex<NewAuthorizedIndex> {
+ private final IndexType.IndexMainType mainType;
+
+ public NewAuthorizedIndex(Index index, SettingsConfiguration settingsConfiguration) {
+ super(index, settingsConfiguration);
+ checkArgument(index.acceptsRelations(), "Index must accept relations");
+
+ this.mainType = IndexType.main(index, TYPE_AUTHORIZATION);
+ super.createTypeMapping(mainType)
+ .createLongField(FIELD_GROUP_IDS)
+ .createLongField(FIELD_USER_IDS)
+ .createBooleanField(FIELD_ALLOW_ANYONE);
+ }
+
+ @Override
+ public IndexType.IndexMainType getMainType() {
+ return mainType;
+ }
+
+ @Override
+ public TypeMapping createTypeMapping(IndexRelationType relationType) {
+ checkArgument(relationType.getMainType().equals(mainType), "mainType of relation must be %s", mainType);
+ return super.createTypeMapping(relationType);
+ }
+
+ @Override
+ public BuiltIndex<NewAuthorizedIndex> build() {
+ checkState(!getRelations().isEmpty(), "At least one relation mapping must be defined");
+ return new BuiltIndex<>(this);
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableSortedMap;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Stream;
+import javax.annotation.CheckForNull;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.sonar.api.config.Configuration;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.IndexType.IndexRelationType;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import static java.lang.String.valueOf;
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
+import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS;
+
+public abstract class NewIndex<T extends NewIndex<T>> {
+
+ private static final String ENABLED = "enabled";
+ private final Index index;
+ private final Map<String, IndexRelationType> relations = new LinkedHashMap<>();
+ private final Settings.Builder settings = DefaultIndexSettings.defaults();
+ private final Map<String, Object> attributes = new TreeMap<>();
+ private final Map<String, Object> properties = new TreeMap<>();
+
+ public NewIndex(Index index, SettingsConfiguration settingsConfiguration) {
+ this.index = index;
+ applySettingsConfiguration(settingsConfiguration);
+ configureDefaultAttributes();
+ }
+
+ private void applySettingsConfiguration(SettingsConfiguration settingsConfiguration) {
+ settings.put("index.mapper.dynamic", valueOf(false));
+ settings.put("index.refresh_interval", refreshInterval(settingsConfiguration));
+ settings.put("mapping.single_type", valueOf(true));
+
+ Configuration config = settingsConfiguration.getConfiguration();
+ boolean clusterMode = config.getBoolean(CLUSTER_ENABLED.getKey()).orElse(false);
+ int shards = config.getInt(String.format("sonar.search.%s.shards", index.getName()))
+ .orElse(settingsConfiguration.getDefaultNbOfShards());
+ int replicas = clusterMode ? config.getInt(SEARCH_REPLICAS.getKey()).orElse(1) : 0;
+
+ settings.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shards);
+ settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas);
+ }
+
+ private void configureDefaultAttributes() {
+ attributes.put("dynamic", valueOf(false));
+ attributes.put("_all", ImmutableSortedMap.of(ENABLED, false));
+ attributes.put("_source", ImmutableSortedMap.of(ENABLED, true));
+ }
+
+ private static String refreshInterval(SettingsConfiguration settingsConfiguration) {
+ int refreshInterval = settingsConfiguration.getRefreshInterval();
+ if (refreshInterval == -1) {
+ return "-1";
+ }
+ return refreshInterval + "s";
+ }
+
+ protected Index getIndex() {
+ return index;
+ }
+
+ public abstract IndexMainType getMainType();
+
+ Collection<IndexRelationType> getRelations() {
+ return relations.values();
+ }
+
+ /**
+ * Public, read-only version of {@link #getRelations()}
+ */
+ public Stream<IndexRelationType> getRelationsStream() {
+ return relations.values().stream();
+ }
+
+ Settings.Builder getSettings() {
+ return settings;
+ }
+
+ @CheckForNull
+ public String getSetting(String key) {
+ return settings.get(key);
+ }
+
+ protected TypeMapping createTypeMapping(IndexMainType mainType) {
+ checkArgument(mainType.getIndex().equals(index), "Main type must belong to index %s", index);
+ return new TypeMapping(this);
+ }
+
+ protected TypeMapping createTypeMapping(IndexRelationType relationType) {
+ checkAcceptsRelations();
+ IndexMainType mainType = getMainType();
+ checkArgument(relationType.getMainType().equals(mainType), "mainType of relation must be %s", mainType);
+ String relationName = relationType.getName();
+ checkArgument(!relations.containsKey(relationName), "relation %s already exists", relationName);
+ relations.put(relationName, relationType);
+ return new TypeMapping(this);
+ }
+
+ private void checkAcceptsRelations() {
+ checkState(getMainType().getIndex().acceptsRelations(), "Index is not configured to accept relations. Update IndexDefinition.Descriptor instance for this index");
+ }
+
+ /**
+ * Complete the json hash named "properties" in mapping type, usually to declare fields
+ */
+ void setFieldImpl(String fieldName, Object attributes) {
+ properties.put(fieldName, attributes);
+ }
+
+ public T setEnableSource(boolean enableSource) {
+ attributes.put("_source", ImmutableSortedMap.of(ENABLED, enableSource));
+ return castThis();
+ }
+
+ @SuppressWarnings("unchecked")
+ private T castThis() {
+ return (T) this;
+ }
+
+ public Map<String, Object> getAttributes() {
+ return attributes;
+ }
+
+ Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ @CheckForNull
+ public Object getProperty(String key) {
+ return properties.get(key);
+ }
+
+ public abstract BuiltIndex<T> build();
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+public class NewRegularIndex extends NewIndex<NewRegularIndex> {
+ private IndexMainType mainType;
+
+ public NewRegularIndex(Index index, SettingsConfiguration settingsConfiguration) {
+ super(index, settingsConfiguration);
+ }
+
+ @Override
+ public IndexMainType getMainType() {
+ checkState(mainType != null, "Main type has not been defined");
+ return mainType;
+ }
+
+ @Override
+ public TypeMapping createTypeMapping(IndexMainType mainType) {
+ checkState(this.mainType == null, "Main type can only be defined once");
+ this.mainType = mainType;
+ return super.createTypeMapping(mainType);
+ }
+
+ @Override
+ public TypeMapping createTypeMapping(IndexType.IndexRelationType relationType) {
+ checkState(mainType != null, "Mapping for main type must be created first");
+ checkArgument(relationType.getMainType().equals(mainType), "main type of relation must be %s", mainType);
+ return super.createTypeMapping(relationType);
+ }
+
+ @Override
+ public BuiltIndex<NewRegularIndex> build() {
+ checkState(mainType != null, "Mapping for main type must be defined");
+ checkState(!mainType.getIndex().acceptsRelations() || !getRelations().isEmpty(), "At least one relation must be defined when index accepts relations");
+ return new BuiltIndex<>(this);
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import org.sonar.api.config.Configuration;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+public class SettingsConfiguration {
+ public static final int MANUAL_REFRESH_INTERVAL = -1;
+
+ private final Configuration configuration;
+ private final int defaultNbOfShards;
+ private final int refreshInterval;
+
+ private SettingsConfiguration(Builder builder) {
+ this.configuration = builder.configuration;
+ this.defaultNbOfShards = builder.defaultNbOfShards;
+ this.refreshInterval = builder.refreshInterval;
+ }
+
+ public static Builder newBuilder(Configuration configuration) {
+ return new Builder(configuration);
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public int getDefaultNbOfShards() {
+ return defaultNbOfShards;
+ }
+
+ public int getRefreshInterval() {
+ return refreshInterval;
+ }
+
+ public static class Builder {
+ private final Configuration configuration;
+ private int defaultNbOfShards = 1;
+ private int refreshInterval = 30;
+
+ public Builder(Configuration configuration) {
+ this.configuration = requireNonNull(configuration, "configuration can't be null");
+ }
+
+ public Builder setDefaultNbOfShards(int defaultNbOfShards) {
+ checkArgument(defaultNbOfShards >= 1, "defaultNbOfShards must be >= 1");
+ this.defaultNbOfShards = defaultNbOfShards;
+ return this;
+ }
+
+ public Builder setRefreshInterval(int refreshInterval) {
+ checkArgument(refreshInterval == -1 || refreshInterval > 0,
+ "refreshInterval must be either -1 or strictly positive");
+ this.refreshInterval = refreshInterval;
+ return this;
+ }
+
+ public SettingsConfiguration build() {
+ return new SettingsConfiguration(this);
+ }
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.Maps;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import static java.lang.String.valueOf;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELDDATA_ENABLED;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_FIELDDATA;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TERM_VECTOR;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_KEYWORD;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_NOT_SEARCHABLE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_SEARCHABLE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.NORMS;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.STORE;
+
+/**
+ * Helper to define a string field in mapping of index type
+ */
+public abstract class StringFieldBuilder<U extends FieldAware<U>, T extends StringFieldBuilder<U, T>> {
+ private final U parent;
+ private final String fieldName;
+ private boolean disableSearch = false;
+ private boolean disableNorms = false;
+ private boolean termVectorWithPositionOffsets = false;
+ private SortedMap<String, Object> subFields = Maps.newTreeMap();
+ private boolean store = false;
+ protected boolean disabledDocValues = false;
+
+ protected StringFieldBuilder(U parent, String fieldName) {
+ this.parent = parent;
+ this.fieldName = fieldName;
+ }
+
+ /**
+ * Add a sub-field. A {@code SortedMap} is required for consistency of the index settings hash.
+ */
+ private T addSubField(String fieldName, SortedMap<String, String> fieldDefinition) {
+ subFields.put(fieldName, fieldDefinition);
+ return castThis();
+ }
+
+ /**
+ * Add subfields, one for each analyzer.
+ */
+ public T addSubFields(DefaultIndexSettingsElement... analyzers) {
+ Arrays.stream(analyzers)
+ .forEach(analyzer -> addSubField(analyzer.getSubFieldSuffix(), analyzer.fieldMapping()));
+ return castThis();
+ }
+
+ /**
+ * Norms consume useless memory if string field is used for filtering or aggregations.
+ *
+ * https://www.elastic.co/guide/en/elasticsearch/reference/2.3/norms.html
+ * https://www.elastic.co/guide/en/elasticsearch/guide/current/scoring-theory.html#field-norm
+ */
+ public T disableNorms() {
+ this.disableNorms = true;
+ return castThis();
+ }
+
+ /**
+ * Position offset term vectors are required for the fast_vector_highlighter (fvh).
+ */
+ public T termVectorWithPositionOffsets() {
+ this.termVectorWithPositionOffsets = true;
+ return castThis();
+ }
+
+ /**
+ * "index: false" -> Make this field not searchable.
+ * By default field is "true": it is searchable, but index the value exactly
+ * as specified.
+ */
+ public T disableSearch() {
+ this.disableSearch = true;
+ return castThis();
+ }
+
+ public T store() {
+ this.store = true;
+ return castThis();
+ }
+
+ @SuppressWarnings("unchecked")
+ private T castThis() {
+ return (T) this;
+ }
+
+ public U build() {
+ if (subFields.isEmpty()) {
+ return buildWithoutSubfields();
+ }
+ return buildWithSubfields();
+ }
+
+ private U buildWithoutSubfields() {
+ Map<String, Object> hash = new TreeMap<>();
+ hash.put("type", getFieldType());
+ hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE);
+ hash.put(NORMS, valueOf(!disableNorms));
+ hash.put(STORE, valueOf(store));
+ if (FIELD_TYPE_KEYWORD.equals(getFieldType())) {
+ hash.put("doc_values", valueOf(!disabledDocValues));
+ }
+ if (getFieldData()) {
+ hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
+ }
+ return parent.setField(fieldName, hash);
+ }
+
+ private U buildWithSubfields() {
+ Map<String, Object> hash = new TreeMap<>();
+ hash.put("type", getFieldType());
+ hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE);
+ hash.put(NORMS, "false");
+ hash.put(STORE, valueOf(store));
+ if (FIELD_TYPE_KEYWORD.equals(getFieldType())) {
+ hash.put("doc_values", valueOf(!disabledDocValues));
+ }
+ if (getFieldData()) {
+ hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED);
+ }
+ if (termVectorWithPositionOffsets) {
+ hash.put(FIELD_TERM_VECTOR, "with_positions_offsets");
+ }
+ hash.put("fields", configureSubFields());
+ return parent.setField(fieldName, hash);
+ }
+
+ private Map<String, Object> configureSubFields() {
+ Map<String, Object> multiFields = new TreeMap<>(subFields);
+
+ // apply this fields configuration to all subfields
+ multiFields.entrySet().forEach(entry -> {
+ Object subFieldMapping = entry.getValue();
+ if (subFieldMapping instanceof Map) {
+ entry.setValue(configureSubField((Map<String, String>) subFieldMapping));
+ }
+ });
+ return multiFields;
+ }
+
+ private Map<String, String> configureSubField(Map<String, String> subFieldMapping) {
+ Map<String, String> subHash = new TreeMap<>(subFieldMapping);
+ subHash.put(INDEX, INDEX_SEARCHABLE);
+ subHash.put(NORMS, "false");
+ subHash.put(STORE, valueOf(store));
+ if (termVectorWithPositionOffsets) {
+ subHash.put(FIELD_TERM_VECTOR, "with_positions_offsets");
+ }
+ return subHash;
+ }
+
+ protected abstract boolean getFieldData();
+
+ protected abstract String getFieldType();
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_TEXT;
+
+public class TextFieldBuilder<U extends FieldAware<U>> extends StringFieldBuilder<U, TextFieldBuilder<U>> {
+
+ private boolean fieldData = false;
+
+ protected TextFieldBuilder(U indexType, String fieldName) {
+ super(indexType, fieldName);
+ }
+
+ protected String getFieldType() {
+ return FIELD_TYPE_TEXT;
+ }
+
+ /**
+ * Required to enable sorting, aggregation and access to field data on fields of type "text".
+ * <p>Disabled by default as this can have significant memory cost</p>
+ */
+ public StringFieldBuilder withFieldData() {
+ this.fieldData = true;
+ return this;
+ }
+
+ @Override
+ protected boolean getFieldData() {
+ return fieldData;
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+public class TypeMapping extends FieldAware<TypeMapping> {
+
+ private final NewIndex<?> newIndex;
+
+ TypeMapping(NewIndex<?> newIndex) {
+ this.newIndex = newIndex;
+ }
+
+ @Override
+ TypeMapping setFieldImpl(String fieldName, Object attributes) {
+ newIndex.setFieldImpl(fieldName, attributes);
+ return this;
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.server.es.newindex;
+
+import javax.annotation.ParametersAreNonnullByDefault;
+
public String toString() {
StringBuilder message = new StringBuilder();
message.append("ES cluster health request");
- if (request.indices().length > 0) {
- message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ",")));
+ String[] indices = request.indices();
+ if (indices != null && indices.length > 0) {
+ message.append(String.format(" on indices '%s'", StringUtils.join(indices, ",")));
}
return message.toString();
}
*/
package org.sonar.server.es.request;
+import java.io.IOException;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
import org.sonar.api.utils.log.Profiler;
import org.sonar.server.es.EsClient;
throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous");
}
+ public String toJson() {
+ try {
+ XContentBuilder builder = XContentFactory.jsonBuilder();
+ builder.startObject()
+ .field("settings")
+ .startObject();
+ request.settings().toXContent(builder, ToXContent.EMPTY_PARAMS);
+ builder.endObject().endObject();
+ builder.prettyPrint();
+ return builder.string();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override
public String toString() {
return String.format("ES create index '%s'", index);
public String toString() {
StringBuilder message = new StringBuilder();
message.append("ES indices stats request");
- if (request.indices().length > 0) {
- message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ",")));
+ String[] indices = request.indices();
+ if (indices != null && indices.length > 0) {
+ message.append(String.format(" on indices '%s'", StringUtils.join(indices, ",")));
}
return message.toString();
}
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.sonar.core.util.stream.MoreCollectors;
-import org.sonar.server.es.DefaultIndexSettings;
-import org.sonar.server.es.DefaultIndexSettingsElement;
+import org.sonar.server.es.newindex.DefaultIndexSettings;
+import org.sonar.server.es.newindex.DefaultIndexSettingsElement;
import org.sonar.server.es.textsearch.ComponentTextSearchQueryFactory.ComponentTextSearchQuery;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
import static org.sonar.server.es.textsearch.ComponentTextSearchFeature.UseCase.CHANGE_ORDER_OF_RESULTS;
import static org.sonar.server.es.textsearch.ComponentTextSearchFeature.UseCase.GENERATE_RESULTS;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.sonar.core.util.stream.MoreCollectors;
-import org.sonar.server.es.DefaultIndexSettings;
+import org.sonar.server.es.newindex.DefaultIndexSettings;
-import static org.sonar.server.es.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH;
/**
* Splits text queries into their tokens, for to use them in n_gram match queries later.
import org.sonar.api.rules.RuleType;
import org.sonar.api.utils.Duration;
import org.sonar.server.es.BaseDoc;
+import org.sonar.server.permission.index.AuthorizationDoc;
+
+import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE;
public class IssueDoc extends BaseDoc {
public IssueDoc(Map<String, Object> fields) {
- super(fields);
+ super(TYPE_ISSUE, fields);
}
public IssueDoc() {
- super(Maps.newHashMapWithExpectedSize(32));
+ super(TYPE_ISSUE, Maps.newHashMapWithExpectedSize(32));
}
@Override
return key();
}
- @Override
- public String getRouting() {
- return projectUuid();
- }
-
- @Override
- public String getParent() {
- return projectUuid();
- }
-
public String key() {
return getField(IssueIndexDefinition.FIELD_ISSUE_KEY);
}
public IssueDoc setProjectUuid(String s) {
setField(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, s);
+ setParent(AuthorizationDoc.idOf(s));
return this;
}
import org.sonar.api.config.Configuration;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.newindex.NewAuthorizedIndex;
+import org.sonar.server.es.newindex.TypeMapping;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
/**
* Definition of ES index "issues", including settings and fields.
*/
public class IssueIndexDefinition implements IndexDefinition {
- public static final IndexType INDEX_TYPE_ISSUE = new IndexType("issues", "issue");
+ public static final Index DESCRIPTOR = Index.withRelations("issues");
+ public static final IndexType.IndexRelationType TYPE_ISSUE = IndexType.relation(IndexType.main(DESCRIPTOR, TYPE_AUTHORIZATION), "issue");
public static final String FIELD_ISSUE_ASSIGNEE_UUID = "assignee";
public static final String FIELD_ISSUE_AUTHOR_LOGIN = "authorLogin";
public static final String FIELD_ISSUE_COMPONENT_UUID = "component";
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(
- INDEX_TYPE_ISSUE.getIndex(),
+ NewAuthorizedIndex index = context.createWithAuthorization(
+ DESCRIPTOR,
newBuilder(config)
.setRefreshInterval(MANUAL_REFRESH_INTERVAL)
.setDefaultNbOfShards(5)
- .build());
+ .build())
+ .setEnableSource(enableSource);
- NewIndex.NewIndexType type = index.createType(INDEX_TYPE_ISSUE.getType());
- type.requireProjectAuthorization();
- type.setEnableSource(enableSource);
-
- type.keywordFieldBuilder(FIELD_ISSUE_ASSIGNEE_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
- type.keywordFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).disableNorms().build();
- type.createLongField(FIELD_ISSUE_EFFORT);
- type.keywordFieldBuilder(FIELD_ISSUE_FILE_PATH).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
- type.createDateTimeField(FIELD_ISSUE_FUNC_CREATED_AT);
- type.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT);
- type.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT);
- type.keywordFieldBuilder(FIELD_ISSUE_KEY).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
- type.keywordFieldBuilder(FIELD_ISSUE_LANGUAGE).disableNorms().build();
- type.createIntegerField(FIELD_ISSUE_LINE);
- type.keywordFieldBuilder(FIELD_ISSUE_MODULE_UUID).disableNorms().build();
- type.createUuidPathField(FIELD_ISSUE_MODULE_PATH);
- type.keywordFieldBuilder(FIELD_ISSUE_ORGANIZATION_UUID).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_PROJECT_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
- type.keywordFieldBuilder(FIELD_ISSUE_BRANCH_UUID).disableNorms().build();
- type.createBooleanField(FIELD_ISSUE_IS_MAIN_BRANCH);
- type.keywordFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_RESOLUTION).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_RULE_ID).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_SEVERITY).disableNorms().build();
- type.createByteField(FIELD_ISSUE_SEVERITY_VALUE);
- type.keywordFieldBuilder(FIELD_ISSUE_STATUS).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
- type.keywordFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_OWASP_TOP_10).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_SANS_TOP_25).disableNorms().build();
- type.keywordFieldBuilder(FIELD_ISSUE_CWE).disableNorms().build();
+ TypeMapping mapping = index.createTypeMapping(TYPE_ISSUE);
+ mapping.keywordFieldBuilder(FIELD_ISSUE_ASSIGNEE_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).disableNorms().build();
+ mapping.createLongField(FIELD_ISSUE_EFFORT);
+ mapping.keywordFieldBuilder(FIELD_ISSUE_FILE_PATH).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
+ mapping.createDateTimeField(FIELD_ISSUE_FUNC_CREATED_AT);
+ mapping.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT);
+ mapping.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT);
+ mapping.keywordFieldBuilder(FIELD_ISSUE_KEY).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_LANGUAGE).disableNorms().build();
+ mapping.createIntegerField(FIELD_ISSUE_LINE);
+ mapping.keywordFieldBuilder(FIELD_ISSUE_MODULE_UUID).disableNorms().build();
+ mapping.createUuidPathField(FIELD_ISSUE_MODULE_PATH);
+ mapping.keywordFieldBuilder(FIELD_ISSUE_ORGANIZATION_UUID).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_PROJECT_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_BRANCH_UUID).disableNorms().build();
+ mapping.createBooleanField(FIELD_ISSUE_IS_MAIN_BRANCH);
+ mapping.keywordFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_RESOLUTION).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_RULE_ID).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_SEVERITY).disableNorms().build();
+ mapping.createByteField(FIELD_ISSUE_SEVERITY_VALUE);
+ mapping.keywordFieldBuilder(FIELD_ISSUE_STATUS).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_OWASP_TOP_10).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_SANS_TOP_25).disableNorms().build();
+ mapping.keywordFieldBuilder(FIELD_ISSUE_CWE).disableNorms().build();
}
}
import org.sonar.server.es.OneToManyResilientIndexingListener;
import org.sonar.server.es.OneToOneResilientIndexingListener;
import org.sonar.server.es.ProjectIndexer;
+import org.sonar.server.permission.index.AuthorizationDoc;
import org.sonar.server.permission.index.AuthorizationScope;
import org.sonar.server.permission.index.NeedAuthorizationIndexer;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID;
-import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
+import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE;
public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer {
*/
private static final String ID_TYPE_PROJECT_UUID = "projectUuid";
private static final Logger LOGGER = Loggers.get(IssueIndexer.class);
- private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_ISSUE, project -> Qualifiers.PROJECT.equals(project.getQualifier()));
- private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(INDEX_TYPE_ISSUE);
+ private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_ISSUE, project -> Qualifiers.PROJECT.equals(project.getQualifier()));
+ private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(TYPE_ISSUE);
private final EsClient esClient;
private final DbClient dbClient;
// the remaining uuids reference issues that don't exist in db. They must
// be deleted from index.
itemsByIssueKey.values().forEach(
- item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting()));
+ item -> bulkIndexer.addDeletion(TYPE_ISSUE.getMainType(), item.getDocId(), item.getDocRouting()));
return bulkIndexer.stop();
}
BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR);
bulkIndexer.start();
- issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, issueKey, projectUuid));
+ issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(TYPE_ISSUE.getMainType(), issueKey, AuthorizationDoc.idOf(projectUuid)));
bulkIndexer.stop();
}
}
private IndexRequest newIndexRequest(IssueDoc issue) {
- String projectUuid = issue.projectUuid();
- return esClient.prepareIndex(INDEX_TYPE_ISSUE)
- .setId(issue.key())
- .setRouting(projectUuid)
- .setParent(projectUuid)
+ return esClient.prepareIndex(TYPE_ISSUE.getMainType())
+ .setId(issue.getId())
+ .setRouting(issue.getRouting().orElseThrow(() -> new IllegalStateException("IssueDoc should define a routing")))
.setSource(issue.getFields())
.request();
}
private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) {
- SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ISSUE)
- .setRouting(projectUuid)
+ SearchRequestBuilder search = esClient.prepareSearch(TYPE_ISSUE.getMainType())
+ .setRouting(AuthorizationDoc.idOf(projectUuid))
.setQuery(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid)));
bulkIndexer.addDeletion(search);
}
private static EsQueueDto createQueueDto(String docId, String docIdType, String projectUuid) {
- return EsQueueDto.create(INDEX_TYPE_ISSUE.format(), docId, docIdType, projectUuid);
+ return EsQueueDto.create(TYPE_ISSUE.format(), docId, docIdType, projectUuid);
}
private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) {
- return new BulkIndexer(esClient, INDEX_TYPE_ISSUE, size, listener);
+ return new BulkIndexer(esClient, TYPE_ISSUE, size, listener);
}
}
import javax.annotation.Nullable;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.server.es.BaseDoc;
+import org.sonar.server.permission.index.AuthorizationDoc;
import static org.sonar.api.measures.Metric.Level.ERROR;
import static org.sonar.api.measures.Metric.Level.OK;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_UUID;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
public class ProjectMeasuresDoc extends BaseDoc {
public static final Map<String, Integer> QUALITY_GATE_STATUS = ImmutableMap.of(OK.name(), 1, WARN.name(), 2, ERROR.name(), 3);
public ProjectMeasuresDoc() {
- super(new HashMap<>(8));
+ super(TYPE_PROJECT_MEASURES, new HashMap<>(8));
}
@Override
return getField(FIELD_UUID);
}
- @Override
- public String getRouting() {
- return getId();
- }
-
- @Override
- public String getParent() {
- return getId();
- }
-
public ProjectMeasuresDoc setId(String s) {
setField(FIELD_UUID, s);
+ setParent(AuthorizationDoc.idOf(s));
return this;
}
package org.sonar.server.measure.index;
import org.sonar.api.config.Configuration;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType.IndexRelationType;
+import org.sonar.server.es.newindex.NewAuthorizedIndex;
+import org.sonar.server.es.newindex.TypeMapping;
+import org.sonar.server.permission.index.IndexAuthorizationConstants;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
public class ProjectMeasuresIndexDefinition implements IndexDefinition {
- public static final IndexType INDEX_TYPE_PROJECT_MEASURES = new IndexType("projectmeasures", "projectmeasure");
+ public static final Index DESCRIPTOR = Index.withRelations("projectmeasures");
+ public static final IndexType.IndexMainType TYPE_AUTHORIZATION = IndexType.main(DESCRIPTOR, IndexAuthorizationConstants.TYPE_AUTHORIZATION);
+ public static final IndexRelationType TYPE_PROJECT_MEASURES = IndexType.relation(TYPE_AUTHORIZATION, "projectmeasure");
+
public static final String FIELD_UUID = "uuid";
public static final String FIELD_ORGANIZATION_UUID = "organizationUuid";
public static final String FIELD_DISTRIB_NCLOC = "ncloc";
private final Configuration config;
+ private final boolean enableSource;
+
+ private ProjectMeasuresIndexDefinition(Configuration config, boolean enableSource) {
+ this.config = config;
+ this.enableSource = enableSource;
+ }
+
+ public ProjectMeasuresIndexDefinition(Configuration config) {
+ this(config, false);
+ }
- public ProjectMeasuresIndexDefinition(Configuration settings) {
- this.config = settings;
+ /**
+ * Keep the document sources in index so that indexer tests can verify content
+ * of indexed documents.
+ */
+ public static ProjectMeasuresIndexDefinition createForTest() {
+ return new ProjectMeasuresIndexDefinition(new MapSettings().asConfig(), true);
}
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(
- INDEX_TYPE_PROJECT_MEASURES.getIndex(),
+ NewAuthorizedIndex index = context.createWithAuthorization(
+ DESCRIPTOR,
newBuilder(config)
.setRefreshInterval(MANUAL_REFRESH_INTERVAL)
.setDefaultNbOfShards(5)
- .build());
-
- NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_PROJECT_MEASURES.getType())
- .requireProjectAuthorization();
+ .build())
+ .setEnableSource(enableSource);
+ TypeMapping mapping = index.createTypeMapping(TYPE_PROJECT_MEASURES);
mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_ORGANIZATION_UUID).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_KEY).disableNorms().addSubFields(SORTABLE_ANALYZER).build();
.addIntegerField(FIELD_DISTRIB_NCLOC)
.build();
mapping.createDateTimeField(FIELD_ANALYSED_AT);
- mapping.setEnableSource(false);
}
}
import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
-import org.elasticsearch.action.index.IndexRequest;
import org.sonar.api.resources.Qualifiers;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.DbClient;
import org.sonar.server.es.IndexingResult;
import org.sonar.server.es.OneToOneResilientIndexingListener;
import org.sonar.server.es.ProjectIndexer;
+import org.sonar.server.permission.index.AuthorizationDoc;
import org.sonar.server.permission.index.AuthorizationScope;
import org.sonar.server.permission.index.NeedAuthorizationIndexer;
-import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorizationIndexer {
- private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_PROJECT_MEASURES, project -> Qualifiers.PROJECT.equals(project.getQualifier()));
- private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(INDEX_TYPE_PROJECT_MEASURES);
+ private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_PROJECT_MEASURES, project -> Qualifiers.PROJECT.equals(project.getQualifier()));
+ private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(TYPE_PROJECT_MEASURES);
private final DbClient dbClient;
private final EsClient esClient;
case PROJECT_TAGS_UPDATE:
case PROJECT_DELETION:
List<EsQueueDto> items = projectUuids.stream()
- .map(projectUuid -> EsQueueDto.create(INDEX_TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid))
+ .map(projectUuid -> EsQueueDto.create(TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid))
.collect(MoreCollectors.toArrayList(projectUuids.size()));
return dbClient.esQueueDao().insert(dbSession, items);
public IndexingResult commitAndIndex(DbSession dbSession, Collection<String> projectUuids) {
List<EsQueueDto> items = projectUuids.stream()
- .map(projectUuid -> EsQueueDto.create(INDEX_TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid))
+ .map(projectUuid -> EsQueueDto.create(TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid))
.collect(MoreCollectors.toArrayList(projectUuids.size()));
dbClient.esQueueDao().insert(dbSession, items);
String projectUuid = it.next();
try (ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) {
while (rowIt.hasNext()) {
- bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(rowIt.next())));
+ bulkIndexer.add(toProjectMeasuresDoc(rowIt.next()).toIndexRequest());
it.remove();
}
}
// the remaining uuids reference projects that don't exist in db. They must
// be deleted from index.
- projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(INDEX_TYPE_PROJECT_MEASURES, projectUuid, projectUuid));
+ projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(TYPE_PROJECT_MEASURES, projectUuid, AuthorizationDoc.idOf(projectUuid)));
return bulkIndexer.stop();
}
bulkIndexer.start();
while (rowIt.hasNext()) {
ProjectMeasures doc = rowIt.next();
- bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(doc)));
+ bulkIndexer.add(toProjectMeasuresDoc(doc).toIndexRequest());
}
bulkIndexer.stop();
}
}
private BulkIndexer createBulkIndexer(Size bulkSize, IndexingListener listener) {
- return new BulkIndexer(esClient, INDEX_TYPE_PROJECT_MEASURES, bulkSize, listener);
- }
-
- private static IndexRequest newIndexRequest(ProjectMeasuresDoc doc) {
- String projectUuid = doc.getId();
- return new IndexRequest(INDEX_TYPE_PROJECT_MEASURES.getIndex(), INDEX_TYPE_PROJECT_MEASURES.getType(), projectUuid)
- .routing(projectUuid)
- .parent(projectUuid)
- .source(doc.getFields());
+ return new BulkIndexer(esClient, TYPE_PROJECT_MEASURES, bulkSize, listener);
}
private static ProjectMeasuresDoc toProjectMeasuresDoc(ProjectMeasures projectMeasures) {
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.permission.index;
+
+import java.util.List;
+import java.util.Optional;
+import org.sonar.server.es.BaseDoc;
+import org.sonar.server.es.IndexType;
+
+import static java.util.Objects.requireNonNull;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_ALLOW_ANYONE;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_GROUP_IDS;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_USER_IDS;
+
+public class AuthorizationDoc extends BaseDoc {
+ private static final String ID_PREFIX = "auth_";
+ private final String projectUuid;
+
+ private AuthorizationDoc(IndexType indexType, String projectUuid) {
+ super(indexType);
+ this.projectUuid = projectUuid;
+ }
+
+ public static AuthorizationDoc fromDto(IndexType indexType, IndexPermissions dto) {
+ AuthorizationDoc res = new AuthorizationDoc(indexType, dto.getProjectUuid());
+ if (dto.isAllowAnyone()) {
+ return res.setAllowAnyone();
+ }
+ return res.setRestricted(dto.getGroupIds(), dto.getUserIds());
+ }
+
+ @Override
+ public String getId() {
+ return idOf(projectUuid);
+ }
+
+ public static String idOf(String projectUuid) {
+ requireNonNull(projectUuid, "projectUuid can't be null");
+ return ID_PREFIX + projectUuid;
+ }
+
+ public static String projectUuidOf(String id) {
+ if (id.startsWith(ID_PREFIX)) {
+ return id.substring(ID_PREFIX.length());
+ }
+ return id;
+ }
+
+ @Override
+ protected Optional<String> getSimpleMainTypeRouting() {
+ return Optional.of(projectUuid);
+ }
+
+ private AuthorizationDoc setAllowAnyone() {
+ setField(FIELD_ALLOW_ANYONE, true);
+ return this;
+ }
+
+ private AuthorizationDoc setRestricted(List<Integer> groupIds, List<Integer> userIds) {
+ setField(FIELD_ALLOW_ANYONE, false);
+ setField(FIELD_GROUP_IDS, groupIds);
+ setField(FIELD_USER_IDS, userIds);
+ return this;
+ }
+}
import java.util.function.Predicate;
import javax.annotation.concurrent.Immutable;
-import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.IndexType.IndexRelationType;
import static com.google.common.base.Preconditions.checkArgument;
import static java.util.Objects.requireNonNull;
@Immutable
public final class AuthorizationScope {
- private final IndexType indexType;
+ private final IndexMainType indexType;
private final Predicate<IndexPermissions> projectPredicate;
- public AuthorizationScope(IndexType indexType, Predicate<IndexPermissions> projectPredicate) {
- this.indexType = getAuthorizationIndexType(indexType);
+ public AuthorizationScope(IndexRelationType functionalType, Predicate<IndexPermissions> projectPredicate) {
+ this.indexType = getAuthorizationIndexType(functionalType);
this.projectPredicate = requireNonNull(projectPredicate);
}
/**
* @return the identifier of the ElasticSearch type (including it's index name), that corresponds to a certain document type
*/
- private static IndexType getAuthorizationIndexType(IndexType indexType) {
- requireNonNull(indexType);
- requireNonNull(indexType.getIndex());
- checkArgument(!TYPE_AUTHORIZATION.equals(indexType.getType()), "Authorization types do not have authorization on their own.");
- return new IndexType(indexType.getIndex(), TYPE_AUTHORIZATION);
+ private static IndexMainType getAuthorizationIndexType(IndexRelationType functionalType) {
+ requireNonNull(functionalType);
+ IndexMainType mainType = functionalType.getMainType();
+ checkArgument(
+ TYPE_AUTHORIZATION.equals(mainType.getType()),
+ "Index %s doesn't seem to be an authorized index as main type is not %s (got %s)",
+ mainType.getIndex(), TYPE_AUTHORIZATION, mainType.getType());
+ return mainType;
}
/**
* Identifier of the authorization type (in the same index than the original IndexType, passed into the constructor).
*/
- public IndexType getIndexType() {
+ public IndexMainType getIndexType() {
return indexType;
}
package org.sonar.server.permission.index;
public final class IndexAuthorizationConstants {
- public static final String TYPE_AUTHORIZATION = "authorization";
- public static final String FIELD_GROUP_IDS = "groupIds";
- public static final String FIELD_USER_IDS = "userIds";
+ public static final String TYPE_AUTHORIZATION = "auth";
+ public static final String FIELD_GROUP_IDS = TYPE_AUTHORIZATION + "_groupIds";
+ public static final String FIELD_USER_IDS = TYPE_AUTHORIZATION + "_userIds";
/**
* When true, then anybody can access to the project. In that case
* it's useless to store granted groups and users. The related
* fields are empty.
*/
- public static final String FIELD_ALLOW_ANYONE = "allowAnyone";
+ public static final String FIELD_ALLOW_ANYONE = TYPE_AUTHORIZATION + "_allowAnyone";
private IndexAuthorizationConstants() {
// prevents instantiation
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_ID;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID;
-import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_ID;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY;
+import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_ID;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE;
public class ActiveRuleDoc extends BaseDoc {
+ public static final String DOC_ID_PREFIX = "ar_";
+
public ActiveRuleDoc(long id) {
- super(Maps.newHashMapWithExpectedSize(10));
+ super(TYPE_ACTIVE_RULE, Maps.newHashMapWithExpectedSize(10));
setField(FIELD_ACTIVE_RULE_ID, String.valueOf(id));
}
public ActiveRuleDoc(Map<String, Object> source) {
- super(source);
+ super(TYPE_ACTIVE_RULE, source);
}
- @Override
- public String getId() {
- return getField(FIELD_ACTIVE_RULE_ID);
+ public static String docIdOf(long activeRuleId) {
+ return docIdOf(String.valueOf(activeRuleId));
}
- @Override
- public String getRouting() {
- return getRuleIdAsString();
+ public static String docIdOf(String activeRuleId) {
+ return DOC_ID_PREFIX + activeRuleId;
}
- @Override
- public String getParent() {
- return getRuleIdAsString();
+ public static long activeRuleIdOf(String docId) {
+ if (docId.startsWith(DOC_ID_PREFIX)) {
+ return Long.valueOf(docId.substring(DOC_ID_PREFIX.length()));
+ }
+ // support for old active rule docId
+ return Long.valueOf(docId);
}
- private String getRuleIdAsString() {
- return getField(FIELD_ACTIVE_RULE_RULE_ID);
+ @Override
+ public String getId() {
+ return docIdOf(getField(FIELD_ACTIVE_RULE_ID));
}
ActiveRuleDoc setRuleId(int ruleId) {
- setField(FIELD_ACTIVE_RULE_RULE_ID, String.valueOf(ruleId));
+ String parent = String.valueOf(ruleId);
+ setParent(parent);
+ setField(FIELD_RULE_ID, parent);
return this;
}
import org.sonar.server.es.ResilientIndexer;
import org.sonar.server.qualityprofile.ActiveRuleChange;
import org.sonar.server.qualityprofile.ActiveRuleInheritance;
-import org.sonar.server.rule.index.RuleIndexDefinition;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
+import static org.sonar.core.util.stream.MoreCollectors.toSet;
+import static org.sonar.server.qualityprofile.index.ActiveRuleDoc.docIdOf;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE;
public class ActiveRuleIndexer implements ResilientIndexer {
@Override
public Set<IndexType> getIndexTypes() {
- return ImmutableSet.of(INDEX_TYPE_ACTIVE_RULE);
+ return ImmutableSet.of(TYPE_ACTIVE_RULE);
}
public void commitAndIndex(DbSession dbSession, Collection<ActiveRuleChange> changes) {
List<EsQueueDto> items = changes.stream()
.map(ActiveRuleChange::getActiveRule)
- .map(ar -> newQueueDto(String.valueOf(ar.getId()), ID_TYPE_ACTIVE_RULE_ID, String.valueOf(ar.getRuleId())))
+ .map(ar -> newQueueDto(docIdOf(ar.getId()), ID_TYPE_ACTIVE_RULE_ID, String.valueOf(ar.getRuleId())))
.collect(toArrayList());
dbClient.esQueueDao().insert(dbSession, items);
return result;
}
- Map<Long, EsQueueDto> activeRuleItems = new HashMap<>();
+ Map<String, EsQueueDto> activeRuleItems = new HashMap<>();
Map<String, EsQueueDto> ruleProfileItems = new HashMap<>();
items.forEach(i -> {
if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) {
ruleProfileItems.put(i.getDocId(), i);
} else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) {
- activeRuleItems.put(Long.parseLong(i.getDocId()), i);
+ activeRuleItems.put(i.getDocId(), i);
} else {
LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i);
deleteQueueDto(dbSession, i);
return result;
}
- private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) {
+ private IndexingResult doIndexActiveRules(DbSession dbSession, Map<String, EsQueueDto> activeRuleItems) {
OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values());
BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener);
bulkIndexer.start();
- Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems);
- dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(),
+ Map<String, EsQueueDto> remaining = new HashMap<>(activeRuleItems);
+ dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, toActiveRuleIds(activeRuleItems),
i -> {
- remaining.remove(i.getId());
+ remaining.remove(docIdOf(i.getId()));
bulkIndexer.add(newIndexRequest(i));
});
// the remaining ids reference rows that don't exist in db. They must
// be deleted from index.
- remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE,
+ remaining.values().forEach(item -> bulkIndexer.addDeletion(TYPE_ACTIVE_RULE,
item.getDocId(), item.getDocRouting()));
return bulkIndexer.stop();
}
+ private static Collection<Long> toActiveRuleIds(Map<String, EsQueueDto> activeRuleItems) {
+ Set<String> docIds = activeRuleItems.keySet();
+ return docIds.stream()
+ .map(ActiveRuleDoc::activeRuleIdOf)
+ .collect(toSet(docIds.size()));
+ }
+
private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) {
IndexingResult result = new IndexingResult();
RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid);
if (profile == null) {
// profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule
- SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE)
+ SearchRequestBuilder search = esClient.prepareSearch(TYPE_ACTIVE_RULE.getMainType())
.setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid)));
- profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search);
+ profileResult = BulkIndexer.delete(esClient, TYPE_ACTIVE_RULE, search);
} else {
BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR);
}
private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) {
- return new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE, size, listener);
+ return new BulkIndexer(esClient, TYPE_ACTIVE_RULE, size, listener);
}
private static IndexRequest newIndexRequest(IndexedActiveRuleDto dto) {
// all the fields must be present, even if value is null
String inheritance = dto.getInheritance();
doc.setInheritance(inheritance == null ? ActiveRuleInheritance.NONE.name() : inheritance);
- return new IndexRequest(INDEX_TYPE_ACTIVE_RULE.getIndex(), INDEX_TYPE_ACTIVE_RULE.getType())
- .id(doc.getId())
- .parent(doc.getParent())
- .routing(doc.getRouting())
- .source(doc.getFields());
+ return doc.toIndexRequest();
}
private static EsQueueDto newQueueDto(String docId, String docIdType, @Nullable String routing) {
- return EsQueueDto.create(INDEX_TYPE_ACTIVE_RULE.format(), docId, docIdType, routing);
+ return EsQueueDto.create(TYPE_ACTIVE_RULE.format(), docId, docIdType, routing);
}
}
*/
package org.sonar.server.rule.index;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Maps;
+import java.util.HashMap;
import java.util.Map;
+import java.util.Optional;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.apache.commons.lang.builder.ReflectionToStringBuilder;
import org.sonar.markdown.Markdown;
import org.sonar.server.es.BaseDoc;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE;
+
+
/**
* Implementation of Rule based on an Elasticsearch document
*/
public class RuleDoc extends BaseDoc {
+ @VisibleForTesting
public RuleDoc(Map<String, Object> fields) {
- super(fields);
+ super(TYPE_RULE, new HashMap<>(fields));
}
public RuleDoc() {
- super(Maps.newHashMapWithExpectedSize(15));
+ super(TYPE_RULE, Maps.newHashMapWithExpectedSize(16));
}
@Override
}
@Override
- public String getRouting() {
- return idAsString();
- }
-
- @Override
- public String getParent() {
- return null;
+ protected Optional<String> getSimpleMainTypeRouting() {
+ return Optional.of(idAsString());
}
public RuleKey key() {
import org.sonar.db.rule.RuleForIndexingDto;
import org.sonar.server.es.BaseDoc;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION;
+
public class RuleExtensionDoc extends BaseDoc {
public RuleExtensionDoc(Map<String, Object> fields) {
- super(fields);
+ super(TYPE_RULE_EXTENSION, fields);
}
public RuleExtensionDoc() {
- super(Maps.newHashMapWithExpectedSize(4));
+ super(TYPE_RULE_EXTENSION, Maps.newHashMapWithExpectedSize(4));
}
@Override
return idOf(getRuleId(), getScope());
}
- @Override
- public String getRouting() {
- return ruleIdAsString();
- }
-
- @Override
- public String getParent() {
- return ruleIdAsString();
- }
-
public int getRuleId() {
return Integer.valueOf(ruleIdAsString());
}
private String ruleIdAsString() {
- return getField(RuleIndexDefinition.FIELD_RULE_EXTENSION_RULE_ID);
+ return getField(RuleIndexDefinition.FIELD_RULE_ID);
}
public RuleExtensionDoc setRuleId(int ruleId) {
- setField(RuleIndexDefinition.FIELD_RULE_EXTENSION_RULE_ID, String.valueOf(ruleId));
+ String parent = String.valueOf(ruleId);
+ setField(RuleIndexDefinition.FIELD_RULE_ID, parent);
+ setParent(parent);
return this;
}
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.organization.OrganizationDto;
import org.sonar.db.qualityprofile.QProfileDto;
-import org.sonar.server.es.DefaultIndexSettings;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.EsUtils;
import org.sonar.server.es.SearchIdResult;
import org.sonar.server.es.SearchOptions;
import org.sonar.server.es.StickyFacetBuilder;
+import org.sonar.server.es.newindex.DefaultIndexSettings;
import org.sonar.server.es.textsearch.JavaTokenizer;
import static com.google.common.base.Preconditions.checkArgument;
import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
-import static org.sonar.server.es.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES;
-import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars;
import static org.sonar.server.es.EsUtils.optimizeScrollRequest;
import static org.sonar.server.es.EsUtils.scrollIds;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TEMPLATE_KEY;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TYPE;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_UPDATED_AT;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION;
/**
* The unique entry-point to interact with Elasticsearch index "rules".
.map(RuleStatus::toString)
.collect(MoreCollectors.toList());
- private static final String AGGREGATION_NAME = "_ref";
private static final String AGGREGATION_NAME_FOR_TAGS = "tagsAggregation";
private final EsClient client;
public SearchIdResult<Integer> search(RuleQuery query, SearchOptions options) {
SearchRequestBuilder esSearch = client
- .prepareSearch(INDEX_TYPE_RULE);
+ .prepareSearch(TYPE_RULE);
QueryBuilder qb = buildQuery(query);
Map<String, QueryBuilder> filters = buildFilters(query);
*/
public Iterator<Integer> searchAll(RuleQuery query) {
SearchRequestBuilder esSearch = client
- .prepareSearch(INDEX_TYPE_RULE)
+ .prepareSearch(TYPE_RULE)
.setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES));
optimizeScrollRequest(esSearch);
private static Map<String, QueryBuilder> buildFilters(RuleQuery query) {
Map<String, QueryBuilder> filters = new HashMap<>();
+ /* Add enforced filter on main type Rule */
+ filters.put(
+ FIELD_INDEX_TYPE,
+ boolQuery().must(QueryBuilders.termsQuery(FIELD_INDEX_TYPE, TYPE_RULE.getType())));
+
/* Add enforced filter on rules that are REMOVED */
filters.put(FIELD_RULE_STATUS,
boolQuery().mustNot(
if (TRUE.equals(query.getActivation())) {
filters.put("activation",
- JoinQueryBuilders.hasChildQuery(INDEX_TYPE_ACTIVE_RULE.getType(),
+ JoinQueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE.getName(),
childQuery, ScoreMode.None));
} else if (FALSE.equals(query.getActivation())) {
filters.put("activation",
boolQuery().mustNot(
- JoinQueryBuilders.hasChildQuery(INDEX_TYPE_ACTIVE_RULE.getType(),
+ JoinQueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE.getName(),
childQuery, ScoreMode.None)));
}
QProfileDto compareToQProfile = query.getCompareToQProfile();
if (compareToQProfile != null) {
filters.put("comparison",
JoinQueryBuilders.hasChildQuery(
- INDEX_TYPE_ACTIVE_RULE.getType(),
+ TYPE_ACTIVE_RULE.getName(),
boolQuery().must(QueryBuilders.termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, compareToQProfile.getRulesProfileUuid())),
ScoreMode.None));
}
.map(tag -> boolQuery()
.filter(QueryBuilders.termQuery(FIELD_RULE_EXTENSION_TAGS, tag))
.filter(termsQuery(FIELD_RULE_EXTENSION_SCOPE, RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(organization).getScope())))
- .map(childQuery -> JoinQueryBuilders.hasChildQuery(INDEX_TYPE_RULE_EXTENSION.getType(), childQuery, ScoreMode.None))
+ .map(childQuery -> JoinQueryBuilders.hasChildQuery(TYPE_RULE_EXTENSION.getName(), childQuery, ScoreMode.None))
.forEach(q::should);
return q;
}
RuleExtensionScope.organization(query.getOrganization()).getScope()))
.subAggregation(termsAggregation);
- return JoinAggregationBuilders.children("children_for_" + termsAggregation.getName(), INDEX_TYPE_RULE_EXTENSION.getType())
+ return JoinAggregationBuilders.children("children_for_" + termsAggregation.getName(), TYPE_RULE_EXTENSION.getName())
.subAggregation(scopeAggregation);
};
// so the rule filter has to be used as parent filter for active rules
// from which we remove filters that concern active rules ("activation")
HasParentQueryBuilder ruleFilter = JoinQueryBuilders.hasParentQuery(
- INDEX_TYPE_RULE.getType(),
+ TYPE_RULE.getType(),
stickyFacetBuilder.getStickyFacetFilter("activation"),
false);
RuleIndex.addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance());
QueryBuilder activeRuleFilter = childrenFilter.must(ruleFilter);
- AggregationBuilder activeSeverities = JoinAggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children", INDEX_TYPE_ACTIVE_RULE.getType())
+ AggregationBuilder activeSeverities = JoinAggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children", TYPE_ACTIVE_RULE.getName())
.subAggregation(
AggregationBuilders.filter(FACET_ACTIVE_SEVERITIES + "_filter", activeRuleFilter)
.subAggregation(
esSearch.setSize(options.getLimit());
}
- public List<String> terms(String fields) {
- return terms(fields, null, Integer.MAX_VALUE);
- }
-
- public List<String> terms(String fields, @Nullable String query, int size) {
- TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME)
- .field(fields)
- .size(size)
- .minDocCount(1);
- if (query != null) {
- termsAggregation.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(query) + ".*", null));
- }
- SearchRequestBuilder request = client
- .prepareSearch(INDEX_TYPE_RULE, INDEX_TYPE_ACTIVE_RULE)
- .setQuery(matchAllQuery())
- .setSize(0)
- .addAggregation(termsAggregation);
-
- SearchResponse esResponse = request.get();
- return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME));
- }
-
public List<String> listTags(@Nullable OrganizationDto organization, @Nullable String query, int size) {
int maxPageSize = 500;
checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize);
.ifPresent(termsAggregation::includeExclude);
SearchRequestBuilder request = client
- .prepareSearch(INDEX_TYPE_RULE_EXTENSION)
+ .prepareSearch(TYPE_RULE_EXTENSION.getMainType())
.setQuery(boolQuery().filter(scopeFilter))
.setSize(0)
.addAggregation(termsAggregation);
*/
package org.sonar.server.rule.index;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import java.util.Set;
import org.sonar.api.config.Configuration;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.IndexType.IndexRelationType;
+import org.sonar.server.es.newindex.NewRegularIndex;
+import org.sonar.server.es.newindex.TypeMapping;
-import static org.sonar.server.es.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
/**
* Definition of ES index "rules", including settings and fields.
*/
public class RuleIndexDefinition implements IndexDefinition {
- private static final String INDEX = "rules";
-
- public static final IndexType INDEX_TYPE_RULE = new IndexType(INDEX, "rule");
- public static final String FIELD_RULE_ID = "id";
+ public static final Index DESCRIPTOR = Index.withRelations("rules");
+ public static final IndexMainType TYPE_RULE = IndexType.main(DESCRIPTOR, "rule");
+ public static final String FIELD_RULE_ID = "ruleId";
public static final String FIELD_RULE_KEY = "key";
public static final String FIELD_RULE_REPOSITORY = "repo";
public static final String FIELD_RULE_RULE_KEY = "ruleKey";
FIELD_RULE_KEY);
// Rule extension fields
- public static final IndexType INDEX_TYPE_RULE_EXTENSION = new IndexType(INDEX, "ruleExtension");
+ public static final IndexRelationType TYPE_RULE_EXTENSION = IndexType.relation(TYPE_RULE, "ruleExtension");
/**
* The uuid of a {@link RuleExtensionScope}
*/
- public static final String FIELD_RULE_EXTENSION_SCOPE = "scope";
- public static final String FIELD_RULE_EXTENSION_RULE_ID = "ruleId";
- public static final String FIELD_RULE_EXTENSION_TAGS = "tags";
+ public static final String FIELD_RULE_EXTENSION_SCOPE = "ruleExt_scope";
+ public static final String FIELD_RULE_EXTENSION_TAGS = "ruleExt_tags";
// Active rule fields
- public static final IndexType INDEX_TYPE_ACTIVE_RULE = new IndexType(INDEX, "activeRule");
- public static final String FIELD_ACTIVE_RULE_ID = "id";
- public static final String FIELD_ACTIVE_RULE_RULE_ID = "ruleId";
- public static final String FIELD_ACTIVE_RULE_INHERITANCE = "inheritance";
- public static final String FIELD_ACTIVE_RULE_PROFILE_UUID = "ruleProfile";
- public static final String FIELD_ACTIVE_RULE_SEVERITY = "severity";
+ public static final IndexRelationType TYPE_ACTIVE_RULE = IndexType.relation(TYPE_RULE, "activeRule");
+ public static final String FIELD_ACTIVE_RULE_ID = "activeRule_id";
+ public static final String FIELD_ACTIVE_RULE_INHERITANCE = "activeRule_inheritance";
+ public static final String FIELD_ACTIVE_RULE_PROFILE_UUID = "activeRule_ruleProfile";
+ public static final String FIELD_ACTIVE_RULE_SEVERITY = "activeRule_severity";
private final Configuration config;
private final boolean enableSource;
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(
- INDEX_TYPE_RULE.getIndex(),
+ NewRegularIndex index = context.create(
+ DESCRIPTOR,
newBuilder(config)
.setRefreshInterval(MANUAL_REFRESH_INTERVAL)
// Default nb of shards should be greater than 1 in order to
// easily detect routing misconfiguration.
// See https://jira.sonarsource.com/browse/SONAR-9489
.setDefaultNbOfShards(2)
- .build());
-
- // Active rule type
- NewIndex.NewIndexType activeRuleMapping = index.createType(INDEX_TYPE_ACTIVE_RULE.getType());
- activeRuleMapping.setEnableSource(enableSource);
- activeRuleMapping.setAttribute("_parent", ImmutableMap.of("type", INDEX_TYPE_RULE.getType()));
-
- activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_ID).disableNorms().build();
- activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_RULE_ID).disableNorms().build();
- activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_PROFILE_UUID).disableNorms().build();
- activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().build();
- activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_SEVERITY).disableNorms().build();
-
- // Rule extension type
- NewIndex.NewIndexType ruleExtensionType = index.createType(INDEX_TYPE_RULE_EXTENSION.getType());
- ruleExtensionType.setEnableSource(enableSource);
- ruleExtensionType.setAttribute("_parent", ImmutableMap.of("type", INDEX_TYPE_RULE.getType()));
-
- ruleExtensionType.keywordFieldBuilder(FIELD_RULE_EXTENSION_SCOPE).disableNorms().build();
- ruleExtensionType.keywordFieldBuilder(FIELD_RULE_EXTENSION_TAGS).build();
+ .build())
+ .setEnableSource(enableSource);
// Rule type
- NewIndex.NewIndexType ruleMapping = index.createType(INDEX_TYPE_RULE.getType());
- ruleMapping.setEnableSource(enableSource);
-
+ TypeMapping ruleMapping = index.createTypeMapping(TYPE_RULE);
ruleMapping.keywordFieldBuilder(FIELD_RULE_ID).disableNorms().build();
ruleMapping.keywordFieldBuilder(FIELD_RULE_KEY).addSubFields(SORTABLE_ANALYZER).build();
ruleMapping.keywordFieldBuilder(FIELD_RULE_RULE_KEY).addSubFields(SORTABLE_ANALYZER).build();
ruleMapping.createLongField(FIELD_RULE_CREATED_AT);
ruleMapping.createLongField(FIELD_RULE_UPDATED_AT);
+
+ // Active rule
+ index.createTypeMapping(TYPE_ACTIVE_RULE)
+ .keywordFieldBuilder(FIELD_ACTIVE_RULE_ID).disableNorms().build()
+ .keywordFieldBuilder(FIELD_ACTIVE_RULE_PROFILE_UUID).disableNorms().build()
+ .keywordFieldBuilder(FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().build()
+ .keywordFieldBuilder(FIELD_ACTIVE_RULE_SEVERITY).disableNorms().build();
+
+ // Rule extension
+ index.createTypeMapping(TYPE_RULE_EXTENSION)
+ .keywordFieldBuilder(FIELD_RULE_EXTENSION_SCOPE).disableNorms().build()
+ .keywordFieldBuilder(FIELD_RULE_EXTENSION_TAGS).build();
}
}
import com.google.common.collect.ListMultimap;
import java.util.Collection;
import java.util.List;
-import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
-import org.elasticsearch.action.index.IndexRequest;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.db.es.EsQueueDto;
import org.sonar.db.es.RuleExtensionId;
import org.sonar.db.organization.OrganizationDto;
-import org.sonar.db.rule.RuleExtensionForIndexingDto;
-import org.sonar.db.rule.RuleForIndexingDto;
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.BulkIndexer.Size;
import org.sonar.server.es.EsClient;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION;
public class RuleIndexer implements ResilientIndexer {
@Override
public Set<IndexType> getIndexTypes() {
- return ImmutableSet.of(INDEX_TYPE_RULE, INDEX_TYPE_RULE_EXTENSION);
+ return ImmutableSet.of(TYPE_RULE, TYPE_RULE_EXTENSION);
}
@Override
bulk.start();
// index all definitions and system extensions
- if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE)) {
+ if (uninitializedIndexTypes.contains(TYPE_RULE)) {
dbClient.ruleDao().scrollIndexingRules(dbSession, dto -> {
- bulk.add(newRuleDocIndexRequest(dto));
- bulk.add(newRuleExtensionDocIndexRequest(dto));
+ bulk.add(RuleDoc.of(dto).toIndexRequest());
+ bulk.add(RuleExtensionDoc.of(dto).toIndexRequest());
});
}
// index all organization extensions
- if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE_EXTENSION)) {
- dbClient.ruleDao().scrollIndexingRuleExtensions(dbSession, dto -> bulk.add(newRuleExtensionDocIndexRequest(dto)));
+ if (uninitializedIndexTypes.contains(TYPE_RULE_EXTENSION)) {
+ dbClient.ruleDao().scrollIndexingRuleExtensions(dbSession, dto -> bulk.add(RuleExtensionDoc.of(dto).toIndexRequest()));
}
bulk.stop();
public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
IndexingResult result = new IndexingResult();
if (!items.isEmpty()) {
- ListMultimap<IndexType, EsQueueDto> itemsByType = groupItemsByType(items);
- result.add(doIndexRules(dbSession, itemsByType.get(INDEX_TYPE_RULE)));
- result.add(doIndexRuleExtensions(dbSession, itemsByType.get(INDEX_TYPE_RULE_EXTENSION)));
+ ListMultimap<String, EsQueueDto> itemsByType = groupItemsByIndexTypeFormat(items);
+ doIndexRules(dbSession, itemsByType.get(TYPE_RULE.format())).ifPresent(result::add);
+ doIndexRuleExtensions(dbSession, itemsByType.get(TYPE_RULE_EXTENSION.format())).ifPresent(result::add);
}
return result;
}
- private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
+ private Optional<IndexingResult> doIndexRules(DbSession dbSession, List<EsQueueDto> items) {
+ if (items.isEmpty()) {
+ return Optional.empty();
+ }
+
BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items));
bulkIndexer.start();
dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleIds,
r -> {
- bulkIndexer.add(newRuleDocIndexRequest(r));
- bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
+ bulkIndexer.add(RuleDoc.of(r).toIndexRequest());
+ bulkIndexer.add(RuleExtensionDoc.of(r).toIndexRequest());
ruleIds.remove(r.getId());
});
// the remaining items reference rows that don't exist in db. They must
// be deleted from index.
ruleIds.forEach(ruleId -> {
- bulkIndexer.addDeletion(INDEX_TYPE_RULE, ruleId.toString(), ruleId.toString());
- bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString());
+ bulkIndexer.addDeletion(TYPE_RULE, ruleId.toString(), ruleId.toString());
+ bulkIndexer.addDeletion(TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString());
});
- return bulkIndexer.stop();
+ return Optional.of(bulkIndexer.stop());
}
- private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) {
+ private Optional<IndexingResult> doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) {
+ if (items.isEmpty()) {
+ return Optional.empty();
+ }
+
BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items));
bulkIndexer.start();
r -> {
RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getRuleId());
docIds.remove(docId);
- bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
+ bulkIndexer.add(RuleExtensionDoc.of(r).toIndexRequest());
});
// the remaining items reference rows that don't exist in db. They must
// be deleted from index.
- docIds.forEach(docId -> bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, docId.getId(), String.valueOf(docId.getRuleId())));
-
- return bulkIndexer.stop();
- }
-
- private static IndexRequest newRuleDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
- RuleDoc doc = RuleDoc.of(ruleForIndexingDto);
-
- return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType())
- .id(doc.getId())
- .routing(doc.getRouting())
- .source(doc.getFields());
- }
-
- private static IndexRequest newRuleExtensionDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) {
- RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleForIndexingDto);
-
- return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
- .id(ruleExtensionDoc.getId())
- .routing(ruleExtensionDoc.getRouting())
- .parent(ruleExtensionDoc.getParent())
- .source(ruleExtensionDoc.getFields());
- }
+ docIds.forEach(docId -> bulkIndexer.addDeletion(TYPE_RULE_EXTENSION, docId.getId(), String.valueOf(docId.getRuleId())));
- private static IndexRequest newRuleExtensionDocIndexRequest(RuleExtensionForIndexingDto ruleExtensionForIndexingDto) {
- RuleExtensionDoc doc = RuleExtensionDoc.of(ruleExtensionForIndexingDto);
- return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType())
- .id(doc.getId())
- .routing(doc.getRouting())
- .parent(doc.getParent())
- .source(doc.getFields());
+ return Optional.of(bulkIndexer.stop());
}
private BulkIndexer createBulkIndexer(Size bulkSize, IndexingListener listener) {
- return new BulkIndexer(esClient, INDEX_TYPE_RULE, bulkSize, listener);
+ return new BulkIndexer(esClient, TYPE_RULE, bulkSize, listener);
}
- private static ListMultimap<IndexType, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) {
- return items.stream().collect(MoreCollectors.index(i -> IndexType.parse(i.getDocType())));
+ private static ListMultimap<String, EsQueueDto> groupItemsByIndexTypeFormat(Collection<EsQueueDto> items) {
+ return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType));
}
private static RuleExtensionId explodeRuleExtensionDocId(EsQueueDto esQueueDto) {
- checkArgument(Objects.equals(esQueueDto.getDocType(), "rules/ruleExtension"));
+ checkArgument(TYPE_RULE_EXTENSION.format().equals(esQueueDto.getDocType()));
return new RuleExtensionId(esQueueDto.getDocId());
}
private static EsQueueDto createQueueDtoForRule(int ruleId) {
String docId = String.valueOf(ruleId);
- return EsQueueDto.create("rules/rule", docId, null, docId);
+ return EsQueueDto.create(TYPE_RULE.format(), docId, null, docId);
}
private static EsQueueDto createQueueDtoForRuleExtension(int ruleId, OrganizationDto organization) {
String docId = RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.organization(organization));
- return EsQueueDto.create("rules/ruleExtension", docId, null, String.valueOf(ruleId));
+ return EsQueueDto.create(TYPE_RULE_EXTENSION.format(), docId, null, String.valueOf(ruleId));
}
}
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
-import org.sonar.api.user.User;
import org.sonar.server.es.BaseDoc;
import static org.sonar.server.user.index.UserIndexDefinition.FIELD_ORGANIZATION_UUIDS;
+import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER;
-public class UserDoc extends BaseDoc implements User {
+public class UserDoc extends BaseDoc {
public UserDoc(Map<String, Object> fields) {
- super(fields);
+ super(TYPE_USER, fields);
}
public UserDoc() {
return uuid();
}
- @Override
- public String getRouting() {
- return null;
- }
-
- @Override
- public String getParent() {
- return null;
- }
-
public String uuid() {
return getField(UserIndexDefinition.FIELD_UUID);
}
- @Override
public String login() {
return getField(UserIndexDefinition.FIELD_LOGIN);
}
- @Override
public String name() {
return getField(UserIndexDefinition.FIELD_NAME);
}
- @Override
@Nullable
public String email() {
return getNullableField(UserIndexDefinition.FIELD_EMAIL);
}
- @Override
public boolean active() {
return (Boolean) getField(UserIndexDefinition.FIELD_ACTIVE);
}
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER;
import static org.sonar.server.user.index.UserIndexDefinition.FIELD_ACTIVE;
import static org.sonar.server.user.index.UserIndexDefinition.FIELD_EMAIL;
import static org.sonar.server.user.index.UserIndexDefinition.FIELD_LOGIN;
public List<UserDoc> getAtMostThreeActiveUsersForScmAccount(String scmAccount, String organizationUuid) {
List<UserDoc> result = new ArrayList<>();
if (!StringUtils.isEmpty(scmAccount)) {
- SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX_TYPE_USER)
+ SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER)
.setQuery(boolQuery().must(matchAllQuery()).filter(
boolQuery()
.must(termQuery(FIELD_ACTIVE, true))
}
public SearchResult<UserDoc> search(UserQuery userQuery, SearchOptions options) {
- SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX_TYPE_USER)
+ SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER)
.setSize(options.getLimit())
.setFrom(options.getOffset())
.addSort(FIELD_NAME, SortOrder.ASC);
package org.sonar.server.user.index;
import org.sonar.api.config.Configuration;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.newindex.NewRegularIndex;
+import org.sonar.server.es.newindex.TypeMapping;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
/**
* Definition of ES index "users", including settings and fields.
*/
public class UserIndexDefinition implements IndexDefinition {
- public static final IndexType INDEX_TYPE_USER = new IndexType("users", "user");
+ public static final Index DESCRIPTOR = Index.simple("users");
+ public static final IndexMainType TYPE_USER = IndexType.main(DESCRIPTOR, "user");
public static final String FIELD_UUID = "uuid";
public static final String FIELD_LOGIN = "login";
public static final String FIELD_NAME = "name";
this.config = config;
}
+ public static UserIndexDefinition createForTest() {
+ return new UserIndexDefinition(new MapSettings().asConfig());
+ }
+
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(INDEX_TYPE_USER.getIndex(),
+ NewRegularIndex index = context.create(
+ DESCRIPTOR,
newBuilder(config)
.setDefaultNbOfShards(1)
- .build());
+ .build())
+ // all information is retrieved from the index, not only IDs
+ .setEnableSource(true);
- // type "user"
- NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_USER.getType());
+ TypeMapping mapping = index.createTypeMapping(TYPE_USER);
mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_LOGIN).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build();
mapping.keywordFieldBuilder(FIELD_NAME).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build();
import static java.util.Collections.singletonList;
import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
import static org.sonar.core.util.stream.MoreCollectors.toList;
-import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
+import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER;
public class UserIndexer implements ResilientIndexer {
@Override
public Set<IndexType> getIndexTypes() {
- return ImmutableSet.of(INDEX_TYPE_USER);
+ return ImmutableSet.of(TYPE_USER);
}
@Override
public void commitAndIndex(DbSession dbSession, Collection<UserDto> users) {
List<String> uuids = users.stream().map(UserDto::getUuid).collect(toList());
List<EsQueueDto> items = uuids.stream()
- .map(uuid -> EsQueueDto.create(INDEX_TYPE_USER.format(), uuid))
+ .map(uuid -> EsQueueDto.create(TYPE_USER.format(), uuid))
.collect(MoreCollectors.toArrayList());
dbClient.esQueueDao().insert(dbSession, items);
// the remaining uuids reference rows that don't exist in db. They must
// be deleted from index.
- uuids.forEach(uuid -> bulkIndexer.addDeletion(INDEX_TYPE_USER, uuid));
+ uuids.forEach(uuid -> bulkIndexer.addDeletion(TYPE_USER, uuid));
return bulkIndexer.stop();
}
private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) {
- return new BulkIndexer(esClient, INDEX_TYPE_USER, bulkSize, listener);
+ return new BulkIndexer(esClient, TYPE_USER, bulkSize, listener);
}
private static IndexRequest newIndexRequest(UserDto user, ListMultimap<String, String> organizationUuidsByUserUuid) {
doc.setScmAccounts(UserDto.decodeScmAccounts(user.getScmAccounts()));
doc.setOrganizationUuids(organizationUuidsByUserUuid.get(user.getUuid()));
- return new IndexRequest(INDEX_TYPE_USER.getIndex(), INDEX_TYPE_USER.getType())
- .id(doc.getId())
- .routing(doc.getRouting())
- .source(doc.getFields());
+ return doc.toIndexRequest();
}
}
import java.util.Map;
import org.sonar.server.es.BaseDoc;
+import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW;
+
public class ViewDoc extends BaseDoc {
public ViewDoc(Map<String, Object> fields) {
- super(fields);
+ super(TYPE_VIEW, fields);
}
public ViewDoc() {
- super(Maps.newHashMap());
+ super(TYPE_VIEW, Maps.newHashMap());
}
@Override
return uuid();
}
- @Override
- public String getRouting() {
- return null;
- }
-
- @Override
- public String getParent() {
- return null;
- }
-
public String uuid() {
return getField(ViewIndexDefinition.FIELD_UUID);
}
}
public List<String> findAllViewUuids() {
- SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.INDEX_TYPE_VIEW)
+ SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.TYPE_VIEW)
.addSort("_doc", SortOrder.ASC)
.setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES))
.setFetchSource(false)
package org.sonar.server.view.index;
import org.sonar.api.config.Configuration;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.newindex.NewRegularIndex;
+import org.sonar.server.es.newindex.TypeMapping;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
/**
* Definition of ES index "views", including settings and fields.
*/
public class ViewIndexDefinition implements IndexDefinition {
- public static final IndexType INDEX_TYPE_VIEW = new IndexType("views", "view");
+ public static final Index DESCRIPTOR = Index.simple("views");
+ public static final IndexMainType TYPE_VIEW = IndexType.main(DESCRIPTOR, "view");
public static final String FIELD_UUID = "uuid";
public static final String FIELD_PROJECTS = "projects";
this.config = config;
}
+ /**
+ * Keep the document sources in index so that indexer tests can verify content
+ * of indexed documents.
+ */
+ public static ViewIndexDefinition createForTest() {
+ return new ViewIndexDefinition(new MapSettings().asConfig());
+ }
+
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(
- INDEX_TYPE_VIEW.getIndex(),
+ NewRegularIndex index = context.create(
+ DESCRIPTOR,
newBuilder(config)
.setDefaultNbOfShards(5)
- .build());
+ .build())
+ // storing source is required because some search queries on issue index use terms lookup query onto the view index
+ // and this requires source to be stored (https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html#query-dsl-terms-lookup)
+ .setEnableSource(true);
// type "view"
- NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_VIEW.getType());
+ TypeMapping mapping = index.createTypeMapping(TYPE_VIEW);
mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build();
mapping.keywordFieldBuilder(FIELD_PROJECTS).disableNorms().build();
}
import static com.google.common.collect.Maps.newHashMap;
import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
-import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW;
+import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW;
public class ViewIndexer implements ResilientIndexer {
@Override
public Set<IndexType> getIndexTypes() {
- return ImmutableSet.of(INDEX_TYPE_VIEW);
+ return ImmutableSet.of(TYPE_VIEW);
}
@Override
* The views lookup cache will be cleared
*/
public void index(ViewDoc viewDoc) {
- BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, Size.REGULAR);
+ BulkIndexer bulk = new BulkIndexer(esClient, TYPE_VIEW, Size.REGULAR);
bulk.start();
doIndex(bulk, viewDoc, true);
bulk.stop();
}
private void index(DbSession dbSession, Map<String, String> viewAndProjectViewUuidMap, boolean needClearCache, Size bulkSize) {
- BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, bulkSize);
+ BulkIndexer bulk = new BulkIndexer(esClient, TYPE_VIEW, bulkSize);
bulk.start();
for (Map.Entry<String, String> entry : viewAndProjectViewUuidMap.entrySet()) {
String viewUuid = entry.getKey();
}
private static IndexRequest newIndexRequest(ViewDoc doc) {
- return new IndexRequest(ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex(), ViewIndexDefinition.INDEX_TYPE_VIEW.getType(), doc.uuid())
+ IndexType.IndexMainType mainType = TYPE_VIEW;
+ return new IndexRequest(mainType.getIndex().getName(), mainType.getType())
+ .id(doc.getId())
+ .routing(doc.getRouting().orElse(null))
.source(doc.getFields());
}
// Safety check to remove all views that may not have been deleted
views.removeAll(dbClient.componentDao().selectExistingUuids(dbSession, views));
- views.forEach(v -> bulkIndexer.addDeletion(INDEX_TYPE_VIEW, v));
+ views.forEach(v -> bulkIndexer.addDeletion(TYPE_VIEW, v));
return bulkIndexer.stop();
}
public void delete(DbSession dbSession, Collection<String> viewUuids) {
List<EsQueueDto> items = viewUuids.stream()
- .map(l -> EsQueueDto.create(INDEX_TYPE_VIEW.format(), l))
+ .map(l -> EsQueueDto.create(TYPE_VIEW.format(), l))
.collect(MoreCollectors.toArrayList());
dbClient.esQueueDao().insert(dbSession, items);
}
private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) {
- return new BulkIndexer(esClient, INDEX_TYPE_VIEW, bulkSize, listener);
+ return new BulkIndexer(esClient, TYPE_VIEW, bulkSize, listener);
}
}
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.sonar.db.component.ComponentTesting.newFileDto;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME;
-import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_TYPE_COMPONENT;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION;
import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION;
@Test
public void test_getIndexTypes() {
- assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_COMPONENT);
+ assertThat(underTest.getIndexTypes()).containsExactly(TYPE_COMPONENT);
}
@Test
underTest.indexOnStartup(emptySet());
assertThatIndexContainsOnly(project);
- ComponentDoc doc = es.getDocuments(INDEX_TYPE_COMPONENT, ComponentDoc.class).get(0);
+ ComponentDoc doc = es.getDocuments(TYPE_COMPONENT, ComponentDoc.class).get(0);
assertThat(doc.getId()).isEqualTo(project.uuid());
assertThat(doc.getKey()).isEqualTo(project.getDbKey());
assertThat(doc.getProjectUuid()).isEqualTo(project.projectUuid());
public void errors_during_indexing_are_recovered() {
ComponentDto project = db.components().insertPrivateProject();
ComponentDto file = db.components().insertComponent(newFileDto(project));
- es.lockWrites(INDEX_TYPE_COMPONENT);
+ es.lockWrites(TYPE_COMPONENT);
IndexingResult result = indexProject(project, PROJECT_CREATION);
assertThat(result.getTotal()).isEqualTo(2L);
result = recover();
assertThat(result.getTotal()).isEqualTo(2L);
assertThat(result.getFailures()).isEqualTo(2L);
- assertThat(es.countDocuments(INDEX_TYPE_COMPONENT)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_COMPONENT)).isEqualTo(0);
- es.unlockWrites(INDEX_TYPE_COMPONENT);
+ es.unlockWrites(TYPE_COMPONENT);
result = recover();
assertThat(result.getTotal()).isEqualTo(2L);
}
private void assertThatIndexHasSize(int expectedSize) {
- assertThat(es.countDocuments(INDEX_TYPE_COMPONENT)).isEqualTo(expectedSize);
+ assertThat(es.countDocuments(TYPE_COMPONENT)).isEqualTo(expectedSize);
}
private void assertThatIndexContainsOnly(ComponentDto... expectedComponents) {
- assertThat(es.getIds(INDEX_TYPE_COMPONENT)).containsExactlyInAnyOrder(
+ assertThat(es.getIds(TYPE_COMPONENT)).containsExactlyInAnyOrder(
Arrays.stream(expectedComponents).map(ComponentDto::uuid).toArray(String[]::new));
}
private void assertThatComponentHasName(ComponentDto component, String expectedName) {
SearchHit[] hits = es.client()
- .prepareSearch(INDEX_TYPE_COMPONENT)
+ .prepareSearch(TYPE_COMPONENT.getMainType())
.setQuery(matchQuery(SORTABLE_ANALYZER.subField(FIELD_NAME), expectedName))
.get()
.getHits()
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.db.DbTester;
import org.sonar.server.es.BulkIndexer.Size;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static java.util.Collections.emptyMap;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.es.FakeIndexDefinition.INDEX;
-import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE;
+import static org.sonar.server.es.newindex.FakeIndexDefinition.INDEX;
+import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE;
public class BulkIndexerTest {
@Test
public void index_nothing() {
- BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR);
+ BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR);
indexer.start();
indexer.stop();
@Test
public void index_documents() {
- BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR);
+ BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR);
indexer.start();
indexer.add(newIndexRequest(42));
indexer.add(newIndexRequest(78));
// index has one replica
assertThat(replicas()).isEqualTo(1);
- BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.LARGE);
+ BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.LARGE);
indexer.start();
// replicas are temporarily disabled
for (int i = 0; i < max; i++) {
docs[i] = FakeIndexDefinition.newDoc(i);
}
- es.putDocuments(INDEX_TYPE_FAKE, docs);
+ es.putDocuments(TYPE_FAKE, docs);
assertThat(count()).isEqualTo(max);
- SearchRequestBuilder req = es.client().prepareSearch(INDEX_TYPE_FAKE)
+ SearchRequestBuilder req = es.client().prepareSearch(TYPE_FAKE)
.setQuery(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom));
- BulkIndexer.delete(es.client(), INDEX_TYPE_FAKE, req);
+ BulkIndexer.delete(es.client(), TYPE_FAKE, req);
assertThat(count()).isEqualTo(removeFrom);
}
@Test
public void listener_is_called_on_successful_requests() {
FakeListener listener = new FakeListener();
- BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener);
+ BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener);
indexer.start();
- indexer.addDeletion(INDEX_TYPE_FAKE, "foo");
+ indexer.addDeletion(TYPE_FAKE, "foo");
indexer.stop();
assertThat(listener.calledDocIds)
- .containsExactlyInAnyOrder(new DocId(INDEX_TYPE_FAKE, "foo"));
+ .containsExactlyInAnyOrder(newDocId(TYPE_FAKE, "foo"));
assertThat(listener.calledResult.getSuccess()).isEqualTo(1);
assertThat(listener.calledResult.getTotal()).isEqualTo(1);
}
@Test
public void listener_is_called_even_if_deleting_a_doc_that_does_not_exist() {
FakeListener listener = new FakeListener();
- BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener);
+ BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener);
indexer.start();
indexer.add(newIndexRequestWithDocId("foo"));
indexer.add(newIndexRequestWithDocId("bar"));
indexer.stop();
assertThat(listener.calledDocIds)
- .containsExactlyInAnyOrder(new DocId(INDEX_TYPE_FAKE, "foo"), new DocId(INDEX_TYPE_FAKE, "bar"));
+ .containsExactlyInAnyOrder(newDocId(TYPE_FAKE, "foo"), newDocId(TYPE_FAKE, "bar"));
assertThat(listener.calledResult.getSuccess()).isEqualTo(2);
assertThat(listener.calledResult.getTotal()).isEqualTo(2);
}
@Test
public void listener_is_not_called_with_errors() {
FakeListener listener = new FakeListener();
- BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener);
+ BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener);
indexer.start();
indexer.add(newIndexRequestWithDocId("foo"));
indexer.add(new IndexRequest("index_does_not_exist", "index_does_not_exist", "bar").source(emptyMap()));
indexer.stop();
- assertThat(listener.calledDocIds).containsExactly(new DocId(INDEX_TYPE_FAKE, "foo"));
+ assertThat(listener.calledDocIds).containsExactly(newDocId(TYPE_FAKE, "foo"));
assertThat(listener.calledResult.getSuccess()).isEqualTo(1);
assertThat(listener.calledResult.getTotal()).isEqualTo(2);
}
public void log_requests_when_TRACE_level_is_enabled() {
logTester.setLevel(LoggerLevel.TRACE);
- BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, new FakeListener());
+ BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, new FakeListener());
indexer.start();
indexer.add(newIndexRequestWithDocId("foo"));
- indexer.addDeletion(INDEX_TYPE_FAKE, "foo");
+ indexer.addDeletion(TYPE_FAKE, "foo");
indexer.add(newIndexRequestWithDocId("bar"));
indexer.stop();
}
private long count() {
- return es.countDocuments("fakes", "fake");
+ return es.countDocuments(IndexType.main(Index.simple("fakes"), "fake"));
}
private int replicas() {
}
private IndexRequest newIndexRequest(int intField) {
- return new IndexRequest(INDEX, INDEX_TYPE_FAKE.getType())
+ return new IndexRequest(INDEX, TYPE_FAKE.getType())
.source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, intField));
}
private IndexRequest newIndexRequestWithDocId(String id) {
- return new IndexRequest(INDEX, INDEX_TYPE_FAKE.getType())
+ return new IndexRequest(INDEX, TYPE_FAKE.getType())
.id(id)
.source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, 42));
}
+
+ private static DocId newDocId(IndexType.IndexMainType mainType, String id) {
+ return new DocId(TYPE_FAKE.getIndex().getName(), mainType.getType(), id);
+ }
}
import java.util.Map;
import org.junit.Test;
+import org.sonar.server.es.newindex.DefaultIndexSettings;
import org.sonar.test.TestUtils;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
public class DefaultIndexSettingsTest {
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.junit.Test;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class DocIdTest {
+ @Test
+ public void equals_is_based_on_index_type_and_id() {
+ String index = randomAlphabetic(5);
+ String type = randomAlphabetic(6);
+ String id = randomAlphabetic(7);
+ DocId underTest = new DocId(index, type, id);
+
+ assertThat(underTest)
+ .isEqualTo(new DocId(index, type, id))
+ .isNotEqualTo(new DocId(randomAlphabetic(7), type, id))
+ .isNotEqualTo(new DocId(index, type, randomAlphabetic(7)))
+ .isNotEqualTo(new DocId(index, randomAlphabetic(7), id))
+ .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), id))
+ .isNotEqualTo(new DocId(randomAlphabetic(7), type, randomAlphabetic(8)))
+ .isNotEqualTo(new DocId(index, randomAlphabetic(7), randomAlphabetic(8)))
+ .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), randomAlphabetic(9)));
+ }
+
+ @Test
+ public void hashcode_is_based_on_index_type_and_id() {
+ String index = randomAlphabetic(5);
+ String type = randomAlphabetic(6);
+ String id = randomAlphabetic(7);
+ DocId underTest = new DocId(index, type, id);
+
+ assertThat(underTest.hashCode())
+ .isEqualTo(new DocId(index, type, id).hashCode())
+ .isNotEqualTo(new DocId(randomAlphabetic(7), type, id).hashCode())
+ .isNotEqualTo(new DocId(index, type, randomAlphabetic(7)).hashCode())
+ .isNotEqualTo(new DocId(index, randomAlphabetic(7), id).hashCode())
+ .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), id).hashCode())
+ .isNotEqualTo(new DocId(randomAlphabetic(7), type, randomAlphabetic(8)).hashCode())
+ .isNotEqualTo(new DocId(index, randomAlphabetic(7), randomAlphabetic(8)).hashCode())
+ .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), randomAlphabetic(9)).hashCode());
+ }
+}
import org.junit.Rule;
import org.junit.Test;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder;
import org.sonar.server.es.request.ProxyClusterStateRequestBuilder;
import org.sonar.server.es.request.ProxyClusterStatsRequestBuilder;
@Test
public void proxify_requests() {
+ Index fakesIndex = Index.simple("fakes");
+ IndexType.IndexMainType fakeMainType = IndexType.main(fakesIndex, "fake");
+
EsClient underTest = es.client();
assertThat(underTest.nativeClient()).isNotNull();
assertThat(underTest.prepareClusterStats()).isInstanceOf(ProxyClusterStatsRequestBuilder.class);
- assertThat(underTest.prepareCreate("fakes")).isInstanceOf(ProxyCreateIndexRequestBuilder.class);
- assertThat(underTest.prepareDelete("fakes", "fake", "my_id")).isInstanceOf(ProxyDeleteRequestBuilder.class);
- assertThat(underTest.prepareIndicesExist()).isInstanceOf(ProxyIndicesExistsRequestBuilder.class);
- assertThat(underTest.prepareGet(new IndexType("fakes", "fake"), "1")).isInstanceOf(ProxyGetRequestBuilder.class);
+ assertThat(underTest.prepareCreate(fakesIndex)).isInstanceOf(ProxyCreateIndexRequestBuilder.class);
+ assertThat(underTest.prepareDelete(fakeMainType, "my_id")).isInstanceOf(ProxyDeleteRequestBuilder.class);
+ assertThat(underTest.prepareIndicesExist(fakesIndex)).isInstanceOf(ProxyIndicesExistsRequestBuilder.class);
+ assertThat(underTest.prepareGet(fakeMainType, "1")).isInstanceOf(ProxyGetRequestBuilder.class);
assertThat(underTest.prepareHealth()).isInstanceOf(ProxyClusterHealthRequestBuilder.class);
assertThat(underTest.prepareNodesStats()).isInstanceOf(ProxyNodesStatsRequestBuilder.class);
- assertThat(underTest.preparePutMapping()).isInstanceOf(ProxyPutMappingRequestBuilder.class);
- assertThat(underTest.prepareRefresh()).isInstanceOf(ProxyRefreshRequestBuilder.class);
- assertThat(underTest.prepareSearch(new IndexType[0])).isInstanceOf(ProxySearchRequestBuilder.class);
+ assertThat(underTest.preparePutMapping(fakesIndex)).isInstanceOf(ProxyPutMappingRequestBuilder.class);
+ assertThat(underTest.prepareRefresh(fakesIndex)).isInstanceOf(ProxyRefreshRequestBuilder.class);
+ assertThat(underTest.prepareSearch(fakesIndex)).isInstanceOf(ProxySearchRequestBuilder.class);
assertThat(underTest.prepareSearchScroll("1234")).isInstanceOf(ProxySearchScrollRequestBuilder.class);
assertThat(underTest.prepareState()).isInstanceOf(ProxyClusterStateRequestBuilder.class);
- assertThat(underTest.prepareStats()).isInstanceOf(ProxyIndicesStatsRequestBuilder.class);
+ assertThat(underTest.prepareStats(fakesIndex)).isInstanceOf(ProxyIndicesStatsRequestBuilder.class);
underTest.close();
}
import com.google.common.base.Throwables;
import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import java.nio.file.Files;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.indices.recovery.RecoverySettings;
+import org.elasticsearch.join.ParentJoinPlugin;
+import org.elasticsearch.node.InternalSettingsPreparer;
import org.elasticsearch.node.Node;
import org.elasticsearch.search.SearchHit;
import org.junit.rules.ExternalResource;
-import org.sonar.api.config.Configuration;
-import org.sonar.api.config.internal.MapSettings;
import org.sonar.server.component.index.ComponentIndexDefinition;
+import org.sonar.server.es.IndexDefinition.IndexDefinitionContext;
+import org.sonar.server.es.IndexType.IndexRelationType;
+import org.sonar.server.es.newindex.BuiltIndex;
+import org.sonar.server.es.newindex.NewIndex;
import org.sonar.server.issue.index.IssueIndexDefinition;
import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
import org.sonar.server.rule.index.RuleIndexDefinition;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Lists.newArrayList;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.sonar.server.es.DefaultIndexSettings.REFRESH_IMMEDIATE;
+import static org.sonar.server.es.Index.ALL_INDICES;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.REFRESH_IMMEDIATE;
public class EsTester extends ExternalResource {
*/
public static EsTester create() {
if (!CORE_INDICES_CREATED.get()) {
- Configuration config = new MapSettings().asConfig();
- List<IndexDefinition.Index> createdIndices = createIndices(
- new ComponentIndexDefinition(config),
+ List<BuiltIndex> createdIndices = createIndices(
+ ComponentIndexDefinition.createForTest(),
IssueIndexDefinition.createForTest(),
- new ProjectMeasuresIndexDefinition(config),
+ ProjectMeasuresIndexDefinition.createForTest(),
RuleIndexDefinition.createForTest(),
- new UserIndexDefinition(config),
- new ViewIndexDefinition(config));
+ UserIndexDefinition.createForTest(),
+ ViewIndexDefinition.createForTest());
CORE_INDICES_CREATED.set(true);
- createdIndices.stream().map(IndexDefinition.Index::getName).forEach(CORE_INDICES_NAMES::add);
+ createdIndices.stream().map(t -> t.getMainType().getIndex().getName()).forEach(CORE_INDICES_NAMES::add);
}
return new EsTester(false);
}
.filter(i -> !CORE_INDICES_NAMES.contains(i))
.forEach(EsTester::deleteIndexIfExists);
}
- BulkIndexer.delete(client(), new IndexType("_all", ""), client().prepareSearch("_all").setQuery(matchAllQuery()));
+ BulkIndexer.delete(client(), IndexType.main(ALL_INDICES, "dummy"), client().prepareSearch(ALL_INDICES).setQuery(matchAllQuery()));
}
public EsClient client() {
return new EsClient(SHARED_NODE.client());
}
- public void putDocuments(String index, String type, BaseDoc... docs) {
- putDocuments(new IndexType(index, type), docs);
- }
-
public void putDocuments(IndexType indexType, BaseDoc... docs) {
try {
BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk()
.setRefreshPolicy(REFRESH_IMMEDIATE);
for (BaseDoc doc : docs) {
- bulk.add(new IndexRequest(indexType.getIndex(), indexType.getType(), doc.getId())
- .parent(doc.getParent())
- .routing(doc.getRouting())
- .source(doc.getFields()));
+ bulk.add(doc.toIndexRequest());
}
BulkResponse bulkResponse = bulk.get();
if (bulkResponse.hasFailures()) {
BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk()
.setRefreshPolicy(REFRESH_IMMEDIATE);
for (Map<String, Object> doc : docs) {
- bulk.add(new IndexRequest(indexType.getIndex(), indexType.getType())
+ IndexType.IndexMainType mainType = indexType.getMainType();
+ bulk.add(new IndexRequest(mainType.getIndex().getName(), mainType.getType())
.source(doc));
}
BulkResponse bulkResponse = bulk.get();
}
}
- public long countDocuments(String index, String type) {
- return countDocuments(new IndexType(index, type));
+ public long countDocuments(Index index) {
+ return client().prepareSearch(index)
+ .setQuery(matchAllQuery())
+ .setSize(0).get().getHits().getTotalHits();
}
public long countDocuments(IndexType indexType) {
- return client().prepareSearch(indexType).setSize(0).get().getHits().getTotalHits();
+ return client().prepareSearch(indexType.getMainType())
+ .setQuery(getDocumentsQuery(indexType))
+ .setSize(0).get().getHits().getTotalHits();
}
/**
}
/**
- * Get all the indexed documents (no paginated results). Results are not sorted.
+ * Get all the indexed documents (no paginated results) in the specified index, whatever their type. Results are not sorted.
+ */
+ public List<SearchHit> getDocuments(Index index) {
+ SearchRequestBuilder req = SHARED_NODE.client()
+ .prepareSearch(index.getName())
+ .setQuery(matchAllQuery());
+ return getDocuments(req);
+ }
+
+ /**
+ * Get all the indexed documents (no paginated results) of the specified type. Results are not sorted.
*/
public List<SearchHit> getDocuments(IndexType indexType) {
- SearchRequestBuilder req = SHARED_NODE.client().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(matchAllQuery());
+ IndexType.IndexMainType mainType = indexType.getMainType();
+ SearchRequestBuilder req = SHARED_NODE.client()
+ .prepareSearch(mainType.getIndex().getName())
+ .setQuery(getDocumentsQuery(indexType));
+ return getDocuments(req);
+ }
+
+ private List<SearchHit> getDocuments(SearchRequestBuilder req) {
EsUtils.optimizeScrollRequest(req);
req.setScroll(new TimeValue(60000))
.setSize(100);
return result;
}
+ private QueryBuilder getDocumentsQuery(IndexType indexType) {
+ if (!indexType.getMainType().getIndex().acceptsRelations()) {
+ return matchAllQuery();
+ }
+
+ if (indexType instanceof IndexRelationType) {
+ return new TermQueryBuilder(FIELD_INDEX_TYPE, ((IndexRelationType) indexType).getName());
+ }
+ if (indexType instanceof IndexType.IndexMainType) {
+ return new TermQueryBuilder(FIELD_INDEX_TYPE, ((IndexType.IndexMainType) indexType).getType());
+ }
+ throw new IllegalArgumentException("Unsupported IndexType " + indexType.getClass());
+ }
+
/**
* Get a list of a specific field from all indexed documents.
*/
}
public void lockWrites(IndexType index) {
- setIndexSettings(index.getIndex(), ImmutableMap.of("index.blocks.write", "true"));
+ setIndexSettings(index.getMainType().getIndex().getName(), ImmutableMap.of("index.blocks.write", "true"));
}
public void unlockWrites(IndexType index) {
- setIndexSettings(index.getIndex(), ImmutableMap.of("index.blocks.write", "false"));
+ setIndexSettings(index.getMainType().getIndex().getName(), ImmutableMap.of("index.blocks.write", "false"));
}
private void setIndexSettings(String index, Map<String, Object> settings) {
}
}
- private static List<IndexDefinition.Index> createIndices(IndexDefinition... definitions) {
- IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
+ private static List<BuiltIndex> createIndices(IndexDefinition... definitions) {
+ IndexDefinitionContext context = new IndexDefinitionContext();
Stream.of(definitions).forEach(d -> d.define(context));
- List<IndexDefinition.Index> result = new ArrayList<>();
+ List<BuiltIndex> result = new ArrayList<>();
for (NewIndex newIndex : context.getIndices().values()) {
- IndexDefinition.Index index = new IndexDefinition.Index(newIndex);
+ BuiltIndex index = newIndex.build();
- deleteIndexIfExists(index.getName());
+ String indexName = index.getMainType().getIndex().getName();
+ deleteIndexIfExists(indexName);
// create index
Settings.Builder settings = Settings.builder();
settings.put(index.getSettings());
CreateIndexResponse indexResponse = SHARED_NODE.client().admin().indices()
- .prepareCreate(index.getName())
+ .prepareCreate(indexName)
.setSettings(settings)
.get();
if (!indexResponse.isAcknowledged()) {
- throw new IllegalStateException("Failed to create index " + index.getName());
+ throw new IllegalStateException("Failed to create index " + indexName);
}
- SHARED_NODE.client().admin().cluster().prepareHealth(index.getName()).setWaitForStatus(ClusterHealthStatus.YELLOW).get();
+ SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get();
// create types
- for (Map.Entry<String, IndexDefinition.Type> entry : index.getTypes().entrySet()) {
- PutMappingResponse mappingResponse = SHARED_NODE.client().admin().indices().preparePutMapping(index.getName())
- .setType(entry.getKey())
- .setSource(entry.getValue().getAttributes())
- .get();
- if (!mappingResponse.isAcknowledged()) {
- throw new IllegalStateException("Failed to create type " + entry.getKey());
- }
+ String typeName = index.getMainType().getType();
+ PutMappingResponse mappingResponse = SHARED_NODE.client().admin().indices().preparePutMapping(indexName)
+ .setType(typeName)
+ .setSource(index.getAttributes())
+ .get();
+ if (!mappingResponse.isAcknowledged()) {
+ throw new IllegalStateException("Failed to create type " + typeName);
}
- SHARED_NODE.client().admin().cluster().prepareHealth(index.getName()).setWaitForStatus(ClusterHealthStatus.YELLOW).get();
+ SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get();
result.add(index);
}
return result;
.put(NetworkModule.HTTP_ENABLED.getKey(), false)
.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "single-node")
.build();
- Node node = new Node(settings);
+ Node node = new TesterNode(settings);
return node.start();
} catch (Exception e) {
throw new IllegalStateException("Fail to start embedded Elasticsearch", e);
}
}
+
+ private static class TesterNode extends Node {
+ public TesterNode(Settings preparedSettings) {
+ super(
+ InternalSettingsPreparer.prepareEnvironment(preparedSettings, null),
+ ImmutableList.of(
+ // install ParentJoin plugin required to create field of type "join"
+ ParentJoinPlugin.class));
+ }
+ }
}
package org.sonar.server.es;
import com.google.common.collect.Maps;
-import java.util.Map;
-import static org.sonar.server.es.FakeIndexDefinition.INT_FIELD;
+import static org.sonar.server.es.newindex.FakeIndexDefinition.INT_FIELD;
+import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE;
public class FakeDoc extends BaseDoc {
- public FakeDoc(Map<String, Object> fields) {
- super(fields);
+ public FakeDoc() {
+ super(TYPE_FAKE, Maps.newHashMap());
}
@Override
return null;
}
- @Override
- public String getRouting() {
- return null;
- }
-
- @Override
- public String getParent() {
- return null;
- }
-
- public FakeDoc() {
- super(Maps.newHashMap());
- }
-
public int getInt() {
return getField(INT_FIELD);
}
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.sonar.api.config.internal.MapSettings;
-
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
-
-public class FakeIndexDefinition implements IndexDefinition {
-
- public static final String INDEX = "fakes";
- public static final String TYPE = "fake";
- public static final IndexType INDEX_TYPE_FAKE = new IndexType("fakes", "fake");
- public static final String INT_FIELD = "intField";
-
- private int replicas = 0;
-
- public FakeIndexDefinition setReplicas(int replicas) {
- this.replicas = replicas;
- return this;
- }
-
- @Override
- public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(INDEX, newBuilder(new MapSettings().asConfig()).build());
- index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas);
- index.getSettings().put("index.refresh_interval", "-1");
- NewIndex.NewIndexType type = index.createType(INDEX_TYPE_FAKE.getType());
- type.createIntegerField(INT_FIELD);
- }
-
- public static FakeDoc newDoc(int value) {
- return new FakeDoc().setInt(value);
- }
-}
*/
package org.sonar.server.es;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.util.Locale;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.newindex.SettingsConfiguration;
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.fail;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+@RunWith(DataProviderRunner.class)
public class IndexDefinitionContextTest {
- private NewIndex.SettingsConfiguration emptySettingsConfiguration = newBuilder(new MapSettings().asConfig()).build();
+ private SettingsConfiguration emptySettingsConfiguration = newBuilder(new MapSettings().asConfig()).build();
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
@Test
public void create_indices() {
IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
- context.create("issues", emptySettingsConfiguration);
- context.create("measures", emptySettingsConfiguration);
- assertThat(context.getIndices().keySet()).containsOnly("issues", "measures");
+ context.create(Index.withRelations("issues"), emptySettingsConfiguration);
+ context.create(Index.simple("users"), emptySettingsConfiguration);
+ assertThat(context.getIndices().keySet())
+ .containsOnly("issues", "users");
}
@Test
- public void fail_to_create_twice_the_same_index() {
+ @UseDataProvider("paarOfIndicesWithSameName")
+ public void fail_to_create_twice_index_with_given_name(Index index1, Index index2) {
IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
- context.create("issues", emptySettingsConfiguration);
- try {
- context.create("issues", emptySettingsConfiguration);
- fail();
- } catch (IllegalArgumentException ok) {
- assertThat(ok).hasMessage("Index already exists: issues");
- }
+ context.create(index1, emptySettingsConfiguration);
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index already exists: " + index1.getName());
+
+ context.create(index2, emptySettingsConfiguration);
+ }
+
+ @DataProvider
+ public static Object[][] paarOfIndicesWithSameName() {
+ String indexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH);
+ return new Object[][] {
+ {Index.simple(indexName), Index.simple(indexName)},
+ {Index.withRelations(indexName), Index.withRelations(indexName)},
+ {Index.simple(indexName), Index.withRelations(indexName)},
+ {Index.withRelations(indexName), Index.simple(indexName)},
+ };
}
}
*/
package org.sonar.server.es;
-import org.junit.Test;
-
import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Consumer;
+import org.junit.Test;
+import org.sonar.api.config.Configuration;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.newindex.SettingsConfiguration;
+import org.sonar.server.es.newindex.TestNewIndex;
+import org.sonar.server.es.newindex.TypeMapping;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
public class IndexDefinitionHashTest {
+ private final SettingsConfiguration settingsConfiguration = settingsConfigurationOf(new MapSettings());
+
+ @Test
+ public void hash_changes_if_mainType_is_different() {
+ Index simpleIndex = Index.simple("foo");
+ Index withRelationsIndex = Index.withRelations("foo");
+ IndexMainType mainTypeBar = IndexMainType.main(simpleIndex, "bar");
+ TestNewIndex indexSimpleBar = new TestNewIndex(mainTypeBar, settingsConfiguration);
+ TestNewIndex indexSimpleDonut = new TestNewIndex(IndexMainType.main(simpleIndex, "donut"), settingsConfiguration);
+ TestNewIndex indexWithRelationsBar = new TestNewIndex(IndexMainType.main(withRelationsIndex, "bar"), settingsConfiguration);
+
+ assertThat(hashOf(indexSimpleBar))
+ .isEqualTo(hashOf(new TestNewIndex(mainTypeBar, settingsConfiguration)))
+ .isNotEqualTo(hashOf(indexSimpleDonut))
+ .isNotEqualTo(hashOf(indexWithRelationsBar));
+ assertThat(hashOf(indexSimpleDonut))
+ .isNotEqualTo(hashOf(indexWithRelationsBar));
+ }
+
+ @Test
+ public void hash_changes_if_relations_are_different() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ TestNewIndex indexNoRelation = new TestNewIndex(mainType, settingsConfiguration);
+ TestNewIndex indexOneRelation = new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut1");
+ TestNewIndex indexOneOtherRelation = new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut2");
+ TestNewIndex indexTwoRelations = new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut1")
+ .addRelation("donut2");
+ TestNewIndex indexTwoOtherRelations = new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut1")
+ .addRelation("donut3");
+
+ assertThat(hashOf(indexNoRelation))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration)))
+ .isNotEqualTo(hashOf(indexOneRelation))
+ .isNotEqualTo(hashOf(indexOneOtherRelation))
+ .isNotEqualTo(hashOf(indexTwoRelations))
+ .isNotEqualTo(hashOf(indexTwoOtherRelations));
+ assertThat(hashOf(indexOneRelation))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration).addRelation("donut1")))
+ .isNotEqualTo(hashOf(indexOneOtherRelation))
+ .isNotEqualTo(hashOf(indexTwoRelations))
+ .isNotEqualTo(hashOf(indexTwoOtherRelations));
+ assertThat(hashOf(indexTwoRelations))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut1")
+ .addRelation("donut2")))
+ .isNotEqualTo(hashOf(indexOneRelation))
+ .isNotEqualTo(hashOf(indexOneOtherRelation))
+ .isNotEqualTo(hashOf(indexTwoOtherRelations));
+ }
+
+ @Test
+ public void hash_is_the_same_if_only_relations_order_changes() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ TestNewIndex indexTwoRelations = new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut1")
+ .addRelation("donut2")
+ .addRelation("donut3");
+ TestNewIndex indexTwoRelationsOtherOrder = new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut2")
+ .addRelation("donut1")
+ .addRelation("donut3");
+ TestNewIndex indexTwoRelationsOtherOrder2 = new TestNewIndex(mainType, settingsConfiguration)
+ .addRelation("donut2")
+ .addRelation("donut3")
+ .addRelation("donut1");
+
+ assertThat(hashOf(indexTwoRelations))
+ .isEqualTo(hashOf(indexTwoRelationsOtherOrder))
+ .isEqualTo(hashOf(indexTwoRelationsOtherOrder2));
+ }
+
+ @Test
+ public void hash_changes_if_fields_on_main_type_mapping_are_different() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ TestNewIndex indexNoField = new TestNewIndex(mainType, settingsConfiguration);
+ TestNewIndex indexOneField = new TestNewIndex(mainType, settingsConfiguration);
+ indexOneField.getMainTypeMapping()
+ .createIntegerField("field1");
+ TestNewIndex indexOneFieldAgain = new TestNewIndex(mainType, settingsConfiguration);
+ indexOneFieldAgain.getMainTypeMapping()
+ .createIntegerField("field1");
+ TestNewIndex indexOneOtherField = new TestNewIndex(mainType, settingsConfiguration);
+ indexOneOtherField.getMainTypeMapping()
+ .createIntegerField("field2");
+ TestNewIndex indexTwoFields = new TestNewIndex(mainType, settingsConfiguration);
+ indexTwoFields.getMainTypeMapping()
+ .createIntegerField("field1")
+ .createIntegerField("field2");
+ TestNewIndex indexTwoFieldsAgain = new TestNewIndex(mainType, settingsConfiguration);
+ indexTwoFieldsAgain.getMainTypeMapping()
+ .createIntegerField("field1")
+ .createIntegerField("field2");
+ TestNewIndex indexTwoOtherFields = new TestNewIndex(mainType, settingsConfiguration);
+ indexTwoOtherFields.getMainTypeMapping()
+ .createIntegerField("field1")
+ .createIntegerField("field3");
+
+ assertThat(hashOf(indexNoField))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration)))
+ .isNotEqualTo(hashOf(indexOneField))
+ .isNotEqualTo(hashOf(indexOneOtherField))
+ .isNotEqualTo(hashOf(indexTwoFields))
+ .isNotEqualTo(hashOf(indexTwoOtherFields));
+ assertThat(hashOf(indexOneField))
+ .isEqualTo(hashOf(indexOneFieldAgain))
+ .isNotEqualTo(hashOf(indexOneOtherField))
+ .isNotEqualTo(hashOf(indexTwoFields))
+ .isNotEqualTo(hashOf(indexTwoOtherFields));
+ assertThat(hashOf(indexTwoFields))
+ .isEqualTo(hashOf(indexTwoFieldsAgain))
+ .isNotEqualTo(hashOf(indexOneField))
+ .isNotEqualTo(hashOf(indexOneOtherField))
+ .isNotEqualTo(hashOf(indexTwoOtherFields));
+ }
+
+ @Test
+ public void hash_is_the_same_if_only_fields_order_changes() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ TestNewIndex indexTwoFields = new TestNewIndex(mainType, settingsConfiguration);
+ indexTwoFields.getMainTypeMapping()
+ .createBooleanField("donut1")
+ .createBooleanField("donut2")
+ .createBooleanField("donut3");
+ TestNewIndex indexTwoFieldsOtherOrder = new TestNewIndex(mainType, settingsConfiguration);
+ indexTwoFieldsOtherOrder.getMainTypeMapping()
+ .createBooleanField("donut2")
+ .createBooleanField("donut1")
+ .createBooleanField("donut3");
+ TestNewIndex indexTwoFieldsOtherOrder2 = new TestNewIndex(mainType, settingsConfiguration);
+ indexTwoFieldsOtherOrder2.getMainTypeMapping()
+ .createBooleanField("donut2")
+ .createBooleanField("donut3")
+ .createBooleanField("donut1");
+
+ assertThat(hashOf(indexTwoFields))
+ .isEqualTo(hashOf(indexTwoFieldsOtherOrder))
+ .isEqualTo(hashOf(indexTwoFieldsOtherOrder2));
+ }
+
+ @Test
+ public void hash_changes_if_field_type_changes() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ String fieldName = "field1";
+
+ computeAndVerifyAllDifferentHashesOnMapping(mainType,
+ (mapping) -> mapping.createBooleanField(fieldName),
+ (mapping) -> mapping.createIntegerField(fieldName),
+ (mapping) -> mapping.createByteField(fieldName),
+ (mapping) -> mapping.createDateTimeField(fieldName),
+ (mapping) -> mapping.createDoubleField(fieldName),
+ (mapping) -> mapping.createLongField(fieldName),
+ (mapping) -> mapping.createShortField(fieldName),
+ (mapping) -> mapping.createUuidPathField(fieldName),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).build(),
+ (mapping) -> mapping.textFieldBuilder(fieldName).build(),
+ (mapping) -> mapping.nestedFieldBuilder(fieldName).addKeywordField("bar").build());
+ }
+
+ @Test
+ public void hash_changes_if_keyword_options_change() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ String fieldName = "field1";
+
+ computeAndVerifyAllDifferentHashesOnMapping(mainType,
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableNorms().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableSearch().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableNorms().disableSearch().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSearch().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().build());
+ }
+
+ @Test
+ public void hash_is_the_same_if_only_order_of_keyword_options_change() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ String fieldName = "field1";
+
+ computeAndVerifyAllSameHashesOnMapping(mainType,
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableNorms().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSortingAndAggregating().build());
+ computeAndVerifyAllSameHashesOnMapping(mainType,
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableSearch().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableSortingAndAggregating().build());
+ computeAndVerifyAllSameHashesOnMapping(mainType,
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableNorms().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSearch().build());
+ computeAndVerifyAllSameHashesOnMapping(mainType,
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableSearch().disableNorms().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableNorms().disableSortingAndAggregating().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSearch().disableSortingAndAggregating().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSortingAndAggregating().disableSearch().build(),
+ (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableSortingAndAggregating().disableNorms().build());
+ }
+
+ @Test
+ public void hash_changes_if_textFieldBuilder_options_change() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ String fieldName = "field1";
+
+ computeAndVerifyAllDifferentHashesOnMapping(mainType,
+ (mapping) -> mapping.textFieldBuilder(fieldName).build(),
+ (mapping) -> mapping.textFieldBuilder(fieldName).disableSearch().build(),
+ (mapping) -> mapping.textFieldBuilder(fieldName).disableNorms().build(),
+ (mapping) -> mapping.textFieldBuilder(fieldName).disableNorms().disableSearch().build());
+ }
+
+ @Test
+ public void hash_is_the_same_if_only_order_of_textFieldBuilder_options_change() {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ String fieldName = "field1";
+
+ computeAndVerifyAllSameHashesOnMapping(mainType,
+ (mapping) -> mapping.textFieldBuilder(fieldName).disableSearch().disableNorms().build(),
+ (mapping) -> mapping.textFieldBuilder(fieldName).disableNorms().disableSearch().build());
+ }
+
+ @SafeVarargs
+ private final void computeAndVerifyAllSameHashesOnMapping(IndexMainType mainType, Consumer<TypeMapping>... fieldTypes) {
+ List<Consumer<TypeMapping>> fieldTypes1 = Arrays.asList(fieldTypes);
+ List<TestNewIndex> mainIndices = fieldTypes1.stream()
+ .map(consumer -> {
+ TestNewIndex mainTypeMapping = new TestNewIndex(mainType, settingsConfiguration);
+ consumer.accept(mainTypeMapping.getMainTypeMapping());
+ return mainTypeMapping;
+ })
+ .collect(toList());
+ List<TestNewIndex> relationIndices = fieldTypes1.stream()
+ .map(consumer -> {
+ TestNewIndex relationTypeMapping = new TestNewIndex(mainType, settingsConfiguration);
+ consumer.accept(relationTypeMapping.createRelationMapping("donut"));
+ return relationTypeMapping;
+ })
+ .collect(toList());
+
+ Set<String> mainHashes = mainIndices.stream()
+ .map(IndexDefinitionHashTest::hashOf)
+ .collect(toSet());
+ Set<String> relationHashes = relationIndices.stream()
+ .map(IndexDefinitionHashTest::hashOf)
+ .collect(toSet());
+
+ assertThat(mainHashes)
+ // verify hashing is stable
+ .isEqualTo(mainIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet()))
+ .doesNotContainAnyElementsOf(relationHashes)
+ .hasSize(1);
+ assertThat(relationHashes)
+ // verify hashing is stable
+ .isEqualTo(relationIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet()))
+ .doesNotContainAnyElementsOf(mainHashes)
+ .hasSize(1);
+ }
+
+ @SafeVarargs
+ private final void computeAndVerifyAllDifferentHashesOnMapping(IndexMainType mainType, Consumer<TypeMapping>... fieldTypes) {
+ List<TestNewIndex> mainIndices = Arrays.stream(fieldTypes)
+ .map(consumer -> {
+ TestNewIndex mainTypeMapping = new TestNewIndex(mainType, settingsConfiguration);
+ consumer.accept(mainTypeMapping.getMainTypeMapping());
+ return mainTypeMapping;
+ })
+ .collect(toList());
+ List<TestNewIndex> relationIndices = Arrays.stream(fieldTypes)
+ .map(consumer -> {
+ TestNewIndex relationTypeMapping = new TestNewIndex(mainType, settingsConfiguration);
+ consumer.accept(relationTypeMapping.createRelationMapping("donut"));
+ return relationTypeMapping;
+ })
+ .collect(toList());
+
+ Set<String> mainHashes = mainIndices.stream()
+ .map(IndexDefinitionHashTest::hashOf)
+ .collect(toSet());
+ Set<String> relationHashes = relationIndices.stream()
+ .map(IndexDefinitionHashTest::hashOf)
+ .collect(toSet());
+
+ assertThat(mainHashes)
+ // verify hashing is stable
+ .isEqualTo(mainIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet()))
+ .doesNotContainAnyElementsOf(relationHashes)
+ .hasSize(fieldTypes.length);
+ assertThat(relationHashes)
+ // verify hashing is stable
+ .isEqualTo(relationIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet()))
+ .doesNotContainAnyElementsOf(mainHashes)
+ .hasSize(fieldTypes.length);
+ }
@Test
- public void of() {
- IndexDefinition.Index indexV1 = new IndexDefinition.Index(createIndex());
- String hashV1 = IndexDefinitionHash.of(indexV1);
- assertThat(hashV1).isNotEmpty();
- // always the same
- assertThat(hashV1).isEqualTo(IndexDefinitionHash.of(indexV1));
+ public void hash_changes_if_clustering_is_enabled_or_not() {
+ Index index = Index.simple("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ MapSettings empty = new MapSettings();
+ MapSettings clusterDisabled = new MapSettings().setProperty(CLUSTER_ENABLED.getKey(), false);
+ MapSettings clusterEnabled = new MapSettings().setProperty(CLUSTER_ENABLED.getKey(), true);
- NewIndex newIndexV2 = createIndex();
- newIndexV2.getTypes().get("fake").createIntegerField("max");
- String hashV2 = IndexDefinitionHash.of(new IndexDefinition.Index(newIndexV2));
- assertThat(hashV2).isNotEmpty().isNotEqualTo(hashV1);
+ assertThat(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(empty))))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(empty))))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(clusterDisabled))))
+ .isNotEqualTo(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(clusterEnabled))));
}
- private NewIndex createIndex() {
- NewIndex newIndex = new NewIndex("fakes", newBuilder(new MapSettings().asConfig()).build());
- NewIndex.NewIndexType mapping = newIndex.createType("fake");
- mapping.setAttribute("list_attr", Arrays.asList("foo", "bar"));
- mapping.keywordFieldBuilder("key").build();
- mapping.createDateTimeField("updatedAt");
- return newIndex;
+ @Test
+ public void hash_changes_if_number_of_shards_changes() {
+ Index index = Index.simple("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ Configuration emptySettings = new MapSettings().asConfig();
+ SettingsConfiguration defaultNbOfShards = SettingsConfiguration.newBuilder(emptySettings)
+ .build();
+ SettingsConfiguration specifiedDefaultNbOfShards = SettingsConfiguration.newBuilder(emptySettings)
+ .setDefaultNbOfShards(5)
+ .build();
+ SettingsConfiguration specifyDefaultNbOfShards = SettingsConfiguration.newBuilder(new MapSettings()
+ .setProperty("sonar.search." + index.getName() + ".shards", 1)
+ .asConfig())
+ .setDefaultNbOfShards(1)
+ .build();
+ SettingsConfiguration specifiedNbOfShards = SettingsConfiguration.newBuilder(new MapSettings()
+ .setProperty("sonar.search." + index.getName() + ".shards", 10)
+ .asConfig())
+ .setDefaultNbOfShards(5)
+ .build();
+
+ assertThat(hashOf(new TestNewIndex(mainType, defaultNbOfShards)))
+ // verify hash is stable
+ .isEqualTo(hashOf(new TestNewIndex(mainType, defaultNbOfShards)))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, specifyDefaultNbOfShards)))
+ .isNotEqualTo(hashOf(new TestNewIndex(mainType, specifiedDefaultNbOfShards)))
+ .isNotEqualTo(hashOf(new TestNewIndex(mainType, specifiedNbOfShards)));
+ assertThat(hashOf(new TestNewIndex(mainType, specifiedDefaultNbOfShards)))
+ // verify hash is stable
+ .isEqualTo(hashOf(new TestNewIndex(mainType, specifiedDefaultNbOfShards)))
+ .isNotEqualTo(hashOf(new TestNewIndex(mainType, specifyDefaultNbOfShards)));
}
+ @Test
+ public void hash_changes_if_refreshInterval_changes() {
+ Index index = Index.simple("foo");
+ IndexMainType mainType = IndexMainType.main(index, "bar");
+ Configuration emptySettings = new MapSettings().asConfig();
+ SettingsConfiguration defaultRefreshInterval = SettingsConfiguration.newBuilder(emptySettings)
+ .build();
+ SettingsConfiguration noRefreshInterval = SettingsConfiguration.newBuilder(emptySettings)
+ .setRefreshInterval(-1)
+ .build();
+ SettingsConfiguration refreshInterval30 = SettingsConfiguration.newBuilder(emptySettings)
+ .setRefreshInterval(30)
+ .build();
+ SettingsConfiguration someRefreshInterval = SettingsConfiguration.newBuilder(emptySettings)
+ .setRefreshInterval(56)
+ .build();
+
+ assertThat(hashOf(new TestNewIndex(mainType, defaultRefreshInterval)))
+ // verify hash is stable
+ .isEqualTo(hashOf(new TestNewIndex(mainType, defaultRefreshInterval)))
+ .isEqualTo(hashOf(new TestNewIndex(mainType, refreshInterval30)))
+ .isNotEqualTo(hashOf(new TestNewIndex(mainType, noRefreshInterval)))
+ .isNotEqualTo(hashOf(new TestNewIndex(mainType, someRefreshInterval)));
+ assertThat(hashOf(new TestNewIndex(mainType, noRefreshInterval)))
+ .isNotEqualTo(hashOf(new TestNewIndex(mainType, someRefreshInterval)));
+ }
+
+ private static SettingsConfiguration settingsConfigurationOf(MapSettings settings) {
+ return SettingsConfiguration.newBuilder(settings.asConfig()).build();
+ }
+
+ private static String hashOf(TestNewIndex newIndex) {
+ return IndexDefinitionHash.of(newIndex.build());
+ }
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.util.Locale;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+
+@RunWith(DataProviderRunner.class)
+public class IndexTest {
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ @UseDataProvider("nullOrEmpty")
+ public void simple_index_constructor_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index name can't be null nor empty");
+
+ Index.simple(nullOrEmpty);
+ }
+
+ @Test
+ @UseDataProvider("nullOrEmpty")
+ public void withRelations_index_constructor_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index name can't be null nor empty");
+
+ Index.withRelations(nullOrEmpty);
+
+ }
+
+ @DataProvider
+ public static Object[][] nullOrEmpty() {
+ return new Object[][] {
+ {null},
+ {""}
+ };
+ }
+
+ @Test
+ public void simple_index_name_must_not_contain_upper_case_char() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index name must be lower-case letters or '_all': Issues");
+
+ Index.simple("Issues");
+ }
+
+ @Test
+ public void withRelations_index_name_must_not_contain_upper_case_char() {
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index name must be lower-case letters or '_all': Issues");
+
+ Index.withRelations("Issues");
+ }
+
+ @Test
+ public void simple_index_name_can_not_contain_underscore_except__all_keyword() {
+ // doesn't fail
+ Index.simple("_all");
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index name must be lower-case letters or '_all': _");
+
+ Index.simple("_");
+ }
+
+ @Test
+ public void withRelations_index_name_can_not_contain_underscore_except__all_keyword() {
+ // doesn't fail
+ Index.withRelations("_all");
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index name must be lower-case letters or '_all': _");
+
+ Index.withRelations("_");
+ }
+
+ @Test
+ public void simple_index_does_not_accept_relations() {
+ Index underTest = Index.simple("foo");
+
+ assertThat(underTest.acceptsRelations()).isFalse();
+ }
+
+ @Test
+ public void withRelations_index_does_not_accept_relations() {
+ Index underTest = Index.withRelations("foo");
+
+ assertThat(underTest.acceptsRelations()).isTrue();
+ }
+
+ @Test
+ public void getName_returns_constructor_parameter() {
+ String indexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH);
+
+ assertThat(Index.simple(indexName).getName()).isEqualTo(indexName);
+ assertThat(Index.withRelations(indexName).getName()).isEqualTo(indexName);
+ }
+
+ @Test
+ public void getJoinField_throws_ISE_on_simple_index() {
+ Index underTest = Index.simple("foo");
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("Only index accepting relations has a join field");
+
+ underTest.getJoinField();
+ }
+
+ @Test
+ public void getJoinField_returns_name_based_on_index_name() {
+ String indexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH);
+ Index underTest = Index.withRelations(indexName);
+
+ assertThat(underTest.getJoinField()).isEqualTo("join_" + indexName);
+ }
+
+ @Test
+ public void equals_is_based_on_name_and_acceptRelations_flag() {
+ assertThat(Index.simple("foo"))
+ .isEqualTo(Index.simple("foo"))
+ .isNotEqualTo(Index.simple("bar"))
+ .isNotEqualTo(Index.withRelations("foo"));
+ }
+
+ @Test
+ public void hashcode_is_based_on_name_and_acceptRelations_flag() {
+ assertThat(Index.simple("foo").hashCode())
+ .isEqualTo(Index.simple("foo").hashCode())
+ .isNotEqualTo(Index.simple("bar").hashCode())
+ .isNotEqualTo(Index.withRelations("foo").hashCode());
+ }
+
+}
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
- package org.sonar.server.es;
+package org.sonar.server.es;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.IndexType.IndexRelationType;
+import org.sonar.server.es.IndexType.SimpleIndexMainType;
import static org.assertj.core.api.Assertions.assertThat;
+@RunWith(DataProviderRunner.class)
public class IndexTypeTest {
@Rule
public ExpectedException expectedException = ExpectedException.none();
@Test
- public void format_and_parse() {
- IndexType type1 = new IndexType("foo", "bar");
+ public void parseMainType_from_main_type_without_relations() {
+ IndexMainType type1 = IndexType.main(Index.simple("foo"), "bar");
assertThat(type1.format()).isEqualTo("foo/bar");
- IndexType type2 = IndexType.parse(type1.format());
- assertThat(type2.equals(type1)).isTrue();
+ SimpleIndexMainType type2 = IndexType.parseMainType(type1.format());
+ assertThat(type2)
+ .extracting(SimpleIndexMainType::getIndex, SimpleIndexMainType::getType)
+ .containsExactly("foo", "bar");
+ }
+
+ @Test
+ public void parseMainType_from_maintype_with_relations() {
+ IndexMainType type1 = IndexType.main(Index.withRelations("foo"), "bar");
+ assertThat(type1.format()).isEqualTo("foo/bar");
+
+ SimpleIndexMainType type2 = IndexType.parseMainType(type1.format());
+ assertThat(type2)
+ .extracting(SimpleIndexMainType::getIndex, SimpleIndexMainType::getType)
+ .containsExactly("foo", "bar");
+ }
+
+ @Test
+ public void parseMainType_from_relationtype() {
+ IndexMainType mainType = IndexType.main(Index.withRelations("foo"), "bar");
+ IndexRelationType type1 = IndexType.relation(mainType, "donut");
+ assertThat(type1.format()).isEqualTo("foo/bar/donut");
+
+ SimpleIndexMainType type2 = IndexType.parseMainType(type1.format());
+ assertThat(type2)
+ .extracting(SimpleIndexMainType::getIndex, SimpleIndexMainType::getType)
+ .containsExactly("foo", "bar");
}
@Test
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Unsupported IndexType value: foo");
- IndexType.parse("foo");
+ IndexType.parseMainType("foo");
+ }
+
+ @Test
+ @UseDataProvider("nullOrEmpty")
+ public void main_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) {
+ Index index = Index.simple("foo");
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("type name can't be null nor empty");
+
+ IndexType.main(index, nullOrEmpty);
+ }
+
+ @Test
+ @UseDataProvider("nullOrEmpty")
+ public void relation_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) {
+ Index index = Index.withRelations("foo");
+ IndexMainType mainType = IndexType.main(index, "foobar");
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("type name can't be null nor empty");
+
+ IndexType.relation(mainType, nullOrEmpty);
+ }
+
+ @DataProvider
+ public static Object[][] nullOrEmpty() {
+ return new Object[][] {
+ {null},
+ {""}
+ };
+ }
+
+ @Test
+ public void IndexMainType_equals_and_hashCode() {
+ IndexMainType type1 = IndexType.main(Index.simple("foo"), "bar");
+ IndexMainType type1b = IndexType.main(Index.simple("foo"), "bar");
+ IndexMainType type1c = IndexType.main(Index.withRelations("foo"), "bar");
+ IndexMainType type2 = IndexType.main(Index.simple("foo"), "baz");
+
+ assertThat(type1)
+ .isEqualTo(type1)
+ .isEqualTo(type1b)
+ .isNotEqualTo(type1c)
+ .isNotEqualTo(type2);
+
+ assertThat(type1.hashCode()).isEqualTo(type1.hashCode());
+ assertThat(type1.hashCode()).isEqualTo(type1b.hashCode());
+ assertThat(type1.hashCode()).isNotEqualTo(type1c.hashCode());
+ assertThat(type2.hashCode()).isNotEqualTo(type1.hashCode());
}
@Test
- public void equals_and_hashCode() {
- IndexType type1 = new IndexType("foo", "bar");
- IndexType type1b = new IndexType("foo", "bar");
- IndexType type2 = new IndexType("foo", "baz");
+ public void IndexRelationType_equals_and_hashCode() {
+ IndexMainType mainType1 = IndexType.main(Index.withRelations("foo"), "bar");
+ IndexMainType mainType2 = IndexType.main(Index.withRelations("foo"), "baz");
+ IndexRelationType type1 = IndexType.relation(mainType1, "donut");
+ IndexRelationType type1b = IndexType.relation(mainType1, "donut");
+ IndexRelationType type2 = IndexType.relation(mainType1, "donuz");
+ IndexRelationType type3 = IndexType.relation(mainType2, "donut");
- assertThat(type1.equals(type1)).isTrue();
- assertThat(type1.equals(type1b)).isTrue();
- assertThat(type1.equals(type2)).isFalse();
+ assertThat(type1)
+ .isEqualTo(type1)
+ .isEqualTo(type1b)
+ .isNotEqualTo(type2)
+ .isNotEqualTo(type3);
assertThat(type1.hashCode()).isEqualTo(type1.hashCode());
assertThat(type1.hashCode()).isEqualTo(type1b.hashCode());
+ assertThat(type2.hashCode()).isNotEqualTo(type1.hashCode());
+ assertThat(type3.hashCode())
+ .isNotEqualTo(type2.hashCode())
+ .isNotEqualTo(type1.hashCode());
}
}
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.sonar.api.config.Configuration;
+import org.sonar.server.es.newindex.SettingsConfiguration;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
public class NewIndexSettingsConfigurationTest {
@Rule
@Test
public void setDefaultNbOfShards_fails_with_IAE_if_argument_is_zero() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("defaultNbOfShards must be >= 1");
@Test
public void setDefaultNbOfShards_fails_with_IAE_if_argument_is_less_than_zero() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("defaultNbOfShards must be >= 1");
@Test
public void setDefaultNbOfShards_accepts_1() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
assertThat(underTest.setDefaultNbOfShards(1).build().getDefaultNbOfShards()).isEqualTo(1);
}
@Test
public void setDefaultNbOfShards_accepts_any_int_greater_than_1() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
int value = 1 + new Random().nextInt(200);
@Test
public void setRefreshInterval_fails_with_IAE_if_argument_is_zero() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("refreshInterval must be either -1 or strictly positive");
@Test
public void setRefreshInterval_fails_with_IAE_if_argument_is_less_than_minus_1() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("refreshInterval must be either -1 or strictly positive");
@Test
public void setRefreshInterval_accepts_minus_1() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
assertThat(underTest.setRefreshInterval(-1).build().getRefreshInterval()).isEqualTo(-1);
}
@Test
public void setRefreshInterval_accepts_any_int_greater_than_1() {
- NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
+ SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration);
int value = 1 + new Random().nextInt(200);
+++ /dev/null
-/*
- * SonarQube
- * Copyright (C) 2009-2019 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.server.es;
-
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.sonar.api.config.internal.MapSettings;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.data.MapEntry.entry;
-import static org.junit.Assert.fail;
-import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
-import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
-
-public class NewIndexTest {
-
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- private MapSettings settings = new MapSettings();
- private NewIndex.SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build();
-
- @Test
- public void getName_returns_constructor_argument() {
- assertThat(new NewIndex("foo", defaultSettingsConfiguration).getName()).isEqualTo("foo");
- }
-
- @Test
- public void no_types_of_none_are_specified() {
- assertThat(new NewIndex("foo", defaultSettingsConfiguration).getTypes()).isEmpty();
- }
-
- @Test
- public void verify_default_index_settings_in_standalone() {
- Settings underTest = new NewIndex("issues", defaultSettingsConfiguration).getSettings().build();
-
- assertThat(underTest.get("index.number_of_shards")).isNotEmpty();
- assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false");
- assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s");
- assertThat(underTest.get("index.number_of_shards")).isEqualTo("1");
- assertThat(underTest.get("index.number_of_replicas")).isEqualTo("0");
- }
-
- @Test
- public void verify_default_index_settings_in_cluster() {
- settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
- Settings underTest = new NewIndex("issues", defaultSettingsConfiguration).getSettings().build();
-
- assertThat(underTest.get("index.number_of_shards")).isNotEmpty();
- assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false");
- assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s");
- assertThat(underTest.get("index.number_of_shards")).isEqualTo("1");
- assertThat(underTest.get("index.number_of_replicas")).isEqualTo("1");
- }
-
- @Test
- public void index_name_is_lower_case() {
- try {
- new NewIndex("Issues", defaultSettingsConfiguration);
- fail();
- } catch (IllegalArgumentException e) {
- assertThat(e).hasMessage("Index name must be lower-case: Issues");
- }
- }
-
- @Test
- public void define_fields() {
- NewIndex index = new NewIndex("issues", defaultSettingsConfiguration);
- NewIndex.NewIndexType mapping = index.createType("issue");
- mapping.setAttribute("dynamic", "true");
- mapping.setProperty("foo_field", ImmutableMap.of("type", "keyword"));
- mapping.createBooleanField("boolean_field");
- mapping.createByteField("byte_field");
- mapping.createDateTimeField("dt_field");
- mapping.createDoubleField("double_field");
- mapping.createIntegerField("int_field");
- mapping.createLongField("long_field");
- mapping.createShortField("short_field");
- mapping.createUuidPathField("uuid_path_field");
-
- mapping = index.getTypes().get("issue");
- assertThat(mapping).isNotNull();
- assertThat(mapping.getAttributes().get("dynamic")).isEqualTo("true");
- assertThat(mapping.getProperty("foo_field")).isInstanceOf(Map.class);
- assertThat((Map) mapping.getProperty("foo_field")).containsEntry("type", "keyword");
- assertThat((Map) mapping.getProperty("byte_field")).isNotEmpty();
- assertThat((Map) mapping.getProperty("double_field")).isNotEmpty();
- assertThat((Map) mapping.getProperty("dt_field")).isNotEmpty();
- assertThat((Map) mapping.getProperty("int_field")).containsEntry("type", "integer");
- assertThat((Map) mapping.getProperty("long_field")).isNotEmpty();
- assertThat((Map) mapping.getProperty("short_field")).isNotEmpty();
- assertThat((Map) mapping.getProperty("uuid_path_field")).isNotEmpty();
- assertThat((Map) mapping.getProperty("unknown")).isNull();
- }
-
- @Test
- public void define_string_field() {
- NewIndex index = new NewIndex("issues", defaultSettingsConfiguration);
- NewIndex.NewIndexType mapping = index.createType("issue");
- mapping.keywordFieldBuilder("basic_field").build();
- mapping.keywordFieldBuilder("not_searchable_field").disableSearch().build();
- mapping.keywordFieldBuilder("all_capabilities_field")
- .addSubFields(
- DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER,
- DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER,
- DefaultIndexSettingsElement.SORTABLE_ANALYZER)
- .build();
- mapping.keywordFieldBuilder("dumb_text_storage")
- .disableSearch()
- .disableNorms()
- .disableSortingAndAggregating()
- .build();
-
- Map<String, Object> props = (Map) mapping.getProperty("basic_field");
- assertThat(props.get("type")).isEqualTo("keyword");
- assertThat(props.get("index")).isEqualTo("true");
- assertThat(props.get("fields")).isNull();
-
- props = (Map) mapping.getProperty("not_searchable_field");
- assertThat(props.get("type")).isEqualTo("keyword");
- assertThat(props.get("index")).isEqualTo("false");
- assertThat(props.get("norms")).isEqualTo("true");
- assertThat(props.get("store")).isEqualTo("false");
- assertThat(props.get("doc_values")).isEqualTo("true");
- assertThat(props.get("fields")).isNull();
-
- props = (Map) mapping.getProperty("all_capabilities_field");
- assertThat(props.get("type")).isEqualTo("keyword");
- // no need to test values, it's not the scope of this test
- assertThat((Map) props.get("fields")).isNotEmpty();
-
- props = (Map) mapping.getProperty("dumb_text_storage");
- assertThat(props.get("type")).isEqualTo("keyword");
- assertThat(props.get("index")).isEqualTo("false");
- assertThat(props.get("norms")).isEqualTo("false");
- assertThat(props.get("store")).isEqualTo("false");
- assertThat(props.get("doc_values")).isEqualTo("false");
- assertThat(props.get("fields")).isNull();
- }
-
- @Test
- public void define_nested_field() {
- NewIndex index = new NewIndex("projectmeasures", defaultSettingsConfiguration);
- NewIndex.NewIndexType mapping = index.createType("projectmeasures");
-
- mapping.nestedFieldBuilder("measures")
- .addKeywordField("key")
- .addDoubleField("value")
- .build();
- Map<String, Object> result = (Map) mapping.getProperty("measures");
-
- assertThat(result.get("type")).isEqualTo("nested");
- Map<String, Map<String, Object>> subProperties = (Map) result.get("properties");
- assertThat(subProperties.get("key").get("type")).isEqualTo("keyword");
- assertThat(subProperties.get("value").get("type")).isEqualTo("double");
- }
-
- @Test
- public void fail_when_nested_with_no_field() {
- expectedException.expect(IllegalArgumentException.class);
- expectedException.expectMessage("At least one sub-field must be declared in nested property 'measures'");
-
- NewIndex index = new NewIndex("projectmeasures", defaultSettingsConfiguration);
- NewIndex.NewIndexType mapping = index.createType("project_measures");
-
- mapping.nestedFieldBuilder("measures").build();
- }
-
- @Test
- public void use_doc_values_by_default() {
- NewIndex index = new NewIndex("issues", defaultSettingsConfiguration);
- NewIndex.NewIndexType mapping = index.createType("issue");
- mapping.keywordFieldBuilder("the_doc_value").build();
-
- Map<String, Object> props = (Map) mapping.getProperty("the_doc_value");
- assertThat(props.get("type")).isEqualTo("keyword");
- assertThat(props.get("doc_values")).isEqualTo("true");
- }
-
- @Test
- public void default_shards_and_replicas() {
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5");
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
- }
-
- @Test
- public void five_shards_and_one_replica_by_default_on_cluster() {
- settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5");
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1");
- }
-
- @Test
- public void customize_number_of_shards() {
- settings.setProperty("sonar.search.issues.shards", "3");
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3");
- // keep default value
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
- }
-
- @Test
- public void default_number_of_replicas_on_standalone_instance_must_be_0() {
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
- }
-
- @Test
- public void default_number_of_replicas_on_non_enabled_cluster_must_be_0() {
- settings.setProperty(CLUSTER_ENABLED.getKey(), "false");
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
- }
-
- @Test
- public void default_number_of_replicas_on_cluster_instance_must_be_1() {
- settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1");
- }
-
- @Test
- public void when_number_of_replicas_on_cluster_is_specified_to_zero_default_value_must_not_be_used() {
- settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
- settings.setProperty(SEARCH_REPLICAS.getKey(), "0");
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
- }
-
- @Test
- public void index_defined_with_specified_number_of_replicas_when_cluster_enabled() {
- settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
- settings.setProperty(SEARCH_REPLICAS.getKey(), "3");
- NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
-
- assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3");
- }
-
- @Test
- public void fail_when_replica_customization_cant_be_parsed() {
- settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
- settings.setProperty(SEARCH_REPLICAS.getKey(), "ꝱꝲꝳପ");
- NewIndex.SettingsConfiguration settingsConfiguration = newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build();
-
- expectedException.expect(IllegalStateException.class);
- expectedException.expectMessage("The property 'sonar.search.replicas' is not an int value: For input string: \"ꝱꝲꝳପ\"");
-
- new NewIndex("issues", settingsConfiguration);
- }
-
- @Test
- public void in_standalone_searchReplicas_is_not_overridable() {
- settings.setProperty(SEARCH_REPLICAS.getKey(), "5");
- NewIndex index = new NewIndex("issues", defaultSettingsConfiguration);
-
- assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0");
- }
-
- @Test
- public void index_with_source() {
- NewIndex index = new NewIndex("issues", defaultSettingsConfiguration);
- NewIndex.NewIndexType mapping = index.createType("issue");
- mapping.setEnableSource(true);
-
- mapping = index.getTypes().get("issue");
- assertThat(mapping).isNotNull();
- assertThat(getAttributeAsMap(mapping, "_source")).containsExactly(entry("enabled", true));
- }
-
- @Test
- public void index_without_source() {
- NewIndex index = new NewIndex("issues", defaultSettingsConfiguration);
- NewIndex.NewIndexType mapping = index.createType("issue");
- mapping.setEnableSource(false);
-
- mapping = index.getTypes().get("issue");
- assertThat(mapping).isNotNull();
- assertThat(getAttributeAsMap(mapping, "_source")).containsExactly(entry("enabled", false));
- }
-
- @Test
- public void index_requires_project_authorization() {
- NewIndex index = new NewIndex("issues", defaultSettingsConfiguration);
- index.createType("issue")
- // creates a second type "authorization" and configures _parent and _routing fields
- .requireProjectAuthorization();
-
- // issue type
- NewIndex.NewIndexType issueType = index.getTypes().get("issue");
- assertThat(getAttributeAsMap(issueType, "_parent")).containsExactly(entry("type", "authorization"));
- assertThat(getAttributeAsMap(issueType, "_routing")).containsExactly(entry("required", true));
-
- // authorization type
- NewIndex.NewIndexType authorizationType = index.getTypes().get("authorization");
- assertThat(getAttributeAsMap(authorizationType, "_parent")).isNull();
- assertThat(getAttributeAsMap(authorizationType, "_routing")).containsExactly(entry("required", true));
- }
-
- private static Map<String, Object> getAttributeAsMap(NewIndex.NewIndexType type, String attributeKey) {
- return (Map<String, Object>) type.getAttributes().get(attributeKey);
- }
-}
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
+import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE;
public class OneToManyResilientIndexingListenerTest {
@Test
public void ES_QUEUE_rows_are_deleted_when_all_docs_are_successfully_indexed() {
- EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "P1");
- EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, "P2");
- EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, "P1");
+ EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "P1");
+ EsQueueDto item2 = insertInQueue(TYPE_ISSUE, "P2");
+ EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.TYPE_COMPONENT, "P1");
db.commit();
// does not contain outOfScopeItem
IndexingListener underTest = newListener(asList(item1, item2));
- DocId issue1 = newDocId(INDEX_TYPE_ISSUE, "I1");
- DocId issue2 = newDocId(INDEX_TYPE_ISSUE, "I2");
+ DocId issue1 = newDocId(TYPE_ISSUE, "I1");
+ DocId issue2 = newDocId(TYPE_ISSUE, "I2");
underTest.onSuccess(asList(issue1, issue2));
assertThatEsTableContainsOnly(item1, item2, outOfScopeItem);
@Test
public void ES_QUEUE_rows_are_not_deleted_on_partial_error() {
- EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "P1");
- EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, "P2");
- EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, "P1");
+ EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "P1");
+ EsQueueDto item2 = insertInQueue(TYPE_ISSUE, "P2");
+ EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.TYPE_COMPONENT, "P1");
db.commit();
// does not contain outOfScopeItem
IndexingListener underTest = newListener(asList(item1, item2));
- DocId issue1 = newDocId(INDEX_TYPE_ISSUE, "I1");
- DocId issue2 = newDocId(INDEX_TYPE_ISSUE, "I2");
+ DocId issue1 = newDocId(TYPE_ISSUE, "I1");
+ DocId issue2 = newDocId(TYPE_ISSUE, "I2");
underTest.onSuccess(asList(issue1, issue2));
assertThatEsTableContainsOnly(item1, item2, outOfScopeItem);
assertThatEsTableContainsOnly(item1, item2, outOfScopeItem);
}
- private static DocId newDocId(IndexType indexType, String id) {
- return new DocId(indexType, id);
+ private static DocId newDocId(IndexType.IndexRelationType indexType, String id) {
+ IndexType.IndexMainType mainType = indexType.getMainType();
+ return new DocId(mainType.getIndex().getName(), mainType.getType(), id);
}
private IndexingListener newListener(Collection<EsQueueDto> items) {
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
+import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE;
public class OneToOneResilientIndexingListenerTest {
@Test
public void onSuccess_deletes_rows_from_ES_QUEUE_table() {
- EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "foo");
- EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, "bar");
- EsQueueDto item3 = insertInQueue(INDEX_TYPE_ISSUE, "baz");
+ EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "foo");
+ EsQueueDto item2 = insertInQueue(TYPE_ISSUE, "bar");
+ EsQueueDto item3 = insertInQueue(TYPE_ISSUE, "baz");
db.commit();
IndexingListener underTest = newListener(asList(item1, item2, item3));
*/
@Test
public void onSuccess_deletes_all_the_rows_with_same_doc_id() {
- EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "foo");
+ EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "foo");
// same id as item1
- EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, item1.getDocId());
- EsQueueDto item3 = insertInQueue(INDEX_TYPE_ISSUE, "bar");
+ EsQueueDto item2 = insertInQueue(TYPE_ISSUE, item1.getDocId());
+ EsQueueDto item3 = insertInQueue(TYPE_ISSUE, "bar");
db.commit();
IndexingListener underTest = newListener(asList(item1, item2, item3));
}
private static DocId toDocId(EsQueueDto dto) {
- return new DocId(IndexType.parse(dto.getDocType()), dto.getDocId());
+ IndexType.SimpleIndexMainType mainType = IndexType.parseMainType(dto.getDocType());
+ return new DocId(mainType.getIndex(), mainType.getType(), dto.getDocId());
}
private IndexingListener newListener(Collection<EsQueueDto> items) {
*/
package org.sonar.server.es.metadata;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.util.Locale;
+import java.util.Random;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.runner.RunWith;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric;
import static org.assertj.core.api.Assertions.assertThat;
+@RunWith(DataProviderRunner.class)
public class MetadataIndexTest {
@Rule
public EsTester es = EsTester.createCustom(new MetadataIndexDefinitionBridge(), new FakeIndexDefinition());
private final MetadataIndex underTest = new MetadataIndex(es.client());
- private final String index = randomAlphanumeric(20);
+ private final String indexName = randomAlphabetic(20).toLowerCase(Locale.ENGLISH);
+ private final Index index = new Random().nextBoolean() ? Index.simple(indexName) : Index.withRelations(indexName);
@Test
- public void type_should_be_not_initialized_by_default() {
- IndexType indexType = new IndexType("examples", "example");
+ @UseDataProvider("mainOrRelationType")
+ public void type_should_be_not_initialized_by_default(IndexType indexType) {
assertThat(underTest.getInitialized(indexType)).isFalse();
}
@Test
- public void type_should_be_initialized_after_explicitly_set_to_initialized() {
- IndexType indexType = new IndexType("examples", "example");
+ @UseDataProvider("mainOrRelationType")
+ public void type_should_be_initialized_after_explicitly_set_to_initialized(IndexType indexType) {
+
underTest.setInitialized(indexType, true);
assertThat(underTest.getInitialized(indexType)).isTrue();
}
+ @DataProvider
+ public static Object[][] mainOrRelationType() {
+ IndexMainType mainType = IndexType.main(Index.withRelations("examples"), "example");
+ return new Object[][] {
+ {mainType},
+ {IndexType.relation(mainType, "doo")}
+ };
+ }
+
@Test
public void hash_should_be_empty_by_default() {
assertThat(underTest.getHash(index)).isEmpty();
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.FakeDoc;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexDefinition;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
+
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+
+public class FakeIndexDefinition implements IndexDefinition {
+
+ public static final String INDEX = "fakes";
+ public static final String TYPE = "fake";
+ public static final Index DESCRIPTOR = Index.simple(INDEX);
+ public static final IndexMainType TYPE_FAKE = IndexType.main(DESCRIPTOR, TYPE);
+ public static final String INT_FIELD = "intField";
+
+ private int replicas = 0;
+
+ public FakeIndexDefinition setReplicas(int replicas) {
+ this.replicas = replicas;
+ return this;
+ }
+
+ @Override
+ public void define(IndexDefinitionContext context) {
+ NewIndex index = context.create(DESCRIPTOR, newBuilder(new MapSettings().asConfig()).build());
+ index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas);
+ index.getSettings().put("index.refresh_interval", "-1");
+ index.createTypeMapping(TYPE_FAKE)
+ .createIntegerField(INT_FIELD);
+ }
+
+ public static FakeDoc newDoc(int value) {
+ return new FakeDoc().setInt(value);
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import java.util.Random;
+import java.util.function.BiConsumer;
+import java.util.stream.Stream;
+import org.junit.Test;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Fail.fail;
+
+public class FieldAwareTest {
+
+ @Test
+ public void indexType_is_a_reserved_field_name_whatever_the_case() {
+ Stream<BiConsumer<TestFieldAware, String>> fieldSetters = Stream.of(
+ (testFieldAware, fieldName) -> testFieldAware.createBooleanField(fieldName),
+ (testFieldAware, fieldName) -> testFieldAware.createByteField(fieldName),
+ (testFieldAware, fieldName) -> testFieldAware.createDateTimeField(fieldName),
+ (testFieldAware, fieldName) -> testFieldAware.createDoubleField(fieldName),
+ (testFieldAware, fieldName) -> testFieldAware.createIntegerField(fieldName),
+ (testFieldAware, fieldName) -> testFieldAware.createLongField(fieldName),
+ (testFieldAware, fieldName) -> testFieldAware.keywordFieldBuilder(fieldName).build(),
+ (testFieldAware, fieldName) -> testFieldAware.textFieldBuilder(fieldName).build(),
+ (testFieldAware, fieldName) -> testFieldAware.nestedFieldBuilder(fieldName).addKeywordField("foo").build()
+ );
+
+ fieldSetters.forEach(c -> {
+ TestFieldAware underTest = new TestFieldAware();
+ // should not fail for other field name
+ c.accept(underTest, randomAlphabetic(1 + new Random().nextInt(10)));
+ // fails whatever the case
+ Stream.of("indexType", "indextype", "InDexType", "INDEXTYPE")
+ .forEach(illegalFieldName -> {
+ try {
+ c.accept(underTest, illegalFieldName);
+ fail("should have thrown a IllegalArgumentException");
+ } catch (IllegalArgumentException e) {
+ assertThat(e).hasMessage("indexType is a reserved field name");
+ }
+ });
+ });
+ }
+
+ private static class TestFieldAware extends FieldAware<TestFieldAware> {
+ private String fieldName;
+ private Object attributes;
+
+ @Override
+ TestFieldAware setFieldImpl(String fieldName, Object attributes) {
+ this.fieldName = fieldName;
+ this.attributes = attributes;
+ return this;
+ }
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Locale;
+import java.util.Map;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.entry;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+
+public class NewAuthorizedIndexTest {
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private String someIndexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH);
+ private Index someIndex = Index.withRelations(someIndexName);
+ private MapSettings settings = new MapSettings();
+ private SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build();
+
+ @Test
+ public void constructor_fails_with_IAE_if_index_does_not_support_relations() {
+ Index simpleIndex = Index.simple(someIndexName);
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("Index must accept relations");
+
+ new NewAuthorizedIndex(simpleIndex, defaultSettingsConfiguration);
+ }
+
+ @Test
+ public void getMainType_returns_main_type_of_authorization_for_index_of_constructor() {
+ NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration);
+
+ assertThat(underTest.getMainType()).isEqualTo(IndexType.main(someIndex, "auth"));
+ }
+
+ @Test
+ public void build_fails_if_no_relation_mapping_has_been_created() {
+ NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration);
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("At least one relation mapping must be defined");
+
+ underTest.build();
+ }
+
+ @Test
+ public void build_enforces_routing() {
+ NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "donut"));
+
+ BuiltIndex<NewAuthorizedIndex> builtIndex = underTest.build();
+
+ assertThat(getAttributeAsMap(builtIndex, "_routing"))
+ .containsOnly(entry("required", true));
+ }
+
+ @Test
+ public void build_defines_type_field() {
+ NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "donut"));
+
+ BuiltIndex<NewAuthorizedIndex> builtIndex = underTest.build();
+
+ Map<String, Object> properties = getProperties(builtIndex);
+ assertThat(getFieldAsMap(properties, "indexType"))
+ .isEqualTo(ImmutableMap.of(
+ "type", "keyword",
+ "norms", false,
+ "store", false,
+ "doc_values", false));
+ }
+
+ @Test
+ public void constructor_creates_mapping_for_authorization_type() {
+ NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "donut"));
+
+ BuiltIndex<NewAuthorizedIndex> builtIndex = underTest.build();
+
+ Map<String, Object> properties = getProperties(builtIndex);
+ assertThat(getFieldAsMap(properties, "auth_groupIds"))
+ .containsOnly(entry("type", "long"));
+ assertThat(getFieldAsMap(properties, "auth_userIds"))
+ .containsOnly(entry("type", "long"));
+ assertThat(getFieldAsMap(properties, "auth_allowAnyone"))
+ .containsOnly(entry("type", "boolean"));
+ }
+
+ private static Map<String, Object> getProperties(BuiltIndex<?> index) {
+ return getAttributeAsMap(index, "properties");
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> getAttributeAsMap(BuiltIndex<?> index, String attributeKey) {
+ return (Map<String, Object>) index.getAttributes().get(attributeKey);
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> getFieldAsMap(Map<String, Object> properties, String fieldName) {
+ return (Map<String, Object>) properties.get(fieldName);
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableMap;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.util.Map;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.data.MapEntry.entry;
+import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
+import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+
+@RunWith(DataProviderRunner.class)
+public class NewIndexTest {
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private static final String someIndexName = randomAlphabetic(5).toLowerCase();
+ private MapSettings settings = new MapSettings();
+ private SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build();
+
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void getRelations_returns_empty_if_no_relation_added(Index index) {
+ NewIndex<?> newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration);
+
+ assertThat(newIndex.getRelations()).isEmpty();
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void verify_default_index_settings_in_standalone(Index index) {
+ Settings underTest = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration)
+ .getSettings().build();
+
+ assertThat(underTest.get("index.number_of_shards")).isNotEmpty();
+ assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false");
+ assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s");
+ assertThat(underTest.get("index.number_of_shards")).isEqualTo("1");
+ assertThat(underTest.get("index.number_of_replicas")).isEqualTo("0");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void verify_default_index_settings_in_cluster(Index index) {
+ settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
+ Settings underTest = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration).getSettings().build();
+
+ assertThat(underTest.get("index.number_of_shards")).isNotEmpty();
+ assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false");
+ assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s");
+ assertThat(underTest.get("index.number_of_shards")).isEqualTo("1");
+ assertThat(underTest.get("index.number_of_replicas")).isEqualTo("1");
+ }
+
+ @Test
+ @UseDataProvider("indexAndTypeMappings")
+ public void define_fields(NewIndex<?> newIndex, TypeMapping typeMapping) {
+ typeMapping.setField("foo_field", ImmutableMap.of("type", "keyword"));
+ typeMapping.createBooleanField("boolean_field");
+ typeMapping.createByteField("byte_field");
+ typeMapping.createDateTimeField("dt_field");
+ typeMapping.createDoubleField("double_field");
+ typeMapping.createIntegerField("int_field");
+ typeMapping.createLongField("long_field");
+ typeMapping.createShortField("short_field");
+ typeMapping.createUuidPathField("uuid_path_field");
+
+ assertThat(newIndex.getProperty("foo_field")).isInstanceOf(Map.class);
+ assertThat((Map) newIndex.getProperty("foo_field")).containsEntry("type", "keyword");
+ assertThat((Map) newIndex.getProperty("byte_field")).isNotEmpty();
+ assertThat((Map) newIndex.getProperty("double_field")).isNotEmpty();
+ assertThat((Map) newIndex.getProperty("dt_field")).isNotEmpty();
+ assertThat((Map) newIndex.getProperty("int_field")).containsEntry("type", "integer");
+ assertThat((Map) newIndex.getProperty("long_field")).isNotEmpty();
+ assertThat((Map) newIndex.getProperty("short_field")).isNotEmpty();
+ assertThat((Map) newIndex.getProperty("uuid_path_field")).isNotEmpty();
+ assertThat((Map) newIndex.getProperty("unknown")).isNull();
+ }
+
+ @Test
+ @UseDataProvider("indexAndTypeMappings")
+ public void define_string_field(NewIndex<?> newIndex, TypeMapping typeMapping) {
+ typeMapping.keywordFieldBuilder("basic_field").build();
+ typeMapping.keywordFieldBuilder("not_searchable_field").disableSearch().build();
+ typeMapping.keywordFieldBuilder("all_capabilities_field")
+ .addSubFields(
+ DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER,
+ DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER,
+ DefaultIndexSettingsElement.SORTABLE_ANALYZER)
+ .build();
+ typeMapping.keywordFieldBuilder("dumb_text_storage")
+ .disableSearch()
+ .disableNorms()
+ .disableSortingAndAggregating()
+ .build();
+
+ Map<String, Object> props = (Map) newIndex.getProperty("basic_field");
+ assertThat(props.get("type")).isEqualTo("keyword");
+ assertThat(props.get("index")).isEqualTo("true");
+ assertThat(props.get("fields")).isNull();
+
+ props = (Map) newIndex.getProperty("not_searchable_field");
+ assertThat(props.get("type")).isEqualTo("keyword");
+ assertThat(props.get("index")).isEqualTo("false");
+ assertThat(props.get("norms")).isEqualTo("true");
+ assertThat(props.get("store")).isEqualTo("false");
+ assertThat(props.get("doc_values")).isEqualTo("true");
+ assertThat(props.get("fields")).isNull();
+
+ props = (Map) newIndex.getProperty("all_capabilities_field");
+ assertThat(props.get("type")).isEqualTo("keyword");
+ // no need to test values, it's not the scope of this test
+ assertThat((Map) props.get("fields")).isNotEmpty();
+
+ props = (Map) newIndex.getProperty("dumb_text_storage");
+ assertThat(props.get("type")).isEqualTo("keyword");
+ assertThat(props.get("index")).isEqualTo("false");
+ assertThat(props.get("norms")).isEqualTo("false");
+ assertThat(props.get("store")).isEqualTo("false");
+ assertThat(props.get("doc_values")).isEqualTo("false");
+ assertThat(props.get("fields")).isNull();
+ }
+
+ @Test
+ @UseDataProvider("indexAndTypeMappings")
+ public void define_nested_field(NewIndex<?> newIndex, TypeMapping typeMapping) {
+ typeMapping.nestedFieldBuilder("measures")
+ .addKeywordField("key")
+ .addDoubleField("value")
+ .build();
+ Map<String, Object> result = (Map) newIndex.getProperty("measures");
+
+ assertThat(result.get("type")).isEqualTo("nested");
+ Map<String, Map<String, Object>> subProperties = (Map) result.get("properties");
+ assertThat(subProperties.get("key").get("type")).isEqualTo("keyword");
+ assertThat(subProperties.get("value").get("type")).isEqualTo("double");
+ }
+
+ @Test
+ @UseDataProvider("indexAndTypeMappings")
+ public void fail_when_nested_with_no_field(NewIndex<?> newIndex, TypeMapping typeMapping) {
+ NestedFieldBuilder<TypeMapping> nestedFieldBuilder = typeMapping.nestedFieldBuilder("measures");
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("At least one sub-field must be declared in nested property 'measures'");
+
+ nestedFieldBuilder.build();
+ }
+
+ @Test
+ @UseDataProvider("indexAndTypeMappings")
+ public void use_doc_values_by_default(NewIndex<?> newIndex, TypeMapping typeMapping) {
+ typeMapping.keywordFieldBuilder("the_doc_value").build();
+
+ Map<String, Object> props = (Map) newIndex.getProperty("the_doc_value");
+ assertThat(props.get("type")).isEqualTo("keyword");
+ assertThat(props.get("doc_values")).isEqualTo("true");
+ }
+
+ @DataProvider
+ public static Object[][] indexAndTypeMappings() {
+ String indexName = randomAlphabetic(5).toLowerCase();
+ MapSettings settings = new MapSettings();
+ SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build();
+ Index index = Index.withRelations(indexName);
+ IndexMainType mainType = IndexType.main(index, "foo");
+ SimplestNewIndex newIndex = new SimplestNewIndex(mainType, defaultSettingsConfiguration);
+ TypeMapping mainMapping = newIndex.createTypeMapping(mainType);
+ TypeMapping relationMapping = newIndex.createTypeMapping(IndexType.relation(mainType, "bar"));
+ return new Object[][] {
+ {newIndex, mainMapping},
+ {newIndex, relationMapping},
+ };
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void default_shards_and_replicas(Index index) {
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5");
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void five_shards_and_one_replica_by_default_on_cluster(Index index) {
+ settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5");
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void customize_number_of_shards(Index index) {
+ settings.setProperty("sonar.search." + index.getName() + ".shards", "3");
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3");
+ // keep default value
+ assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void default_number_of_replicas_on_standalone_instance_must_be_0(Index index) {
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void default_number_of_replicas_on_non_enabled_cluster_must_be_0(Index index) {
+ settings.setProperty(CLUSTER_ENABLED.getKey(), "false");
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void default_number_of_replicas_on_cluster_instance_must_be_1(Index index) {
+ settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void when_number_of_replicas_on_cluster_is_specified_to_zero_default_value_must_not_be_used(Index index) {
+ settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
+ settings.setProperty(SEARCH_REPLICAS.getKey(), "0");
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void index_defined_with_specified_number_of_replicas_when_cluster_enabled(Index index) {
+ settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
+ settings.setProperty(SEARCH_REPLICAS.getKey(), "3");
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build());
+
+ assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void fail_when_replica_customization_cant_be_parsed(Index index) {
+ settings.setProperty(CLUSTER_ENABLED.getKey(), "true");
+ settings.setProperty(SEARCH_REPLICAS.getKey(), "ꝱꝲꝳପ");
+ SettingsConfiguration settingsConfiguration = newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build();
+ IndexMainType mainType = IndexType.main(index, "foo");
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("The property 'sonar.search.replicas' is not an int value: For input string: \"ꝱꝲꝳପ\"");
+
+ new SimplestNewIndex(mainType, settingsConfiguration);
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void in_standalone_searchReplicas_is_not_overridable(Index index) {
+ settings.setProperty(SEARCH_REPLICAS.getKey(), "5");
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration);
+
+ assertThat(newIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0");
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void index_with_source(Index index) {
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration);
+ newIndex.setEnableSource(true);
+
+ assertThat(newIndex).isNotNull();
+ assertThat(getAttributeAsMap(newIndex, "_source")).containsExactly(entry("enabled", true));
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void index_without_source(Index index) {
+ NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration);
+ newIndex.setEnableSource(false);
+
+ assertThat(getAttributeAsMap(newIndex, "_source")).containsExactly(entry("enabled", false));
+ }
+
+ @Test
+ public void createTypeMapping_with_IndexRelationType_fails_with_ISE_if_index_does_not_allow_relations() {
+ IndexType.IndexRelationType indexRelationType = IndexType.relation(IndexType.main(Index.withRelations(someIndexName), "bar"), "bar");
+
+ Index index = Index.simple(someIndexName);
+ IndexMainType mainType = IndexType.main(index, "foo");
+ NewIndex underTest = new NewIndex(index, defaultSettingsConfiguration) {
+ @Override
+ public IndexMainType getMainType() {
+ return mainType;
+ }
+
+ @Override
+ public BuiltIndex build() {
+ throw new UnsupportedOperationException("build not implemented");
+ }
+ };
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("Index is not configured to accept relations. Update IndexDefinition.Descriptor instance for this index");
+
+ underTest.createTypeMapping(indexRelationType);
+ }
+
+ @DataProvider
+ public static Object[][] indexWithAndWithoutRelations() {
+ return new Object[][] {
+ {Index.simple(someIndexName)},
+ {Index.withRelations(someIndexName)}
+ };
+ }
+
+ private static Map<String, Object> getAttributeAsMap(NewIndex newIndex, String attributeKey) {
+ return (Map<String, Object>) newIndex.getAttributes().get(attributeKey);
+ }
+
+ private static final class SimplestNewIndex extends NewIndex<SimplestNewIndex> {
+ private final IndexMainType mainType;
+
+ public SimplestNewIndex(IndexMainType mainType, SettingsConfiguration settingsConfiguration) {
+ super(mainType.getIndex(), settingsConfiguration);
+ this.mainType = mainType;
+ }
+
+ @Override
+ public IndexMainType getMainType() {
+ return mainType;
+ }
+
+ @Override
+ public BuiltIndex<SimplestNewIndex> build() {
+ return new BuiltIndex<>(this);
+ }
+ }
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import com.google.common.collect.ImmutableMap;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.util.Locale;
+import java.util.Map;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.entry;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.NORMS;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.STORE;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
+
+@RunWith(DataProviderRunner.class)
+public class NewRegularIndexTest {
+ private static final String SOME_INDEX_NAME = randomAlphabetic(10).toLowerCase(Locale.ENGLISH);
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private MapSettings settings = new MapSettings();
+ private SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build();
+
+ @Test
+ @UseDataProvider("indexes")
+ public void getMainType_fails_with_ISE_if_createTypeMapping_with_IndexMainType_has_not_been_called(Index index) {
+ NewRegularIndex newIndex = new NewRegularIndex(index, defaultSettingsConfiguration);
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("Main type has not been defined");
+
+ newIndex.getMainType();
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void createTypeMapping_with_IndexMainType_fails_with_ISE_if_called_twice(Index index) {
+ NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.main(index, "foo"));
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("Main type can only be defined once");
+
+ underTest.createTypeMapping(IndexType.main(index, "foo"));
+ }
+
+ @Test
+ public void createTypeMapping_with_IndexRelationType_fails_with_ISE_if_called_before_createType_with_IndexMainType() {
+ Index index = Index.withRelations(SOME_INDEX_NAME);
+ NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration);
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("Mapping for main type must be created first");
+
+ underTest.createTypeMapping(IndexType.relation(IndexType.main(index, "foo"), "bar"));
+ }
+
+ @Test
+ public void createTypeMapping_with_IndexRelationType_fails_with_IAE_if_mainType_does_not_match_defined_one() {
+ Index index = Index.withRelations(SOME_INDEX_NAME);
+ IndexType.IndexMainType mainType = IndexType.main(index, "foo");
+ NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration);
+ underTest.createTypeMapping(mainType);
+
+ expectedException.expect(IllegalArgumentException.class);
+ expectedException.expectMessage("main type of relation must be "+ mainType);
+
+ underTest.createTypeMapping(IndexType.relation(IndexType.main(index, "donut"), "bar"));
+ }
+
+ @Test
+ @UseDataProvider("indexWithAndWithoutRelations")
+ public void build_fails_with_ISE_if_no_mainType_is_defined(Index index) {
+ NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration);
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("Mapping for main type must be defined");
+
+ underTest.build();
+ }
+
+ @DataProvider
+ public static Object[][] indexWithAndWithoutRelations() {
+ return new Object[][] {
+ {Index.simple(SOME_INDEX_NAME)},
+ {Index.withRelations(SOME_INDEX_NAME)}
+ };
+ }
+
+ @Test
+ public void build_fails_with_ISE_if_index_accepts_relations_and_none_is_defined() {
+ Index index = Index.withRelations(SOME_INDEX_NAME);
+ NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.main(index, "foo"));
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("At least one relation must be defined when index accepts relations");
+
+ underTest.build();
+ }
+
+ @Test
+ public void build_does_not_enforce_routing_if_mainType_does_not_accepts_relations() {
+ Index someIndex = Index.simple(SOME_INDEX_NAME);
+ NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.main(someIndex, "foo"));
+
+ BuiltIndex<NewRegularIndex> builtIndex = underTest.build();
+
+ assertThat(builtIndex.getAttributes().get("_routing"))
+ .isNull();
+ }
+
+ @Test
+ public void build_enforces_routing_if_mainType_accepts_relations() {
+ Index someIndex = Index.withRelations(SOME_INDEX_NAME);
+ NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.main(someIndex, "foo"));
+ underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "bar"));
+
+ BuiltIndex<NewRegularIndex> builtIndex = underTest.build();
+
+ assertThat((Map<String, Object>) builtIndex.getAttributes().get("_routing"))
+ .contains(entry("required", true));
+ }
+
+ @Test
+ public void build_does_not_define_type_field_if_index_does_not_accept_relations() {
+ Index someIndex = Index.simple(SOME_INDEX_NAME);
+ NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.main(someIndex, "foo"));
+
+ BuiltIndex<NewRegularIndex> builtIndex = underTest.build();
+
+ Map<String, Object> properties = (Map<String, Object>) builtIndex.getAttributes().get("properties");
+ assertThat(properties.get("indexType"))
+ .isNull();
+ }
+
+ @Test
+ public void build_defines_type_field_if_index_accepts_relations() {
+ Index someIndex = Index.withRelations(SOME_INDEX_NAME);
+ NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration);
+ underTest.createTypeMapping(IndexType.main(someIndex, "foo"));
+ underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "bar"));
+
+ BuiltIndex<NewRegularIndex> builtIndex = underTest.build();
+
+ Map<String, Object> properties = (Map<String, Object>) builtIndex.getAttributes().get("properties");
+ assertThat((Map) properties.get("indexType"))
+ .isEqualTo(ImmutableMap.of(
+ TYPE, "keyword",
+ NORMS, false,
+ STORE, false,
+ "doc_values", false));
+ }
+
+ @DataProvider
+ public static Object[][] indexes() {
+ String someIndexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH);
+ return new Object[][] {
+ {Index.simple(someIndexName)},
+ {Index.withRelations(someIndexName)}
+ };
+ }
+
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es.newindex;
+
+import org.sonar.server.es.IndexType;
+
+public final class TestNewIndex extends NewIndex<TestNewIndex> {
+ private final IndexType.IndexMainType mainType;
+ private final TypeMapping mainTypeMapping;
+
+ public TestNewIndex(IndexType.IndexMainType mainType, SettingsConfiguration settingsConfiguration) {
+ super(mainType.getIndex(), settingsConfiguration);
+ this.mainType = mainType;
+ mainTypeMapping = super.createTypeMapping(mainType);
+ }
+
+ @Override
+ public IndexType.IndexMainType getMainType() {
+ return mainType;
+ }
+
+ public TypeMapping getMainTypeMapping() {
+ return mainTypeMapping;
+ }
+
+ @Override
+ public BuiltIndex<TestNewIndex> build() {
+ return new BuiltIndex<>(this);
+ }
+
+ public TestNewIndex addRelation(String name) {
+ super.createTypeMapping(IndexType.relation(mainType, name));
+ return this;
+ }
+
+ public TypeMapping createRelationMapping(String name) {
+ return super.createTypeMapping(IndexType.relation(mainType, name));
+ }
+}
*/
package org.sonar.server.es.request;
+import java.util.Locale;
+import java.util.Random;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
import org.junit.Rule;
import org.junit.Test;
-import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
+import org.sonar.server.es.Index;
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
@Test
public void create_index() {
- CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndexName());
+ CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex());
requestBuilder.get();
}
@Test
public void to_string() {
- String indexName = generateNewIndexName();
- assertThat(es.client().prepareCreate(indexName).toString()).contains("ES create index '" + indexName + "'");
+ Index index = generateNewIndex();
+ assertThat(es.client().prepareCreate(index).toString()).contains("ES create index '" + index.getName() + "'");
}
@Test
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndexName());
+ CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex());
requestBuilder.get();
assertThat(logTester.logs()).hasSize(1);
}
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().prepareCreate(generateNewIndexName()).get("1");
+ es.client().prepareCreate(generateNewIndex()).get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareCreate(generateNewIndexName()).get(TimeValue.timeValueMinutes(1));
+ es.client().prepareCreate(generateNewIndex()).get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().prepareCreate(generateNewIndexName()).execute();
+ es.client().prepareCreate(generateNewIndex()).execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
}
}
- private static String generateNewIndexName(){
- return "index_" + Long.toString(System2.INSTANCE.now());
+ private static Index generateNewIndex(){
+ String name = randomAlphabetic(10).toLowerCase(Locale.ENGLISH);
+ return new Random().nextBoolean() ? Index.simple(name) : Index.withRelations(name);
}
}
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
@Rule
public EsTester es = EsTester.createCustom(new FakeIndexDefinition());
-
@Rule
public LogTester logTester = new LogTester();
+ private final Index index = Index.simple("fakes");
+ private final IndexType.IndexMainType mainType = IndexType.main(index, "fake");
+
@Test
public void delete() {
- es.client().prepareDelete("fakes", "fake", "the_id").get();
+ es.client().prepareDelete(mainType, "the_id").get();
}
@Test
public void to_string() {
- assertThat(es.client().prepareDelete("fakes", "fake", "the_id").toString()).isEqualTo("ES delete request of doc the_id in index fakes/fake");
+ assertThat(es.client().prepareDelete(mainType, "the_id").toString()).isEqualTo("ES delete request of doc the_id in index fakes/fake");
}
@Test
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- es.client().prepareDelete("fakes", "fake", "the_id").get();
+ es.client().prepareDelete(mainType, "the_id").get();
assertThat(logTester.logs()).hasSize(1);
}
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().prepareDelete("fakes", "fake", "the_id").get("1");
+ es.client().prepareDelete(mainType, "the_id").get("1");
fail();
} catch (UnsupportedOperationException e) {
assertThat(e).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareDelete("fakes", "fake", "the_id").get(TimeValue.timeValueMinutes(1));
+ es.client().prepareDelete(mainType, "the_id").get(TimeValue.timeValueMinutes(1));
fail();
} catch (UnsupportedOperationException e) {
assertThat(e).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().prepareDelete("fakes", "fake", "the_id").execute();
+ es.client().prepareDelete(mainType, "the_id").execute();
fail();
} catch (UnsupportedOperationException e) {
assertThat(e).hasMessage("execute() should not be called as it's used for asynchronous");
*/
package org.sonar.server.es.request;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.common.unit.TimeValue;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.runner.RunWith;
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import org.sonar.server.es.IndexType;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
-import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE;
+import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE;
+@RunWith(DataProviderRunner.class)
public class ProxyGetRequestBuilderTest {
@Rule
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey")
+ es.client().prepareGet(TYPE_FAKE, "ruleKey")
.get();
assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1);
}
@Test
- public void fail_to_get_bad_query() {
- GetRequestBuilder requestBuilder = es.client().prepareGet(new IndexType("unknown", "test"), "rule1");
+ @UseDataProvider("mainAndRelationWithUnknownIndex")
+ public void prepareGet_fails_if_index_unknown(IndexType indexType) {
+ GetRequestBuilder requestBuilder = es.client().prepareGet(indexType, "rule1");
try {
requestBuilder.get();
fail();
}
}
+ @DataProvider
+ public static Object[][] mainAndRelationWithUnknownIndex() {
+ IndexType.IndexMainType mainType = IndexType.main(Index.withRelations("unknown"), "test");
+ return new Object[][] {
+ {mainType},
+ {IndexType.relation(mainType, "donut")}
+ };
+ }
+
@Test
public void get_with_string_timeout_is_not_implemented() {
try {
- es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey").get("1");
+ es.client().prepareGet(TYPE_FAKE, "ruleKey").get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey").get(TimeValue.timeValueMinutes(1));
+ es.client().prepareGet(TYPE_FAKE, "ruleKey").get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey").execute();
+ es.client().prepareGet(TYPE_FAKE, "ruleKey").execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
*/
package org.sonar.server.es.request;
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
import org.elasticsearch.action.DocWriteResponse.Result;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.runner.RunWith;
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
+import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE;
+@RunWith(DataProviderRunner.class)
public class ProxyIndexRequestBuilderTest {
@Rule
@Test
public void index_with_index_type_and_id() {
- IndexResponse response = es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE)
+ IndexResponse response = es.client().prepareIndex(TYPE_FAKE)
.setSource(FakeIndexDefinition.newDoc(42).getFields())
.get();
assertThat(response.getResult()).isSameAs(Result.CREATED);
@Test
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- IndexResponse response = es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE)
+ IndexResponse response = es.client().prepareIndex(TYPE_FAKE)
.setSource(FakeIndexDefinition.newDoc(42).getFields())
.get();
assertThat(response.getResult()).isSameAs(Result.CREATED);
}
@Test
- public void fail_if_bad_query() {
- IndexRequestBuilder requestBuilder = es.client().prepareIndex(new IndexType("unknownIndex", "unknownType"));
+ @UseDataProvider("mainOrRelationType")
+ public void fail_if_bad_query(IndexType indexType) {
+ IndexRequestBuilder requestBuilder = es.client().prepareIndex(indexType);
try {
requestBuilder.get();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class);
- assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'unknownIndex' on type 'unknownType'");
+ assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'foo' on type 'bar'");
}
}
- @Test
- public void fail_if_bad_query_with_basic_profiling() {
- IndexRequestBuilder requestBuilder = es.client().prepareIndex(new IndexType("unknownIndex", "unknownType"));
- try {
- requestBuilder.get();
- fail();
- } catch (Exception e) {
- assertThat(e).isInstanceOf(IllegalStateException.class);
- assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'unknownIndex' on type 'unknownType'");
- }
+ @DataProvider
+ public static Object[][] mainOrRelationType() {
+ IndexMainType mainType = IndexType.main(Index.withRelations("foo"), "bar");
+ return new Object[][] {
+ {mainType},
+ {IndexType.relation(mainType, "donut")}
+ };
}
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).get("1");
+ es.client().prepareIndex(TYPE_FAKE).get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).get(TimeValue.timeValueMinutes(1));
+ es.client().prepareIndex(TYPE_FAKE).get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void do_not_support_execute_method() {
try {
- es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).execute();
+ es.client().prepareIndex(TYPE_FAKE).execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
@Test
public void exists() {
- assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.INDEX).get().isExists()).isTrue();
- assertThat(es.client().prepareIndicesExist("unknown").get().isExists()).isFalse();
+ assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get().isExists()).isTrue();
+ assertThat(es.client().prepareIndicesExist(Index.simple("unknown")).get().isExists()).isFalse();
}
@Test
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- es.client().prepareIndicesExist(FakeIndexDefinition.INDEX).get();
+ es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get();
assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1);
}
- @Test
- public void fail_to_exists() {
- try {
- es.client().prepareIndicesExist().get();
-
- // expected to fail because elasticsearch is not correctly configured, but that does not matter
- fail();
- } catch (Exception e) {
- assertThat(e).isInstanceOf(IllegalStateException.class);
- assertThat(e.getMessage()).contains("Fail to execute ES indices exists request");
- }
- }
-
@Test
public void to_string() {
- assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.INDEX).toString()).isEqualTo("ES indices exists request on indices 'fakes'");
+ assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES indices exists request on indices 'fakes'");
}
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().prepareIndicesExist().get("1");
+ es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareIndicesExist().get(TimeValue.timeValueMinutes(1));
+ es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().prepareIndicesExist().execute();
+ es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
@Test
public void stats() {
- es.client().prepareStats(FakeIndexDefinition.INDEX).get();
+ es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get();
}
@Test
public void to_string() {
- assertThat(es.client().prepareStats(FakeIndexDefinition.INDEX).setIndices("rules").toString()).isEqualTo("ES indices stats request on indices 'rules'");
+ assertThat(es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).setIndices("rules").toString()).isEqualTo("ES indices stats request on indices 'rules'");
assertThat(es.client().prepareStats().toString()).isEqualTo("ES indices stats request");
}
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- es.client().prepareStats(FakeIndexDefinition.INDEX).get();
+ es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get();
assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1);
}
@Test
public void fail_to_stats() {
try {
- es.client().prepareStats("unknown").get();
+ es.client().prepareStats(Index.simple("unknown")).get();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class);
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().prepareStats(FakeIndexDefinition.INDEX).get("1");
+ es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareStats(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1));
+ es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().prepareStats(FakeIndexDefinition.INDEX).execute();
+ es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
@Test
public void put_mapping() {
- PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.INDEX)
+ PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR)
.setType(FakeIndexDefinition.TYPE)
.setSource(mapDomain());
requestBuilder.get();
@Test
public void to_string() {
- assertThat(es.client().preparePutMapping(FakeIndexDefinition.INDEX).setSource(mapDomain()).toString())
+ assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setSource(mapDomain()).toString())
.isEqualTo("ES put mapping request on indices 'fakes' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'");
- assertThat(es.client().preparePutMapping(FakeIndexDefinition.INDEX).setType(FakeIndexDefinition.TYPE).setSource(mapDomain()).toString())
+ assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setType(FakeIndexDefinition.TYPE).setSource(mapDomain()).toString())
.isEqualTo("ES put mapping request on indices 'fakes' on type 'fake' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'");
}
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.INDEX)
+ PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR)
.setType(FakeIndexDefinition.TYPE)
.setSource(mapDomain());
requestBuilder.get();
@Test
public void fail_on_bad_query() {
try {
- es.client().preparePutMapping().get();
+ es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class);
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().preparePutMapping().get("1");
+ es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().preparePutMapping().get(TimeValue.timeValueMinutes(1));
+ es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().preparePutMapping().execute();
+ es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
@Test
public void refresh() {
- RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.INDEX);
+ RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR);
requestBuilder.get();
}
@Test
public void to_string() {
- assertThat(es.client().prepareRefresh(FakeIndexDefinition.INDEX).toString()).isEqualTo("ES refresh request on indices 'fakes'");
- assertThat(es.client().prepareRefresh().toString()).isEqualTo("ES refresh request");
+ assertThat(es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES refresh request on indices 'fakes'");
}
@Test
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.INDEX);
+ RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR);
requestBuilder.get();
assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1);
}
@Test
public void fail_to_refresh() {
try {
- es.client().prepareRefresh("unknown").get();
+ es.client().prepareRefresh(Index.simple("unknown")).get();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class);
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().prepareRefresh(FakeIndexDefinition.INDEX).get("1");
+ es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareRefresh(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1));
+ es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().prepareRefresh(FakeIndexDefinition.INDEX).execute();
+ es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
-import org.sonar.server.es.IndexType;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
@Test
public void search() {
- es.client().prepareSearch(FakeIndexDefinition.INDEX).get();
+ es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get();
}
@Test
public void to_string() {
- assertThat(es.client().prepareSearch(FakeIndexDefinition.INDEX).setTypes(FakeIndexDefinition.TYPE).toString()).contains("ES search request '").contains(
- "' on indices '[fakes]' on types '[fake]'");
- assertThat(es.client().prepareSearch(FakeIndexDefinition.INDEX).toString()).contains("ES search request '").contains("' on indices '[fakes]'");
- assertThat(es.client().prepareSearch(new IndexType[0]).toString()).contains("ES search request");
+ assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).setTypes(FakeIndexDefinition.TYPE).toString()).contains("ES search request '")
+ .contains("' on indices '[fakes]' on types '[fake]'");
+ assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).toString())
+ .contains("ES search request '")
+ .contains("' on indices '[fakes]'");
}
@Test
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- es.client().prepareSearch(FakeIndexDefinition.INDEX).get();
+ es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get();
assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1);
}
@Test
public void fail_to_search_bad_query() {
try {
- es.client().prepareSearch("non-existing-index").get();
+ es.client().prepareSearch(Index.simple("unknown")).get();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class);
- assertThat(e.getMessage()).contains("Fail to execute ES search request 'SearchRequest{").contains("}' on indices '[non-existing-index]'");
+ assertThat(e.getMessage())
+ .contains("Fail to execute ES search request 'SearchRequest{")
+ .contains("}' on indices '[unknown]'");
}
}
@Test
public void get_with_string_timeout_is_not_yet_implemented() {
try {
- es.client().prepareSearch(FakeIndexDefinition.INDEX).get("1");
+ es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get("1");
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void get_with_time_value_timeout_is_not_yet_implemented() {
try {
- es.client().prepareSearch(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1));
+ es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1));
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented");
@Test
public void execute_should_throw_an_unsupported_operation_exception() {
try {
- es.client().prepareSearch(FakeIndexDefinition.INDEX).execute();
+ es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).execute();
fail();
} catch (Exception e) {
assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous");
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
public void trace_logs() {
logTester.setLevel(LoggerLevel.TRACE);
- SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.INDEX)
+ SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR)
.setScroll(TimeValue.timeValueMinutes(1))
.get();
logTester.clear();
public void no_trace_logs() {
logTester.setLevel(LoggerLevel.DEBUG);
- SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.INDEX)
+ SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR)
.setScroll(TimeValue.timeValueMinutes(1))
.get();
logTester.clear();
@Test
public void to_string() {
- assertThat(es.client().prepareHealth("rules").toString()).isEqualTo("ES cluster health request on indices 'rules'");
assertThat(es.client().prepareHealth().toString()).isEqualTo("ES cluster health request");
}
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.newindex.NewIndex;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
public class IssueIndexDefinitionTest {
def.define(underTest);
assertThat(underTest.getIndices()).hasSize(1);
- NewIndex issuesIndex = underTest.getIndices().get("issues");
- assertThat(issuesIndex).isNotNull();
- assertThat(issuesIndex.getTypes().keySet()).containsOnly("issue", "authorization");
+ NewIndex<?> issuesIndex = underTest.getIndices().get("issues");
+ IndexType.IndexMainType mainType = IndexType.main(Index.withRelations("issues"), TYPE_AUTHORIZATION);
+ assertThat(issuesIndex.getMainType()).isEqualTo(mainType);
+ assertThat(issuesIndex.getRelationsStream())
+ .containsOnly(IndexType.relation(mainType, "issue"));
// no cluster by default
- assertThat(issuesIndex.getSettings().get("index.number_of_shards")).isEqualTo("5");
- assertThat(issuesIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0");
+ assertThat(issuesIndex.getSetting("index.number_of_shards")).isEqualTo("5");
+ assertThat(issuesIndex.getSetting("index.number_of_replicas")).isEqualTo("0");
}
}
import static org.junit.rules.ExpectedException.none;
import static org.sonar.db.component.ComponentTesting.newFileDto;
import static org.sonar.server.issue.IssueDocTesting.newDoc;
-import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
+import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE;
import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_POROUS_DEFENSES;
import static org.sonar.server.issue.index.SecurityStandardHelper.UNKNOWN_STANDARD;
import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION;
@Test
public void test_getIndexTypes() {
- assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_ISSUE);
+ assertThat(underTest.getIndexTypes()).containsExactly(TYPE_ISSUE);
}
@Test
public void test_getAuthorizationScope() {
AuthorizationScope scope = underTest.getAuthorizationScope();
- assertThat(scope.getIndexType().getIndex()).isEqualTo(INDEX_TYPE_ISSUE.getIndex());
+ assertThat(scope.getIndexType().getIndex()).isEqualTo(IssueIndexDefinition.DESCRIPTOR);
assertThat(scope.getIndexType().getType()).isEqualTo(TYPE_AUTHORIZATION);
Predicate<IndexPermissions> projectPredicate = scope.getProjectPredicate();
underTest.indexOnStartup(emptySet());
- IssueDoc doc = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class).get(0);
+ IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0);
assertThat(doc.getId()).isEqualTo(issue.getKey());
assertThat(doc.organizationUuid()).isEqualTo(organization.getUuid());
assertThat(doc.assigneeUuid()).isEqualTo(issue.getAssigneeUuid());
assertThat(doc.creationDate()).isEqualToIgnoringMillis(issue.getIssueCreationDate());
assertThat(doc.directoryPath()).isEqualTo(dir.path());
assertThat(doc.filePath()).isEqualTo(file.path());
- assertThat(doc.getParent()).isEqualTo(project.uuid());
- assertThat(doc.getRouting()).isEqualTo(project.uuid());
assertThat(doc.language()).isEqualTo(issue.getLanguage());
assertThat(doc.line()).isEqualTo(issue.getLine());
// functional date
underTest.indexOnStartup(emptySet());
- IssueDoc doc = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class).get(0);
+ IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0);
assertThat(doc.getCwe()).containsExactlyInAnyOrder("123", "863");
assertThat(doc.getOwaspTop10()).containsExactlyInAnyOrder("a3");
assertThat(doc.getSansTop25()).containsExactlyInAnyOrder(SANS_TOP_25_POROUS_DEFENSES);
@Test
public void indexOnStartup_does_not_fail_on_errors_and_does_enable_recovery_mode() {
- es.lockWrites(INDEX_TYPE_ISSUE);
+ es.lockWrites(TYPE_ISSUE);
db.issues().insertIssue(organization);
try {
} finally {
assertThatIndexHasSize(0);
assertThatEsQueueTableHasSize(0);
- es.unlockWrites(INDEX_TYPE_ISSUE);
+ es.unlockWrites(TYPE_ISSUE);
}
}
underTest.indexOnAnalysis(project.uuid());
- assertThat(es.getDocuments(INDEX_TYPE_ISSUE))
+ assertThat(es.getDocuments(TYPE_ISSUE))
.extracting(SearchHit::getId)
.containsExactlyInAnyOrder(issue.getKey(), "orphan");
}
*/
@Test
public void indexOnAnalysis_does_not_fail_on_errors_and_does_not_enable_recovery_mode() {
- es.lockWrites(INDEX_TYPE_ISSUE);
+ es.lockWrites(TYPE_ISSUE);
IssueDto issue = db.issues().insertIssue(organization);
try {
} finally {
assertThatIndexHasSize(0);
assertThatEsQueueTableHasSize(0);
- es.unlockWrites(INDEX_TYPE_ISSUE);
+ es.unlockWrites(TYPE_ISSUE);
}
}
public void errors_during_project_deletion_are_recovered() {
addIssueToIndex("P1", "I1");
assertThatIndexHasSize(1);
- es.lockWrites(INDEX_TYPE_ISSUE);
+ es.lockWrites(TYPE_ISSUE);
IndexingResult result = indexProject("P1", ProjectIndexer.Cause.PROJECT_DELETION);
assertThat(result.getTotal()).isEqualTo(1L);
assertThat(result.getFailures()).isEqualTo(1L);
assertThatIndexHasSize(1);
- es.unlockWrites(INDEX_TYPE_ISSUE);
+ es.unlockWrites(TYPE_ISSUE);
result = recover();
assertThat(result.getTotal()).isEqualTo(1L);
db.getDbClient().issueDao().insert(db.getSession(), issue1, issue2);
// index is read-only
- es.lockWrites(INDEX_TYPE_ISSUE);
+ es.lockWrites(TYPE_ISSUE);
underTest.commitAndIndexIssues(db.getSession(), asList(issue1, issue2));
assertThatEsQueueTableHasSize(2);
// re-enable write on index
- es.unlockWrites(INDEX_TYPE_ISSUE);
+ es.unlockWrites(TYPE_ISSUE);
// emulate the recovery daemon
IndexingResult result = recover();
@Test
public void recovery_does_not_fail_if_unsupported_docIdType() {
- EsQueueDto item = EsQueueDto.create(INDEX_TYPE_ISSUE.format(), "I1", "unknown", "P1");
+ EsQueueDto item = EsQueueDto.create(TYPE_ISSUE.format(), "I1", "unknown", "P1");
db.getDbClient().esQueueDao().insert(db.getSession(), item);
db.commit();
@Test
public void indexing_recovers_multiple_errors_on_the_same_issue() {
- es.lockWrites(INDEX_TYPE_ISSUE);
+ es.lockWrites(TYPE_ISSUE);
IssueDto issue = db.issues().insertIssue(organization);
// three changes on the same issue
// three attempts of indexing are stored in es_queue recovery table
assertThatEsQueueTableHasSize(3);
- es.unlockWrites(INDEX_TYPE_ISSUE);
+ es.unlockWrites(TYPE_ISSUE);
recover();
assertThatIndexHasOnly(issue);
IssueDto issue1 = db.issues().insertIssue(IssueTesting.newIssue(rule, project, file));
IssueDto issue2 = db.issues().insertIssue(IssueTesting.newIssue(rule, project, file));
- es.lockWrites(INDEX_TYPE_ISSUE);
+ es.lockWrites(TYPE_ISSUE);
IndexingResult result = indexProject(project.uuid(), ProjectIndexer.Cause.PROJECT_DELETION);
assertThat(result.getTotal()).isEqualTo(2L);
assertThat(result.getFailures()).isEqualTo(2L);
assertThatIndexHasSize(0);
- es.unlockWrites(INDEX_TYPE_ISSUE);
+ es.unlockWrites(TYPE_ISSUE);
result = recover();
assertThat(result.getTotal()).isEqualTo(2L);
@Test
public void deleteByKeys_does_not_recover_from_errors() {
addIssueToIndex("P1", "Issue1");
- es.lockWrites(INDEX_TYPE_ISSUE);
+ es.lockWrites(TYPE_ISSUE);
try {
// FIXME : test also message
} finally {
assertThatIndexHasOnly("Issue1");
assertThatEsQueueTableHasSize(0);
- es.unlockWrites(INDEX_TYPE_ISSUE);
+ es.unlockWrites(TYPE_ISSUE);
}
}
new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient()))
.index(asList(issueDoc).iterator());
- assertThat(es.countDocuments(INDEX_TYPE_ISSUE)).isEqualTo(1L);
+ assertThat(es.countDocuments(TYPE_ISSUE)).isEqualTo(1L);
}
@Test
underTest.indexOnStartup(emptySet());
- IssueDoc doc = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class).get(0);
+ IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0);
assertThat(doc.getId()).isEqualTo(issue.getKey());
assertThat(doc.organizationUuid()).isEqualTo(organization.getUuid());
assertThat(doc.componentUuid()).isEqualTo(file.uuid());
}
private void addIssueToIndex(String projectUuid, String issueKey) {
- es.putDocuments(INDEX_TYPE_ISSUE,
+ es.putDocuments(TYPE_ISSUE,
newDoc().setKey(issueKey).setProjectUuid(projectUuid));
}
private void assertThatIndexHasSize(long expectedSize) {
- assertThat(es.countDocuments(INDEX_TYPE_ISSUE)).isEqualTo(expectedSize);
+ assertThat(es.countDocuments(TYPE_ISSUE)).isEqualTo(expectedSize);
}
private void assertThatIndexHasOnly(IssueDto... expectedIssues) {
- assertThat(es.getDocuments(INDEX_TYPE_ISSUE))
+ assertThat(es.getDocuments(TYPE_ISSUE))
.extracting(SearchHit::getId)
.containsExactlyInAnyOrder(Arrays.stream(expectedIssues).map(IssueDto::getKey).toArray(String[]::new));
}
private void assertThatIndexHasOnly(String... expectedKeys) {
- List<IssueDoc> issues = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class);
+ List<IssueDoc> issues = es.getDocuments(TYPE_ISSUE, IssueDoc.class);
assertThat(issues).extracting(IssueDoc::key).containsOnly(expectedKeys);
}
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION;
import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION;
import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_KEY_UPDATE;
import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_TAGS_UPDATE;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
-import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
public class ProjectMeasuresIndexerTest {
public void index_nothing() {
underTest.indexOnStartup(emptySet());
- assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isZero();
+ assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero();
}
@Test
db.getDbClient().componentDao().delete(db.getSession(), project.getId());
IndexingResult result = indexProject(project, PROJECT_DELETION);
- assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
assertThat(result.getTotal()).isEqualTo(1L);
assertThat(result.getSuccess()).isEqualTo(1L);
}
underTest.index(db.getSession(), emptyList());
- assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
}
@Test
public void errors_during_indexing_are_recovered() {
ComponentDto project = db.components().insertPrivateProject();
- es.lockWrites(INDEX_TYPE_PROJECT_MEASURES);
+ es.lockWrites(TYPE_PROJECT_MEASURES);
IndexingResult result = indexProject(project, PROJECT_CREATION);
assertThat(result.getTotal()).isEqualTo(1L);
result = recover();
assertThat(result.getTotal()).isEqualTo(1L);
assertThat(result.getFailures()).isEqualTo(1L);
- assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
assertThatEsQueueTableHasSize(1);
- es.unlockWrites(INDEX_TYPE_PROJECT_MEASURES);
+ es.unlockWrites(TYPE_PROJECT_MEASURES);
result = recover();
assertThat(result.getTotal()).isEqualTo(1L);
underTest.indexOnAnalysis(branch.uuid());
- assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0);
}
private IndexingResult indexProject(ComponentDto project, ProjectIndexer.Cause cause) {
private void assertThatProjectHasTag(ComponentDto project, String expectedTag) {
SearchRequestBuilder request = es.client()
- .prepareSearch(INDEX_TYPE_PROJECT_MEASURES)
- .setQuery(boolQuery().filter(termQuery(FIELD_TAGS, expectedTag)));
+ .prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
+ .setQuery(boolQuery()
+ .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()))
+ .filter(termQuery(FIELD_TAGS, expectedTag)));
assertThat(request.get().getHits().getHits())
.extracting(SearchHit::getId)
.contains(project.uuid());
}
private void assertThatIndexContainsOnly(SnapshotDto... expectedProjects) {
- assertThat(es.getIds(INDEX_TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
+ assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
Arrays.stream(expectedProjects).map(SnapshotDto::getComponentUuid).toArray(String[]::new));
}
private void assertThatIndexContainsOnly(ComponentDto... expectedProjects) {
- assertThat(es.getIds(INDEX_TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
+ assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder(
Arrays.stream(expectedProjects).map(ComponentDto::uuid).toArray(String[]::new));
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.permission.index;
+
+import com.tngtech.java.junit.dataprovider.DataProvider;
+import com.tngtech.java.junit.dataprovider.DataProviderRunner;
+import com.tngtech.java.junit.dataprovider.UseDataProvider;
+import java.util.List;
+import java.util.Random;
+import java.util.stream.IntStream;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
+
+import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Fail.fail;
+
+@RunWith(DataProviderRunner.class)
+public class AuthorizationDocTest {
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void idOf_returns_argument_with_a_prefix() {
+ String s = randomAlphabetic(12);
+
+ assertThat(AuthorizationDoc.idOf(s)).isEqualTo("auth_" + s);
+ }
+
+ @Test
+ public void idOf_fails_with_NPE_if_argument_is_null() {
+ expectedException.expect(NullPointerException.class);
+ expectedException.expectMessage("projectUuid can't be null");
+
+ AuthorizationDoc.idOf(null);
+ }
+
+ @Test
+ public void projectUuidOf_fails_with_NPE_if_argument_is_null() {
+ expectedException.expect(NullPointerException.class);
+
+ AuthorizationDoc.projectUuidOf(null);
+ }
+
+ @Test
+ public void projectUuidOf_returns_substring_if_starts_with_id_prefix() {
+ assertThat(AuthorizationDoc.projectUuidOf("auth_")).isEmpty();
+
+ String id = randomAlphabetic(1 + new Random().nextInt(10));
+ assertThat(AuthorizationDoc.projectUuidOf("auth_" + id)).isEqualTo(id);
+ }
+
+ @Test
+ public void projectUuidOf_returns_argument_if_does_not_starts_with_id_prefix() {
+ String id = randomAlphabetic(1 + new Random().nextInt(10));
+ assertThat(AuthorizationDoc.projectUuidOf(id)).isEqualTo(id);
+ assertThat(AuthorizationDoc.projectUuidOf("")).isEqualTo("");
+ }
+
+ @Test
+ public void getId_fails_with_NPE_if_IndexPermissions_has_null_projectUuid() {
+ IndexPermissions dto = new IndexPermissions(null, null);
+ IndexType.IndexMainType mainType = IndexType.main(Index.simple("foo"), "bar");
+ AuthorizationDoc underTest = AuthorizationDoc.fromDto(mainType, dto);
+
+ expectedException.expect(NullPointerException.class);
+ expectedException.expectMessage("projectUuid can't be null");
+
+ underTest.getId();
+ }
+
+ @Test
+ @UseDataProvider("dtos")
+ public void getId_returns_projectUuid_with_a_prefix(IndexPermissions dto) {
+ AuthorizationDoc underTest = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), dto);
+
+ assertThat(underTest.getId()).isEqualTo("auth_" + dto.getProjectUuid());
+ }
+
+ @Test
+ @UseDataProvider("dtos")
+ public void getRouting_returns_projectUuid(IndexPermissions dto) {
+ AuthorizationDoc underTest = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), dto);
+
+ assertThat(underTest.getRouting()).contains(dto.getProjectUuid());
+ }
+
+ @Test
+ public void fromDto_of_allowAnyone_is_false_and_no_user_nor_group() {
+ IndexPermissions underTest = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4));
+
+ AuthorizationDoc doc = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), underTest);
+
+ boolean auth_allowAnyone = doc.getField("auth_allowAnyone");
+ assertThat(auth_allowAnyone).isFalse();
+ List<Integer> userIds = doc.getField("auth_userIds");
+ assertThat(userIds).isEmpty();
+ List<Integer> groupIds = doc.getField("auth_groupIds");
+ assertThat(groupIds).isEmpty();
+ }
+
+ @Test
+ public void fromDto_defines_userIds_and_groupIds_if_allowAnyone_is_false() {
+ IndexPermissions underTest = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4));
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addUserId);
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addGroupId);
+
+ AuthorizationDoc doc = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), underTest);
+
+ boolean auth_allowAnyone = doc.getField("auth_allowAnyone");
+ assertThat(auth_allowAnyone).isFalse();
+ List<Integer> userIds = doc.getField("auth_userIds");
+ assertThat(userIds).isEqualTo(underTest.getUserIds());
+ List<Integer> groupIds = doc.getField("auth_groupIds");
+ assertThat(groupIds).isEqualTo(underTest.getGroupIds());
+ }
+
+ @Test
+ public void fromDto_ignores_userIds_and_groupIds_if_allowAnyone_is_true() {
+ IndexPermissions underTest = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4));
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addUserId);
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addGroupId);
+ underTest.allowAnyone();
+
+ AuthorizationDoc doc = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), underTest);
+
+ boolean auth_allowAnyone = doc.getField("auth_allowAnyone");
+ assertThat(auth_allowAnyone).isTrue();
+ try {
+ doc.getField("auth_userIds");
+ fail("should have thrown IllegalStateException");
+ } catch (IllegalStateException e) {
+ assertThat(e).hasMessage("Field auth_userIds not specified in query options");
+ }
+ try {
+ doc.getField("auth_groupIds");
+ fail("should have thrown IllegalStateException");
+ } catch (IllegalStateException e) {
+ assertThat(e).hasMessage("Field auth_groupIds not specified in query options");
+ }
+ }
+
+ @DataProvider
+ public static Object[][] dtos() {
+ IndexPermissions allowAnyone = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4));
+ allowAnyone.allowAnyone();
+ IndexPermissions someUserIds = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4));
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someUserIds::addUserId);
+ IndexPermissions someGroupIds = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4));
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someGroupIds::addGroupId);
+ IndexPermissions someGroupIdAndUserIs = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4));
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someGroupIdAndUserIs::addUserId);
+ IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someGroupIdAndUserIs::addGroupId);
+ return new Object[][] {
+ {allowAnyone},
+ {someUserIds},
+ {someGroupIds},
+ {someGroupIdAndUserIs}
+ };
+ }
+}
import static java.util.Collections.emptySet;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE;
public class ActiveRuleIndexerTest {
@Test
public void getIndexTypes() {
- assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_ACTIVE_RULE);
+ assertThat(underTest.getIndexTypes()).containsExactly(TYPE_ACTIVE_RULE);
}
@Test
public void indexOnStartup_does_nothing_if_no_data() {
underTest.indexOnStartup(emptySet());
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isZero();
+ assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isZero();
}
@Test
underTest.indexOnStartup(emptySet());
- List<ActiveRuleDoc> docs = es.getDocuments(INDEX_TYPE_ACTIVE_RULE, ActiveRuleDoc.class);
+ List<ActiveRuleDoc> docs = es.getDocuments(TYPE_ACTIVE_RULE, ActiveRuleDoc.class);
assertThat(docs).hasSize(1);
verify(docs.get(0), profile1, activeRule);
assertThatEsQueueTableIsEmpty();
underTest.commitAndIndex(db.getSession(), Collections.emptyList());
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(0);
assertThatEsQueueTableIsEmpty();
}
@Test
public void commitAndIndex_keeps_elements_to_recover_in_ES_QUEUE_on_errors() {
ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
- es.lockWrites(INDEX_TYPE_ACTIVE_RULE);
+ es.lockWrites(TYPE_ACTIVE_RULE);
commitAndIndex(rule1, ar);
- EsQueueDto expectedItem = EsQueueDto.create(INDEX_TYPE_ACTIVE_RULE.format(), "" + ar.getId(), "activeRuleId", valueOf(ar.getRuleId()));
+ EsQueueDto expectedItem = EsQueueDto.create(TYPE_ACTIVE_RULE.format(), "ar_" + ar.getId(), "activeRuleId", valueOf(ar.getRuleId()));
assertThatEsQueueContainsExactly(expectedItem);
- es.unlockWrites(INDEX_TYPE_ACTIVE_RULE);
+ es.unlockWrites(TYPE_ACTIVE_RULE);
}
@Test
public void commitAndIndex_deletes_the_documents_that_dont_exist_in_database() {
ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1);
indexAll();
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(1);
db.getDbClient().activeRuleDao().delete(db.getSession(), ar.getKey());
commitAndIndex(rule1, ar);
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(0);
assertThatEsQueueTableIsEmpty();
}
@Test
public void index_fails_and_deletes_doc_if_docIdType_is_unsupported() {
- EsQueueDto item = EsQueueDto.create(INDEX_TYPE_ACTIVE_RULE.format(), "the_id", "unsupported", "the_routing");
+ EsQueueDto item = EsQueueDto.create(TYPE_ACTIVE_RULE.format(), "the_id", "unsupported", "the_routing");
db.getDbClient().esQueueDao().insert(db.getSession(), item);
underTest.index(db.getSession(), asList(item));
assertThatEsQueueTableIsEmpty();
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0);
+ assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(0);
}
@Test
public void commitDeletionOfProfiles_does_nothing_if_profiles_are_not_indexed() {
db.qualityProfiles().activateRule(profile1, rule1);
indexAll();
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(1);
underTest.commitDeletionOfProfiles(db.getSession(), singletonList(profile2));
- assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(1);
}
private void assertThatEsQueueTableIsEmpty() {
}
private void verifyOnlyIndexed(ActiveRuleDto... expected) {
- List<String> docs = es.getIds(INDEX_TYPE_ACTIVE_RULE);
+ List<String> docs = es.getIds(TYPE_ACTIVE_RULE);
assertThat(docs).hasSize(expected.length);
for (ActiveRuleDto activeRuleDto : expected) {
- assertThat(docs).contains(activeRuleDto.getId().toString());
+ assertThat(docs).contains("ar_" + activeRuleDto.getId());
}
}
private void verify(ActiveRuleDoc doc1, QProfileDto profile, ActiveRuleDto activeRule) {
assertThat(doc1)
- .matches(doc -> doc.getId().equals("" + activeRule.getId()))
+ .matches(doc -> doc.getId().equals("ar_" + activeRule.getId()))
.matches(doc -> doc.getRuleProfileUuid().equals(profile.getRulesProfileUuid()))
.matches(doc -> doc.getSeverity().equals(activeRule.getSeverityString()));
}
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.server.es.EsTester;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.newindex.NewIndex;
import static org.assertj.core.api.Assertions.assertThat;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED;
-import static org.sonar.server.es.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_ID;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_KEY;
import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_REPOSITORY;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
public class RuleIndexDefinitionTest {
underTest.define(context);
assertThat(context.getIndices()).hasSize(1);
- NewIndex ruleIndex = context.getIndices().get("rules");
- assertThat(ruleIndex).isNotNull();
- assertThat(ruleIndex.getTypes().keySet()).containsOnly("activeRule", "ruleExtension", "rule");
+ NewIndex<?> ruleIndex = context.getIndices().get("rules");
+ assertThat(ruleIndex.getMainType())
+ .isEqualTo(IndexType.main(Index.withRelations("rules"), "rule"));
+ assertThat(ruleIndex.getRelationsStream())
+ .extracting(IndexType.IndexRelationType::getName)
+ .containsOnly("activeRule", "ruleExtension");
// no cluster by default
- assertThat(ruleIndex.getSettings().get("index.number_of_shards")).isEqualTo("2");
- assertThat(ruleIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0");
+ assertThat(ruleIndex.getSetting("index.number_of_shards")).isEqualTo("2");
+ assertThat(ruleIndex.getSetting("index.number_of_replicas")).isEqualTo("0");
}
@Test
underTest.define(context);
NewIndex ruleIndex = context.getIndices().get("rules");
- assertThat(ruleIndex.getSettings().get("index.number_of_replicas")).isEqualTo("1");
+ assertThat(ruleIndex.getSetting("index.number_of_replicas")).isEqualTo("1");
}
@Test
"quick", "brown", "fox", "jump", "over", "lazi", "dog");
// the following method fails if PUT fails
- tester.putDocuments(INDEX_TYPE_RULE, new RuleDoc(ImmutableMap.of(
+ tester.putDocuments(TYPE_RULE, new RuleDoc(ImmutableMap.of(
FIELD_RULE_ID, "123",
FIELD_RULE_HTML_DESCRIPTION, longText,
FIELD_RULE_REPOSITORY, "squid",
FIELD_RULE_KEY, "squid:S001")));
- assertThat(tester.countDocuments(INDEX_TYPE_RULE)).isEqualTo(1);
- assertThat(tester.client().prepareSearch(INDEX_TYPE_RULE.getIndex()).setQuery(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy"))
+ assertThat(tester.countDocuments(TYPE_RULE)).isEqualTo(1);
+ assertThat(tester.client().prepareSearch(TYPE_RULE.getIndex()).setQuery(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy"))
.get().getHits().getTotalHits()).isEqualTo(1);
}
}
private List<AnalyzeResponse.AnalyzeToken> analyzeIndexedTokens(String text) {
- return tester.client().nativeClient().admin().indices().prepareAnalyze(INDEX_TYPE_RULE.getIndex(),
+ return tester.client().nativeClient().admin().indices().prepareAnalyze(TYPE_RULE.getIndex().getName(),
text)
.setField(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION))
.execute().actionGet().getTokens();
import static org.sonar.server.rule.index.RuleIndex.FACET_REPOSITORIES;
import static org.sonar.server.rule.index.RuleIndex.FACET_TAGS;
import static org.sonar.server.rule.index.RuleIndex.FACET_TYPES;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
-import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION;
public class RuleIndexTest {
createRuleMetadata(rule2, organization, setTags("tag2"));
index();
- assertThat(es.countDocuments(INDEX_TYPE_RULE_EXTENSION)).isEqualTo(4);
+ assertThat(es.countDocuments(TYPE_RULE_EXTENSION)).isEqualTo(4);
// tag2s in filter
RuleQuery query = new RuleQuery().setOrganization(organization).setTags(of("tag2s"));
verifySearch(query, rule2);
}
private void index() {
- ruleIndexer.indexOnStartup(Sets.newHashSet(INDEX_TYPE_RULE, INDEX_TYPE_RULE_EXTENSION));
- activeRuleIndexer.indexOnStartup(Sets.newHashSet(INDEX_TYPE_ACTIVE_RULE));
+ ruleIndexer.indexOnStartup(Sets.newHashSet(TYPE_RULE, TYPE_RULE_EXTENSION));
+ activeRuleIndexer.indexOnStartup(Sets.newHashSet(TYPE_ACTIVE_RULE));
}
private RuleQuery newRuleQuery() {
index();
// inactive rules on profile
- List<SearchHit> ruleDocs = es.getDocuments(INDEX_TYPE_RULE);
- List<SearchHit> activeRuleDocs = es.getDocuments(INDEX_TYPE_ACTIVE_RULE);
+ List<SearchHit> ruleDocs = es.getDocuments(TYPE_RULE);
+ List<SearchHit> activeRuleDocs = es.getDocuments(TYPE_ACTIVE_RULE);
assertThat(underTest.searchAll(new RuleQuery().setActivation(false).setQProfile(profile2)))
.containsOnly(rule2.getId(), rule3.getId());
import static java.util.Collections.emptySet;
import static org.assertj.core.api.Assertions.assertThat;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE;
+import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION;
public class RuleIndexerTest {
@Test
public void index_nothing() {
underTest.index(dbSession, emptyList());
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(0L);
+ assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(0L);
}
@Test
dbClient.ruleDao().insert(dbSession, rule);
underTest.commitAndIndex(dbSession, rule.getId());
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1);
}
@Test
dbClient.ruleDao().insert(dbSession, rule.setStatus(RuleStatus.READY));
dbSession.commit();
underTest.commitAndIndex(dbTester.getSession(), rule.getId());
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1);
// Remove rule
dbTester.getDbClient().ruleDao().update(dbTester.getSession(), rule.setStatus(RuleStatus.READY).setUpdatedAt(2000000000000L));
underTest.commitAndIndex(dbTester.getSession(), rule.getId());
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1);
}
@Test
.setScope(RuleExtensionScope.organization(organization.getUuid()));
assertThat(
es.client()
- .prepareSearch(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION)
+ .prepareSearch(TYPE_RULE_EXTENSION.getMainType())
.setQuery(termQuery("_id", doc.getId()))
.get()
.getHits()
RuleExtensionDoc doc = new RuleExtensionDoc()
.setRuleId(rule.getId())
.setScope(RuleExtensionScope.organization(organization.getUuid()));
- assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION)).contains(doc.getId());
+ assertThat(es.getIds(TYPE_RULE_EXTENSION)).contains(doc.getId());
// update db table "rules_metadata" with empty tags and delete tags from index
metadata = RuleTesting.newRuleMetadata(rule, organization).setTags(emptySet());
dbTester.getDbClient().ruleDao().insertOrUpdate(dbTester.getSession(), metadata);
underTest.commitAndIndex(dbTester.getSession(), rule.getId(), organization);
- assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION)).doesNotContain(doc.getId());
+ assertThat(es.getIds(TYPE_RULE_EXTENSION)).doesNotContain(doc.getId());
}
@Test
RuleDefinitionDto rule = dbTester.rules().insert(r -> r.setDescription(description));
underTest.commitAndIndex(dbTester.getSession(), rule.getId());
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1);
}
}
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.newindex.NewIndex;
import static org.assertj.core.api.Assertions.assertThat;
assertThat(underTest.getIndices()).hasSize(1);
NewIndex index = underTest.getIndices().get("users");
- assertThat(index).isNotNull();
- assertThat(index.getTypes().keySet()).containsOnly("user");
+ assertThat(index.getMainType())
+ .isEqualTo(IndexType.main(Index.simple("users"), "user"));
+ assertThat(index.getRelationsStream()).isEmpty();
// no cluster by default
- assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("1");
- assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0");
+ assertThat(index.getSetting("index.number_of_shards")).isEqualTo("1");
+ assertThat(index.getSetting("index.number_of_replicas")).isEqualTo("0");
}
}
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
+import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER;
public class UserIndexTest {
UserDoc user1 = newUser("user1", asList("user_1", "u1"));
UserDoc user2 = newUser("user_with_same_email_as_user1", asList("user_2")).setEmail(user1.email());
UserDoc user3 = newUser("inactive_user_with_same_scm_as_user1", user1.scmAccounts()).setActive(false);
- es.putDocuments(INDEX_TYPE_USER, user1);
- es.putDocuments(INDEX_TYPE_USER, user2);
- es.putDocuments(INDEX_TYPE_USER, user3);
+ es.putDocuments(TYPE_USER, user1);
+ es.putDocuments(TYPE_USER, user2);
+ es.putDocuments(TYPE_USER, user3);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(user1.scmAccounts().get(0), ORGANIZATION_UUID)).extractingResultOf("login").containsOnly(user1.login());
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(user1.login(), ORGANIZATION_UUID)).extractingResultOf("login").containsOnly(user1.login());
public void getAtMostThreeActiveUsersForScmAccount_ignores_inactive_user() {
String scmAccount = "scmA";
UserDoc user = newUser(USER1_LOGIN, singletonList(scmAccount)).setActive(false);
- es.putDocuments(INDEX_TYPE_USER, user);
+ es.putDocuments(TYPE_USER, user);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(user.login(), ORGANIZATION_UUID)).isEmpty();
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(scmAccount, ORGANIZATION_UUID)).isEmpty();
UserDoc user2 = newUser("user2", Collections.emptyList()).setEmail(email);
UserDoc user3 = newUser("user3", Collections.emptyList()).setEmail(email);
UserDoc user4 = newUser("user4", Collections.emptyList()).setEmail(email);
- es.putDocuments(INDEX_TYPE_USER, user1);
- es.putDocuments(INDEX_TYPE_USER, user2);
- es.putDocuments(INDEX_TYPE_USER, user3);
- es.putDocuments(INDEX_TYPE_USER, user4);
+ es.putDocuments(TYPE_USER, user1);
+ es.putDocuments(TYPE_USER, user2);
+ es.putDocuments(TYPE_USER, user3);
+ es.putDocuments(TYPE_USER, user4);
// restrict results to 3 users
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(email, ORGANIZATION_UUID)).hasSize(3);
@Test
public void getAtMostThreeActiveUsersForScmAccount_is_case_sensitive_for_login() {
UserDoc user = newUser("the_login", singletonList("John.Smith"));
- es.putDocuments(INDEX_TYPE_USER, user);
+ es.putDocuments(TYPE_USER, user);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_login", ORGANIZATION_UUID)).hasSize(1);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_Login", ORGANIZATION_UUID)).isEmpty();
@Test
public void getAtMostThreeActiveUsersForScmAccount_is_case_insensitive_for_email() {
UserDoc user = newUser("the_login", "the_EMAIL@corp.com", singletonList("John.Smith"));
- es.putDocuments(INDEX_TYPE_USER, user);
+ es.putDocuments(TYPE_USER, user);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_EMAIL@corp.com", ORGANIZATION_UUID)).hasSize(1);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_email@corp.com", ORGANIZATION_UUID)).hasSize(1);
@Test
public void getAtMostThreeActiveUsersForScmAccount_is_case_insensitive_for_scm_account() {
UserDoc user = newUser("the_login", singletonList("John.Smith"));
- es.putDocuments(INDEX_TYPE_USER, user);
+ es.putDocuments(TYPE_USER, user);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("John.Smith", ORGANIZATION_UUID)).hasSize(1);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("JOHN.SMIth", ORGANIZATION_UUID)).hasSize(1);
public void getAtMostThreeActiveUsersForScmAccount_search_only_user_within_given_organization() {
UserDoc user1 = newUser("user1", singletonList("same_scm")).setOrganizationUuids(singletonList(ORGANIZATION_UUID));
UserDoc user2 = newUser("user2", singletonList("same_scm")).setOrganizationUuids(singletonList("another_organization"));
- es.putDocuments(INDEX_TYPE_USER, user1);
- es.putDocuments(INDEX_TYPE_USER, user2);
+ es.putDocuments(TYPE_USER, user1);
+ es.putDocuments(TYPE_USER, user2);
assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("same_scm", ORGANIZATION_UUID)).extractingResultOf("login").containsOnly(user1.login());
}
@Test
public void searchUsers() {
- es.putDocuments(INDEX_TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1"));
- es.putDocuments(INDEX_TYPE_USER, newUser(USER2_LOGIN, Collections.emptyList()).setEmail("email2"));
+ es.putDocuments(TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1"));
+ es.putDocuments(TYPE_USER, newUser(USER2_LOGIN, Collections.emptyList()).setEmail("email2"));
assertThat(underTest.search(userQuery.build(), new SearchOptions()).getDocs()).hasSize(2);
assertThat(underTest.search(userQuery.setTextQuery("user").build(), new SearchOptions()).getDocs()).hasSize(2);
@Test
public void search_users_filter_by_organization_uuid() {
- es.putDocuments(INDEX_TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1")
+ es.putDocuments(TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1")
.setOrganizationUuids(newArrayList("O1", "O2")));
- es.putDocuments(INDEX_TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2")
+ es.putDocuments(TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2")
.setOrganizationUuids(newArrayList("O2")));
assertThat(underTest.search(userQuery.setOrganizationUuid("O42").build(), new SearchOptions()).getDocs()).isEmpty();
@Test
public void search_users_filter_by_excluded_organization_uuid() {
- es.putDocuments(INDEX_TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1")
+ es.putDocuments(TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1")
.setOrganizationUuids(newArrayList("O1", "O2")));
- es.putDocuments(INDEX_TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2")
+ es.putDocuments(TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2")
.setOrganizationUuids(newArrayList("O2")));
assertThat(underTest.search(userQuery.setExcludedOrganizationUuid("O42").build(), new SearchOptions()).getDocs()).hasSize(2);
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.tuple;
+import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER;
public class UserIndexerTest {
public void index_nothing_on_startup() {
underTest.indexOnStartup(new HashSet<>());
- assertThat(es.countDocuments(UserIndexDefinition.INDEX_TYPE_USER)).isEqualTo(0L);
+ assertThat(es.countDocuments(TYPE_USER)).isEqualTo(0L);
}
@Test
underTest.indexOnStartup(new HashSet<>());
- List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
+ List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class);
assertThat(docs).hasSize(1);
UserDoc doc = docs.get(0);
assertThat(doc.uuid()).isEqualTo(user.getUuid());
underTest.indexOnStartup(new HashSet<>());
- List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
+ List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class);
assertThat(docs).hasSize(1);
UserDoc doc = docs.get(0);
assertThat(doc.uuid()).isEqualTo(user.getUuid());
underTest.commitAndIndex(db.getSession(), user);
- List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
+ List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class);
assertThat(docs)
.extracting(UserDoc::uuid)
.containsExactlyInAnyOrder(user.getUuid())
underTest.commitAndIndex(db.getSession(), user);
- List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
+ List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class);
assertThat(docs)
.extracting(UserDoc::uuid, UserDoc::organizationUuids)
.containsExactlyInAnyOrder(tuple(user.getUuid(), asList(organization1.getUuid(), organization2.getUuid())));
underTest.commitAndIndex(db.getSession(), asList(user1, user2));
- List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class);
+ List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class);
assertThat(docs)
.extracting(UserDoc::login, UserDoc::organizationUuids)
.containsExactlyInAnyOrder(
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.IndexType;
+import org.sonar.server.es.newindex.NewIndex;
import static org.assertj.core.api.Assertions.assertThat;
assertThat(underTest.getIndices()).hasSize(1);
NewIndex index = underTest.getIndices().get("views");
- assertThat(index).isNotNull();
- assertThat(index.getTypes().keySet()).containsOnly("view");
+ assertThat(index.getMainType())
+ .isEqualTo(IndexType.main(Index.simple("views"), "view"));
+ assertThat(index.getRelationsStream()).isEmpty();
- assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("5");
- assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0");
+ assertThat(index.getSetting("index.number_of_shards")).isEqualTo("5");
+ assertThat(index.getSetting("index.number_of_replicas")).isEqualTo("0");
}
}
import static com.google.common.collect.Lists.newArrayList;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW;
+import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW;
public class ViewIndexTest {
public void find_all_view_uuids() {
ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(singletonList("P1"));
ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(singletonList("P2"));
- es.putDocuments(INDEX_TYPE_VIEW, view1);
- es.putDocuments(INDEX_TYPE_VIEW, view2);
+ es.putDocuments(TYPE_VIEW, view1);
+ es.putDocuments(TYPE_VIEW, view2);
List<String> result = newArrayList(index.findAllViewUuids());
import static org.assertj.core.api.Assertions.tuple;
import static org.sonar.api.resources.Qualifiers.APP;
import static org.sonar.db.component.ComponentTesting.newProjectCopy;
-import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW;
+import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW;
public class ViewIndexerTest {
@Test
public void index_nothing() {
underTest.indexOnStartup(emptySet());
- assertThat(es.countDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW)).isEqualTo(0L);
+ assertThat(es.countDocuments(TYPE_VIEW)).isEqualTo(0L);
}
@Test
underTest.indexOnStartup(emptySet());
- List<ViewDoc> docs = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class);
+ List<ViewDoc> docs = es.getDocuments(TYPE_VIEW, ViewDoc.class);
assertThat(docs).hasSize(4);
Map<String, ViewDoc> viewsByUuid = Maps.uniqueIndex(docs, ViewDoc::uuid);
underTest.index("EFGH");
- List<ViewDoc> docs = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class);
+ List<ViewDoc> docs = es.getDocuments(TYPE_VIEW, ViewDoc.class);
assertThat(docs).hasSize(2);
Map<String, ViewDoc> viewsByUuid = Maps.uniqueIndex(docs, ViewDoc::uuid);
public void index_view_doc() {
underTest.index(new ViewDoc().setUuid("EFGH").setProjects(newArrayList("KLMN", "JKLM")));
- List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class);
+ List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class);
assertThat(result).hasSize(1);
ViewDoc view = result.get(0);
db.components().insertComponent(newProjectCopy("PC1", project, application));
underTest.index(application.uuid());
- List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class);
+ List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class);
assertThat(result).hasSize(1);
ViewDoc resultApp = result.get(0);
db.components().insertComponent(newProjectCopy("PC1", project, application));
underTest.indexOnStartup(emptySet());
- List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class);
+ List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class);
assertThat(result).hasSize(1);
ViewDoc resultApp = result.get(0);
underTest.index(applicationBranch1.uuid());
- List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class);
+ List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class);
assertThat(result)
.extracting(ViewDoc::uuid, ViewDoc::projects)
.containsExactlyInAnyOrder(
ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(asList("P1"));
ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(asList("P2", "P3", "P4"));
ViewDoc view3 = new ViewDoc().setUuid("UUID3").setProjects(asList("P2", "P3", "P4"));
- es.putDocuments(INDEX_TYPE_VIEW, view1);
- es.putDocuments(INDEX_TYPE_VIEW, view2);
- es.putDocuments(INDEX_TYPE_VIEW, view3);
+ es.putDocuments(TYPE_VIEW, view1);
+ es.putDocuments(TYPE_VIEW, view2);
+ es.putDocuments(TYPE_VIEW, view3);
- assertThat(es.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
+ assertThat(es.getDocumentFieldValues(TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
.containsOnly(view1.uuid(), view2.uuid(), view3.uuid());
underTest.delete(dbSession, asList(view1.uuid(), view2.uuid()));
- assertThat(es.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
+ assertThat(es.getDocumentFieldValues(TYPE_VIEW, ViewIndexDefinition.FIELD_UUID))
.containsOnly(view3.uuid());
}
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_ORGANIZATION_UUID;
import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_QUALIFIER;
-import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_TYPE_COMPONENT;
import static org.sonar.server.component.index.ComponentIndexDefinition.NAME_ANALYZERS;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
public class ComponentIndex {
public SearchIdResult<String> search(ComponentQuery query, SearchOptions searchOptions) {
SearchRequestBuilder requestBuilder = client
- .prepareSearch(INDEX_TYPE_COMPONENT)
+ .prepareSearch(TYPE_COMPONENT.getMainType())
.setFetchSource(false)
.setFrom(searchOptions.getOffset())
.setSize(searchOptions.getLimit());
}
SearchRequestBuilder request = client
- .prepareSearch(INDEX_TYPE_COMPONENT)
+ .prepareSearch(TYPE_COMPONENT.getMainType())
.setQuery(createQuery(query, features))
.addAggregation(createAggregation(query))
private QueryBuilder createQuery(SuggestionQuery query, ComponentTextSearchFeature... features) {
BoolQueryBuilder esQuery = boolQuery();
+ esQuery.filter(termQuery(FIELD_INDEX_TYPE, TYPE_COMPONENT.getName()));
esQuery.filter(authorizationTypeSupport.createQueryFilter());
ComponentTextSearchQuery componentTextSearchQuery = ComponentTextSearchQuery.builder()
.setQueryText(query.getQuery())
import org.sonar.server.component.index.ComponentIndex;
import org.sonar.server.component.index.ComponentIndexResults;
import org.sonar.server.component.index.SuggestionQuery;
-import org.sonar.server.es.DefaultIndexSettings;
+import org.sonar.server.es.newindex.DefaultIndexSettings;
import org.sonar.server.favorite.FavoriteFinder;
import org.sonar.server.user.UserSession;
import org.sonarqube.ws.Components.SuggestionsWsResponse;
import static org.sonar.core.util.stream.MoreCollectors.toList;
import static org.sonar.core.util.stream.MoreCollectors.toSet;
import static org.sonar.server.component.index.SuggestionQuery.DEFAULT_LIMIT;
-import static org.sonar.server.es.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH;
+import static org.sonar.server.es.newindex.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH;
import static org.sonar.server.ws.WsUtils.writeProtobuf;
import static org.sonarqube.ws.Common.Organization;
import static org.sonarqube.ws.Components.SuggestionsWsResponse.newBuilder;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.process.ProcessProperties;
-import org.sonar.server.es.IndexDefinition.Index;
import org.sonar.server.es.metadata.EsDbCompatibility;
import org.sonar.server.es.metadata.MetadataIndex;
import org.sonar.server.es.metadata.MetadataIndexDefinition;
+import org.sonar.server.es.newindex.BuiltIndex;
+import org.sonar.server.es.newindex.NewIndex;
+
+import static org.sonar.server.es.metadata.MetadataIndexDefinition.DESCRIPTOR;
+import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA;
/**
* Creates/deletes all indices in Elasticsearch during server startup.
@Override
public void start() {
// create the "metadata" index first
- if (!client.prepareIndicesExist(MetadataIndexDefinition.INDEX_TYPE_METADATA.getIndex()).get().isExists()) {
+ IndexType.IndexMainType metadataMainType = TYPE_METADATA;
+ if (!client.prepareIndicesExist(metadataMainType.getIndex()).get().isExists()) {
IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext();
metadataIndexDefinition.define(context);
NewIndex index = context.getIndices().values().iterator().next();
- createIndex(new Index(index), false);
+ createIndex(index.build(), false);
}
checkDbCompatibility(definitions.getIndices().values());
// create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...)
- for (Index index : definitions.getIndices().values()) {
- boolean exists = client.prepareIndicesExist(index.getName()).get().isExists();
- if (exists && !index.getName().equals(MetadataIndexDefinition.INDEX_TYPE_METADATA.getIndex()) && hasDefinitionChange(index)) {
- verifyNotBlueGreenDeployment(index.getName());
- LOGGER.info("Delete Elasticsearch index {} (structure changed)", index.getName());
- deleteIndex(index.getName());
+ for (BuiltIndex<?> builtIndex : definitions.getIndices().values()) {
+ Index index = builtIndex.getMainType().getIndex();
+ String indexName = index.getName();
+ boolean exists = client.prepareIndicesExist(index).get().isExists();
+ if (exists && !builtIndex.getMainType().equals(metadataMainType) && hasDefinitionChange(builtIndex)) {
+ verifyNotBlueGreenDeployment(indexName);
+ LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName);
+ deleteIndex(indexName);
exists = false;
}
if (!exists) {
- createIndex(index, true);
+ createIndex(builtIndex, true);
}
}
}
// nothing to do
}
- private void createIndex(Index index, boolean useMetadata) {
+ private void createIndex(BuiltIndex<?> builtIndex, boolean useMetadata) {
+ Index index = builtIndex.getMainType().getIndex();
LOGGER.info(String.format("Create index %s", index.getName()));
Settings.Builder settings = Settings.builder();
- settings.put(index.getSettings());
+ settings.put(builtIndex.getSettings());
if (useMetadata) {
- metadataIndex.setHash(index.getName(), IndexDefinitionHash.of(index));
- for (IndexDefinition.Type type : index.getTypes().values()) {
- metadataIndex.setInitialized(new IndexType(index.getName(), type.getName()), false);
- }
+ metadataIndex.setHash(index, IndexDefinitionHash.of(builtIndex));
+ metadataIndex.setInitialized(builtIndex.getMainType(), false);
+ builtIndex.getRelationTypes().forEach(relationType -> metadataIndex.setInitialized(relationType, false));
}
CreateIndexResponse indexResponse = client
- .prepareCreate(index.getName())
+ .prepareCreate(index)
.setSettings(settings)
.get();
if (!indexResponse.isAcknowledged()) {
client.waitForStatus(ClusterHealthStatus.YELLOW);
// create types
- for (Map.Entry<String, IndexDefinition.Type> entry : index.getTypes().entrySet()) {
- LOGGER.info(String.format("Create type %s/%s", index.getName(), entry.getKey()));
- PutMappingResponse mappingResponse = client.preparePutMapping(index.getName())
- .setType(entry.getKey())
- .setSource(entry.getValue().getAttributes())
- .get();
- if (!mappingResponse.isAcknowledged()) {
- throw new IllegalStateException("Failed to create type " + entry.getKey());
- }
+ LOGGER.info("Create type {}", builtIndex.getMainType().format());
+ PutMappingResponse mappingResponse = client.preparePutMapping(index)
+ .setType(builtIndex.getMainType().getType())
+ .setSource(builtIndex.getAttributes())
+ .get();
+ if (!mappingResponse.isAcknowledged()) {
+ throw new IllegalStateException("Failed to create type " + builtIndex.getMainType().getType());
}
client.waitForStatus(ClusterHealthStatus.YELLOW);
}
client.nativeClient().admin().indices().prepareDelete(indexName).get();
}
- private boolean hasDefinitionChange(Index index) {
- return metadataIndex.getHash(index.getName())
+ private boolean hasDefinitionChange(BuiltIndex<?> index) {
+ return metadataIndex.getHash(index.getMainType().getIndex())
.map(hash -> {
String defHash = IndexDefinitionHash.of(index);
return !StringUtils.equals(hash, defHash);
}).orElse(true);
}
- private void checkDbCompatibility(Collection<Index> definitions) {
+ private void checkDbCompatibility(Collection<BuiltIndex> definitions) {
List<String> existingIndices = loadExistingIndicesExceptMetadata(definitions);
if (!existingIndices.isEmpty()) {
boolean delete = false;
esDbCompatibility.markAsCompatible();
}
- private List<String> loadExistingIndicesExceptMetadata(Collection<Index> definitions) {
- Set<String> definedNames = definitions.stream().map(IndexDefinition.Index::getName).collect(Collectors.toSet());
+ private List<String> loadExistingIndicesExceptMetadata(Collection<BuiltIndex> definitions) {
+ Set<String> definedNames = definitions.stream()
+ .map(t -> t.getMainType().getIndex().getName())
+ .collect(Collectors.toSet());
return Arrays.stream(client.nativeClient().admin().indices().prepareGetIndex().get().getIndices())
.filter(definedNames::contains)
- .filter(index -> !MetadataIndexDefinition.INDEX_TYPE_METADATA.getIndex().equals(index))
+ .filter(index -> !DESCRIPTOR.getName().equals(index))
.collect(Collectors.toList());
}
}
import org.picocontainer.Startable;
import org.sonar.api.config.Configuration;
import org.sonar.api.server.ServerSide;
+import org.sonar.server.es.newindex.BuiltIndex;
+import org.sonar.server.es.newindex.NewIndex;
/**
* This class collects definitions of all Elasticsearch indices during server startup
@ServerSide
public class IndexDefinitions implements Startable {
- private final Map<String, IndexDefinition.Index> byKey = Maps.newHashMap();
+ private final Map<String, BuiltIndex> byKey = Maps.newHashMap();
private final IndexDefinition[] defs;
private final Configuration config;
this.config = config;
}
- public Map<String, IndexDefinition.Index> getIndices() {
+ public Map<String, BuiltIndex> getIndices() {
return byKey;
}
}
for (Map.Entry<String, NewIndex> entry : context.getIndices().entrySet()) {
- byKey.put(entry.getKey(), new IndexDefinition.Index(entry.getValue()));
+ byKey.put(entry.getKey(), entry.getValue().build());
}
}
}
}
private Set<IndexType> getUninitializedTypes(StartupIndexer indexer) {
- return indexer.getIndexTypes().stream().filter(indexType -> !metadataIndex.getInitialized(indexType)).collect(toSet());
+ return indexer.getIndexTypes().stream()
+ .filter(indexType -> !metadataIndex.getInitialized(indexType))
+ .collect(toSet());
}
private void setInitialized(IndexType indexType) {
- String index = indexType.getIndex();
- waitForIndexYellow(index);
+ waitForIndexYellow(indexType.getMainType().getIndex().getName());
metadataIndex.setInitialized(indexType, true);
}
private final System2 system2;
private final Configuration config;
private final DbClient dbClient;
- private final Map<IndexType, ResilientIndexer> indexersByType;
+ private final Map<String, Indexer> indexersByType;
private final long minAgeInMs;
private final long loopLimit;
this.config = config;
this.dbClient = dbClient;
this.indexersByType = new HashMap<>();
- Arrays.stream(indexers).forEach(i -> i.getIndexTypes().forEach(indexType -> indexersByType.put(indexType, i)));
+ Arrays.stream(indexers).forEach(i -> i.getIndexTypes().forEach(indexType -> indexersByType.put(indexType.format(), new Indexer(indexType, i))));
this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS);
this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT);
}
+ private static final class Indexer {
+ private final IndexType indexType;
+ private final ResilientIndexer delegate;
+
+ private Indexer(IndexType indexType, ResilientIndexer delegate) {
+ this.indexType = indexType;
+ this.delegate = delegate;
+ }
+
+ public IndexType getIndexType() {
+ return indexType;
+ }
+
+ public ResilientIndexer getDelegate() {
+ return delegate;
+ }
+ }
+
@Override
public void start() {
long delayInMs = getSetting(PROPERTY_DELAY, DEFAULT_DELAY_IN_MS);
while (!items.isEmpty()) {
IndexingResult loopResult = new IndexingResult();
- groupItemsByType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems)));
+ groupItemsByDocType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems)));
result.add(loopResult);
if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) {
}
}
- private IndexingResult doIndex(DbSession dbSession, IndexType type, Collection<EsQueueDto> typeItems) {
- LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type);
+ private IndexingResult doIndex(DbSession dbSession, String docType, Collection<EsQueueDto> typeItems) {
+ LOGGER.trace(LOG_PREFIX + "processing {} [{}]", typeItems.size(), docType);
- ResilientIndexer indexer = indexersByType.get(type);
+ Indexer indexer = indexersByType.get(docType);
if (indexer == null) {
- LOGGER.error(LOG_PREFIX + "ignore {} items with unsupported type {}", typeItems.size(), type);
+ LOGGER.error(LOG_PREFIX + "ignore {} items with unsupported type [{}]", typeItems.size(), docType);
return new IndexingResult();
}
- return indexer.index(dbSession, typeItems);
+ return indexer.delegate.index(dbSession, typeItems);
}
- private static ListMultimap<IndexType, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) {
- return items.stream().collect(MoreCollectors.index(i -> IndexType.parse(i.getDocType())));
+ private static ListMultimap<String, EsQueueDto> groupItemsByDocType(Collection<EsQueueDto> items) {
+ return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType));
}
private long getSetting(String key, long defaultValue) {
import org.sonar.server.es.BaseDoc;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.EsUtils;
+import org.sonar.server.es.IndexType;
import org.sonar.server.es.SearchOptions;
import org.sonar.server.es.Sorting;
import org.sonar.server.es.StickyFacetBuilder;
import org.sonar.server.issue.index.IssueQuery.PeriodStart;
+import org.sonar.server.permission.index.AuthorizationDoc;
import org.sonar.server.permission.index.WebAuthorizationTypeSupport;
import org.sonar.server.user.UserSession;
import org.sonar.server.view.index.ViewIndexDefinition;
import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex;
import static org.sonar.server.es.BaseDoc.epochMillisToEpochSeconds;
import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNED_TO_ME;
import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNEES;
import static org.sonar.server.issue.index.IssueIndex.Facet.AUTHOR;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_STATUS;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TAGS;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TYPE;
-import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
+import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE;
import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_INSECURE_INTERACTION;
import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_POROUS_DEFENSES;
import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_RISKY_RESOURCE;
import static org.sonar.server.issue.index.SecurityStandardHelper.UNKNOWN_STANDARD;
+import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW;
import static org.sonarqube.ws.client.issue.IssuesWsParameters.DEPRECATED_PARAM_AUTHORS;
import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT;
import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_ASSIGNEES;
}
public SearchResponse search(IssueQuery query, SearchOptions options) {
- SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX_TYPE_ISSUE);
+ SearchRequestBuilder requestBuilder = client.prepareSearch(TYPE_ISSUE.getMainType());
configureSorting(query, requestBuilder);
configurePagination(options, requestBuilder);
private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequestBuilder requestBuilder) {
Collection<String> uuids = query.projectUuids();
if (!uuids.isEmpty() && options.getFacets().isEmpty()) {
- requestBuilder.setRouting(uuids.toArray(new String[uuids.size()]));
+ requestBuilder.setRouting(uuids.stream().map(AuthorizationDoc::idOf).toArray(String[]::new));
}
}
private Map<String, QueryBuilder> createFilters(IssueQuery query) {
Map<String, QueryBuilder> filters = new HashMap<>();
+ filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_ISSUE.getName()));
filters.put("__authorization", createAuthorizationFilter());
// Issue is assigned Filter
BoolQueryBuilder viewsFilter = boolQuery();
for (String viewUuid : viewUuids) {
+ IndexType.IndexMainType mainType = TYPE_VIEW;
viewsFilter.should(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID,
new TermsLookup(
- ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex(),
- ViewIndexDefinition.INDEX_TYPE_VIEW.getType(),
+ mainType.getIndex().getName(),
+ mainType.getType(),
viewUuid,
ViewIndexDefinition.FIELD_PROJECTS)));
}
private OptionalLong getMinCreatedAt(Map<String, QueryBuilder> filters, QueryBuilder esQuery) {
String facetNameAndField = CREATED_AT.getFieldName();
SearchRequestBuilder esRequest = client
- .prepareSearch(INDEX_TYPE_ISSUE)
+ .prepareSearch(TYPE_ISSUE.getMainType())
.setSize(0);
BoolQueryBuilder esFilter = boolQuery();
filters.values().stream().filter(Objects::nonNull).forEach(esFilter::must);
private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable String textQuery, Terms.Order termsOrder, int size) {
SearchRequestBuilder requestBuilder = client
- .prepareSearch(INDEX_TYPE_ISSUE)
+ .prepareSearch(TYPE_ISSUE.getMainType())
// Avoids returning search hits
.setSize(0);
if (projectUuids.isEmpty()) {
return Collections.emptyList();
}
- SearchRequestBuilder request = client.prepareSearch(INDEX_TYPE_ISSUE)
+ SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType())
.setQuery(
boolQuery()
.mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))
return Collections.emptyList();
}
- SearchRequestBuilder request = client.prepareSearch(INDEX_TYPE_ISSUE)
- .setRouting(projectUuid)
+ SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType())
+ .setRouting(AuthorizationDoc.idOf(projectUuid))
.setQuery(
boolQuery()
.must(termsQuery(FIELD_ISSUE_BRANCH_UUID, branchUuids))
private SearchRequestBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) {
BoolQueryBuilder componentFilter = boolQuery();
if (isViewOrApp) {
+ IndexType.IndexMainType mainType = TYPE_VIEW;
componentFilter.filter(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID,
new TermsLookup(
- ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex(),
- ViewIndexDefinition.INDEX_TYPE_VIEW.getType(),
+ mainType.getIndex().getName(),
+ mainType.getType(),
projectUuid,
ViewIndexDefinition.FIELD_PROJECTS)));
} else {
componentFilter.filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectUuid));
}
- return client.prepareSearch(INDEX_TYPE_ISSUE)
+ return client.prepareSearch(TYPE_ISSUE.getMainType())
.setQuery(
componentFilter
.filter(termsQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name(), RuleType.VULNERABILITY.name()))
import org.sonar.api.server.ServerSide;
import org.sonar.api.utils.System2;
import org.sonar.core.util.stream.MoreCollectors;
-import org.sonar.server.es.DefaultIndexSettingsElement;
+import org.sonar.server.es.newindex.DefaultIndexSettingsElement;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.SearchIdResult;
import org.sonar.server.es.SearchOptions;
import static org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY;
import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars;
import static org.sonar.server.es.EsUtils.termsToMap;
+import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE;
import static org.sonar.server.measure.index.ProjectMeasuresDoc.QUALITY_GATE_STATUS;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ORGANIZATION_UUID;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
-import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_LAST_ANALYSIS_DATE;
import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_NAME;
import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_LANGUAGES;
public SearchIdResult<String> search(ProjectMeasuresQuery query, SearchOptions searchOptions) {
SearchRequestBuilder requestBuilder = client
- .prepareSearch(INDEX_TYPE_PROJECT_MEASURES)
+ .prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
.setFetchSource(false)
.setFrom(searchOptions.getOffset())
.setSize(searchOptions.getLimit());
public ProjectMeasuresStatistics searchTelemetryStatistics() {
SearchRequestBuilder request = client
- .prepareSearch(INDEX_TYPE_PROJECT_MEASURES)
+ .prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
.setFetchSource(false)
.setSize(0);
private Map<String, QueryBuilder> createFilters(ProjectMeasuresQuery query) {
Map<String, QueryBuilder> filters = new HashMap<>();
+ filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName()));
filters.put("__authorization", authorizationTypeSupport.createQueryFilter());
Multimap<String, MetricCriterion> metricCriterionMultimap = ArrayListMultimap.create();
query.getMetricCriteria().forEach(metricCriterion -> metricCriterionMultimap.put(metricCriterion.getMetricKey(), metricCriterion));
}
SearchRequestBuilder searchQuery = client
- .prepareSearch(INDEX_TYPE_PROJECT_MEASURES)
+ .prepareSearch(TYPE_PROJECT_MEASURES.getMainType())
.setQuery(authorizationTypeSupport.createQueryFilter())
.setFetchSource(false)
.setSize(0)
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
-import org.sonar.server.es.DefaultIndexSettings;
+import org.sonar.server.es.newindex.DefaultIndexSettings;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
-import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER;
+import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME;
package org.sonar.server.permission.index;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.elasticsearch.action.index.IndexRequest;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import static java.util.Collections.emptyList;
import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
-import static org.sonar.core.util.stream.MoreCollectors.toSet;
/**
* Populates the types "authorization" of each index requiring project
private final DbClient dbClient;
private final EsClient esClient;
private final Collection<AuthorizationScope> authorizationScopes;
- private final Set<IndexType> indexTypes;
+ private final Map<String, IndexType> indexTypeByFormat;
public PermissionIndexer(DbClient dbClient, EsClient esClient, NeedAuthorizationIndexer... needAuthorizationIndexers) {
this(dbClient, esClient, Arrays.stream(needAuthorizationIndexers)
this.dbClient = dbClient;
this.esClient = esClient;
this.authorizationScopes = authorizationScopes;
- this.indexTypes = authorizationScopes.stream()
+ this.indexTypeByFormat = authorizationScopes.stream()
.map(AuthorizationScope::getIndexType)
- .collect(toSet(authorizationScopes.size()));
+ .collect(MoreCollectors.uniqueIndex(IndexType.IndexMainType::format, t -> t, authorizationScopes.size()));
}
@Override
public Set<IndexType> getIndexTypes() {
- return indexTypes;
+ return ImmutableSet.copyOf(indexTypeByFormat.values());
}
@Override
}
private Collection<EsQueueDto> insertIntoEsQueue(DbSession dbSession, Collection<String> projectUuids) {
- List<EsQueueDto> items = indexTypes.stream()
- .flatMap(indexType -> projectUuids.stream().map(projectUuid -> EsQueueDto.create(indexType.format(), projectUuid, null, projectUuid)))
+ List<EsQueueDto> items = indexTypeByFormat.values().stream()
+ .flatMap(indexType -> projectUuids.stream().map(projectUuid -> EsQueueDto.create(indexType.format(), AuthorizationDoc.idOf(projectUuid), null, projectUuid)))
.collect(toArrayList());
dbClient.esQueueDao().insert(dbSession, items);
authorizations.stream()
.filter(scope.getProjectPredicate())
- .map(dto -> newIndexRequest(dto, indexType))
+ .map(dto -> AuthorizationDoc.fromDto(indexType, dto).toIndexRequest())
.forEach(bulkIndexer::add);
bulkIndexer.stop();
List<BulkIndexer> bulkIndexers = items.stream()
.map(EsQueueDto::getDocType)
.distinct()
- .map(IndexType::parse)
- .filter(indexTypes::contains)
+ .map(indexTypeByFormat::get)
+ .filter(Objects::nonNull)
.map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)))
.collect(Collectors.toList());
bulkIndexers.forEach(BulkIndexer::start);
PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao();
- Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet());
+ Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId)
+ .map(AuthorizationDoc::projectUuidOf)
+ .collect(MoreCollectors.toHashSet());
permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> {
remainingProjectUuids.remove(p.getProjectUuid());
- bulkIndexers.forEach(bi -> bi.add(newIndexRequest(p, bi.getIndexType())));
+ bulkIndexers.forEach(bi -> bi.add(AuthorizationDoc.fromDto(bi.getIndexType(), p).toIndexRequest()));
});
// the remaining references on projects that don't exist in db. They must
// be deleted from index.
- remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> bi.addDeletion(bi.getIndexType(), projectUuid, projectUuid)));
+ remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> {
+ String authorizationDocId = AuthorizationDoc.idOf(projectUuid);
+ bi.addDeletion(bi.getIndexType(), authorizationDocId, authorizationDocId);
+ }));
bulkIndexers.forEach(b -> result.add(b.stop()));
return result;
}
- private static IndexRequest newIndexRequest(IndexPermissions dto, IndexType indexType) {
- Map<String, Object> doc = new HashMap<>();
- if (dto.isAllowAnyone()) {
- doc.put(IndexAuthorizationConstants.FIELD_ALLOW_ANYONE, true);
- // no need to feed users and groups
- } else {
- doc.put(IndexAuthorizationConstants.FIELD_ALLOW_ANYONE, false);
- doc.put(IndexAuthorizationConstants.FIELD_GROUP_IDS, dto.getGroupIds());
- doc.put(IndexAuthorizationConstants.FIELD_USER_IDS, dto.getUserIds());
- }
- return new IndexRequest(indexType.getIndex(), indexType.getType())
- .id(dto.getProjectUuid())
- .routing(dto.getProjectUuid())
- .source(doc);
- }
-
private Stream<AuthorizationScope> getScopes(Set<IndexType> indexTypes) {
return authorizationScopes.stream()
.filter(scope -> indexTypes.contains(scope.getIndexType()));
import org.sonar.server.component.index.ComponentIndexDefinition;
import org.sonar.server.es.BulkIndexer;
import org.sonar.server.es.EsClient;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexType;
import org.sonar.server.issue.index.IssueIndexDefinition;
import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition;
esClient.prepareClearCache().get();
for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) {
- clearIndex(new IndexType(index, index));
+ /*under the hood, type is not used to perform clearIndex, so it's ok it does not match any existing index*/
+ clearIndex(Index.simple(index));
}
} catch (Exception e) {
throw new IllegalStateException("Unable to clear indexes", e);
throw new IllegalStateException("Fail to reset data", e);
}
- clearIndex(IssueIndexDefinition.INDEX_TYPE_ISSUE);
- clearIndex(ViewIndexDefinition.INDEX_TYPE_VIEW);
- clearIndex(ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES);
- clearIndex(ComponentIndexDefinition.INDEX_TYPE_COMPONENT);
+ clearIndex(IssueIndexDefinition.DESCRIPTOR);
+ clearIndex(ViewIndexDefinition.DESCRIPTOR);
+ clearIndex(ProjectMeasuresIndexDefinition.DESCRIPTOR);
+ clearIndex(ComponentIndexDefinition.DESCRIPTOR);
}
private void truncateAnalysisTables(Connection connection) throws SQLException {
/**
* Completely remove a index with all types
*/
- public void clearIndex(IndexType indexType) {
- BulkIndexer.delete(esClient, indexType, esClient.prepareSearch(indexType.getIndex()).setQuery(matchAllQuery()));
+ public void clearIndex(Index index) {
+ BulkIndexer.delete(esClient, IndexType.main(index, index.getName()), esClient.prepareSearch(index).setQuery(matchAllQuery()));
}
@FunctionalInterface
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
+import org.sonar.server.es.IndexType.IndexMainType;
import org.sonar.server.es.metadata.MetadataIndex;
import org.sonar.server.es.metadata.MetadataIndexDefinition;
+import org.sonar.server.es.newindex.NewRegularIndex;
+import org.sonar.server.es.newindex.SettingsConfiguration;
import static org.assertj.core.api.Assertions.assertThat;
import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.IndexType.main;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
public class IndexCreatorTest {
- private static final NewIndex.SettingsConfiguration SETTINGS_CONFIGURATION = newBuilder(new MapSettings().asConfig()).build();
+ private static final SettingsConfiguration SETTINGS_CONFIGURATION = newBuilder(new MapSettings().asConfig()).build();
private static final String LOG_DB_VENDOR_CHANGED = "Delete Elasticsearch indices (DB vendor changed)";
private static final String LOG_DB_SCHEMA_CHANGED = "Delete Elasticsearch indices (DB schema changed)";
@Test
public void mark_all_non_existing_index_types_as_uninitialized() {
+ Index fakesIndex = Index.simple("fakes");
+ Index fakersIndex = Index.simple("fakers");
startNewCreator(context -> {
- NewIndex i = context.create("fakes", SETTINGS_CONFIGURATION);
- i.createType("t1");
- i.createType("t2");
+ context.create(fakesIndex, SETTINGS_CONFIGURATION)
+ .createTypeMapping(IndexType.main(fakesIndex, "fake"));
+ context.create(fakersIndex, SETTINGS_CONFIGURATION)
+ .createTypeMapping(IndexType.main(fakersIndex, "faker"));
});
- assertThat(metadataIndex.getHash("fakes")).isNotEmpty();
- assertThat(metadataIndex.getInitialized(new IndexType("fakes", "t1"))).isFalse();
- assertThat(metadataIndex.getInitialized(new IndexType("fakes", "t2"))).isFalse();
+ assertThat(metadataIndex.getHash(fakesIndex)).isNotEmpty();
+ assertThat(metadataIndex.getHash(fakersIndex)).isNotEmpty();
+ assertThat(metadataIndex.getInitialized(main(fakesIndex, "fake"))).isFalse();
+ assertThat(metadataIndex.getInitialized(main(fakersIndex, "faker"))).isFalse();
}
@Test
// v1
startNewCreator(new FakeIndexDefinition());
- IndexType fakeIndexType = new IndexType("fakes", "fake");
+ IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake");
String id = "1";
es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue();
public void do_not_recreate_index_on_unchanged_definition() {
// v1
startNewCreator(new FakeIndexDefinition());
- IndexType fakeIndexType = new IndexType("fakes", "fake");
+ IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake");
String id = "1";
es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(IMMEDIATE).get();
assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue();
private static class FakeIndexDefinition implements IndexDefinition {
- private static final IndexType INDEX_TYPE = new IndexType("fakes", "fake");
+ private static final IndexMainType INDEX_TYPE = main(Index.simple("fakes"), "fake");
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create("fakes", SETTINGS_CONFIGURATION);
- NewIndex.NewIndexType mapping = index.createType("fake");
- mapping.keywordFieldBuilder("key").build();
- mapping.createDateTimeField("updatedAt");
+ Index index = Index.simple("fakes");
+ NewRegularIndex newIndex = context.create(index, SETTINGS_CONFIGURATION);
+ newIndex.createTypeMapping(IndexType.main(index, "fake"))
+ .keywordFieldBuilder("key").build()
+ .createDateTimeField("updatedAt");
}
}
private static class FakeIndexDefinitionV2 implements IndexDefinition {
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create("fakes", SETTINGS_CONFIGURATION);
- NewIndex.NewIndexType mapping = index.createType("fake");
- mapping.keywordFieldBuilder("key").build();
- mapping.createDateTimeField("updatedAt");
- mapping.createIntegerField("newField");
+ Index index = Index.simple("fakes");
+ NewRegularIndex newIndex = context.create(index, SETTINGS_CONFIGURATION);
+ newIndex.createTypeMapping(IndexType.main(index, "fake"))
+ .keywordFieldBuilder("key").build()
+ .createDateTimeField("updatedAt")
+ .createIntegerField("newField");
}
}
}
import org.mockito.Mockito;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.server.es.metadata.MetadataIndex;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE;
+import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE;
public class IndexerStartupTaskTest {
@Before
public void setUp() throws Exception {
- doReturn(ImmutableSet.of(INDEX_TYPE_FAKE)).when(indexer).getIndexTypes();
+ doReturn(ImmutableSet.of(TYPE_FAKE)).when(indexer).getIndexTypes();
}
@Test
public void index_if_not_initialized() {
- doReturn(false).when(metadataIndex).getInitialized(INDEX_TYPE_FAKE);
+ doReturn(false).when(metadataIndex).getInitialized(TYPE_FAKE);
underTest.execute();
verify(indexer).getIndexTypes();
- verify(indexer).indexOnStartup(Mockito.eq(ImmutableSet.of(INDEX_TYPE_FAKE)));
+ verify(indexer).indexOnStartup(Mockito.eq(ImmutableSet.of(TYPE_FAKE)));
}
@Test
public void set_initialized_after_indexation() {
- doReturn(false).when(metadataIndex).getInitialized(INDEX_TYPE_FAKE);
+ doReturn(false).when(metadataIndex).getInitialized(TYPE_FAKE);
underTest.execute();
- verify(metadataIndex).setInitialized(eq(INDEX_TYPE_FAKE), eq(true));
+ verify(metadataIndex).setInitialized(eq(TYPE_FAKE), eq(true));
}
@Test
public void do_not_index_if_already_initialized() {
- doReturn(true).when(metadataIndex).getInitialized(INDEX_TYPE_FAKE);
+ doReturn(true).when(metadataIndex).getInitialized(TYPE_FAKE);
underTest.execute();
@Test
public void do_not_index_if_indexes_are_disabled() {
settings.setProperty("sonar.internal.es.disableIndexes", "true");
- es.putDocuments(INDEX_TYPE_FAKE, new FakeDoc());
+ es.putDocuments(TYPE_FAKE, new FakeDoc());
underTest.execute();
package org.sonar.server.es;
import java.util.Iterator;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;
import static org.assertj.core.api.Assertions.assertThat;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
public class MigrationEsClientImplTest {
@Rule
public LogTester logTester = new LogTester();
@Rule
public EsTester es = EsTester.createCustom(
- new SimpleIndexDefinition("a"),
- new SimpleIndexDefinition("b"),
- new SimpleIndexDefinition("c"));
+ new SimpleIndexDefinition("as"),
+ new SimpleIndexDefinition("bs"),
+ new SimpleIndexDefinition("cs"));
private MigrationEsClient underTest = new MigrationEsClientImpl(es.client());
@Test
public void delete_existing_index() {
- underTest.deleteIndexes("a");
+ underTest.deleteIndexes("as");
assertThat(loadExistingIndices())
- .doesNotContain("a")
- .contains("b", "c");
+ .doesNotContain("as")
+ .contains("bs", "cs");
assertThat(logTester.logs(LoggerLevel.INFO))
- .contains("Drop Elasticsearch index [a]");
+ .contains("Drop Elasticsearch index [as]");
}
@Test
public void ignore_indices_that_do_not_exist() {
- underTest.deleteIndexes("a", "xxx", "c");
+ underTest.deleteIndexes("as", "xxx", "cs");
assertThat(loadExistingIndices())
- .doesNotContain("a", "c")
- .contains("b");
+ .doesNotContain("as", "cs")
+ .contains("bs");
assertThat(logTester.logs(LoggerLevel.INFO))
- .contains("Drop Elasticsearch index [a]", "Drop Elasticsearch index [c]")
+ .contains("Drop Elasticsearch index [as]", "Drop Elasticsearch index [cs]")
.doesNotContain("Drop Elasticsearch index [xxx]");
}
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(indexName, newBuilder(new MapSettings().asConfig()).build());
- index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
- index.getSettings().put("index.refresh_interval", "-1");
+ IndexType.IndexMainType mainType = IndexType.main(Index.simple(indexName), indexName.substring(1));
+ context.create(
+ mainType.getIndex(),
+ newBuilder(new MapSettings().asConfig()).build())
+ .createTypeMapping(mainType);
}
}
}
import org.sonar.db.DbSession;
import org.sonar.db.DbTester;
import org.sonar.db.es.EsQueueDto;
+import org.sonar.server.es.IndexType.IndexMainType;
import org.sonar.server.rule.index.RuleIndexer;
import org.sonar.server.user.index.UserIndexer;
public class RecoveryIndexerTest {
private static final long PAST = 1_000L;
- private static final IndexType FOO_TYPE = new IndexType("foos", "foo");
+ private static final IndexMainType FOO_TYPE = IndexType.main(Index.simple("foos"), "foo");
private TestSystem2 system2 = new TestSystem2().setNow(PAST);
private MapSettings emptySettings = new MapSettings();
@Test
public void successfully_recover_indexing_requests() {
- IndexType type1 = new IndexType("foos", "foo");
- EsQueueDto item1a = insertItem(type1, "f1");
- EsQueueDto item1b = insertItem(type1, "f2");
- IndexType type2 = new IndexType("bars", "bar");
+ EsQueueDto item1a = insertItem(FOO_TYPE, "f1");
+ EsQueueDto item1b = insertItem(FOO_TYPE, "f2");
+ IndexMainType type2 = IndexType.main(Index.simple("bars"), "bar");
EsQueueDto item2 = insertItem(type2, "b1");
- SuccessfulFakeIndexer indexer1 = new SuccessfulFakeIndexer(type1);
+ SuccessfulFakeIndexer indexer1 = new SuccessfulFakeIndexer(FOO_TYPE);
SuccessfulFakeIndexer indexer2 = new SuccessfulFakeIndexer(type2);
advanceInTime();
public void unsupported_types_are_kept_in_queue_for_manual_fix_operation() {
insertItem(FOO_TYPE, "f1");
- ResilientIndexer indexer = new SuccessfulFakeIndexer(new IndexType("bars", "bar"));
+ ResilientIndexer indexer = new SuccessfulFakeIndexer(IndexType.main(Index.simple("bars"), "bar"));
advanceInTime();
underTest = newRecoveryIndexer(indexer);
import org.mockito.Mockito;
import org.sonar.db.DbClient;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
private TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db);
- private WebIssueStorage underTest = new WebIssueStorage(system2, dbClient, new DefaultRuleFinder(db.getDbClient(), defaultOrganizationProvider), mock(IssueIndexer.class));
+ private IssueIndexer issueIndexer = mock(IssueIndexer.class);
+ private WebIssueStorage underTest = new WebIssueStorage(system2, dbClient, new DefaultRuleFinder(db.getDbClient(), defaultOrganizationProvider), issueIndexer);
@Test
public void load_component_id_from_db() {
// project2 can be seen by group2
indexIssue(newDoc("I2", file2));
authorizationIndexer.allowOnlyGroup(project2, group2);
- // project3 can be seen by nobody
+ // project3 can be seen by nobody but root
indexIssue(newDoc("I3", file3));
userSessionRule.logIn().setGroups(group1);
userSessionRule.logIn().setGroups(group1, group2);
assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList(project3.uuid())));
+
+ userSessionRule.setRoot();
+ assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2", "I3");
}
@Test
// another user
userSessionRule.logIn(newUserDto());
assertThatSearchReturnsEmpty(IssueQuery.builder());
+
+ userSessionRule.setRoot();
+ assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2", "I3");
}
@Test
import static org.sonar.db.user.GroupTesting.newGroupDto;
import static org.sonar.db.user.UserTesting.newUserDto;
import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS;
-import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
import static org.sonar.server.measure.index.ProjectMeasuresQuery.Operator;
@RunWith(DataProviderRunner.class)
@Test
public void search_statistics() {
- es.putDocuments(INDEX_TYPE_PROJECT_MEASURES,
+ es.putDocuments(TYPE_PROJECT_MEASURES,
newDoc("lines", 10, "coverage", 80)
.setLanguages(Arrays.asList("java", "cs", "js"))
.setNclocLanguageDistributionFromMap(ImmutableMap.of("java", 200, "cs", 250, "js", 50)),
}
private void index(ProjectMeasuresDoc... docs) {
- es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs);
+ es.putDocuments(TYPE_PROJECT_MEASURES, docs);
authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).allowAnyone()).collect(toList()));
}
private void indexForUser(UserDto user, ProjectMeasuresDoc... docs) {
- es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs);
+ es.putDocuments(TYPE_PROJECT_MEASURES, docs);
authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).addUserId(user.getId())).collect(toList()));
}
private void indexForGroup(GroupDto group, ProjectMeasuresDoc... docs) {
- es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs);
+ es.putDocuments(TYPE_PROJECT_MEASURES, docs);
authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).addGroupId(group.getId())).collect(toList()));
}
import static org.assertj.core.api.Assertions.entry;
import static org.sonar.api.resources.Qualifiers.PROJECT;
import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto;
-import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
import static org.sonar.server.measure.index.ProjectMeasuresQuery.Operator.GT;
import static org.sonar.server.measure.index.ProjectMeasuresQuery.Operator.LT;
}
private void index(ProjectMeasuresDoc... docs) {
- es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs);
+ es.putDocuments(TYPE_PROJECT_MEASURES, docs);
authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).allowAnyone()).collect(toList()));
}
private void assertUserIsNotMember(OrganizationDto organization, UserDto user) {
db.organizations().assertUserIsNotMemberOfOrganization(organization, user);
- SearchRequestBuilder request = es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER)
+ SearchRequestBuilder request = es.client().prepareSearch(UserIndexDefinition.TYPE_USER)
.setQuery(boolQuery()
.must(termQuery(FIELD_ORGANIZATION_UUIDS, organization.getUuid()))
.must(termQuery(FIELD_UUID, user.getUuid())));
import org.sonar.server.permission.PermissionServiceImpl;
import org.sonar.server.qualityprofile.BuiltInQProfileRepository;
import org.sonar.server.tester.UserSessionRule;
-import org.sonar.server.user.index.UserIndexDefinition;
import org.sonar.server.user.index.UserIndexer;
import org.sonar.server.usergroups.DefaultGroupCreatorImpl;
import org.sonar.server.ws.TestRequest;
import static org.sonar.server.organization.ws.OrganizationsWsSupport.PARAM_NAME;
import static org.sonar.server.organization.ws.OrganizationsWsTestSupport.STRING_257_CHARS_LONG;
import static org.sonar.server.organization.ws.OrganizationsWsTestSupport.STRING_65_CHARS_LONG;
+import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER;
import static org.sonar.server.user.index.UserIndexDefinition.FIELD_ORGANIZATION_UUIDS;
import static org.sonar.server.user.index.UserIndexDefinition.FIELD_UUID;
import static org.sonar.test.JsonAssert.assertJson;
OrganizationDto organization = dbClient.organizationDao().selectByKey(dbSession, "bar").get();
assertThat(dbClient.organizationMemberDao().select(dbSession, organization.getUuid(), user.getId())).isPresent();
- assertThat(es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER)
+ assertThat(es.client().prepareSearch(TYPE_USER)
.setQuery(boolQuery()
.must(termQuery(FIELD_ORGANIZATION_UUIDS, organization.getUuid()))
.must(termQuery(FIELD_UUID, user.getUuid())))
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.server.es.EsClient;
-import static org.sonar.server.permission.index.FooIndexDefinition.FOO_INDEX;
-import static org.sonar.server.permission.index.FooIndexDefinition.FOO_TYPE;
+import static org.sonar.server.permission.index.FooIndexDefinition.DESCRIPTOR;
+import static org.sonar.server.permission.index.FooIndexDefinition.TYPE_AUTHORIZATION;
public class FooIndex {
}
public boolean hasAccessToProject(String projectUuid) {
- SearchHits hits = esClient.prepareSearch(FOO_INDEX)
- .setTypes(FOO_TYPE)
+ SearchHits hits = esClient.prepareSearch(DESCRIPTOR)
+ .setTypes(TYPE_AUTHORIZATION.getType())
.setQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid))
.filter(authorizationTypeSupport.createQueryFilter()))
package org.sonar.server.permission.index;
import org.sonar.api.config.internal.MapSettings;
+import org.sonar.server.es.Index;
import org.sonar.server.es.IndexDefinition;
import org.sonar.server.es.IndexType;
-import org.sonar.server.es.NewIndex;
+import org.sonar.server.es.newindex.NewAuthorizedIndex;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
-import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL;
+import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;
public class FooIndexDefinition implements IndexDefinition {
- public static final String FOO_INDEX = "foos";
+ public static final Index DESCRIPTOR = Index.withRelations("foos");
public static final String FOO_TYPE = "foo";
- public static final IndexType INDEX_TYPE_FOO = new IndexType(FOO_INDEX, FOO_TYPE);
+ public static final IndexType.IndexMainType TYPE_AUTHORIZATION = IndexType.main(DESCRIPTOR, IndexAuthorizationConstants.TYPE_AUTHORIZATION);
+ public static final IndexType.IndexRelationType TYPE_FOO = IndexType.relation(TYPE_AUTHORIZATION, FOO_TYPE);
public static final String FIELD_NAME = "name";
public static final String FIELD_PROJECT_UUID = "projectUuid";
@Override
public void define(IndexDefinitionContext context) {
- NewIndex index = context.create(FOO_INDEX, newBuilder(new MapSettings().asConfig()).setRefreshInterval(MANUAL_REFRESH_INTERVAL).build());
+ NewAuthorizedIndex newIndex = context.createWithAuthorization(
+ DESCRIPTOR,
+ newBuilder(new MapSettings().asConfig())
+ .setRefreshInterval(MANUAL_REFRESH_INTERVAL)
+ .build());
- NewIndex.NewIndexType type = index.createType(FOO_TYPE)
- .requireProjectAuthorization();
-
- type.keywordFieldBuilder(FIELD_NAME).build();
- type.keywordFieldBuilder(FIELD_PROJECT_UUID).build();
+ newIndex.createTypeMapping(TYPE_FOO)
+ .keywordFieldBuilder(FIELD_NAME).build()
+ .keywordFieldBuilder(FIELD_PROJECT_UUID).build();
}
}
*/
package org.sonar.server.permission.index;
-import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import java.util.Collection;
import java.util.Set;
import org.sonar.db.DbSession;
import org.sonar.db.es.EsQueueDto;
+import org.sonar.server.es.BaseDoc;
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexType;
import org.sonar.server.es.IndexingResult;
import org.sonar.server.es.ProjectIndexer;
-import static org.sonar.server.permission.index.FooIndexDefinition.INDEX_TYPE_FOO;
+import static org.sonar.server.permission.index.FooIndexDefinition.TYPE_FOO;
public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer {
- private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_FOO, p -> true);
+ private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_FOO, p -> true);
private final EsClient esClient;
}
private void addToIndex(String projectUuid, String name) {
- esClient.prepareIndex(INDEX_TYPE_FOO)
- .setRouting(projectUuid)
- .setParent(projectUuid)
- .setSource(ImmutableMap.of(
- FooIndexDefinition.FIELD_NAME, name,
- FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid))
+ FooDoc fooDoc = new FooDoc(projectUuid, name);
+ esClient.prepareIndex(TYPE_FOO)
+ .setId(fooDoc.getId())
+ .setRouting(fooDoc.getRouting().orElse(null))
+ .setSource(fooDoc.getFields())
.get();
}
+ private static final class FooDoc extends BaseDoc {
+ private final String projectUuid;
+ private final String name;
+
+ private FooDoc(String projectUuid, String name) {
+ super(TYPE_FOO);
+ this.projectUuid = projectUuid;
+ this.name = name;
+ setField(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid);
+ setField(FooIndexDefinition.FIELD_NAME, name);
+ setParent(AuthorizationDoc.idOf(projectUuid));
+ }
+
+ @Override
+ public String getId() {
+ return projectUuid + "_" + name;
+ }
+
+ }
+
@Override
public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
throw new UnsupportedOperationException();
@Override
public Set<IndexType> getIndexTypes() {
- return ImmutableSet.of(INDEX_TYPE_FOO);
+ return ImmutableSet.of(TYPE_FOO);
}
@Override
import org.sonar.db.user.UserDto;
import org.sonar.server.es.EsTester;
import org.sonar.server.es.IndexType;
+import org.sonar.server.es.IndexType.IndexMainType;
import org.sonar.server.es.IndexingResult;
import org.sonar.server.es.ProjectIndexer;
import org.sonar.server.tester.UserSessionRule;
public class PermissionIndexerTest {
- private static final IndexType INDEX_TYPE_FOO_AUTH = new IndexType(FooIndexDefinition.INDEX_TYPE_FOO.getIndex(), TYPE_AUTHORIZATION);
+ private static final IndexMainType INDEX_TYPE_FOO_AUTH = IndexType.main(FooIndexDefinition.DESCRIPTOR, TYPE_AUTHORIZATION);
@Rule
public ExpectedException expectedException = ExpectedException.none();
}
private void assertThatAuthIndexHasSize(int expectedSize) {
- IndexType authIndexType = underTest.getIndexTypes().iterator().next();
- assertThat(es.countDocuments(authIndexType)).isEqualTo(expectedSize);
+ assertThat(es.countDocuments(FooIndexDefinition.TYPE_AUTHORIZATION)).isEqualTo(expectedSize);
}
private void indexOnStartup() {
" \"bool\" : {" +
" \"should\" : [{" +
" \"term\" : {" +
- " \"allowAnyone\" : {\"value\": true}" +
+ " \"auth_allowAnyone\" : {\"value\": true}" +
" }" +
" }]" +
" }" +
" }]" +
" }" +
" }," +
- " \"parent_type\" : \"authorization\"" +
+ " \"parent_type\" : \"auth\"" +
" }" +
"}");
}
" \"should\": [" +
" {" +
" \"term\": {" +
- " \"allowAnyone\": {\"value\": true}" +
+ " \"auth_allowAnyone\": {\"value\": true}" +
" }" +
" }," +
" {" +
" \"term\": {" +
- " \"userIds\": {\"value\": 1234}" +
+ " \"auth_userIds\": {\"value\": 1234}" +
" }" +
" }" +
" ]" +
" }]" +
" }" +
" }," +
- " \"parent_type\": \"authorization\"" +
+ " \"parent_type\": \"auth\"" +
" }" +
"}");
}
" \"should\": [" +
" {" +
" \"term\": {" +
- " \"allowAnyone\": {\"value\": true}" +
+ " \"auth_allowAnyone\": {\"value\": true}" +
" }" +
" }," +
" {" +
" \"term\": {" +
- " \"userIds\": {\"value\": 1234}" +
+ " \"auth_userIds\": {\"value\": 1234}" +
" }" +
" }," +
" {" +
" \"term\": {" +
- " \"groupIds\": {\"value\": 10}" +
+ " \"auth_groupIds\": {\"value\": 10}" +
" }" +
" }," +
" {" +
" \"term\": {" +
- " \"groupIds\": {\"value\": 11}" +
+ " \"auth_groupIds\": {\"value\": 11}" +
" }" +
" }" +
" ]" +
" }]" +
" }" +
" }," +
- " \"parent_type\": \"authorization\"" +
+ " \"parent_type\": \"auth\"" +
" }" +
"}");
}
@Test
public void clear_indexes() {
- es.putDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE, IssueDocTesting.newDoc());
- es.putDocuments(RuleIndexDefinition.INDEX_TYPE_RULE, newRuleDoc());
- es.putDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, newComponentDoc());
+ es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+ es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+ es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
underTest.clearIndexes();
- assertThat(es.countDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE)).isEqualTo(0);
- assertThat(es.countDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT)).isEqualTo(0);
+ assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
+ assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
}
@Test
public void clear_all() {
dbTester.prepareDbUnit(getClass(), "shared.xml");
- es.putDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE, IssueDocTesting.newDoc());
- es.putDocuments(RuleIndexDefinition.INDEX_TYPE_RULE, newRuleDoc());
- es.putDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, newComponentDoc());
+ es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+ es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+ es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
underTest.clearAll();
- assertThat(es.countDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE)).isEqualTo(0);
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(0);
- assertThat(es.countDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT)).isEqualTo(0);
+ assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0);
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(0);
+ assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0);
assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0);
assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0);
@Test
public void reset_data() {
dbTester.prepareDbUnit(getClass(), "shared.xml");
- es.putDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE, IssueDocTesting.newDoc());
- es.putDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG")));
- es.putDocuments(RuleIndexDefinition.INDEX_TYPE_RULE, newRuleDoc());
- es.putDocuments(ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc()
+ es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc());
+ es.putDocuments(ViewIndexDefinition.TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG")));
+ es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc());
+ es.putDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc()
.setId("PROJECT")
.setKey("Key")
.setName("Name"));
- es.putDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, newComponentDoc());
+ es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc());
underTest.resetData();
assertThat(dbTester.countRowsOfTable("projects")).isZero();
assertThat(dbTester.countRowsOfTable("snapshots")).isZero();
assertThat(dbTester.countRowsOfTable("properties")).isZero();
- assertThat(es.countDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE)).isZero();
- assertThat(es.countDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW)).isZero();
- assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES)).isZero();
- assertThat(es.countDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT)).isZero();
+ assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isZero();
+ assertThat(es.countDocuments(ViewIndexDefinition.TYPE_VIEW)).isZero();
+ assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).isZero();
+ assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isZero();
// Rules should not be removed
assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(1);
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1);
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1);
}
private static RuleDoc newRuleDoc() {
import org.junit.Test;
import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.FakeIndexDefinition;
+import org.sonar.server.es.newindex.FakeIndexDefinition;
import static org.assertj.core.api.Assertions.assertThat;
import static java.util.stream.Collectors.toList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.sonar.api.resources.Qualifiers.PROJECT;
-import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES;
+import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES;
import static org.sonar.test.JsonAssert.assertJson;
public class SearchActionTest {
}
private void index(ProjectMeasuresDoc... docs) {
- es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs);
+ es.putDocuments(TYPE_PROJECT_MEASURES, docs);
authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).allowAnyone()).collect(toList()));
}
.containsOnly(READY);
// verify index
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(numberOfRules);
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(numberOfRules);
assertThat(ruleIndex.search(new RuleQuery(), new SearchOptions()).getIds())
.isNotEmpty();
.containsOnly(REMOVED);
// verify index (documents are still in the index, but all are removed)
- assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(numberOfRules);
+ assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(numberOfRules);
assertThat(ruleIndex.search(new RuleQuery(), new SearchOptions()).getIds())
.isEmpty();
}
RuleDto rule1 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY1);
RuleDto rule2 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY2);
RuleDto hotspotRule = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, HOTSPOT_RULE_KEY);
- assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId()));
+ assertThat(es.getIds(RuleIndexDefinition.TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId()));
// user adds tags and sets markdown note
rule1.setTags(newHashSet("usertag1", "usertag2"));
RuleDto rule1 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY1);
RuleDto rule2 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY2);
RuleDto hotspotRule = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, HOTSPOT_RULE_KEY);
- assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId()));
+ assertThat(es.getIds(RuleIndexDefinition.TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId()));
assertThat(rule2.getStatus()).isEqualTo(READY);
execute(new BigRepository());
assertThat(db.countRowsOfTable("rules")).isEqualTo(BigRepository.SIZE);
assertThat(db.countRowsOfTable("rules_parameters")).isEqualTo(BigRepository.SIZE * 20);
- assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE)).hasSize(BigRepository.SIZE);
+ assertThat(es.getIds(RuleIndexDefinition.TYPE_RULE)).hasSize(BigRepository.SIZE);
}
@Test
*/
package org.sonar.server.search;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.util.Collections;
import java.util.Date;
import java.util.Map;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.sonar.server.es.BaseDoc;
import org.sonar.server.es.EsUtils;
+import org.sonar.server.es.Index;
+import org.sonar.server.es.IndexType;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
public class BaseDocTest {
+ private final IndexType.IndexMainType someType = IndexType.main(Index.simple("bar"), "donut");
+
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
@Test
public void getField() {
fields.put("a_string", "foo");
fields.put("a_int", 42);
fields.put("a_null", null);
- BaseDoc doc = new BaseDoc(fields) {
+ BaseDoc doc = new BaseDoc(someType, fields) {
@Override
public String getId() {
return null;
}
- @Override
- public String getRouting() {
- return null;
- }
-
- @Override
- public String getParent() {
- return null;
- }
};
assertThat((String) doc.getNullableField("a_string")).isEqualTo("foo");
@Test
public void getField_fails_if_missing_field() {
Map<String, Object> fields = Collections.emptyMap();
- BaseDoc doc = new BaseDoc(fields) {
+ BaseDoc doc = new BaseDoc(someType, fields) {
@Override
public String getId() {
return null;
}
- @Override
- public String getRouting() {
- return null;
- }
-
- @Override
- public String getParent() {
- return null;
- }
};
try {
@Test
public void getFieldAsDate() {
- BaseDoc doc = new BaseDoc(Maps.newHashMap()) {
+ BaseDoc doc = new BaseDoc(someType, Maps.newHashMap()) {
@Override
public String getId() {
return null;
}
- @Override
- public String getRouting() {
- return null;
- }
-
- @Override
- public String getParent() {
- return null;
- }
};
Date now = new Date();
doc.setField("javaDate", now);
@Test
public void getNullableFieldAsDate() {
- BaseDoc doc = new BaseDoc(Maps.newHashMap()) {
+ BaseDoc doc = new BaseDoc(someType, Maps.newHashMap()) {
@Override
public String getId() {
return null;
}
- @Override
- public String getRouting() {
- return null;
- }
-
- @Override
- public String getParent() {
- return null;
- }
};
Date now = new Date();
doc.setField("javaDate", now);
doc.setField("noValue", null);
assertThat(doc.getNullableFieldAsDate("noValue")).isNull();
}
+
+ @Test
+ public void getFields_fails_with_ISE_if_setParent_has_not_been_called_on_IndexRelationType() {
+ IndexType.IndexRelationType relationType = IndexType.relation(IndexType.main(Index.withRelations("foo"), "bar"), "donut");
+ BaseDoc doc = new BaseDoc(relationType) {
+
+ @Override
+ public String getId() {
+ throw new UnsupportedOperationException("getId not implemented");
+ }
+
+ };
+
+ expectedException.expect(IllegalStateException.class);
+ expectedException.expectMessage("parent must be set on a doc associated to a IndexRelationType (see BaseDoc#setParent(String))");
+
+ doc.getFields();
+ }
+
+ @Test
+ public void getFields_contains_join_field_and_indexType_field_when_setParent_has_been_called_on_IndexRelationType() {
+ Index index = Index.withRelations("foo");
+ IndexType.IndexRelationType relationType = IndexType.relation(IndexType.main(index, "bar"), "donut");
+ BaseDoc doc = new BaseDoc(relationType) {
+ {
+ setParent("miam");
+ }
+
+ @Override
+ public String getId() {
+ throw new UnsupportedOperationException("getId not implemented");
+ }
+
+ };
+
+ Map<String, Object> fields = doc.getFields();
+
+ assertThat((Map) fields.get(index.getJoinField()))
+ .isEqualTo(ImmutableMap.of("name", relationType.getName(), "parent", "miam"));
+ assertThat(fields.get("indexType")).isEqualTo(relationType.getName());
+ }
}
.isEqualTo(dto.getUpdatedAt());
assertThat(dbClient.userDao().selectByLogin(session, "user").getId()).isEqualTo(dto.getId());
- List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER);
+ List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.TYPE_USER);
assertThat(indexUsers).hasSize(1);
assertThat(indexUsers.get(0).getSource())
.contains(
.build(), u -> {
}, otherUser);
- assertThat(es.getIds(UserIndexDefinition.INDEX_TYPE_USER)).containsExactlyInAnyOrder(created.getUuid(), otherUser.getUuid());
+ assertThat(es.getIds(UserIndexDefinition.TYPE_USER)).containsExactlyInAnyOrder(created.getUuid(), otherUser.getUuid());
}
@Test
assertThat(updatedUser.getCreatedAt()).isEqualTo(user.getCreatedAt());
assertThat(updatedUser.getUpdatedAt()).isGreaterThan(user.getCreatedAt());
- List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER);
+ List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.TYPE_USER);
assertThat(indexUsers).hasSize(1);
assertThat(indexUsers.get(0).getSource())
.contains(
.setLogin("new_login"), u -> {
});
- List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER);
+ List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.TYPE_USER);
assertThat(indexUsers).hasSize(1);
assertThat(indexUsers.get(0).getSource())
.contains(entry("login", "new_login"));
.setScmAccounts(asList("ma2")), u -> {
}, otherUser);
- assertThat(es.getIds(UserIndexDefinition.INDEX_TYPE_USER)).containsExactlyInAnyOrder(user.getUuid(), otherUser.getUuid());
+ assertThat(es.getIds(UserIndexDefinition.TYPE_USER)).containsExactlyInAnyOrder(user.getUuid(), otherUser.getUuid());
}
@Test
.containsOnly("john", "John", "john@email.com", singletonList("jn"), true);
// exists in index
- assertThat(es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER)
+ assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER)
.setQuery(boolQuery()
.must(termQuery(FIELD_LOGIN, "john"))
.must(termQuery(FIELD_NAME, "John"))
deactivate(user.getLogin());
verifyThatUserIsDeactivated(user.getLogin());
- assertThat(es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER)
+ assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER)
.setQuery(boolQuery()
.must(termQuery(FIELD_UUID, user.getUuid()))
.must(termQuery(FIELD_ACTIVE, "false")))
projects / sonarqube.git / commitdiff