diff options
| author | Sébastien Lesaint <sebastien.lesaint@sonarsource.com> | 2019-02-20 18:46:05 +0100 |
|---|---|---|
| committer | SonarTech <sonartech@sonarsource.com> | 2019-03-19 20:21:19 +0100 |
| commit | a8f011b7f52fedd8f0294db46d3f02334383391d (patch) | |
| tree | 3f52e5670ddf9bb8d10cc61f42d6f76edc02fccb /server | |
| parent | 277774059792a8e27971243e46d0961fdd44bac8 (diff) | |
| download | sonarqube-a8f011b7f52fedd8f0294db46d3f02334383391d.tar.gz sonarqube-a8f011b7f52fedd8f0294db46d3f02334383391d.zip | |
SONAR-11791 use single type ES indices
Diffstat (limited to 'server')
143 files changed, 4684 insertions, 2316 deletions
diff --git a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistAdHocRulesStepTest.java b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistAdHocRulesStepTest.java index 78da2be751e..e01c4259c91 100644 --- a/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistAdHocRulesStepTest.java +++ b/server/sonar-ce-task-projectanalysis/src/test/java/org/sonar/ce/task/projectanalysis/step/PersistAdHocRulesStepTest.java @@ -94,8 +94,8 @@ public class PersistAdHocRulesStepTest extends BaseStepTest { assertThat(reloaded.getSeverity()).isNull(); assertThat(reloaded.getName()).isEqualTo("eslint:no-cond-assign"); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1l); - assertThat(es.getDocuments(RuleIndexDefinition.INDEX_TYPE_RULE).iterator().next().getId()).isEqualTo(Integer.toString(reloaded.getId())); + assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1l); + assertThat(es.getDocuments(RuleIndexDefinition.TYPE_RULE).iterator().next().getId()).isEqualTo(Integer.toString(reloaded.getId())); } @Test @@ -108,7 +108,7 @@ public class PersistAdHocRulesStepTest extends BaseStepTest { RuleDao ruleDao = dbClient.ruleDao(); assertThat(ruleDao.selectAllDefinitions(dbClient.openSession(false))).hasSize(1); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isZero(); + assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isZero(); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentDoc.java index 0a6128ef1fa..5ed6873b580 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentDoc.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.server.es.BaseDoc; +import org.sonar.server.permission.index.AuthorizationDoc; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_KEY; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_LANGUAGE; @@ -32,15 +33,16 @@ import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_OR import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_PROJECT_UUID; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_QUALIFIER; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_UUID; +import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; public class ComponentDoc extends BaseDoc { public ComponentDoc() { - super(new HashMap<>(6)); + super(TYPE_COMPONENT, new HashMap<>(6)); } public ComponentDoc(Map<String, Object> fields) { - super(fields); + super(TYPE_COMPONENT, fields); } @Override @@ -48,16 +50,6 @@ public class ComponentDoc extends BaseDoc { return getField(FIELD_UUID); } - @Override - public String getRouting() { - return getProjectUuid(); - } - - @Override - public String getParent() { - return getProjectUuid(); - } - public ComponentDoc setId(String s) { setField(FIELD_UUID, s); return this; @@ -69,6 +61,7 @@ public class ComponentDoc extends BaseDoc { public ComponentDoc setProjectUuid(String s) { setField(FIELD_PROJECT_UUID, s); + setParent(AuthorizationDoc.idOf(s)); return this; } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java index 6593e57f35d..42916f12d29 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexDefinition.java @@ -20,21 +20,26 @@ package org.sonar.server.component.index; import org.sonar.api.config.Configuration; -import org.sonar.server.es.DefaultIndexSettingsElement; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.newindex.DefaultIndexSettingsElement; +import org.sonar.server.es.newindex.NewAuthorizedIndex; +import org.sonar.server.es.newindex.TypeMapping; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; public class ComponentIndexDefinition implements IndexDefinition { - public static final IndexType INDEX_TYPE_COMPONENT = new IndexType("components", "component"); + public static final Index DESCRIPTOR = Index.withRelations("components"); + public static final IndexType.IndexRelationType TYPE_COMPONENT = IndexType.relation(IndexType.main(DESCRIPTOR, TYPE_AUTHORIZATION), "component"); public static final String FIELD_UUID = "uuid"; public static final String FIELD_PROJECT_UUID = "project_uuid"; public static final String FIELD_ORGANIZATION_UUID = "organization_uuid"; @@ -48,23 +53,36 @@ public class ComponentIndexDefinition implements IndexDefinition { static final DefaultIndexSettingsElement[] NAME_ANALYZERS = {SORTABLE_ANALYZER, SEARCH_PREFIX_ANALYZER, SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER, SEARCH_GRAMS_ANALYZER}; private final Configuration config; + private final boolean enableSource; - public ComponentIndexDefinition(Configuration config) { + private ComponentIndexDefinition(Configuration config, boolean enableSource) { this.config = config; + this.enableSource = enableSource; + } + + public ComponentIndexDefinition(Configuration config) { + this(config, false); + } + + /** + * Keep the document sources in index so that indexer tests can verify content + * of indexed documents. + */ + public static ComponentIndexDefinition createForTest() { + return new ComponentIndexDefinition(new MapSettings().asConfig(), true); } @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create( - INDEX_TYPE_COMPONENT.getIndex(), + NewAuthorizedIndex index = context.createWithAuthorization( + DESCRIPTOR, newBuilder(config) .setRefreshInterval(MANUAL_REFRESH_INTERVAL) .setDefaultNbOfShards(DEFAULT_NUMBER_OF_SHARDS) - .build()); - - NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_COMPONENT.getType()) - .requireProjectAuthorization(); + .build()) + .setEnableSource(enableSource); + TypeMapping mapping = index.createTypeMapping(TYPE_COMPONENT); mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_PROJECT_UUID).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_KEY).addSubFields(SORTABLE_ANALYZER).build(); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index d4f4901aa9b..15a400b3797 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -27,7 +27,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import javax.annotation.Nullable; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.sonar.core.util.stream.MoreCollectors; @@ -35,6 +34,7 @@ import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.component.ComponentDto; import org.sonar.db.es.EsQueueDto; +import org.sonar.server.es.BaseDoc; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; @@ -42,16 +42,17 @@ import org.sonar.server.es.IndexType; import org.sonar.server.es.IndexingResult; import org.sonar.server.es.OneToManyResilientIndexingListener; import org.sonar.server.es.ProjectIndexer; +import org.sonar.server.permission.index.AuthorizationDoc; import org.sonar.server.permission.index.AuthorizationScope; import org.sonar.server.permission.index.NeedAuthorizationIndexer; import static java.util.Collections.emptyList; -import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_TYPE_COMPONENT; +import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexer { - private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_COMPONENT, project -> true); - private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(INDEX_TYPE_COMPONENT); + private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_COMPONENT, project -> true); + private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(TYPE_COMPONENT); private final DbClient dbClient; private final EsClient esClient; @@ -94,7 +95,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe case PROJECT_DELETION: case PROJECT_KEY_UPDATE: List<EsQueueDto> items = projectUuids.stream() - .map(branchUuid -> EsQueueDto.create(INDEX_TYPE_COMPONENT.format(), branchUuid, null, branchUuid)) + .map(branchUuid -> EsQueueDto.create(TYPE_COMPONENT.format(), branchUuid, null, branchUuid)) .collect(MoreCollectors.toArrayList(projectUuids.size())); return dbClient.esQueueDao().insert(dbSession, items); @@ -111,7 +112,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe } OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items); - BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR, listener); + BulkIndexer bulkIndexer = new BulkIndexer(esClient, TYPE_COMPONENT, Size.REGULAR, listener); bulkIndexer.start(); Set<String> branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size())); Set<String> remaining = new HashSet<>(branchUuids); @@ -120,7 +121,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe // TODO allow scrolling multiple projects at the same time dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> { ComponentDto dto = context.getResultObject(); - bulkIndexer.add(newIndexRequest(toDocument(dto))); + bulkIndexer.add(toDocument(dto).toIndexRequest()); remaining.remove(dto.projectUuid()); }); } @@ -137,51 +138,44 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe * <b>Warning:</b> only use {@code null} during startup. */ private void doIndexByProjectUuid(@Nullable String projectUuid, Size bulkSize) { - BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, bulkSize); + BulkIndexer bulk = new BulkIndexer(esClient, TYPE_COMPONENT, bulkSize); bulk.start(); try (DbSession dbSession = dbClient.openSession(false)) { dbClient.componentDao() .scrollForIndexing(dbSession, projectUuid, context -> { ComponentDto dto = context.getResultObject(); - bulk.add(newIndexRequest(toDocument(dto))); + bulk.add(toDocument(dto).toIndexRequest()); }); } bulk.stop(); } private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { - SearchRequestBuilder searchRequest = esClient.prepareSearch(INDEX_TYPE_COMPONENT) + SearchRequestBuilder searchRequest = esClient.prepareSearch(TYPE_COMPONENT.getMainType()) .setQuery(QueryBuilders.termQuery(ComponentIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) - .setRouting(projectUuid); + .setRouting(AuthorizationDoc.idOf(projectUuid)); bulkIndexer.addDeletion(searchRequest); } public void delete(String projectUuid, Collection<String> disabledComponentUuids) { - BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR); + BulkIndexer bulk = new BulkIndexer(esClient, TYPE_COMPONENT, Size.REGULAR); bulk.start(); - disabledComponentUuids.forEach(uuid -> bulk.addDeletion(INDEX_TYPE_COMPONENT, uuid, projectUuid)); + disabledComponentUuids.forEach(uuid -> bulk.addDeletion(TYPE_COMPONENT, uuid, AuthorizationDoc.idOf(projectUuid))); bulk.stop(); } @VisibleForTesting void index(ComponentDto... docs) { - BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR); + BulkIndexer bulk = new BulkIndexer(esClient, TYPE_COMPONENT, Size.REGULAR); bulk.start(); Arrays.stream(docs) .map(ComponentIndexer::toDocument) - .map(ComponentIndexer::newIndexRequest) + .map(BaseDoc::toIndexRequest) .forEach(bulk::add); bulk.stop(); } - private static IndexRequest newIndexRequest(ComponentDoc doc) { - return new IndexRequest(INDEX_TYPE_COMPONENT.getIndex(), INDEX_TYPE_COMPONENT.getType(), doc.getId()) - .routing(doc.getRouting()) - .parent(doc.getParent()) - .source(doc.getFields()); - } - public static ComponentDoc toDocument(ComponentDto component) { return new ComponentDoc() .setId(component.uuid()) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java index 7fa5f275f68..512f151d0f3 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BaseDoc.java @@ -19,34 +19,77 @@ */ package org.sonar.server.es; +import com.google.common.collect.ImmutableMap; import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import javax.annotation.CheckForNull; import javax.annotation.Nullable; +import org.elasticsearch.action.index.IndexRequest; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.IndexType.IndexRelationType; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; /** * Base implementation for business objects based on elasticsearch document */ public abstract class BaseDoc { + private static final String SETPARENT_NOT_CALLED = "parent must be set on a doc associated to a IndexRelationType (see BaseDoc#setParent(String))"; + private final IndexType indexType; + private String parentId = null; protected final Map<String, Object> fields; - protected BaseDoc() { - this.fields = new HashMap<>(); + protected BaseDoc(IndexType indexType) { + this(indexType, new HashMap<>()); } - protected BaseDoc(Map<String, Object> fields) { + protected BaseDoc(IndexType indexType, Map<String, Object> fields) { + this.indexType = indexType; this.fields = fields; + if (indexType instanceof IndexMainType) { + IndexMainType mainType = (IndexMainType) indexType; + if (mainType.getIndex().acceptsRelations()) { + setField(mainType.getIndex().getJoinField(), ImmutableMap.of("name", mainType.getType())); + setField(FIELD_INDEX_TYPE, mainType.getType()); + } + } + } + + protected void setParent(String parentId) { + checkState(this.indexType instanceof IndexRelationType, "Doc must be associated to a IndexRelationType to set a parent"); + checkArgument(parentId != null && !parentId.isEmpty(), "parentId can't be null nor empty"); + this.parentId = parentId; + IndexRelationType indexRelationType = (IndexRelationType) this.indexType; + setField(indexRelationType.getMainType().getIndex().getJoinField(), ImmutableMap.of("name", indexRelationType.getName(), "parent", parentId)); + setField(FIELD_INDEX_TYPE, indexRelationType.getName()); } public abstract String getId(); - @CheckForNull - public abstract String getRouting(); + public Optional<String> getRouting() { + // when using relations, routing MUST be defined and MUST be the id of the parent + if (this.indexType instanceof IndexRelationType) { + ensureSetParentCalled(); + return Optional.of(this.parentId); + } + if (this.indexType instanceof IndexMainType && indexType.getMainType().getIndex().acceptsRelations()) { + return Optional.of(getId()); + } + return getSimpleMainTypeRouting(); + } - @CheckForNull - public abstract String getParent(); + /** + * Intended to be overridden by subclass which wants to define a routing and which indexType is a {@link IndexMainType} + * (if not a {@link IndexMainType}, this method will never be called). + */ + protected Optional<String> getSimpleMainTypeRouting() { + return Optional.empty(); + } /** * Use this method when field value can be null. See warning in {@link #getField(String)} @@ -64,7 +107,7 @@ public abstract class BaseDoc { Object val = getNullableField(key); if (val != null) { if (val instanceof Date) { - return (Date)val; + return (Date) val; } if (val instanceof Number) { return epochSecondsToDate((Number) val); @@ -94,22 +137,37 @@ public abstract class BaseDoc { public Date getFieldAsDate(String key) { Object value = getField(key); if (value instanceof Date) { - return (Date)value; + return (Date) value; } if (value instanceof Number) { return epochSecondsToDate((Number) value); } - return EsUtils.parseDateTime((String)value); + return EsUtils.parseDateTime((String) value); } public void setField(String key, @Nullable Object value) { fields.put(key, value); } - public Map<String, Object> getFields() { + public final Map<String, Object> getFields() { + if (indexType instanceof IndexRelationType) { + ensureSetParentCalled(); + } return fields; } + private void ensureSetParentCalled() { + checkState(this.parentId != null, SETPARENT_NOT_CALLED); + } + + public IndexRequest toIndexRequest() { + IndexMainType mainType = this.indexType.getMainType(); + return new IndexRequest(mainType.getIndex().getName(), mainType.getType()) + .id(getId()) + .routing(getRouting().orElse(null)) + .source(getFields()); + } + public static long epochMillisToEpochSeconds(long epochMillis) { return epochMillis / 1000L; } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java index 671530504bd..3db7df268b9 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -117,7 +117,7 @@ public class BulkIndexer { Thread.currentThread().interrupt(); throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e); } - client.prepareRefresh(indexType.getIndex()).get(); + client.prepareRefresh(indexType.getMainType().getIndex()).get(); sizeHandler.afterStop(this); indexingListener.onFinish(result); return result; @@ -384,21 +384,22 @@ public class BulkIndexer { @Override void beforeStart(BulkIndexer bulkIndexer) { - this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", bulkIndexer.indexType.getIndex()), bulkIndexer.result.total, LOGGER) + String index = bulkIndexer.indexType.getMainType().getIndex().getName(); + this.progress = new ProgressLogger(format("Progress[BulkIndexer[%s]]", index), bulkIndexer.result.total, LOGGER) .setPluralLabel("requests"); this.progress.start(); Map<String, Object> temporarySettings = new HashMap<>(); - GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(bulkIndexer.indexType.getIndex()).get(); + GetSettingsResponse settingsResp = bulkIndexer.client.nativeClient().admin().indices().prepareGetSettings(index).get(); // deactivate replicas - int initialReplicas = Integer.parseInt(settingsResp.getSetting(bulkIndexer.indexType.getIndex(), IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); + int initialReplicas = Integer.parseInt(settingsResp.getSetting(index, IndexMetaData.SETTING_NUMBER_OF_REPLICAS)); if (initialReplicas > 0) { initialSettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, initialReplicas); temporarySettings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); } // deactivate periodical refresh - String refreshInterval = settingsResp.getSetting(bulkIndexer.indexType.getIndex(), REFRESH_INTERVAL_SETTING); + String refreshInterval = settingsResp.getSetting(index, REFRESH_INTERVAL_SETTING); initialSettings.put(REFRESH_INTERVAL_SETTING, refreshInterval); temporarySettings.put(REFRESH_INTERVAL_SETTING, "-1"); @@ -410,14 +411,14 @@ public class BulkIndexer { // optimize lucene segments and revert index settings // Optimization must be done before re-applying replicas: // http://www.elasticsearch.org/blog/performance-considerations-elasticsearch-indexing/ - bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getIndex()).get(); + bulkIndexer.client.prepareForceMerge(bulkIndexer.indexType.getMainType().getIndex().getName()).get(); updateSettings(bulkIndexer, initialSettings); this.progress.stop(); } private static void updateSettings(BulkIndexer bulkIndexer, Map<String, Object> settings) { - UpdateSettingsRequestBuilder req = bulkIndexer.client.nativeClient().admin().indices().prepareUpdateSettings(bulkIndexer.indexType.getIndex()); + UpdateSettingsRequestBuilder req = bulkIndexer.client.nativeClient().admin().indices().prepareUpdateSettings(bulkIndexer.indexType.getMainType().getIndex().getName()); req.setSettings(settings); req.get(); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/DocId.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/DocId.java index 26326c4f8ef..c78efb4529c 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/DocId.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/DocId.java @@ -21,6 +21,8 @@ package org.sonar.server.es; import javax.annotation.concurrent.Immutable; +import static java.util.Objects.requireNonNull; + @Immutable class DocId { @@ -28,14 +30,10 @@ class DocId { private final String indexType; private final String id; - DocId(IndexType indexType, String id) { - this(indexType.getIndex(), indexType.getType(), id); - } - DocId(String index, String indexType, String id) { - this.index = index; - this.indexType = indexType; - this.id = id; + this.index = requireNonNull(index, "index can't be null"); + this.indexType = requireNonNull(indexType,"type can't be null"); + this.id = requireNonNull(id, "id can't be null"); } @Override @@ -48,13 +46,7 @@ class DocId { } DocId docId = (DocId) o; - if (!index.equals(docId.index)) { - return false; - } - if (!indexType.equals(docId.indexType)) { - return false; - } - return id.equals(docId.id); + return index.equals(docId.index) && indexType.equals(docId.indexType) && id.equals(docId.id); } @Override @@ -64,4 +56,9 @@ class DocId { result = 31 * result + id.hashCode(); return result; } + + @Override + public String toString() { + return "DocId{" + index + '/' + indexType + '/' + id + '}'; + } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java index 03edf91a500..bfd37f0b76a 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java @@ -41,6 +41,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Priority; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; +import org.sonar.server.es.IndexType.IndexMainType; import org.sonar.server.es.request.ProxyClearCacheRequestBuilder; import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder; import org.sonar.server.es.request.ProxyClusterStateRequestBuilder; @@ -77,12 +78,16 @@ public class EsClient implements Closeable { this.nativeClient = null; } - public RefreshRequestBuilder prepareRefresh(String... indices) { - return new ProxyRefreshRequestBuilder(nativeClient()).setIndices(indices); + public RefreshRequestBuilder prepareRefresh(Index index) { + return new ProxyRefreshRequestBuilder(nativeClient()).setIndices(index.getName()); } - public IndicesStatsRequestBuilder prepareStats(String... indices) { - return new ProxyIndicesStatsRequestBuilder(nativeClient()).setIndices(indices); + public IndicesStatsRequestBuilder prepareStats() { + return new ProxyIndicesStatsRequestBuilder(nativeClient()); + } + + public IndicesStatsRequestBuilder prepareStats(Index index) { + return new ProxyIndicesStatsRequestBuilder(nativeClient()).setIndices(index.getName()); } public NodesStatsRequestBuilder prepareNodesStats(String... nodesIds) { @@ -97,34 +102,34 @@ public class EsClient implements Closeable { return new ProxyClusterStateRequestBuilder(nativeClient()); } - public ClusterHealthRequestBuilder prepareHealth(String... indices) { - return new ProxyClusterHealthRequestBuilder(nativeClient()).setIndices(indices); + public ClusterHealthRequestBuilder prepareHealth() { + return new ProxyClusterHealthRequestBuilder(nativeClient()); } public void waitForStatus(ClusterHealthStatus status) { prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForStatus(status).get(); } - public IndicesExistsRequestBuilder prepareIndicesExist(String... indices) { - return new ProxyIndicesExistsRequestBuilder(nativeClient(), indices); + public IndicesExistsRequestBuilder prepareIndicesExist(Index index) { + return new ProxyIndicesExistsRequestBuilder(nativeClient(), index.getName()); } - public CreateIndexRequestBuilder prepareCreate(String index) { - return new ProxyCreateIndexRequestBuilder(nativeClient(), index); + public CreateIndexRequestBuilder prepareCreate(Index index) { + return new ProxyCreateIndexRequestBuilder(nativeClient(), index.getName()); } - public PutMappingRequestBuilder preparePutMapping(String... indices) { - return new ProxyPutMappingRequestBuilder(nativeClient()).setIndices(indices); + public PutMappingRequestBuilder preparePutMapping(Index index) { + return new ProxyPutMappingRequestBuilder(nativeClient()).setIndices(index.getName()); } - public SearchRequestBuilder prepareSearch(String... indices) { - return new ProxySearchRequestBuilder(nativeClient()).setIndices(indices); + public SearchRequestBuilder prepareSearch(Index index) { + return new ProxySearchRequestBuilder(nativeClient()).setIndices(index.getName()); } - public SearchRequestBuilder prepareSearch(IndexType... indexType) { + public SearchRequestBuilder prepareSearch(IndexMainType indexType) { return new ProxySearchRequestBuilder(nativeClient()) - .setIndices(IndexType.getIndices(indexType)) - .setTypes(IndexType.getTypes(indexType)); + .setIndices(indexType.getIndex().getName()) + .setTypes(indexType.getType()); } public SearchScrollRequestBuilder prepareSearchScroll(String scrollId) { @@ -132,19 +137,22 @@ public class EsClient implements Closeable { } public GetRequestBuilder prepareGet(IndexType indexType, String id) { - return new ProxyGetRequestBuilder(nativeClient()).setIndex(indexType.getIndex()).setType(indexType.getType()).setId(id); + IndexMainType mainType = indexType.getMainType(); + return new ProxyGetRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType()).setId(id); } public DeleteRequestBuilder prepareDelete(IndexType indexType, String id) { - return new ProxyDeleteRequestBuilder(nativeClient(), indexType.getIndex()).setType(indexType.getType()).setId(id); + IndexMainType mainType = indexType.getMainType(); + return new ProxyDeleteRequestBuilder(nativeClient(), mainType.getIndex().getName()).setType(mainType.getType()).setId(id); } - public DeleteRequestBuilder prepareDelete(String index, String type, String id) { + DeleteRequestBuilder prepareDelete(String index, String type, String id) { return new ProxyDeleteRequestBuilder(nativeClient(), index).setType(type).setId(id); } public IndexRequestBuilder prepareIndex(IndexType indexType) { - return new ProxyIndexRequestBuilder(nativeClient()).setIndex(indexType.getIndex()).setType(indexType.getType()); + IndexMainType mainType = indexType.getMainType(); + return new ProxyIndexRequestBuilder(nativeClient()).setIndex(mainType.getIndex().getName()).setType(mainType.getType()); } public ForceMergeRequestBuilder prepareForceMerge(String indexName) { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/Index.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/Index.java new file mode 100644 index 00000000000..77f19814029 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/Index.java @@ -0,0 +1,88 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.Objects; +import org.apache.commons.lang.StringUtils; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +public final class Index { + public static final Index ALL_INDICES = Index.simple("_all"); + + private final String name; + private final boolean relations; + + private Index(String name, boolean acceptsRelations) { + checkArgument(name != null && !name.isEmpty(), "Index name can't be null nor empty"); + checkArgument("_all".equals(name) || StringUtils.isAllLowerCase(name), "Index name must be lower-case letters or '_all': %s", name); + this.name = name; + this.relations = acceptsRelations; + } + + public static Index simple(String name) { + return new Index(name, false); + } + + public static Index withRelations(String name) { + return new Index(name, true); + } + + public String getName() { + return name; + } + + public boolean acceptsRelations() { + return relations; + } + + /** + * @return the name of the join field for this index if it accepts relations + * @throws IllegalStateException if index does not accept relations + * @see #acceptsRelations() + */ + public String getJoinField() { + checkState(relations, "Only index accepting relations has a join field"); + return "join_" + name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Index index = (Index) o; + return relations == index.relations && name.equals(index.name); + } + + @Override + public int hashCode() { + return Objects.hash(name, relations); + } + + @Override + public String toString() { + return "[" + name + (relations ? "|*" : "|") + ']'; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinition.java index 94faba81a52..09005233a0e 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinition.java @@ -19,13 +19,15 @@ */ package org.sonar.server.es; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import org.elasticsearch.common.settings.Settings; +import java.util.Map; import org.sonar.api.server.ServerSide; +import org.sonar.server.es.newindex.NewAuthorizedIndex; +import org.sonar.server.es.newindex.NewIndex; +import org.sonar.server.es.newindex.NewRegularIndex; +import org.sonar.server.es.newindex.SettingsConfiguration; -import java.util.Map; +import static com.google.common.base.Preconditions.checkArgument; @ServerSide public interface IndexDefinition { @@ -33,70 +35,29 @@ public interface IndexDefinition { class IndexDefinitionContext { private final Map<String, NewIndex> byKey = Maps.newHashMap(); - public NewIndex create(String key, NewIndex.SettingsConfiguration settingsConfiguration) { - Preconditions.checkArgument(!byKey.containsKey(key), String.format("Index already exists: %s", key)); - NewIndex index = new NewIndex(key, settingsConfiguration); - byKey.put(key, index); - return index; - } - - public Map<String, NewIndex> getIndices() { - return byKey; - } - } - - void define(IndexDefinitionContext context); - - /** - * Immutable copy of {@link NewIndex} - */ - class Index { - private final String name; - private final Settings settings; - private final Map<String, Type> types; - - Index(NewIndex newIndex) { - this.name = newIndex.getName(); - this.settings = newIndex.getSettings().build(); - ImmutableMap.Builder<String, Type> builder = ImmutableMap.builder(); - for (NewIndex.NewIndexType newIndexType : newIndex.getTypes().values()) { - Type type = new Type(newIndexType); - builder.put(type.getName(), type); - } - this.types = builder.build(); + public NewRegularIndex create(Index index, SettingsConfiguration settingsConfiguration) { + String indexName = index.getName(); + checkArgument(!byKey.containsKey(indexName), String.format("Index already exists: %s", indexName)); + NewRegularIndex newIndex = new NewRegularIndex(index, settingsConfiguration); + byKey.put(indexName, newIndex); + return newIndex; } - public String getName() { - return name; - } + public NewAuthorizedIndex createWithAuthorization(Index index, SettingsConfiguration settingsConfiguration) { + checkArgument(index.acceptsRelations(), "Index with authorization must accept relations"); + String indexName = index.getName(); + checkArgument(!byKey.containsKey(indexName), String.format("Index already exists: %s", indexName)); - public Settings getSettings() { - return settings; + NewAuthorizedIndex newIndex = new NewAuthorizedIndex(index, settingsConfiguration); + byKey.put(indexName, newIndex); + return newIndex; } - public Map<String, Type> getTypes() { - return types; + public Map<String, NewIndex> getIndices() { + return byKey; } } - /** - * Immutable copy of {@link NewIndex.NewIndexType} - */ - class Type { - private final String name; - private final Map<String, Object> attributes; - - private Type(NewIndex.NewIndexType newType) { - this.name = newType.getName(); - this.attributes = ImmutableMap.copyOf(newType.getAttributes()); - } - - public String getName() { - return name; - } + void define(IndexDefinitionContext context); - public Map<String, Object> getAttributes() { - return attributes; - } - } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinitionHash.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinitionHash.java index d8dba45db8b..8be1677ede3 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinitionHash.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexDefinitionHash.java @@ -19,14 +19,15 @@ */ package org.sonar.server.es; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; -import com.google.common.collect.Lists; -import org.apache.commons.codec.digest.DigestUtils; - -import java.util.Collections; -import java.util.List; +import java.util.Arrays; import java.util.Map; import java.util.SortedMap; +import org.apache.commons.codec.digest.DigestUtils; +import org.sonar.server.es.newindex.BuiltIndex; + +import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; /** * Hash of index definition is stored in the index itself in order to detect changes of mappings @@ -34,14 +35,18 @@ import java.util.SortedMap; * and re-populated from scratch. There's no attempt to migrate existing data. */ class IndexDefinitionHash { - private static final char DELIMITER = ','; private IndexDefinitionHash() { } - static String of(IndexDefinition.Index index) { - return of(index.getSettings().getAsMap(), index.getTypes()); + static String of(BuiltIndex<?> index) { + IndexType.IndexMainType mainType = index.getMainType(); + return of( + ImmutableMap.of(mainType.getIndex(), mainType), + index.getRelationTypes().stream().collect(uniqueIndex(IndexType.IndexRelationType::getName, t -> t)), + index.getSettings().getAsMap(), + index.getAttributes()); } private static String of(Map... maps) { @@ -52,22 +57,6 @@ class IndexDefinitionHash { return DigestUtils.sha256Hex(sb.toString()); } - private static void appendObject(StringBuilder sb, Object value) { - if (value instanceof IndexDefinition.Type) { - appendIndexType(sb, (IndexDefinition.Type) value); - } else if (value instanceof Map) { - appendMap(sb, (Map) value); - } else if (value instanceof Iterable) { - appendIterable(sb, (Iterable) value); - } else { - sb.append(String.valueOf(value)); - } - } - - private static void appendIndexType(StringBuilder sb, IndexDefinition.Type type) { - appendMap(sb, type.getAttributes()); - } - private static void appendMap(StringBuilder sb, Map attributes) { for (Object entry : sort(attributes).entrySet()) { sb.append(((Map.Entry) entry).getKey()); @@ -77,16 +66,22 @@ class IndexDefinitionHash { } } - private static void appendIterable(StringBuilder sb, Iterable value) { - List sorted = Lists.newArrayList(value); - Collections.sort(sorted); - for (Object o : sorted) { - appendObject(sb, o); - sb.append(DELIMITER); + private static void appendObject(StringBuilder sb, Object value) { + if (value instanceof Object[]) { + sb.append(Arrays.toString((Object[]) value)); + } else if (value instanceof Map) { + appendMap(sb, (Map) value); + } else if (value instanceof IndexType) { + sb.append(((IndexType) value).format()); + } else { + sb.append(String.valueOf(value)); } } private static SortedMap sort(Map map) { + if (map instanceof ImmutableSortedMap) { + return (ImmutableSortedMap) map; + } return ImmutableSortedMap.copyOf(map); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java index 9aa62543b73..e773f5904bd 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/IndexType.java @@ -20,83 +20,193 @@ package org.sonar.server.es; import com.google.common.base.Splitter; -import java.util.Arrays; import java.util.List; -import java.util.function.Function; -import org.sonar.core.util.stream.MoreCollectors; +import java.util.Objects; +import javax.annotation.concurrent.Immutable; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; -public class IndexType { +public abstract class IndexType { + + public static final String FIELD_INDEX_TYPE = "indexType"; private static final String SEPARATOR = "/"; private static final Splitter SEPARATOR_SPLITTER = Splitter.on(SEPARATOR); - private final String index; - private final String type; - private final String key; + public abstract IndexMainType getMainType(); - public IndexType(String index, String type) { - this.index = requireNonNull(index); - this.type = requireNonNull(type); - this.key = index + SEPARATOR + type; - } + public abstract String format(); - public String getIndex() { - return index; - } + /** + * Parse a String generated by {@link #format()} to extract the simple details of the main type. + * <p> + * Note: neither {@link IndexMainType} nor {@link IndexRelationType} can be parsed from the string generated by + * {@link #format()} as the generated string does not contain the {@link Index#acceptsRelations() acceptsRelations} + * flag). + */ + public static SimpleIndexMainType parseMainType(String s) { + List<String> split = SEPARATOR_SPLITTER.splitToList(s); + checkArgument(split.size() >= 2, "Unsupported IndexType value: %s", s); - public String getType() { - return type; + return new SimpleIndexMainType(split.get(0), split.get(1)); } - public static String[] getIndices(IndexType... indexTypes) { - return getDetails(IndexType::getIndex, indexTypes); - } + @Immutable + public static final class SimpleIndexMainType { + private final String index; + private final String type; + + private SimpleIndexMainType(String index, String type) { + this.index = index; + this.type = type; + } + + public String getIndex() { + return index; + } + + public String getType() { + return type; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SimpleIndexMainType that = (SimpleIndexMainType) o; + return index.equals(that.index) && type.equals(that.type); + } - public static String[] getTypes(IndexType... indexTypes) { - return getDetails(IndexType::getType, indexTypes); + @Override + public int hashCode() { + return Objects.hash(index, type); + } + + @Override + public String toString() { + return "[" + index + '/' + type + ']'; + } } - private static String[] getDetails(Function<? super IndexType, ? extends String> function, IndexType... indexTypes) { - return Arrays.stream(indexTypes).map(function).collect(MoreCollectors.toSet(indexTypes.length)).toArray(new String[0]); + public static IndexMainType main(Index index, String type) { + return new IndexMainType(index, type); } - public String format() { - return key; + public static IndexRelationType relation(IndexMainType mainType, String name) { + checkArgument(mainType.getIndex().acceptsRelations(), "Index must define a join field to have relations"); + + return new IndexRelationType(mainType, name); } - /** - * Parse a String generated by {@link #format()} - */ - public static IndexType parse(String s) { - List<String> split = SEPARATOR_SPLITTER.splitToList(s); - if (split.size() != 2) { - throw new IllegalArgumentException("Unsupported IndexType value: " + s); + @Immutable + public static final class IndexMainType extends IndexType { + private final Index index; + private final String type; + private final String key; + + private IndexMainType(Index index, String type) { + this.index = requireNonNull(index); + checkArgument(type != null && !type.isEmpty(), "type name can't be null nor empty"); + this.type = type; + this.key = index.getName() + SEPARATOR + type; } - return new IndexType(split.get(0), split.get(1)); - } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; + @Override + public IndexMainType getMainType() { + return this; } - if (o == null || getClass() != o.getClass()) { - return false; + + public Index getIndex() { + return index; } - IndexType indexType = (IndexType) o; - return key.equals(indexType.key); - } + public String getType() { + return type; + } + + @Override + public String format() { + return key; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + IndexMainType indexType = (IndexMainType) o; + return index.equals(indexType.index) && type.equals(indexType.type); + } + + @Override + public int hashCode() { + return Objects.hash(index, type); + } + + @Override + public String toString() { + return "[" + key + "]"; + } - @Override - public int hashCode() { - return key.hashCode(); } - @Override - public String toString() { - return "[" + index + "/" + type + "]"; + @Immutable + public static final class IndexRelationType extends IndexType { + private final IndexMainType mainType; + private final String name; + private final String key; + + private IndexRelationType(IndexMainType mainType, String name) { + this.mainType = mainType; + checkArgument(name != null && !name.isEmpty(), "type name can't be null nor empty"); + this.name = name; + this.key = mainType.index.getName() + "/" + mainType.type + "/" + name; + } + + @Override + public IndexMainType getMainType() { + return mainType; + } + + public String getName() { + return name; + } + + @Override + public String format() { + return key; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + IndexRelationType indexType = (IndexRelationType) o; + return mainType.equals(indexType.mainType) && name.equals(indexType.name); + } + + @Override + public int hashCode() { + return Objects.hash(mainType, name); + } + + @Override + public String toString() { + return "[" + key + "]"; + } } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/NewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/NewIndex.java deleted file mode 100644 index 88276a2753c..00000000000 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/NewIndex.java +++ /dev/null @@ -1,540 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedMap; -import com.google.common.collect.Maps; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; -import javax.annotation.CheckForNull; -import org.apache.commons.lang.StringUtils; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.Settings; -import org.sonar.api.config.Configuration; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.lang.String.format; -import static java.lang.String.valueOf; -import static java.util.Objects.requireNonNull; -import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; -import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS; -import static org.sonar.server.es.DefaultIndexSettings.ANALYZER; -import static org.sonar.server.es.DefaultIndexSettings.FIELDDATA_ENABLED; -import static org.sonar.server.es.DefaultIndexSettings.FIELD_FIELDDATA; -import static org.sonar.server.es.DefaultIndexSettings.FIELD_TERM_VECTOR; -import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_KEYWORD; -import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT; -import static org.sonar.server.es.DefaultIndexSettings.INDEX; -import static org.sonar.server.es.DefaultIndexSettings.INDEX_NOT_SEARCHABLE; -import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE; -import static org.sonar.server.es.DefaultIndexSettings.NORMS; -import static org.sonar.server.es.DefaultIndexSettings.STORE; -import static org.sonar.server.es.DefaultIndexSettings.TYPE; -import static org.sonar.server.es.DefaultIndexSettingsElement.UUID_MODULE_ANALYZER; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_ALLOW_ANYONE; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_GROUP_IDS; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_USER_IDS; -import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; - -public class NewIndex { - - private final String indexName; - private final Settings.Builder settings = DefaultIndexSettings.defaults(); - private final Map<String, NewIndexType> types = new LinkedHashMap<>(); - - NewIndex(String indexName, SettingsConfiguration settingsConfiguration) { - checkArgument(StringUtils.isAllLowerCase(indexName), "Index name must be lower-case: " + indexName); - this.indexName = indexName; - applySettingsConfiguration(settingsConfiguration); - } - - private void applySettingsConfiguration(SettingsConfiguration settingsConfiguration) { - settings.put("index.mapper.dynamic", valueOf(false)); - settings.put("index.refresh_interval", refreshInterval(settingsConfiguration)); - - Configuration config = settingsConfiguration.getConfiguration(); - boolean clusterMode = config.getBoolean(CLUSTER_ENABLED.getKey()).orElse(false); - int shards = config.getInt(format("sonar.search.%s.shards", indexName)) - .orElse(settingsConfiguration.getDefaultNbOfShards()); - int replicas = clusterMode ? config.getInt(SEARCH_REPLICAS.getKey()).orElse(1) : 0; - - settings.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shards); - settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); - } - - private static String refreshInterval(SettingsConfiguration settingsConfiguration) { - int refreshInterval = settingsConfiguration.getRefreshInterval(); - if (refreshInterval == -1) { - return "-1"; - } - return refreshInterval + "s"; - } - - public static class SettingsConfiguration { - public static final int MANUAL_REFRESH_INTERVAL = -1; - - private final Configuration configuration; - private final int defaultNbOfShards; - private final int refreshInterval; - - private SettingsConfiguration(Builder builder) { - this.configuration = builder.configuration; - this.defaultNbOfShards = builder.defaultNbOfShards; - this.refreshInterval = builder.refreshInterval; - } - - public static Builder newBuilder(Configuration configuration) { - return new Builder(configuration); - } - - public Configuration getConfiguration() { - return configuration; - } - - public int getDefaultNbOfShards() { - return defaultNbOfShards; - } - - public int getRefreshInterval() { - return refreshInterval; - } - - public static class Builder { - private final Configuration configuration; - private int defaultNbOfShards = 1; - private int refreshInterval = 30; - - public Builder(Configuration configuration) { - this.configuration = requireNonNull(configuration, "configuration can't be null"); - } - - public Builder setDefaultNbOfShards(int defaultNbOfShards) { - checkArgument(defaultNbOfShards >= 1, "defaultNbOfShards must be >= 1"); - this.defaultNbOfShards = defaultNbOfShards; - return this; - } - - public Builder setRefreshInterval(int refreshInterval) { - checkArgument(refreshInterval == -1 || refreshInterval > 0, - "refreshInterval must be either -1 or strictly positive"); - this.refreshInterval = refreshInterval; - return this; - } - - public SettingsConfiguration build() { - return new SettingsConfiguration(this); - } - } - - } - - public String getName() { - return indexName; - } - - public Settings.Builder getSettings() { - return settings; - } - - public NewIndexType createType(String typeName) { - NewIndexType type = new NewIndexType(this, typeName); - types.put(typeName, type); - return type; - } - - public Map<String, NewIndexType> getTypes() { - return types; - } - - public static class NewIndexType { - private final NewIndex index; - private final String name; - private final Map<String, Object> attributes = new TreeMap<>(); - private final Map<String, Object> properties = new TreeMap<>(); - - private NewIndexType(NewIndex index, String typeName) { - this.index = index; - this.name = typeName; - // defaults - attributes.put("dynamic", false); - attributes.put("_all", ImmutableSortedMap.of("enabled", false)); - attributes.put("_source", ImmutableSortedMap.of("enabled", true)); - attributes.put("properties", properties); - } - - public NewIndexType requireProjectAuthorization() { - enableProjectAuthorization(this); - return this; - } - - /** - * Creates a type that requires to verify that user has the read permission - * when searching for documents. - * - * Both types {@code typeName} and "authorization" are created. Documents - * must be created with _parent and _routing having the parent uuid as values. - * - * @see NewIndex.NewIndexType#requireProjectAuthorization() - */ - private static NewIndex.NewIndexType enableProjectAuthorization(NewIndex.NewIndexType type) { - type.setAttribute("_parent", ImmutableMap.of("type", TYPE_AUTHORIZATION)); - type.setAttribute("_routing", ImmutableMap.of("required", true)); - - NewIndex.NewIndexType authType = type.getIndex().createType(TYPE_AUTHORIZATION); - authType.setAttribute("_routing", ImmutableMap.of("required", true)); - authType.createLongField(FIELD_GROUP_IDS); - authType.createLongField(FIELD_USER_IDS); - authType.createBooleanField(FIELD_ALLOW_ANYONE); - authType.setEnableSource(false); - return type; - } - - public NewIndex getIndex() { - return index; - } - - public String getName() { - return name; - } - - /** - * Complete the root json hash of mapping type, for example to set the attribute "_id" - */ - public NewIndexType setAttribute(String key, Object value) { - attributes.put(key, value); - return this; - } - - /** - * Complete the json hash named "properties" in mapping type, usually to declare fields - */ - public NewIndexType setProperty(String key, Object value) { - properties.put(key, value); - return this; - } - - public NewIndexType setEnableSource(boolean enableSource) { - attributes.put("_source", ImmutableSortedMap.of("enabled", enableSource)); - return this; - } - - public KeywordFieldBuilder keywordFieldBuilder(String fieldName) { - return new KeywordFieldBuilder(this, fieldName); - } - - public TextFieldBuilder textFieldBuilder(String fieldName) { - return new TextFieldBuilder(this, fieldName); - } - - public NestedFieldBuilder nestedFieldBuilder(String fieldName) { - return new NestedFieldBuilder(this, fieldName); - } - - public NewIndexType createBooleanField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "boolean")); - } - - public NewIndexType createByteField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "byte")); - } - - public NewIndexType createDateTimeField(String fieldName) { - Map<String, String> hash = new TreeMap<>(); - hash.put("type", "date"); - hash.put("format", "date_time||epoch_second"); - return setProperty(fieldName, hash); - } - - public NewIndexType createDoubleField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "double")); - } - - public NewIndexType createIntegerField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "integer")); - } - - public NewIndexType createLongField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "long")); - } - - public NewIndexType createShortField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "short")); - } - - public NewIndexType createUuidPathField(String fieldName) { - return setProperty(fieldName, ImmutableSortedMap.of( - TYPE, FIELD_TYPE_TEXT, - INDEX, DefaultIndexSettings.INDEX_SEARCHABLE, - ANALYZER, UUID_MODULE_ANALYZER.getName())); - } - - public Map<String, Object> getAttributes() { - return attributes; - } - - @CheckForNull - public Object getProperty(String key) { - return properties.get(key); - } - } - - /** - * Helper to define a string field in mapping of index type - */ - public abstract static class StringFieldBuilder<T extends StringFieldBuilder<T>> { - private final NewIndexType indexType; - private final String fieldName; - private boolean disableSearch = false; - private boolean disableNorms = false; - private boolean termVectorWithPositionOffsets = false; - private SortedMap<String, Object> subFields = Maps.newTreeMap(); - private boolean store = false; - protected boolean disabledDocValues = false; - - private StringFieldBuilder(NewIndexType indexType, String fieldName) { - this.indexType = indexType; - this.fieldName = fieldName; - } - - /** - * Add a sub-field. A {@code SortedMap} is required for consistency of the index settings hash. - * @see IndexDefinitionHash - */ - private T addSubField(String fieldName, SortedMap<String, String> fieldDefinition) { - subFields.put(fieldName, fieldDefinition); - return castThis(); - } - - /** - * Add subfields, one for each analyzer. - */ - public T addSubFields(DefaultIndexSettingsElement... analyzers) { - Arrays.stream(analyzers) - .forEach(analyzer -> addSubField(analyzer.getSubFieldSuffix(), analyzer.fieldMapping())); - return castThis(); - } - - /** - * Norms consume useless memory if string field is used for filtering or aggregations. - * - * https://www.elastic.co/guide/en/elasticsearch/reference/2.3/norms.html - * https://www.elastic.co/guide/en/elasticsearch/guide/current/scoring-theory.html#field-norm - */ - public T disableNorms() { - this.disableNorms = true; - return castThis(); - } - - /** - * Position offset term vectors are required for the fast_vector_highlighter (fvh). - */ - public T termVectorWithPositionOffsets() { - this.termVectorWithPositionOffsets = true; - return castThis(); - } - - /** - * "index: false" -> Make this field not searchable. - * By default field is "true": it is searchable, but index the value exactly - * as specified. - */ - public T disableSearch() { - this.disableSearch = true; - return castThis(); - } - - public T store() { - this.store = true; - return castThis(); - } - - @SuppressWarnings("unchecked") - private T castThis() { - return (T) this; - } - - public NewIndexType build() { - if (subFields.isEmpty()) { - return buildWithoutSubfields(); - } - return buildWithSubfields(); - } - - private NewIndexType buildWithoutSubfields() { - Map<String, Object> hash = new TreeMap<>(); - hash.put("type", getFieldType()); - hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE); - hash.put(NORMS, valueOf(!disableNorms)); - hash.put(STORE, valueOf(store)); - if (FIELD_TYPE_KEYWORD.equals(getFieldType())) { - hash.put("doc_values", valueOf(!disabledDocValues)); - } - if (getFieldData()) { - hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED); - } - return indexType.setProperty(fieldName, hash); - } - - private NewIndexType buildWithSubfields() { - Map<String, Object> hash = new TreeMap<>(); - hash.put("type", getFieldType()); - hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE); - hash.put(NORMS, "false"); - hash.put(STORE, valueOf(store)); - if (FIELD_TYPE_KEYWORD.equals(getFieldType())) { - hash.put("doc_values", valueOf(!disabledDocValues)); - } - if (getFieldData()) { - hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED); - } - if (termVectorWithPositionOffsets) { - hash.put(FIELD_TERM_VECTOR, "with_positions_offsets"); - } - hash.put("fields", configureSubFields()); - return indexType.setProperty(fieldName, hash); - } - - private Map<String, Object> configureSubFields() { - Map<String, Object> multiFields = new TreeMap<>(subFields); - - // apply this fields configuration to all subfields - multiFields.entrySet().forEach(entry -> { - Object subFieldMapping = entry.getValue(); - if (subFieldMapping instanceof Map) { - entry.setValue(configureSubField((Map<String, String>) subFieldMapping)); - } - }); - return multiFields; - } - - private Map<String, String> configureSubField(Map<String, String> subFieldMapping) { - Map<String, String> subHash = new TreeMap<>(subFieldMapping); - subHash.put(INDEX, INDEX_SEARCHABLE); - subHash.put(NORMS, "false"); - subHash.put(STORE, valueOf(store)); - if (termVectorWithPositionOffsets) { - subHash.put(FIELD_TERM_VECTOR, "with_positions_offsets"); - } - return subHash; - } - - protected abstract boolean getFieldData(); - - protected abstract String getFieldType(); - } - - public static class KeywordFieldBuilder extends StringFieldBuilder<KeywordFieldBuilder> { - - private KeywordFieldBuilder(NewIndexType indexType, String fieldName) { - super(indexType, fieldName); - } - - @Override - protected boolean getFieldData() { - return false; - } - - protected String getFieldType() { - return FIELD_TYPE_KEYWORD; - } - - /** - * By default, field is stored on disk in a column-stride fashion, so that it can later be used for sorting, - * aggregations, or scripting. - * Disabling this reduces the size of the index and drop the constraint of single term max size of - * 32766 bytes (which, if there is no tokenizing enabled on the field, equals the size of the whole data). - */ - public KeywordFieldBuilder disableSortingAndAggregating() { - this.disabledDocValues = true; - return this; - } - } - - public static class TextFieldBuilder extends StringFieldBuilder<TextFieldBuilder> { - - private boolean fieldData = false; - - private TextFieldBuilder(NewIndexType indexType, String fieldName) { - super(indexType, fieldName); - } - - protected String getFieldType() { - return FIELD_TYPE_TEXT; - } - - /** - * Required to enable sorting, aggregation and access to field data on fields of type "text". - * <p>Disabled by default as this can have significant memory cost</p> - */ - public StringFieldBuilder withFieldData() { - this.fieldData = true; - return this; - } - - @Override - protected boolean getFieldData() { - return fieldData; - } - } - - public static class NestedFieldBuilder { - private final NewIndexType indexType; - private final String fieldName; - private final Map<String, Object> properties = new TreeMap<>(); - - private NestedFieldBuilder(NewIndexType indexType, String fieldName) { - this.indexType = indexType; - this.fieldName = fieldName; - } - - private NestedFieldBuilder setProperty(String fieldName, Object value) { - properties.put(fieldName, value); - - return this; - } - - public NestedFieldBuilder addKeywordField(String fieldName) { - return setProperty(fieldName, ImmutableSortedMap.of( - "type", FIELD_TYPE_KEYWORD, - INDEX, INDEX_SEARCHABLE)); - } - - public NestedFieldBuilder addDoubleField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "double")); - } - - public NestedFieldBuilder addIntegerField(String fieldName) { - return setProperty(fieldName, ImmutableMap.of("type", "integer")); - } - - public NewIndexType build() { - checkArgument(!properties.isEmpty(), "At least one sub-field must be declared in nested property '%s'", fieldName); - - return indexType.setProperty(fieldName, ImmutableSortedMap.of( - "type", "nested", - "properties", properties)); - } - } - -} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java index 732551a30b5..c4dfdbb5bcc 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/OneToOneResilientIndexingListener.java @@ -48,7 +48,10 @@ public class OneToOneResilientIndexingListener implements IndexingListener { this.dbClient = dbClient; this.dbSession = dbSession; this.itemsById = items.stream() - .collect(MoreCollectors.index(i -> new DocId(IndexType.parse(i.getDocType()), i.getDocId()), Function.identity())); + .collect(MoreCollectors.index(i -> { + IndexType.SimpleIndexMainType mainType = IndexType.parseMainType(i.getDocType()); + return new DocId(mainType.getIndex(), mainType.getType(), i.getDocId()); + }, Function.identity())); } @Override diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java index 3d49090fc30..1aaccbc132e 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java @@ -24,9 +24,13 @@ import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.index.get.GetField; import org.sonar.server.es.EsClient; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.IndexType.IndexRelationType; -import static org.sonar.server.es.DefaultIndexSettings.REFRESH_IMMEDIATE; +import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA; +import static org.sonar.server.es.newindex.DefaultIndexSettings.REFRESH_IMMEDIATE; public class MetadataIndex { @@ -38,16 +42,16 @@ public class MetadataIndex { this.esClient = esClient; } - public Optional<String> getHash(String index) { + public Optional<String> getHash(Index index) { return getMetadata(hashId(index)); } - public void setHash(String index, String hash) { + public void setHash(Index index, String hash) { setMetadata(hashId(index), hash); } - private static String hashId(String index) { - return index + ".indexStructure"; + private static String hashId(Index index) { + return index.getName() + ".indexStructure"; } public boolean getInitialized(IndexType indexType) { @@ -59,7 +63,16 @@ public class MetadataIndex { } private static String initializedId(IndexType indexType) { - return indexType.getIndex() + "." + indexType.getType() + ".initialized"; + if (indexType instanceof IndexMainType) { + IndexMainType mainType = (IndexMainType) indexType; + return mainType.getIndex().getName() + "." + mainType.getType() + ".initialized"; + } + if (indexType instanceof IndexRelationType) { + IndexRelationType relationType = (IndexRelationType) indexType; + IndexMainType mainType = relationType.getMainType(); + return mainType.getIndex().getName() + "." + mainType.getType() + "." + relationType.getName() + ".initialized"; + } + throw new IllegalArgumentException("Unsupported IndexType " + indexType.getClass()); } public Optional<String> getDbVendor() { @@ -71,7 +84,7 @@ public class MetadataIndex { } private Optional<String> getMetadata(String id) { - GetRequestBuilder request = esClient.prepareGet(MetadataIndexDefinition.INDEX_TYPE_METADATA, id) + GetRequestBuilder request = esClient.prepareGet(TYPE_METADATA, id) .setStoredFields(MetadataIndexDefinition.FIELD_VALUE); GetResponse response = request.get(); if (response.isExists()) { @@ -83,7 +96,7 @@ public class MetadataIndex { } private void setMetadata(String id, String value) { - esClient.prepareIndex(MetadataIndexDefinition.INDEX_TYPE_METADATA) + esClient.prepareIndex(TYPE_METADATA) .setId(id) .setSource(MetadataIndexDefinition.FIELD_VALUE, value) .setRefreshPolicy(REFRESH_IMMEDIATE) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexDefinition.java index 57c00c12491..391e03254a0 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexDefinition.java @@ -20,16 +20,19 @@ package org.sonar.server.es.metadata; import org.sonar.api.config.Configuration; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition.IndexDefinitionContext; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.newindex.NewRegularIndex; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class MetadataIndexDefinition { - public static final IndexType INDEX_TYPE_METADATA = new IndexType("metadatas", "metadata"); + public static final Index DESCRIPTOR = Index.simple("metadatas"); + public static final IndexMainType TYPE_METADATA = IndexType.main(DESCRIPTOR, "metadata"); public static final String FIELD_VALUE = "value"; private static final int DEFAULT_NUMBER_OF_SHARDS = 1; @@ -41,15 +44,14 @@ public class MetadataIndexDefinition { } public void define(IndexDefinitionContext context) { - NewIndex index = context.create( - INDEX_TYPE_METADATA.getIndex(), + NewRegularIndex index = context.create( + DESCRIPTOR, newBuilder(configuration) .setRefreshInterval(MANUAL_REFRESH_INTERVAL) .setDefaultNbOfShards(DEFAULT_NUMBER_OF_SHARDS) .build()); - NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_METADATA.getType()); - - mapping.keywordFieldBuilder(FIELD_VALUE).disableSearch().store().build(); + index.createTypeMapping(TYPE_METADATA) + .keywordFieldBuilder(FIELD_VALUE).disableSearch().store().build(); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/BuiltIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/BuiltIndex.java new file mode 100644 index 00000000000..bae180d1a93 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/BuiltIndex.java @@ -0,0 +1,124 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; +import java.io.Serializable; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import org.elasticsearch.common.settings.Settings; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexRelationType; + +import static org.sonar.core.util.stream.MoreCollectors.toSet; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.NORMS; +import static org.sonar.server.es.newindex.DefaultIndexSettings.STORE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE; + +/** + * Immutable copy of {@link NewIndex} + */ +public final class BuiltIndex<T extends NewIndex<T>> { + private final IndexType.IndexMainType mainType; + private final Set<IndexRelationType> relationTypes; + private final Settings settings; + private final Map<String, Object> attributes; + + BuiltIndex(T newIndex) { + this.mainType = newIndex.getMainType(); + this.settings = newIndex.getSettings().build(); + this.relationTypes = newIndex.getRelationsStream().collect(toSet()); + this.attributes = buildAttributes(newIndex); + } + + private static Map<String, Object> buildAttributes(NewIndex<?> newIndex) { + Map<String, Object> indexAttributes = new TreeMap<>(newIndex.getAttributes()); + setRouting(indexAttributes, newIndex); + indexAttributes.put("properties", buildProperties(newIndex)); + return ImmutableSortedMap.copyOf(indexAttributes); + } + + private static void setRouting(Map<String, Object> indexAttributes, NewIndex newIndex) { + if (!newIndex.getRelations().isEmpty()) { + indexAttributes.put("_routing", ImmutableMap.of("required", true)); + } + } + + private static TreeMap<String, Object> buildProperties(NewIndex<?> newIndex) { + TreeMap<String, Object> indexProperties = new TreeMap<>(newIndex.getProperties()); + setTypeField(indexProperties, newIndex); + setJoinField(indexProperties, newIndex); + return indexProperties; + } + + private static void setTypeField(TreeMap<String, Object> indexProperties, NewIndex newIndex) { + Collection<IndexRelationType> relations = newIndex.getRelations(); + if (!relations.isEmpty()) { + indexProperties.put( + FIELD_INDEX_TYPE, + ImmutableMap.of( + TYPE, "keyword", + NORMS, false, + STORE, false, + "doc_values", false)); + } + } + + private static void setJoinField(TreeMap<String, Object> indexProperties, NewIndex newIndex) { + Collection<IndexRelationType> relations = newIndex.getRelations(); + IndexType.IndexMainType mainType = newIndex.getMainType(); + if (!relations.isEmpty()) { + indexProperties.put(mainType.getIndex().getJoinField(), ImmutableMap.of( + TYPE, "join", + "relations", ImmutableMap.of(mainType.getType(), namesToStringOrStringArray(relations)))); + } + } + + private static Serializable namesToStringOrStringArray(Collection<IndexRelationType> relations) { + if (relations.size() == 1) { + return relations.iterator().next().getName(); + } + + return relations.stream() + .map(IndexRelationType::getName) + .sorted() + .toArray(String[]::new); + } + + public IndexType.IndexMainType getMainType() { + return mainType; + } + + public Set<IndexRelationType> getRelationTypes() { + return relationTypes; + } + + public Settings getSettings() { + return settings; + } + + public Map<String, Object> getAttributes() { + return attributes; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/DefaultIndexSettings.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java index 2513e338426..aaf2b2e2d64 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/DefaultIndexSettings.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.server.es; +package org.sonar.server.es.newindex; import java.util.Arrays; import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; @@ -80,7 +80,7 @@ public class DefaultIndexSettings { // only static stuff } - static Settings.Builder defaults() { + public static Settings.Builder defaults() { Settings.Builder builder = Settings.builder() .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) .put("index.refresh_interval", "30s") diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java index 31ccdd0757e..c05173149b3 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/DefaultIndexSettingsElement.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java @@ -17,7 +17,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.server.es; +package org.sonar.server.es.newindex; import com.google.common.collect.ImmutableSortedMap; import java.util.Arrays; @@ -26,34 +26,34 @@ import java.util.SortedMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings.Builder; -import static org.sonar.server.es.DefaultIndexSettings.ANALYSIS; -import static org.sonar.server.es.DefaultIndexSettings.ANALYZER; -import static org.sonar.server.es.DefaultIndexSettings.ASCIIFOLDING; -import static org.sonar.server.es.DefaultIndexSettings.CHAR_FILTER; -import static org.sonar.server.es.DefaultIndexSettings.DELIMITER; -import static org.sonar.server.es.DefaultIndexSettings.FIELDDATA_ENABLED; -import static org.sonar.server.es.DefaultIndexSettings.FIELD_FIELDDATA; -import static org.sonar.server.es.DefaultIndexSettings.FIELD_TYPE_TEXT; -import static org.sonar.server.es.DefaultIndexSettings.FILTER; -import static org.sonar.server.es.DefaultIndexSettings.HTML_STRIP; -import static org.sonar.server.es.DefaultIndexSettings.INDEX; -import static org.sonar.server.es.DefaultIndexSettings.INDEX_SEARCHABLE; -import static org.sonar.server.es.DefaultIndexSettings.KEYWORD; -import static org.sonar.server.es.DefaultIndexSettings.LOWERCASE; -import static org.sonar.server.es.DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH; -import static org.sonar.server.es.DefaultIndexSettings.MAX_GRAM; -import static org.sonar.server.es.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH; -import static org.sonar.server.es.DefaultIndexSettings.MIN_GRAM; -import static org.sonar.server.es.DefaultIndexSettings.PATTERN; -import static org.sonar.server.es.DefaultIndexSettings.PORTER_STEM; -import static org.sonar.server.es.DefaultIndexSettings.SEARCH_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettings.STANDARD; -import static org.sonar.server.es.DefaultIndexSettings.STOP; -import static org.sonar.server.es.DefaultIndexSettings.SUB_FIELD_DELIMITER; -import static org.sonar.server.es.DefaultIndexSettings.TOKENIZER; -import static org.sonar.server.es.DefaultIndexSettings.TRIM; -import static org.sonar.server.es.DefaultIndexSettings.TYPE; -import static org.sonar.server.es.DefaultIndexSettings.WHITESPACE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.ANALYSIS; +import static org.sonar.server.es.newindex.DefaultIndexSettings.ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.ASCIIFOLDING; +import static org.sonar.server.es.newindex.DefaultIndexSettings.CHAR_FILTER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.DELIMITER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELDDATA_ENABLED; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_FIELDDATA; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_TEXT; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FILTER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.HTML_STRIP; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_SEARCHABLE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.KEYWORD; +import static org.sonar.server.es.newindex.DefaultIndexSettings.LOWERCASE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.MAXIMUM_NGRAM_LENGTH; +import static org.sonar.server.es.newindex.DefaultIndexSettings.MAX_GRAM; +import static org.sonar.server.es.newindex.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH; +import static org.sonar.server.es.newindex.DefaultIndexSettings.MIN_GRAM; +import static org.sonar.server.es.newindex.DefaultIndexSettings.PATTERN; +import static org.sonar.server.es.newindex.DefaultIndexSettings.PORTER_STEM; +import static org.sonar.server.es.newindex.DefaultIndexSettings.SEARCH_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.STANDARD; +import static org.sonar.server.es.newindex.DefaultIndexSettings.STOP; +import static org.sonar.server.es.newindex.DefaultIndexSettings.SUB_FIELD_DELIMITER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.TOKENIZER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.TRIM; +import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.WHITESPACE; public enum DefaultIndexSettingsElement { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/FieldAware.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/FieldAware.java new file mode 100644 index 00000000000..a1cdd5a87cf --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/FieldAware.java @@ -0,0 +1,97 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; +import java.util.Map; +import java.util.TreeMap; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_TEXT; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX; +import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.UUID_MODULE_ANALYZER; + +public abstract class FieldAware<U extends FieldAware<U>> { + + abstract U setFieldImpl(String fieldName, Object attributes); + + protected final U setField(String fieldName, Object attributes) { + checkArgument(!FIELD_INDEX_TYPE.equalsIgnoreCase(fieldName), "%s is a reserved field name", FIELD_INDEX_TYPE); + return setFieldImpl(fieldName, attributes); + } + + @SuppressWarnings("unchecked") + public KeywordFieldBuilder<U> keywordFieldBuilder(String fieldName) { + return (KeywordFieldBuilder<U>) new KeywordFieldBuilder(this, fieldName); + } + + @SuppressWarnings("unchecked") + public TextFieldBuilder<U> textFieldBuilder(String fieldName) { + return (TextFieldBuilder<U>) new TextFieldBuilder(this, fieldName); + } + + @SuppressWarnings("unchecked") + public NestedFieldBuilder<U> nestedFieldBuilder(String fieldName) { + return (NestedFieldBuilder<U>) new NestedFieldBuilder(this, fieldName); + } + + public U createBooleanField(String fieldName) { + return setField(fieldName, ImmutableMap.of("type", "boolean")); + } + + public U createByteField(String fieldName) { + return setField(fieldName, ImmutableMap.of("type", "byte")); + } + + public U createDateTimeField(String fieldName) { + Map<String, String> hash = new TreeMap<>(); + hash.put("type", "date"); + hash.put("format", "date_time||epoch_second"); + return setField(fieldName, hash); + } + + public U createDoubleField(String fieldName) { + return setField(fieldName, ImmutableMap.of("type", "double")); + } + + public U createIntegerField(String fieldName) { + return setField(fieldName, ImmutableMap.of("type", "integer")); + } + + public U createLongField(String fieldName) { + return setField(fieldName, ImmutableMap.of("type", "long")); + } + + public U createShortField(String fieldName) { + return setField(fieldName, ImmutableMap.of("type", "short")); + } + + public U createUuidPathField(String fieldName) { + return setField(fieldName, ImmutableSortedMap.of( + TYPE, FIELD_TYPE_TEXT, + INDEX, DefaultIndexSettings.INDEX_SEARCHABLE, + ANALYZER, UUID_MODULE_ANALYZER.getName())); + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/KeywordFieldBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/KeywordFieldBuilder.java new file mode 100644 index 00000000000..751952cd33b --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/KeywordFieldBuilder.java @@ -0,0 +1,49 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_KEYWORD; + +public class KeywordFieldBuilder<U extends FieldAware<U>> extends StringFieldBuilder<U, KeywordFieldBuilder<U>> { + + protected KeywordFieldBuilder(U indexType, String fieldName) { + super(indexType, fieldName); + } + + @Override + protected boolean getFieldData() { + return false; + } + + protected String getFieldType() { + return FIELD_TYPE_KEYWORD; + } + + /** + * By default, field is stored on disk in a column-stride fashion, so that it can later be used for sorting, + * aggregations, or scripting. + * Disabling this reduces the size of the index and drop the constraint of single term max size of + * 32766 bytes (which, if there is no tokenizing enabled on the field, equals the size of the whole data). + */ + public KeywordFieldBuilder disableSortingAndAggregating() { + this.disabledDocValues = true; + return this; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NestedFieldBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NestedFieldBuilder.java new file mode 100644 index 00000000000..01167131d7c --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NestedFieldBuilder.java @@ -0,0 +1,69 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; +import java.util.Map; +import java.util.TreeMap; + +import static com.google.common.base.Preconditions.checkArgument; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_KEYWORD; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_SEARCHABLE; + +public class NestedFieldBuilder<U extends FieldAware<U>> { + private final U parent; + private final String fieldName; + private final Map<String, Object> properties = new TreeMap<>(); + + protected NestedFieldBuilder(U parent, String fieldName) { + this.parent = parent; + this.fieldName = fieldName; + } + + private NestedFieldBuilder setProperty(String fieldName, Object value) { + properties.put(fieldName, value); + + return this; + } + + public NestedFieldBuilder addKeywordField(String fieldName) { + return setProperty(fieldName, ImmutableSortedMap.of( + "type", FIELD_TYPE_KEYWORD, + INDEX, INDEX_SEARCHABLE)); + } + + public NestedFieldBuilder addDoubleField(String fieldName) { + return setProperty(fieldName, ImmutableMap.of("type", "double")); + } + + public NestedFieldBuilder addIntegerField(String fieldName) { + return setProperty(fieldName, ImmutableMap.of("type", "integer")); + } + + public U build() { + checkArgument(!properties.isEmpty(), "At least one sub-field must be declared in nested property '%s'", fieldName); + + return parent.setField(fieldName, ImmutableSortedMap.of( + "type", "nested", + "properties", properties)); + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewAuthorizedIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewAuthorizedIndex.java new file mode 100644 index 00000000000..33202b6ef2a --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewAuthorizedIndex.java @@ -0,0 +1,64 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexRelationType; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_ALLOW_ANYONE; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_GROUP_IDS; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_USER_IDS; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; + +public class NewAuthorizedIndex extends NewIndex<NewAuthorizedIndex> { + private final IndexType.IndexMainType mainType; + + public NewAuthorizedIndex(Index index, SettingsConfiguration settingsConfiguration) { + super(index, settingsConfiguration); + checkArgument(index.acceptsRelations(), "Index must accept relations"); + + this.mainType = IndexType.main(index, TYPE_AUTHORIZATION); + super.createTypeMapping(mainType) + .createLongField(FIELD_GROUP_IDS) + .createLongField(FIELD_USER_IDS) + .createBooleanField(FIELD_ALLOW_ANYONE); + } + + @Override + public IndexType.IndexMainType getMainType() { + return mainType; + } + + @Override + public TypeMapping createTypeMapping(IndexRelationType relationType) { + checkArgument(relationType.getMainType().equals(mainType), "mainType of relation must be %s", mainType); + return super.createTypeMapping(relationType); + } + + @Override + public BuiltIndex<NewAuthorizedIndex> build() { + checkState(!getRelations().isEmpty(), "At least one relation mapping must be defined"); + return new BuiltIndex<>(this); + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java new file mode 100644 index 00000000000..6246e8ab345 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java @@ -0,0 +1,163 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.ImmutableSortedMap; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.sonar.api.config.Configuration; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.IndexType.IndexRelationType; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.lang.String.valueOf; +import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; +import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS; + +public abstract class NewIndex<T extends NewIndex<T>> { + + private static final String ENABLED = "enabled"; + private final Index index; + private final Map<String, IndexRelationType> relations = new LinkedHashMap<>(); + private final Settings.Builder settings = DefaultIndexSettings.defaults(); + private final Map<String, Object> attributes = new TreeMap<>(); + private final Map<String, Object> properties = new TreeMap<>(); + + public NewIndex(Index index, SettingsConfiguration settingsConfiguration) { + this.index = index; + applySettingsConfiguration(settingsConfiguration); + configureDefaultAttributes(); + } + + private void applySettingsConfiguration(SettingsConfiguration settingsConfiguration) { + settings.put("index.mapper.dynamic", valueOf(false)); + settings.put("index.refresh_interval", refreshInterval(settingsConfiguration)); + settings.put("mapping.single_type", valueOf(true)); + + Configuration config = settingsConfiguration.getConfiguration(); + boolean clusterMode = config.getBoolean(CLUSTER_ENABLED.getKey()).orElse(false); + int shards = config.getInt(String.format("sonar.search.%s.shards", index.getName())) + .orElse(settingsConfiguration.getDefaultNbOfShards()); + int replicas = clusterMode ? config.getInt(SEARCH_REPLICAS.getKey()).orElse(1) : 0; + + settings.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, shards); + settings.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); + } + + private void configureDefaultAttributes() { + attributes.put("dynamic", valueOf(false)); + attributes.put("_all", ImmutableSortedMap.of(ENABLED, false)); + attributes.put("_source", ImmutableSortedMap.of(ENABLED, true)); + } + + private static String refreshInterval(SettingsConfiguration settingsConfiguration) { + int refreshInterval = settingsConfiguration.getRefreshInterval(); + if (refreshInterval == -1) { + return "-1"; + } + return refreshInterval + "s"; + } + + protected Index getIndex() { + return index; + } + + public abstract IndexMainType getMainType(); + + Collection<IndexRelationType> getRelations() { + return relations.values(); + } + + /** + * Public, read-only version of {@link #getRelations()} + */ + public Stream<IndexRelationType> getRelationsStream() { + return relations.values().stream(); + } + + Settings.Builder getSettings() { + return settings; + } + + @CheckForNull + public String getSetting(String key) { + return settings.get(key); + } + + protected TypeMapping createTypeMapping(IndexMainType mainType) { + checkArgument(mainType.getIndex().equals(index), "Main type must belong to index %s", index); + return new TypeMapping(this); + } + + protected TypeMapping createTypeMapping(IndexRelationType relationType) { + checkAcceptsRelations(); + IndexMainType mainType = getMainType(); + checkArgument(relationType.getMainType().equals(mainType), "mainType of relation must be %s", mainType); + String relationName = relationType.getName(); + checkArgument(!relations.containsKey(relationName), "relation %s already exists", relationName); + relations.put(relationName, relationType); + return new TypeMapping(this); + } + + private void checkAcceptsRelations() { + checkState(getMainType().getIndex().acceptsRelations(), "Index is not configured to accept relations. Update IndexDefinition.Descriptor instance for this index"); + } + + /** + * Complete the json hash named "properties" in mapping type, usually to declare fields + */ + void setFieldImpl(String fieldName, Object attributes) { + properties.put(fieldName, attributes); + } + + public T setEnableSource(boolean enableSource) { + attributes.put("_source", ImmutableSortedMap.of(ENABLED, enableSource)); + return castThis(); + } + + @SuppressWarnings("unchecked") + private T castThis() { + return (T) this; + } + + public Map<String, Object> getAttributes() { + return attributes; + } + + Map<String, Object> getProperties() { + return properties; + } + + @CheckForNull + public Object getProperty(String key) { + return properties.get(key); + } + + public abstract BuiltIndex<T> build(); + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewRegularIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewRegularIndex.java new file mode 100644 index 00000000000..57b1852ee25 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewRegularIndex.java @@ -0,0 +1,62 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +public class NewRegularIndex extends NewIndex<NewRegularIndex> { + private IndexMainType mainType; + + public NewRegularIndex(Index index, SettingsConfiguration settingsConfiguration) { + super(index, settingsConfiguration); + } + + @Override + public IndexMainType getMainType() { + checkState(mainType != null, "Main type has not been defined"); + return mainType; + } + + @Override + public TypeMapping createTypeMapping(IndexMainType mainType) { + checkState(this.mainType == null, "Main type can only be defined once"); + this.mainType = mainType; + return super.createTypeMapping(mainType); + } + + @Override + public TypeMapping createTypeMapping(IndexType.IndexRelationType relationType) { + checkState(mainType != null, "Mapping for main type must be created first"); + checkArgument(relationType.getMainType().equals(mainType), "main type of relation must be %s", mainType); + return super.createTypeMapping(relationType); + } + + @Override + public BuiltIndex<NewRegularIndex> build() { + checkState(mainType != null, "Mapping for main type must be defined"); + checkState(!mainType.getIndex().acceptsRelations() || !getRelations().isEmpty(), "At least one relation must be defined when index accepts relations"); + return new BuiltIndex<>(this); + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/SettingsConfiguration.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/SettingsConfiguration.java new file mode 100644 index 00000000000..260703ae2b3 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/SettingsConfiguration.java @@ -0,0 +1,83 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import org.sonar.api.config.Configuration; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class SettingsConfiguration { + public static final int MANUAL_REFRESH_INTERVAL = -1; + + private final Configuration configuration; + private final int defaultNbOfShards; + private final int refreshInterval; + + private SettingsConfiguration(Builder builder) { + this.configuration = builder.configuration; + this.defaultNbOfShards = builder.defaultNbOfShards; + this.refreshInterval = builder.refreshInterval; + } + + public static Builder newBuilder(Configuration configuration) { + return new Builder(configuration); + } + + public Configuration getConfiguration() { + return configuration; + } + + public int getDefaultNbOfShards() { + return defaultNbOfShards; + } + + public int getRefreshInterval() { + return refreshInterval; + } + + public static class Builder { + private final Configuration configuration; + private int defaultNbOfShards = 1; + private int refreshInterval = 30; + + public Builder(Configuration configuration) { + this.configuration = requireNonNull(configuration, "configuration can't be null"); + } + + public Builder setDefaultNbOfShards(int defaultNbOfShards) { + checkArgument(defaultNbOfShards >= 1, "defaultNbOfShards must be >= 1"); + this.defaultNbOfShards = defaultNbOfShards; + return this; + } + + public Builder setRefreshInterval(int refreshInterval) { + checkArgument(refreshInterval == -1 || refreshInterval > 0, + "refreshInterval must be either -1 or strictly positive"); + this.refreshInterval = refreshInterval; + return this; + } + + public SettingsConfiguration build() { + return new SettingsConfiguration(this); + } + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/StringFieldBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/StringFieldBuilder.java new file mode 100644 index 00000000000..24dd3abc580 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/StringFieldBuilder.java @@ -0,0 +1,181 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.Maps; +import java.util.Arrays; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import static java.lang.String.valueOf; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELDDATA_ENABLED; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_FIELDDATA; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TERM_VECTOR; +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_KEYWORD; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_NOT_SEARCHABLE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.INDEX_SEARCHABLE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.NORMS; +import static org.sonar.server.es.newindex.DefaultIndexSettings.STORE; + +/** + * Helper to define a string field in mapping of index type + */ +public abstract class StringFieldBuilder<U extends FieldAware<U>, T extends StringFieldBuilder<U, T>> { + private final U parent; + private final String fieldName; + private boolean disableSearch = false; + private boolean disableNorms = false; + private boolean termVectorWithPositionOffsets = false; + private SortedMap<String, Object> subFields = Maps.newTreeMap(); + private boolean store = false; + protected boolean disabledDocValues = false; + + protected StringFieldBuilder(U parent, String fieldName) { + this.parent = parent; + this.fieldName = fieldName; + } + + /** + * Add a sub-field. A {@code SortedMap} is required for consistency of the index settings hash. + */ + private T addSubField(String fieldName, SortedMap<String, String> fieldDefinition) { + subFields.put(fieldName, fieldDefinition); + return castThis(); + } + + /** + * Add subfields, one for each analyzer. + */ + public T addSubFields(DefaultIndexSettingsElement... analyzers) { + Arrays.stream(analyzers) + .forEach(analyzer -> addSubField(analyzer.getSubFieldSuffix(), analyzer.fieldMapping())); + return castThis(); + } + + /** + * Norms consume useless memory if string field is used for filtering or aggregations. + * + * https://www.elastic.co/guide/en/elasticsearch/reference/2.3/norms.html + * https://www.elastic.co/guide/en/elasticsearch/guide/current/scoring-theory.html#field-norm + */ + public T disableNorms() { + this.disableNorms = true; + return castThis(); + } + + /** + * Position offset term vectors are required for the fast_vector_highlighter (fvh). + */ + public T termVectorWithPositionOffsets() { + this.termVectorWithPositionOffsets = true; + return castThis(); + } + + /** + * "index: false" -> Make this field not searchable. + * By default field is "true": it is searchable, but index the value exactly + * as specified. + */ + public T disableSearch() { + this.disableSearch = true; + return castThis(); + } + + public T store() { + this.store = true; + return castThis(); + } + + @SuppressWarnings("unchecked") + private T castThis() { + return (T) this; + } + + public U build() { + if (subFields.isEmpty()) { + return buildWithoutSubfields(); + } + return buildWithSubfields(); + } + + private U buildWithoutSubfields() { + Map<String, Object> hash = new TreeMap<>(); + hash.put("type", getFieldType()); + hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE); + hash.put(NORMS, valueOf(!disableNorms)); + hash.put(STORE, valueOf(store)); + if (FIELD_TYPE_KEYWORD.equals(getFieldType())) { + hash.put("doc_values", valueOf(!disabledDocValues)); + } + if (getFieldData()) { + hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED); + } + return parent.setField(fieldName, hash); + } + + private U buildWithSubfields() { + Map<String, Object> hash = new TreeMap<>(); + hash.put("type", getFieldType()); + hash.put(INDEX, disableSearch ? INDEX_NOT_SEARCHABLE : INDEX_SEARCHABLE); + hash.put(NORMS, "false"); + hash.put(STORE, valueOf(store)); + if (FIELD_TYPE_KEYWORD.equals(getFieldType())) { + hash.put("doc_values", valueOf(!disabledDocValues)); + } + if (getFieldData()) { + hash.put(FIELD_FIELDDATA, FIELDDATA_ENABLED); + } + if (termVectorWithPositionOffsets) { + hash.put(FIELD_TERM_VECTOR, "with_positions_offsets"); + } + hash.put("fields", configureSubFields()); + return parent.setField(fieldName, hash); + } + + private Map<String, Object> configureSubFields() { + Map<String, Object> multiFields = new TreeMap<>(subFields); + + // apply this fields configuration to all subfields + multiFields.entrySet().forEach(entry -> { + Object subFieldMapping = entry.getValue(); + if (subFieldMapping instanceof Map) { + entry.setValue(configureSubField((Map<String, String>) subFieldMapping)); + } + }); + return multiFields; + } + + private Map<String, String> configureSubField(Map<String, String> subFieldMapping) { + Map<String, String> subHash = new TreeMap<>(subFieldMapping); + subHash.put(INDEX, INDEX_SEARCHABLE); + subHash.put(NORMS, "false"); + subHash.put(STORE, valueOf(store)); + if (termVectorWithPositionOffsets) { + subHash.put(FIELD_TERM_VECTOR, "with_positions_offsets"); + } + return subHash; + } + + protected abstract boolean getFieldData(); + + protected abstract String getFieldType(); +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/TextFieldBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/TextFieldBuilder.java new file mode 100644 index 00000000000..7817b50f72a --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/TextFieldBuilder.java @@ -0,0 +1,49 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import static org.sonar.server.es.newindex.DefaultIndexSettings.FIELD_TYPE_TEXT; + +public class TextFieldBuilder<U extends FieldAware<U>> extends StringFieldBuilder<U, TextFieldBuilder<U>> { + + private boolean fieldData = false; + + protected TextFieldBuilder(U indexType, String fieldName) { + super(indexType, fieldName); + } + + protected String getFieldType() { + return FIELD_TYPE_TEXT; + } + + /** + * Required to enable sorting, aggregation and access to field data on fields of type "text". + * <p>Disabled by default as this can have significant memory cost</p> + */ + public StringFieldBuilder withFieldData() { + this.fieldData = true; + return this; + } + + @Override + protected boolean getFieldData() { + return fieldData; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/TypeMapping.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/TypeMapping.java new file mode 100644 index 00000000000..40a42d85328 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/TypeMapping.java @@ -0,0 +1,36 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +public class TypeMapping extends FieldAware<TypeMapping> { + + private final NewIndex<?> newIndex; + + TypeMapping(NewIndex<?> newIndex) { + this.newIndex = newIndex; + } + + @Override + TypeMapping setFieldImpl(String fieldName, Object attributes) { + newIndex.setFieldImpl(fieldName, attributes); + return this; + } + +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/package-info.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/package-info.java new file mode 100644 index 00000000000..f3a4f39b0a6 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/package-info.java @@ -0,0 +1,24 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.es.newindex; + +import javax.annotation.ParametersAreNonnullByDefault; + diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java index a52ca3bb90c..c4d9428f693 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java @@ -68,8 +68,9 @@ public class ProxyClusterHealthRequestBuilder extends ClusterHealthRequestBuilde public String toString() { StringBuilder message = new StringBuilder(); message.append("ES cluster health request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); + String[] indices = request.indices(); + if (indices != null && indices.length > 0) { + message.append(String.format(" on indices '%s'", StringUtils.join(indices, ","))); } return message.toString(); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java index d97cd6741d3..51ee8240080 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java @@ -19,12 +19,16 @@ */ package org.sonar.server.es.request; +import java.io.IOException; import org.elasticsearch.action.ListenableActionFuture; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.sonar.api.utils.log.Profiler; import org.sonar.server.es.EsClient; @@ -66,6 +70,21 @@ public class ProxyCreateIndexRequestBuilder extends CreateIndexRequestBuilder { throw new UnsupportedOperationException("execute() should not be called as it's used for asynchronous"); } + public String toJson() { + try { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject() + .field("settings") + .startObject(); + request.settings().toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject().endObject(); + builder.prettyPrint(); + return builder.string(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + @Override public String toString() { return String.format("ES create index '%s'", index); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java index dcba39e44b3..a7bae09c894 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java @@ -68,8 +68,9 @@ public class ProxyIndicesStatsRequestBuilder extends IndicesStatsRequestBuilder public String toString() { StringBuilder message = new StringBuilder(); message.append("ES indices stats request"); - if (request.indices().length > 0) { - message.append(String.format(" on indices '%s'", StringUtils.join(request.indices(), ","))); + String[] indices = request.indices(); + if (indices != null && indices.length > 0) { + message.append(String.format(" on indices '%s'", StringUtils.join(indices, ","))); } return message.toString(); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java index 623a8b0ee45..6e4cdd2452d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/ComponentTextSearchFeatureRepertoire.java @@ -29,17 +29,17 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.server.es.DefaultIndexSettings; -import org.sonar.server.es.DefaultIndexSettingsElement; +import org.sonar.server.es.newindex.DefaultIndexSettings; +import org.sonar.server.es.newindex.DefaultIndexSettingsElement; import org.sonar.server.es.textsearch.ComponentTextSearchQueryFactory.ComponentTextSearchQuery; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_PREFIX_CASE_INSENSITIVE_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; import static org.sonar.server.es.textsearch.ComponentTextSearchFeature.UseCase.CHANGE_ORDER_OF_RESULTS; import static org.sonar.server.es.textsearch.ComponentTextSearchFeature.UseCase.GENERATE_RESULTS; diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/JavaTokenizer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/JavaTokenizer.java index 658f6fff7e6..6decac89ccd 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/JavaTokenizer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/textsearch/JavaTokenizer.java @@ -23,9 +23,9 @@ import java.util.Arrays; import java.util.List; import org.apache.commons.lang.StringUtils; import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.server.es.DefaultIndexSettings; +import org.sonar.server.es.newindex.DefaultIndexSettings; -import static org.sonar.server.es.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH; +import static org.sonar.server.es.newindex.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH; /** * Splits text queries into their tokens, for to use them in n_gram match queries later. diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java index 82f3e279ffb..406133e9657 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueDoc.java @@ -29,15 +29,18 @@ import org.sonar.api.rule.Severity; import org.sonar.api.rules.RuleType; import org.sonar.api.utils.Duration; import org.sonar.server.es.BaseDoc; +import org.sonar.server.permission.index.AuthorizationDoc; + +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; public class IssueDoc extends BaseDoc { public IssueDoc(Map<String, Object> fields) { - super(fields); + super(TYPE_ISSUE, fields); } public IssueDoc() { - super(Maps.newHashMapWithExpectedSize(32)); + super(TYPE_ISSUE, Maps.newHashMapWithExpectedSize(32)); } @Override @@ -45,16 +48,6 @@ public class IssueDoc extends BaseDoc { return key(); } - @Override - public String getRouting() { - return projectUuid(); - } - - @Override - public String getParent() { - return projectUuid(); - } - public String key() { return getField(IssueIndexDefinition.FIELD_ISSUE_KEY); } @@ -181,6 +174,7 @@ public class IssueDoc extends BaseDoc { public IssueDoc setProjectUuid(String s) { setField(IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID, s); + setParent(AuthorizationDoc.idOf(s)); return this; } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java index 0e3db147dfc..04b24d58e79 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexDefinition.java @@ -21,20 +21,24 @@ package org.sonar.server.issue.index; import org.sonar.api.config.Configuration; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.newindex.NewAuthorizedIndex; +import org.sonar.server.es.newindex.TypeMapping; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; /** * Definition of ES index "issues", including settings and fields. */ public class IssueIndexDefinition implements IndexDefinition { - public static final IndexType INDEX_TYPE_ISSUE = new IndexType("issues", "issue"); + public static final Index DESCRIPTOR = Index.withRelations("issues"); + public static final IndexType.IndexRelationType TYPE_ISSUE = IndexType.relation(IndexType.main(DESCRIPTOR, TYPE_AUTHORIZATION), "issue"); public static final String FIELD_ISSUE_ASSIGNEE_UUID = "assignee"; public static final String FIELD_ISSUE_AUTHOR_LOGIN = "authorLogin"; public static final String FIELD_ISSUE_COMPONENT_UUID = "component"; @@ -118,44 +122,42 @@ public class IssueIndexDefinition implements IndexDefinition { @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create( - INDEX_TYPE_ISSUE.getIndex(), + NewAuthorizedIndex index = context.createWithAuthorization( + DESCRIPTOR, newBuilder(config) .setRefreshInterval(MANUAL_REFRESH_INTERVAL) .setDefaultNbOfShards(5) - .build()); + .build()) + .setEnableSource(enableSource); - NewIndex.NewIndexType type = index.createType(INDEX_TYPE_ISSUE.getType()); - type.requireProjectAuthorization(); - type.setEnableSource(enableSource); - - type.keywordFieldBuilder(FIELD_ISSUE_ASSIGNEE_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); - type.keywordFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).disableNorms().build(); - type.createLongField(FIELD_ISSUE_EFFORT); - type.keywordFieldBuilder(FIELD_ISSUE_FILE_PATH).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); - type.createDateTimeField(FIELD_ISSUE_FUNC_CREATED_AT); - type.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT); - type.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT); - type.keywordFieldBuilder(FIELD_ISSUE_KEY).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); - type.keywordFieldBuilder(FIELD_ISSUE_LANGUAGE).disableNorms().build(); - type.createIntegerField(FIELD_ISSUE_LINE); - type.keywordFieldBuilder(FIELD_ISSUE_MODULE_UUID).disableNorms().build(); - type.createUuidPathField(FIELD_ISSUE_MODULE_PATH); - type.keywordFieldBuilder(FIELD_ISSUE_ORGANIZATION_UUID).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_PROJECT_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); - type.keywordFieldBuilder(FIELD_ISSUE_BRANCH_UUID).disableNorms().build(); - type.createBooleanField(FIELD_ISSUE_IS_MAIN_BRANCH); - type.keywordFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_RESOLUTION).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_RULE_ID).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_SEVERITY).disableNorms().build(); - type.createByteField(FIELD_ISSUE_SEVERITY_VALUE); - type.keywordFieldBuilder(FIELD_ISSUE_STATUS).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); - type.keywordFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_OWASP_TOP_10).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_SANS_TOP_25).disableNorms().build(); - type.keywordFieldBuilder(FIELD_ISSUE_CWE).disableNorms().build(); + TypeMapping mapping = index.createTypeMapping(TYPE_ISSUE); + mapping.keywordFieldBuilder(FIELD_ISSUE_ASSIGNEE_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_AUTHOR_LOGIN).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_COMPONENT_UUID).disableNorms().build(); + mapping.createLongField(FIELD_ISSUE_EFFORT); + mapping.keywordFieldBuilder(FIELD_ISSUE_FILE_PATH).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); + mapping.createDateTimeField(FIELD_ISSUE_FUNC_CREATED_AT); + mapping.createDateTimeField(FIELD_ISSUE_FUNC_UPDATED_AT); + mapping.createDateTimeField(FIELD_ISSUE_FUNC_CLOSED_AT); + mapping.keywordFieldBuilder(FIELD_ISSUE_KEY).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_LANGUAGE).disableNorms().build(); + mapping.createIntegerField(FIELD_ISSUE_LINE); + mapping.keywordFieldBuilder(FIELD_ISSUE_MODULE_UUID).disableNorms().build(); + mapping.createUuidPathField(FIELD_ISSUE_MODULE_PATH); + mapping.keywordFieldBuilder(FIELD_ISSUE_ORGANIZATION_UUID).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_PROJECT_UUID).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_BRANCH_UUID).disableNorms().build(); + mapping.createBooleanField(FIELD_ISSUE_IS_MAIN_BRANCH); + mapping.keywordFieldBuilder(FIELD_ISSUE_DIRECTORY_PATH).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_RESOLUTION).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_RULE_ID).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_SEVERITY).disableNorms().build(); + mapping.createByteField(FIELD_ISSUE_SEVERITY_VALUE); + mapping.keywordFieldBuilder(FIELD_ISSUE_STATUS).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_TAGS).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_TYPE).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_OWASP_TOP_10).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_SANS_TOP_25).disableNorms().build(); + mapping.keywordFieldBuilder(FIELD_ISSUE_CWE).disableNorms().build(); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 472498b8de3..37396061bd3 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -46,6 +46,7 @@ import org.sonar.server.es.IndexingResult; import org.sonar.server.es.OneToManyResilientIndexingListener; import org.sonar.server.es.OneToOneResilientIndexingListener; import org.sonar.server.es.ProjectIndexer; +import org.sonar.server.permission.index.AuthorizationDoc; import org.sonar.server.permission.index.AuthorizationScope; import org.sonar.server.permission.index.NeedAuthorizationIndexer; @@ -53,7 +54,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID; -import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { @@ -66,8 +67,8 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { */ private static final String ID_TYPE_PROJECT_UUID = "projectUuid"; private static final Logger LOGGER = Loggers.get(IssueIndexer.class); - private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_ISSUE, project -> Qualifiers.PROJECT.equals(project.getQualifier())); - private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(INDEX_TYPE_ISSUE); + private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_ISSUE, project -> Qualifiers.PROJECT.equals(project.getQualifier())); + private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(TYPE_ISSUE); private final EsClient esClient; private final DbClient dbClient; @@ -187,7 +188,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { // the remaining uuids reference issues that don't exist in db. They must // be deleted from index. itemsByIssueKey.values().forEach( - item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting())); + item -> bulkIndexer.addDeletion(TYPE_ISSUE.getMainType(), item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); } @@ -229,7 +230,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); - issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, issueKey, projectUuid)); + issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(TYPE_ISSUE.getMainType(), issueKey, AuthorizationDoc.idOf(projectUuid))); bulkIndexer.stop(); } @@ -249,27 +250,25 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } private IndexRequest newIndexRequest(IssueDoc issue) { - String projectUuid = issue.projectUuid(); - return esClient.prepareIndex(INDEX_TYPE_ISSUE) - .setId(issue.key()) - .setRouting(projectUuid) - .setParent(projectUuid) + return esClient.prepareIndex(TYPE_ISSUE.getMainType()) + .setId(issue.getId()) + .setRouting(issue.getRouting().orElseThrow(() -> new IllegalStateException("IssueDoc should define a routing"))) .setSource(issue.getFields()) .request(); } private void addProjectDeletionToBulkIndexer(BulkIndexer bulkIndexer, String projectUuid) { - SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ISSUE) - .setRouting(projectUuid) + SearchRequestBuilder search = esClient.prepareSearch(TYPE_ISSUE.getMainType()) + .setRouting(AuthorizationDoc.idOf(projectUuid)) .setQuery(boolQuery().must(termQuery(FIELD_ISSUE_PROJECT_UUID, projectUuid))); bulkIndexer.addDeletion(search); } private static EsQueueDto createQueueDto(String docId, String docIdType, String projectUuid) { - return EsQueueDto.create(INDEX_TYPE_ISSUE.format(), docId, docIdType, projectUuid); + return EsQueueDto.create(TYPE_ISSUE.format(), docId, docIdType, projectUuid); } private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) { - return new BulkIndexer(esClient, INDEX_TYPE_ISSUE, size, listener); + return new BulkIndexer(esClient, TYPE_ISSUE, size, listener); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java index de4ded09982..4d2f68acd0d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresDoc.java @@ -29,6 +29,7 @@ import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.es.BaseDoc; +import org.sonar.server.permission.index.AuthorizationDoc; import static org.sonar.api.measures.Metric.Level.ERROR; import static org.sonar.api.measures.Metric.Level.OK; @@ -47,13 +48,14 @@ import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIEL import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_UUID; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; public class ProjectMeasuresDoc extends BaseDoc { public static final Map<String, Integer> QUALITY_GATE_STATUS = ImmutableMap.of(OK.name(), 1, WARN.name(), 2, ERROR.name(), 3); public ProjectMeasuresDoc() { - super(new HashMap<>(8)); + super(TYPE_PROJECT_MEASURES, new HashMap<>(8)); } @Override @@ -61,18 +63,9 @@ public class ProjectMeasuresDoc extends BaseDoc { return getField(FIELD_UUID); } - @Override - public String getRouting() { - return getId(); - } - - @Override - public String getParent() { - return getId(); - } - public ProjectMeasuresDoc setId(String s) { setField(FIELD_UUID, s); + setParent(AuthorizationDoc.idOf(s)); return this; } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java index 6905b9c2f84..ed1c40ea8b8 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexDefinition.java @@ -20,18 +20,26 @@ package org.sonar.server.measure.index; import org.sonar.api.config.Configuration; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType.IndexRelationType; +import org.sonar.server.es.newindex.NewAuthorizedIndex; +import org.sonar.server.es.newindex.TypeMapping; +import org.sonar.server.permission.index.IndexAuthorizationConstants; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class ProjectMeasuresIndexDefinition implements IndexDefinition { - public static final IndexType INDEX_TYPE_PROJECT_MEASURES = new IndexType("projectmeasures", "projectmeasure"); + public static final Index DESCRIPTOR = Index.withRelations("projectmeasures"); + public static final IndexType.IndexMainType TYPE_AUTHORIZATION = IndexType.main(DESCRIPTOR, IndexAuthorizationConstants.TYPE_AUTHORIZATION); + public static final IndexRelationType TYPE_PROJECT_MEASURES = IndexType.relation(TYPE_AUTHORIZATION, "projectmeasure"); + public static final String FIELD_UUID = "uuid"; public static final String FIELD_ORGANIZATION_UUID = "organizationUuid"; @@ -52,23 +60,36 @@ public class ProjectMeasuresIndexDefinition implements IndexDefinition { public static final String FIELD_DISTRIB_NCLOC = "ncloc"; private final Configuration config; + private final boolean enableSource; + + private ProjectMeasuresIndexDefinition(Configuration config, boolean enableSource) { + this.config = config; + this.enableSource = enableSource; + } + + public ProjectMeasuresIndexDefinition(Configuration config) { + this(config, false); + } - public ProjectMeasuresIndexDefinition(Configuration settings) { - this.config = settings; + /** + * Keep the document sources in index so that indexer tests can verify content + * of indexed documents. + */ + public static ProjectMeasuresIndexDefinition createForTest() { + return new ProjectMeasuresIndexDefinition(new MapSettings().asConfig(), true); } @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create( - INDEX_TYPE_PROJECT_MEASURES.getIndex(), + NewAuthorizedIndex index = context.createWithAuthorization( + DESCRIPTOR, newBuilder(config) .setRefreshInterval(MANUAL_REFRESH_INTERVAL) .setDefaultNbOfShards(5) - .build()); - - NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_PROJECT_MEASURES.getType()) - .requireProjectAuthorization(); + .build()) + .setEnableSource(enableSource); + TypeMapping mapping = index.createTypeMapping(TYPE_PROJECT_MEASURES); mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_ORGANIZATION_UUID).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_KEY).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); @@ -85,6 +106,5 @@ public class ProjectMeasuresIndexDefinition implements IndexDefinition { .addIntegerField(FIELD_DISTRIB_NCLOC) .build(); mapping.createDateTimeField(FIELD_ANALYSED_AT); - mapping.setEnableSource(false); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index 44f24c64a77..851da90388c 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -28,7 +28,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; import javax.annotation.Nullable; -import org.elasticsearch.action.index.IndexRequest; import org.sonar.api.resources.Qualifiers; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; @@ -44,15 +43,16 @@ import org.sonar.server.es.IndexingListener; import org.sonar.server.es.IndexingResult; import org.sonar.server.es.OneToOneResilientIndexingListener; import org.sonar.server.es.ProjectIndexer; +import org.sonar.server.permission.index.AuthorizationDoc; import org.sonar.server.permission.index.AuthorizationScope; import org.sonar.server.permission.index.NeedAuthorizationIndexer; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorizationIndexer { - private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_PROJECT_MEASURES, project -> Qualifiers.PROJECT.equals(project.getQualifier())); - private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(INDEX_TYPE_PROJECT_MEASURES); + private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_PROJECT_MEASURES, project -> Qualifiers.PROJECT.equals(project.getQualifier())); + private static final ImmutableSet<IndexType> INDEX_TYPES = ImmutableSet.of(TYPE_PROJECT_MEASURES); private final DbClient dbClient; private final EsClient esClient; @@ -96,7 +96,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization case PROJECT_TAGS_UPDATE: case PROJECT_DELETION: List<EsQueueDto> items = projectUuids.stream() - .map(projectUuid -> EsQueueDto.create(INDEX_TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid)) + .map(projectUuid -> EsQueueDto.create(TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid)) .collect(MoreCollectors.toArrayList(projectUuids.size())); return dbClient.esQueueDao().insert(dbSession, items); @@ -108,7 +108,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization public IndexingResult commitAndIndex(DbSession dbSession, Collection<String> projectUuids) { List<EsQueueDto> items = projectUuids.stream() - .map(projectUuid -> EsQueueDto.create(INDEX_TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid)) + .map(projectUuid -> EsQueueDto.create(TYPE_PROJECT_MEASURES.format(), projectUuid, null, projectUuid)) .collect(MoreCollectors.toArrayList(projectUuids.size())); dbClient.esQueueDao().insert(dbSession, items); @@ -132,7 +132,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization String projectUuid = it.next(); try (ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { while (rowIt.hasNext()) { - bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(rowIt.next()))); + bulkIndexer.add(toProjectMeasuresDoc(rowIt.next()).toIndexRequest()); it.remove(); } } @@ -140,7 +140,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization // the remaining uuids reference projects that don't exist in db. They must // be deleted from index. - projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(INDEX_TYPE_PROJECT_MEASURES, projectUuid, projectUuid)); + projectUuids.forEach(projectUuid -> bulkIndexer.addDeletion(TYPE_PROJECT_MEASURES, projectUuid, AuthorizationDoc.idOf(projectUuid))); return bulkIndexer.stop(); } @@ -153,22 +153,14 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization bulkIndexer.start(); while (rowIt.hasNext()) { ProjectMeasures doc = rowIt.next(); - bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(doc))); + bulkIndexer.add(toProjectMeasuresDoc(doc).toIndexRequest()); } bulkIndexer.stop(); } } private BulkIndexer createBulkIndexer(Size bulkSize, IndexingListener listener) { - return new BulkIndexer(esClient, INDEX_TYPE_PROJECT_MEASURES, bulkSize, listener); - } - - private static IndexRequest newIndexRequest(ProjectMeasuresDoc doc) { - String projectUuid = doc.getId(); - return new IndexRequest(INDEX_TYPE_PROJECT_MEASURES.getIndex(), INDEX_TYPE_PROJECT_MEASURES.getType(), projectUuid) - .routing(projectUuid) - .parent(projectUuid) - .source(doc.getFields()); + return new BulkIndexer(esClient, TYPE_PROJECT_MEASURES, bulkSize, listener); } private static ProjectMeasuresDoc toProjectMeasuresDoc(ProjectMeasures projectMeasures) { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/AuthorizationDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/AuthorizationDoc.java new file mode 100644 index 00000000000..c9e782b2dc7 --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/AuthorizationDoc.java @@ -0,0 +1,82 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.permission.index; + +import java.util.List; +import java.util.Optional; +import org.sonar.server.es.BaseDoc; +import org.sonar.server.es.IndexType; + +import static java.util.Objects.requireNonNull; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_ALLOW_ANYONE; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_GROUP_IDS; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.FIELD_USER_IDS; + +public class AuthorizationDoc extends BaseDoc { + private static final String ID_PREFIX = "auth_"; + private final String projectUuid; + + private AuthorizationDoc(IndexType indexType, String projectUuid) { + super(indexType); + this.projectUuid = projectUuid; + } + + public static AuthorizationDoc fromDto(IndexType indexType, IndexPermissions dto) { + AuthorizationDoc res = new AuthorizationDoc(indexType, dto.getProjectUuid()); + if (dto.isAllowAnyone()) { + return res.setAllowAnyone(); + } + return res.setRestricted(dto.getGroupIds(), dto.getUserIds()); + } + + @Override + public String getId() { + return idOf(projectUuid); + } + + public static String idOf(String projectUuid) { + requireNonNull(projectUuid, "projectUuid can't be null"); + return ID_PREFIX + projectUuid; + } + + public static String projectUuidOf(String id) { + if (id.startsWith(ID_PREFIX)) { + return id.substring(ID_PREFIX.length()); + } + return id; + } + + @Override + protected Optional<String> getSimpleMainTypeRouting() { + return Optional.of(projectUuid); + } + + private AuthorizationDoc setAllowAnyone() { + setField(FIELD_ALLOW_ANYONE, true); + return this; + } + + private AuthorizationDoc setRestricted(List<Integer> groupIds, List<Integer> userIds) { + setField(FIELD_ALLOW_ANYONE, false); + setField(FIELD_GROUP_IDS, groupIds); + setField(FIELD_USER_IDS, userIds); + return this; + } +} diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/AuthorizationScope.java b/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/AuthorizationScope.java index c5ca1107870..e7337040f3f 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/AuthorizationScope.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/AuthorizationScope.java @@ -21,7 +21,8 @@ package org.sonar.server.permission.index; import java.util.function.Predicate; import javax.annotation.concurrent.Immutable; -import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.IndexType.IndexRelationType; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; @@ -29,28 +30,31 @@ import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE @Immutable public final class AuthorizationScope { - private final IndexType indexType; + private final IndexMainType indexType; private final Predicate<IndexPermissions> projectPredicate; - public AuthorizationScope(IndexType indexType, Predicate<IndexPermissions> projectPredicate) { - this.indexType = getAuthorizationIndexType(indexType); + public AuthorizationScope(IndexRelationType functionalType, Predicate<IndexPermissions> projectPredicate) { + this.indexType = getAuthorizationIndexType(functionalType); this.projectPredicate = requireNonNull(projectPredicate); } /** * @return the identifier of the ElasticSearch type (including it's index name), that corresponds to a certain document type */ - private static IndexType getAuthorizationIndexType(IndexType indexType) { - requireNonNull(indexType); - requireNonNull(indexType.getIndex()); - checkArgument(!TYPE_AUTHORIZATION.equals(indexType.getType()), "Authorization types do not have authorization on their own."); - return new IndexType(indexType.getIndex(), TYPE_AUTHORIZATION); + private static IndexMainType getAuthorizationIndexType(IndexRelationType functionalType) { + requireNonNull(functionalType); + IndexMainType mainType = functionalType.getMainType(); + checkArgument( + TYPE_AUTHORIZATION.equals(mainType.getType()), + "Index %s doesn't seem to be an authorized index as main type is not %s (got %s)", + mainType.getIndex(), TYPE_AUTHORIZATION, mainType.getType()); + return mainType; } /** * Identifier of the authorization type (in the same index than the original IndexType, passed into the constructor). */ - public IndexType getIndexType() { + public IndexMainType getIndexType() { return indexType; } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/IndexAuthorizationConstants.java b/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/IndexAuthorizationConstants.java index 2a0c222f6e8..ff4567b89d6 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/IndexAuthorizationConstants.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/permission/index/IndexAuthorizationConstants.java @@ -20,15 +20,15 @@ package org.sonar.server.permission.index; public final class IndexAuthorizationConstants { - public static final String TYPE_AUTHORIZATION = "authorization"; - public static final String FIELD_GROUP_IDS = "groupIds"; - public static final String FIELD_USER_IDS = "userIds"; + public static final String TYPE_AUTHORIZATION = "auth"; + public static final String FIELD_GROUP_IDS = TYPE_AUTHORIZATION + "_groupIds"; + public static final String FIELD_USER_IDS = TYPE_AUTHORIZATION + "_userIds"; /** * When true, then anybody can access to the project. In that case * it's useless to store granted groups and users. The related * fields are empty. */ - public static final String FIELD_ALLOW_ANYONE = "allowAnyone"; + public static final String FIELD_ALLOW_ANYONE = TYPE_AUTHORIZATION + "_allowAnyone"; private IndexAuthorizationConstants() { // prevents instantiation diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java index 90257da8fff..24ca4ddbd7f 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleDoc.java @@ -29,41 +29,48 @@ import static org.apache.commons.lang.StringUtils.containsIgnoreCase; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_ID; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID; -import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_RULE_ID; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY; +import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_ID; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE; public class ActiveRuleDoc extends BaseDoc { + public static final String DOC_ID_PREFIX = "ar_"; + public ActiveRuleDoc(long id) { - super(Maps.newHashMapWithExpectedSize(10)); + super(TYPE_ACTIVE_RULE, Maps.newHashMapWithExpectedSize(10)); setField(FIELD_ACTIVE_RULE_ID, String.valueOf(id)); } public ActiveRuleDoc(Map<String, Object> source) { - super(source); + super(TYPE_ACTIVE_RULE, source); } - @Override - public String getId() { - return getField(FIELD_ACTIVE_RULE_ID); + public static String docIdOf(long activeRuleId) { + return docIdOf(String.valueOf(activeRuleId)); } - @Override - public String getRouting() { - return getRuleIdAsString(); + public static String docIdOf(String activeRuleId) { + return DOC_ID_PREFIX + activeRuleId; } - @Override - public String getParent() { - return getRuleIdAsString(); + public static long activeRuleIdOf(String docId) { + if (docId.startsWith(DOC_ID_PREFIX)) { + return Long.valueOf(docId.substring(DOC_ID_PREFIX.length())); + } + // support for old active rule docId + return Long.valueOf(docId); } - private String getRuleIdAsString() { - return getField(FIELD_ACTIVE_RULE_RULE_ID); + @Override + public String getId() { + return docIdOf(getField(FIELD_ACTIVE_RULE_ID)); } ActiveRuleDoc setRuleId(int ruleId) { - setField(FIELD_ACTIVE_RULE_RULE_ID, String.valueOf(ruleId)); + String parent = String.valueOf(ruleId); + setParent(parent); + setField(FIELD_RULE_ID, parent); return this; } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java index 5f7f3948462..8ca2cefe867 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java @@ -48,12 +48,13 @@ import org.sonar.server.es.OneToOneResilientIndexingListener; import org.sonar.server.es.ResilientIndexer; import org.sonar.server.qualityprofile.ActiveRuleChange; import org.sonar.server.qualityprofile.ActiveRuleInheritance; -import org.sonar.server.rule.index.RuleIndexDefinition; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.sonar.core.util.stream.MoreCollectors.toArrayList; +import static org.sonar.core.util.stream.MoreCollectors.toSet; +import static org.sonar.server.qualityprofile.index.ActiveRuleDoc.docIdOf; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE; public class ActiveRuleIndexer implements ResilientIndexer { @@ -81,13 +82,13 @@ public class ActiveRuleIndexer implements ResilientIndexer { @Override public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(INDEX_TYPE_ACTIVE_RULE); + return ImmutableSet.of(TYPE_ACTIVE_RULE); } public void commitAndIndex(DbSession dbSession, Collection<ActiveRuleChange> changes) { List<EsQueueDto> items = changes.stream() .map(ActiveRuleChange::getActiveRule) - .map(ar -> newQueueDto(String.valueOf(ar.getId()), ID_TYPE_ACTIVE_RULE_ID, String.valueOf(ar.getRuleId()))) + .map(ar -> newQueueDto(docIdOf(ar.getId()), ID_TYPE_ACTIVE_RULE_ID, String.valueOf(ar.getRuleId()))) .collect(toArrayList()); dbClient.esQueueDao().insert(dbSession, items); @@ -125,13 +126,13 @@ public class ActiveRuleIndexer implements ResilientIndexer { return result; } - Map<Long, EsQueueDto> activeRuleItems = new HashMap<>(); + Map<String, EsQueueDto> activeRuleItems = new HashMap<>(); Map<String, EsQueueDto> ruleProfileItems = new HashMap<>(); items.forEach(i -> { if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) { ruleProfileItems.put(i.getDocId(), i); } else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) { - activeRuleItems.put(Long.parseLong(i.getDocId()), i); + activeRuleItems.put(i.getDocId(), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i); deleteQueueDto(dbSession, i); @@ -147,24 +148,31 @@ public class ActiveRuleIndexer implements ResilientIndexer { return result; } - private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) { + private IndexingResult doIndexActiveRules(DbSession dbSession, Map<String, EsQueueDto> activeRuleItems) { OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); - Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems); - dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(), + Map<String, EsQueueDto> remaining = new HashMap<>(activeRuleItems); + dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, toActiveRuleIds(activeRuleItems), i -> { - remaining.remove(i.getId()); + remaining.remove(docIdOf(i.getId())); bulkIndexer.add(newIndexRequest(i)); }); // the remaining ids reference rows that don't exist in db. They must // be deleted from index. - remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE, + remaining.values().forEach(item -> bulkIndexer.addDeletion(TYPE_ACTIVE_RULE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); } + private static Collection<Long> toActiveRuleIds(Map<String, EsQueueDto> activeRuleItems) { + Set<String> docIds = activeRuleItems.keySet(); + return docIds.stream() + .map(ActiveRuleDoc::activeRuleIdOf) + .collect(toSet(docIds.size())); + } + private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) { IndexingResult result = new IndexingResult(); @@ -176,9 +184,9 @@ public class ActiveRuleIndexer implements ResilientIndexer { RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule - SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE) + SearchRequestBuilder search = esClient.prepareSearch(TYPE_ACTIVE_RULE.getMainType()) .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); - profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search); + profileResult = BulkIndexer.delete(esClient, TYPE_ACTIVE_RULE, search); } else { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); @@ -202,7 +210,7 @@ public class ActiveRuleIndexer implements ResilientIndexer { } private BulkIndexer createBulkIndexer(Size size, IndexingListener listener) { - return new BulkIndexer(esClient, INDEX_TYPE_ACTIVE_RULE, size, listener); + return new BulkIndexer(esClient, TYPE_ACTIVE_RULE, size, listener); } private static IndexRequest newIndexRequest(IndexedActiveRuleDto dto) { @@ -213,14 +221,10 @@ public class ActiveRuleIndexer implements ResilientIndexer { // all the fields must be present, even if value is null String inheritance = dto.getInheritance(); doc.setInheritance(inheritance == null ? ActiveRuleInheritance.NONE.name() : inheritance); - return new IndexRequest(INDEX_TYPE_ACTIVE_RULE.getIndex(), INDEX_TYPE_ACTIVE_RULE.getType()) - .id(doc.getId()) - .parent(doc.getParent()) - .routing(doc.getRouting()) - .source(doc.getFields()); + return doc.toIndexRequest(); } private static EsQueueDto newQueueDto(String docId, String docIdType, @Nullable String routing) { - return EsQueueDto.create(INDEX_TYPE_ACTIVE_RULE.format(), docId, docIdType, routing); + return EsQueueDto.create(TYPE_ACTIVE_RULE.format(), docId, docIdType, routing); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java index 973575f00df..0ab4ce88747 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleDoc.java @@ -19,8 +19,11 @@ */ package org.sonar.server.rule.index; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.commons.lang.builder.ReflectionToStringBuilder; @@ -32,17 +35,21 @@ import org.sonar.db.rule.RuleForIndexingDto; import org.sonar.markdown.Markdown; import org.sonar.server.es.BaseDoc; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; + + /** * Implementation of Rule based on an Elasticsearch document */ public class RuleDoc extends BaseDoc { + @VisibleForTesting public RuleDoc(Map<String, Object> fields) { - super(fields); + super(TYPE_RULE, new HashMap<>(fields)); } public RuleDoc() { - super(Maps.newHashMapWithExpectedSize(15)); + super(TYPE_RULE, Maps.newHashMapWithExpectedSize(16)); } @Override @@ -60,13 +67,8 @@ public class RuleDoc extends BaseDoc { } @Override - public String getRouting() { - return idAsString(); - } - - @Override - public String getParent() { - return null; + protected Optional<String> getSimpleMainTypeRouting() { + return Optional.of(idAsString()); } public RuleKey key() { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java index 950fa811bf2..d75627e3291 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java @@ -27,14 +27,16 @@ import org.sonar.db.rule.RuleExtensionForIndexingDto; import org.sonar.db.rule.RuleForIndexingDto; import org.sonar.server.es.BaseDoc; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION; + public class RuleExtensionDoc extends BaseDoc { public RuleExtensionDoc(Map<String, Object> fields) { - super(fields); + super(TYPE_RULE_EXTENSION, fields); } public RuleExtensionDoc() { - super(Maps.newHashMapWithExpectedSize(4)); + super(TYPE_RULE_EXTENSION, Maps.newHashMapWithExpectedSize(4)); } @Override @@ -42,26 +44,18 @@ public class RuleExtensionDoc extends BaseDoc { return idOf(getRuleId(), getScope()); } - @Override - public String getRouting() { - return ruleIdAsString(); - } - - @Override - public String getParent() { - return ruleIdAsString(); - } - public int getRuleId() { return Integer.valueOf(ruleIdAsString()); } private String ruleIdAsString() { - return getField(RuleIndexDefinition.FIELD_RULE_EXTENSION_RULE_ID); + return getField(RuleIndexDefinition.FIELD_RULE_ID); } public RuleExtensionDoc setRuleId(int ruleId) { - setField(RuleIndexDefinition.FIELD_RULE_EXTENSION_RULE_ID, String.valueOf(ruleId)); + String parent = String.valueOf(ruleId); + setField(RuleIndexDefinition.FIELD_RULE_ID, parent); + setParent(parent); return this; } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java index 385f270bc12..b3a121d6200 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -59,12 +59,12 @@ import org.sonar.api.utils.System2; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.organization.OrganizationDto; import org.sonar.db.qualityprofile.QProfileDto; -import org.sonar.server.es.DefaultIndexSettings; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; import org.sonar.server.es.SearchIdResult; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.StickyFacetBuilder; +import org.sonar.server.es.newindex.DefaultIndexSettings; import org.sonar.server.es.textsearch.JavaTokenizer; import static com.google.common.base.Preconditions.checkArgument; @@ -77,14 +77,14 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.termsQuery; -import static org.sonar.server.es.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; import static org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES; -import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; import static org.sonar.server.es.EsUtils.optimizeScrollRequest; import static org.sonar.server.es.EsUtils.scrollIds; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY; @@ -105,9 +105,9 @@ import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_STATUS; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TEMPLATE_KEY; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TYPE; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_UPDATED_AT; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION; /** * The unique entry-point to interact with Elasticsearch index "rules". @@ -131,7 +131,6 @@ public class RuleIndex { .map(RuleStatus::toString) .collect(MoreCollectors.toList()); - private static final String AGGREGATION_NAME = "_ref"; private static final String AGGREGATION_NAME_FOR_TAGS = "tagsAggregation"; private final EsClient client; @@ -144,7 +143,7 @@ public class RuleIndex { public SearchIdResult<Integer> search(RuleQuery query, SearchOptions options) { SearchRequestBuilder esSearch = client - .prepareSearch(INDEX_TYPE_RULE); + .prepareSearch(TYPE_RULE); QueryBuilder qb = buildQuery(query); Map<String, QueryBuilder> filters = buildFilters(query); @@ -172,7 +171,7 @@ public class RuleIndex { */ public Iterator<Integer> searchAll(RuleQuery query) { SearchRequestBuilder esSearch = client - .prepareSearch(INDEX_TYPE_RULE) + .prepareSearch(TYPE_RULE) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); optimizeScrollRequest(esSearch); @@ -236,6 +235,11 @@ public class RuleIndex { private static Map<String, QueryBuilder> buildFilters(RuleQuery query) { Map<String, QueryBuilder> filters = new HashMap<>(); + /* Add enforced filter on main type Rule */ + filters.put( + FIELD_INDEX_TYPE, + boolQuery().must(QueryBuilders.termsQuery(FIELD_INDEX_TYPE, TYPE_RULE.getType()))); + /* Add enforced filter on rules that are REMOVED */ filters.put(FIELD_RULE_STATUS, boolQuery().mustNot( @@ -323,19 +327,19 @@ public class RuleIndex { if (TRUE.equals(query.getActivation())) { filters.put("activation", - JoinQueryBuilders.hasChildQuery(INDEX_TYPE_ACTIVE_RULE.getType(), + JoinQueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE.getName(), childQuery, ScoreMode.None)); } else if (FALSE.equals(query.getActivation())) { filters.put("activation", boolQuery().mustNot( - JoinQueryBuilders.hasChildQuery(INDEX_TYPE_ACTIVE_RULE.getType(), + JoinQueryBuilders.hasChildQuery(TYPE_ACTIVE_RULE.getName(), childQuery, ScoreMode.None))); } QProfileDto compareToQProfile = query.getCompareToQProfile(); if (compareToQProfile != null) { filters.put("comparison", JoinQueryBuilders.hasChildQuery( - INDEX_TYPE_ACTIVE_RULE.getType(), + TYPE_ACTIVE_RULE.getName(), boolQuery().must(QueryBuilders.termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, compareToQProfile.getRulesProfileUuid())), ScoreMode.None)); } @@ -350,7 +354,7 @@ public class RuleIndex { .map(tag -> boolQuery() .filter(QueryBuilders.termQuery(FIELD_RULE_EXTENSION_TAGS, tag)) .filter(termsQuery(FIELD_RULE_EXTENSION_SCOPE, RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(organization).getScope()))) - .map(childQuery -> JoinQueryBuilders.hasChildQuery(INDEX_TYPE_RULE_EXTENSION.getType(), childQuery, ScoreMode.None)) + .map(childQuery -> JoinQueryBuilders.hasChildQuery(TYPE_RULE_EXTENSION.getName(), childQuery, ScoreMode.None)) .forEach(q::should); return q; } @@ -428,7 +432,7 @@ public class RuleIndex { RuleExtensionScope.organization(query.getOrganization()).getScope())) .subAggregation(termsAggregation); - return JoinAggregationBuilders.children("children_for_" + termsAggregation.getName(), INDEX_TYPE_RULE_EXTENSION.getType()) + return JoinAggregationBuilders.children("children_for_" + termsAggregation.getName(), TYPE_RULE_EXTENSION.getName()) .subAggregation(scopeAggregation); }; @@ -473,7 +477,7 @@ public class RuleIndex { // so the rule filter has to be used as parent filter for active rules // from which we remove filters that concern active rules ("activation") HasParentQueryBuilder ruleFilter = JoinQueryBuilders.hasParentQuery( - INDEX_TYPE_RULE.getType(), + TYPE_RULE.getType(), stickyFacetBuilder.getStickyFacetFilter("activation"), false); @@ -483,7 +487,7 @@ public class RuleIndex { RuleIndex.addTermFilter(childrenFilter, FIELD_ACTIVE_RULE_INHERITANCE, query.getInheritance()); QueryBuilder activeRuleFilter = childrenFilter.must(ruleFilter); - AggregationBuilder activeSeverities = JoinAggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children", INDEX_TYPE_ACTIVE_RULE.getType()) + AggregationBuilder activeSeverities = JoinAggregationBuilders.children(FACET_ACTIVE_SEVERITIES + "_children", TYPE_ACTIVE_RULE.getName()) .subAggregation( AggregationBuilders.filter(FACET_ACTIVE_SEVERITIES + "_filter", activeRuleFilter) .subAggregation( @@ -533,28 +537,6 @@ public class RuleIndex { esSearch.setSize(options.getLimit()); } - public List<String> terms(String fields) { - return terms(fields, null, Integer.MAX_VALUE); - } - - public List<String> terms(String fields, @Nullable String query, int size) { - TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME) - .field(fields) - .size(size) - .minDocCount(1); - if (query != null) { - termsAggregation.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(query) + ".*", null)); - } - SearchRequestBuilder request = client - .prepareSearch(INDEX_TYPE_RULE, INDEX_TYPE_ACTIVE_RULE) - .setQuery(matchAllQuery()) - .setSize(0) - .addAggregation(termsAggregation); - - SearchResponse esResponse = request.get(); - return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME)); - } - public List<String> listTags(@Nullable OrganizationDto organization, @Nullable String query, int size) { int maxPageSize = 500; checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize); @@ -583,7 +565,7 @@ public class RuleIndex { .ifPresent(termsAggregation::includeExclude); SearchRequestBuilder request = client - .prepareSearch(INDEX_TYPE_RULE_EXTENSION) + .prepareSearch(TYPE_RULE_EXTENSION.getMainType()) .setQuery(boolQuery().filter(scopeFilter)) .setSize(0) .addAggregation(termsAggregation); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java index 07075b2c589..4d0226a0229 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexDefinition.java @@ -19,30 +19,32 @@ */ package org.sonar.server.rule.index; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.Set; import org.sonar.api.config.Configuration; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.IndexType.IndexRelationType; +import org.sonar.server.es.newindex.NewRegularIndex; +import org.sonar.server.es.newindex.TypeMapping; -import static org.sonar.server.es.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; /** * Definition of ES index "rules", including settings and fields. */ public class RuleIndexDefinition implements IndexDefinition { - private static final String INDEX = "rules"; - - public static final IndexType INDEX_TYPE_RULE = new IndexType(INDEX, "rule"); - public static final String FIELD_RULE_ID = "id"; + public static final Index DESCRIPTOR = Index.withRelations("rules"); + public static final IndexMainType TYPE_RULE = IndexType.main(DESCRIPTOR, "rule"); + public static final String FIELD_RULE_ID = "ruleId"; public static final String FIELD_RULE_KEY = "key"; public static final String FIELD_RULE_REPOSITORY = "repo"; public static final String FIELD_RULE_RULE_KEY = "ruleKey"; @@ -66,21 +68,19 @@ public class RuleIndexDefinition implements IndexDefinition { FIELD_RULE_KEY); // Rule extension fields - public static final IndexType INDEX_TYPE_RULE_EXTENSION = new IndexType(INDEX, "ruleExtension"); + public static final IndexRelationType TYPE_RULE_EXTENSION = IndexType.relation(TYPE_RULE, "ruleExtension"); /** * The uuid of a {@link RuleExtensionScope} */ - public static final String FIELD_RULE_EXTENSION_SCOPE = "scope"; - public static final String FIELD_RULE_EXTENSION_RULE_ID = "ruleId"; - public static final String FIELD_RULE_EXTENSION_TAGS = "tags"; + public static final String FIELD_RULE_EXTENSION_SCOPE = "ruleExt_scope"; + public static final String FIELD_RULE_EXTENSION_TAGS = "ruleExt_tags"; // Active rule fields - public static final IndexType INDEX_TYPE_ACTIVE_RULE = new IndexType(INDEX, "activeRule"); - public static final String FIELD_ACTIVE_RULE_ID = "id"; - public static final String FIELD_ACTIVE_RULE_RULE_ID = "ruleId"; - public static final String FIELD_ACTIVE_RULE_INHERITANCE = "inheritance"; - public static final String FIELD_ACTIVE_RULE_PROFILE_UUID = "ruleProfile"; - public static final String FIELD_ACTIVE_RULE_SEVERITY = "severity"; + public static final IndexRelationType TYPE_ACTIVE_RULE = IndexType.relation(TYPE_RULE, "activeRule"); + public static final String FIELD_ACTIVE_RULE_ID = "activeRule_id"; + public static final String FIELD_ACTIVE_RULE_INHERITANCE = "activeRule_inheritance"; + public static final String FIELD_ACTIVE_RULE_PROFILE_UUID = "activeRule_ruleProfile"; + public static final String FIELD_ACTIVE_RULE_SEVERITY = "activeRule_severity"; private final Configuration config; private final boolean enableSource; @@ -104,39 +104,19 @@ public class RuleIndexDefinition implements IndexDefinition { @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create( - INDEX_TYPE_RULE.getIndex(), + NewRegularIndex index = context.create( + DESCRIPTOR, newBuilder(config) .setRefreshInterval(MANUAL_REFRESH_INTERVAL) // Default nb of shards should be greater than 1 in order to // easily detect routing misconfiguration. // See https://jira.sonarsource.com/browse/SONAR-9489 .setDefaultNbOfShards(2) - .build()); - - // Active rule type - NewIndex.NewIndexType activeRuleMapping = index.createType(INDEX_TYPE_ACTIVE_RULE.getType()); - activeRuleMapping.setEnableSource(enableSource); - activeRuleMapping.setAttribute("_parent", ImmutableMap.of("type", INDEX_TYPE_RULE.getType())); - - activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_ID).disableNorms().build(); - activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_RULE_ID).disableNorms().build(); - activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_PROFILE_UUID).disableNorms().build(); - activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().build(); - activeRuleMapping.keywordFieldBuilder(FIELD_ACTIVE_RULE_SEVERITY).disableNorms().build(); - - // Rule extension type - NewIndex.NewIndexType ruleExtensionType = index.createType(INDEX_TYPE_RULE_EXTENSION.getType()); - ruleExtensionType.setEnableSource(enableSource); - ruleExtensionType.setAttribute("_parent", ImmutableMap.of("type", INDEX_TYPE_RULE.getType())); - - ruleExtensionType.keywordFieldBuilder(FIELD_RULE_EXTENSION_SCOPE).disableNorms().build(); - ruleExtensionType.keywordFieldBuilder(FIELD_RULE_EXTENSION_TAGS).build(); + .build()) + .setEnableSource(enableSource); // Rule type - NewIndex.NewIndexType ruleMapping = index.createType(INDEX_TYPE_RULE.getType()); - ruleMapping.setEnableSource(enableSource); - + TypeMapping ruleMapping = index.createTypeMapping(TYPE_RULE); ruleMapping.keywordFieldBuilder(FIELD_RULE_ID).disableNorms().build(); ruleMapping.keywordFieldBuilder(FIELD_RULE_KEY).addSubFields(SORTABLE_ANALYZER).build(); ruleMapping.keywordFieldBuilder(FIELD_RULE_RULE_KEY).addSubFields(SORTABLE_ANALYZER).build(); @@ -162,5 +142,17 @@ public class RuleIndexDefinition implements IndexDefinition { ruleMapping.createLongField(FIELD_RULE_CREATED_AT); ruleMapping.createLongField(FIELD_RULE_UPDATED_AT); + + // Active rule + index.createTypeMapping(TYPE_ACTIVE_RULE) + .keywordFieldBuilder(FIELD_ACTIVE_RULE_ID).disableNorms().build() + .keywordFieldBuilder(FIELD_ACTIVE_RULE_PROFILE_UUID).disableNorms().build() + .keywordFieldBuilder(FIELD_ACTIVE_RULE_INHERITANCE).disableNorms().build() + .keywordFieldBuilder(FIELD_ACTIVE_RULE_SEVERITY).disableNorms().build(); + + // Rule extension + index.createTypeMapping(TYPE_RULE_EXTENSION) + .keywordFieldBuilder(FIELD_RULE_EXTENSION_SCOPE).disableNorms().build() + .keywordFieldBuilder(FIELD_RULE_EXTENSION_TAGS).build(); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexer.java index 7ed25632d25..b91a5fff10d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndexer.java @@ -23,17 +23,14 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import java.util.Collection; import java.util.List; -import java.util.Objects; +import java.util.Optional; import java.util.Set; -import org.elasticsearch.action.index.IndexRequest; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.es.EsQueueDto; import org.sonar.db.es.RuleExtensionId; import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.rule.RuleExtensionForIndexingDto; -import org.sonar.db.rule.RuleForIndexingDto; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; @@ -47,8 +44,8 @@ import static com.google.common.base.Preconditions.checkArgument; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.sonar.core.util.stream.MoreCollectors.toHashSet; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION; public class RuleIndexer implements ResilientIndexer { @@ -62,7 +59,7 @@ public class RuleIndexer implements ResilientIndexer { @Override public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(INDEX_TYPE_RULE, INDEX_TYPE_RULE_EXTENSION); + return ImmutableSet.of(TYPE_RULE, TYPE_RULE_EXTENSION); } @Override @@ -72,16 +69,16 @@ public class RuleIndexer implements ResilientIndexer { bulk.start(); // index all definitions and system extensions - if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE)) { + if (uninitializedIndexTypes.contains(TYPE_RULE)) { dbClient.ruleDao().scrollIndexingRules(dbSession, dto -> { - bulk.add(newRuleDocIndexRequest(dto)); - bulk.add(newRuleExtensionDocIndexRequest(dto)); + bulk.add(RuleDoc.of(dto).toIndexRequest()); + bulk.add(RuleExtensionDoc.of(dto).toIndexRequest()); }); } // index all organization extensions - if (uninitializedIndexTypes.contains(INDEX_TYPE_RULE_EXTENSION)) { - dbClient.ruleDao().scrollIndexingRuleExtensions(dbSession, dto -> bulk.add(newRuleExtensionDocIndexRequest(dto))); + if (uninitializedIndexTypes.contains(TYPE_RULE_EXTENSION)) { + dbClient.ruleDao().scrollIndexingRuleExtensions(dbSession, dto -> bulk.add(RuleExtensionDoc.of(dto).toIndexRequest())); } bulk.stop(); @@ -123,14 +120,18 @@ public class RuleIndexer implements ResilientIndexer { public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); if (!items.isEmpty()) { - ListMultimap<IndexType, EsQueueDto> itemsByType = groupItemsByType(items); - result.add(doIndexRules(dbSession, itemsByType.get(INDEX_TYPE_RULE))); - result.add(doIndexRuleExtensions(dbSession, itemsByType.get(INDEX_TYPE_RULE_EXTENSION))); + ListMultimap<String, EsQueueDto> itemsByType = groupItemsByIndexTypeFormat(items); + doIndexRules(dbSession, itemsByType.get(TYPE_RULE.format())).ifPresent(result::add); + doIndexRuleExtensions(dbSession, itemsByType.get(TYPE_RULE_EXTENSION.format())).ifPresent(result::add); } return result; } - private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) { + private Optional<IndexingResult> doIndexRules(DbSession dbSession, List<EsQueueDto> items) { + if (items.isEmpty()) { + return Optional.empty(); + } + BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); @@ -141,22 +142,26 @@ public class RuleIndexer implements ResilientIndexer { dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleIds, r -> { - bulkIndexer.add(newRuleDocIndexRequest(r)); - bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); + bulkIndexer.add(RuleDoc.of(r).toIndexRequest()); + bulkIndexer.add(RuleExtensionDoc.of(r).toIndexRequest()); ruleIds.remove(r.getId()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. ruleIds.forEach(ruleId -> { - bulkIndexer.addDeletion(INDEX_TYPE_RULE, ruleId.toString(), ruleId.toString()); - bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString()); + bulkIndexer.addDeletion(TYPE_RULE, ruleId.toString(), ruleId.toString()); + bulkIndexer.addDeletion(TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString()); }); - return bulkIndexer.stop(); + return Optional.of(bulkIndexer.stop()); } - private IndexingResult doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) { + private Optional<IndexingResult> doIndexRuleExtensions(DbSession dbSession, List<EsQueueDto> items) { + if (items.isEmpty()) { + return Optional.empty(); + } + BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); @@ -171,65 +176,37 @@ public class RuleIndexer implements ResilientIndexer { r -> { RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getRuleId()); docIds.remove(docId); - bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); + bulkIndexer.add(RuleExtensionDoc.of(r).toIndexRequest()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. - docIds.forEach(docId -> bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, docId.getId(), String.valueOf(docId.getRuleId()))); - - return bulkIndexer.stop(); - } - - private static IndexRequest newRuleDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) { - RuleDoc doc = RuleDoc.of(ruleForIndexingDto); - - return new IndexRequest(INDEX_TYPE_RULE.getIndex(), INDEX_TYPE_RULE.getType()) - .id(doc.getId()) - .routing(doc.getRouting()) - .source(doc.getFields()); - } - - private static IndexRequest newRuleExtensionDocIndexRequest(RuleForIndexingDto ruleForIndexingDto) { - RuleExtensionDoc ruleExtensionDoc = RuleExtensionDoc.of(ruleForIndexingDto); - - return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType()) - .id(ruleExtensionDoc.getId()) - .routing(ruleExtensionDoc.getRouting()) - .parent(ruleExtensionDoc.getParent()) - .source(ruleExtensionDoc.getFields()); - } + docIds.forEach(docId -> bulkIndexer.addDeletion(TYPE_RULE_EXTENSION, docId.getId(), String.valueOf(docId.getRuleId()))); - private static IndexRequest newRuleExtensionDocIndexRequest(RuleExtensionForIndexingDto ruleExtensionForIndexingDto) { - RuleExtensionDoc doc = RuleExtensionDoc.of(ruleExtensionForIndexingDto); - return new IndexRequest(INDEX_TYPE_RULE_EXTENSION.getIndex(), INDEX_TYPE_RULE_EXTENSION.getType()) - .id(doc.getId()) - .routing(doc.getRouting()) - .parent(doc.getParent()) - .source(doc.getFields()); + return Optional.of(bulkIndexer.stop()); } private BulkIndexer createBulkIndexer(Size bulkSize, IndexingListener listener) { - return new BulkIndexer(esClient, INDEX_TYPE_RULE, bulkSize, listener); + return new BulkIndexer(esClient, TYPE_RULE, bulkSize, listener); } - private static ListMultimap<IndexType, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) { - return items.stream().collect(MoreCollectors.index(i -> IndexType.parse(i.getDocType()))); + private static ListMultimap<String, EsQueueDto> groupItemsByIndexTypeFormat(Collection<EsQueueDto> items) { + return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType)); } private static RuleExtensionId explodeRuleExtensionDocId(EsQueueDto esQueueDto) { - checkArgument(Objects.equals(esQueueDto.getDocType(), "rules/ruleExtension")); + checkArgument(TYPE_RULE_EXTENSION.format().equals(esQueueDto.getDocType())); return new RuleExtensionId(esQueueDto.getDocId()); } private static EsQueueDto createQueueDtoForRule(int ruleId) { String docId = String.valueOf(ruleId); - return EsQueueDto.create("rules/rule", docId, null, docId); + return EsQueueDto.create(TYPE_RULE.format(), docId, null, docId); } private static EsQueueDto createQueueDtoForRuleExtension(int ruleId, OrganizationDto organization) { String docId = RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.organization(organization)); - return EsQueueDto.create("rules/ruleExtension", docId, null, String.valueOf(ruleId)); + return EsQueueDto.create(TYPE_RULE_EXTENSION.format(), docId, null, String.valueOf(ruleId)); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserDoc.java index ab48d5e1e06..ed435fadfa7 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserDoc.java @@ -23,15 +23,15 @@ import com.google.common.collect.Maps; import java.util.List; import java.util.Map; import javax.annotation.Nullable; -import org.sonar.api.user.User; import org.sonar.server.es.BaseDoc; import static org.sonar.server.user.index.UserIndexDefinition.FIELD_ORGANIZATION_UUIDS; +import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER; -public class UserDoc extends BaseDoc implements User { +public class UserDoc extends BaseDoc { public UserDoc(Map<String, Object> fields) { - super(fields); + super(TYPE_USER, fields); } public UserDoc() { @@ -43,37 +43,23 @@ public class UserDoc extends BaseDoc implements User { return uuid(); } - @Override - public String getRouting() { - return null; - } - - @Override - public String getParent() { - return null; - } - public String uuid() { return getField(UserIndexDefinition.FIELD_UUID); } - @Override public String login() { return getField(UserIndexDefinition.FIELD_LOGIN); } - @Override public String name() { return getField(UserIndexDefinition.FIELD_NAME); } - @Override @Nullable public String email() { return getNullableField(UserIndexDefinition.FIELD_EMAIL); } - @Override public boolean active() { return (Boolean) getField(UserIndexDefinition.FIELD_ACTIVE); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java index 69526173cb8..a654d3fe7f2 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java @@ -41,8 +41,8 @@ import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER; import static org.sonar.server.user.index.UserIndexDefinition.FIELD_ACTIVE; import static org.sonar.server.user.index.UserIndexDefinition.FIELD_EMAIL; import static org.sonar.server.user.index.UserIndexDefinition.FIELD_LOGIN; @@ -69,7 +69,7 @@ public class UserIndex { public List<UserDoc> getAtMostThreeActiveUsersForScmAccount(String scmAccount, String organizationUuid) { List<UserDoc> result = new ArrayList<>(); if (!StringUtils.isEmpty(scmAccount)) { - SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) + SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER) .setQuery(boolQuery().must(matchAllQuery()).filter( boolQuery() .must(termQuery(FIELD_ACTIVE, true)) @@ -86,7 +86,7 @@ public class UserIndex { } public SearchResult<UserDoc> search(UserQuery userQuery, SearchOptions options) { - SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) + SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.TYPE_USER) .setSize(options.getLimit()) .setFrom(options.getOffset()) .addSort(FIELD_NAME, SortOrder.ASC); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java index 852f22663e5..a887b55ffdc 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java @@ -20,20 +20,25 @@ package org.sonar.server.user.index; import org.sonar.api.config.Configuration; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.newindex.NewRegularIndex; +import org.sonar.server.es.newindex.TypeMapping; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.USER_SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; /** * Definition of ES index "users", including settings and fields. */ public class UserIndexDefinition implements IndexDefinition { - public static final IndexType INDEX_TYPE_USER = new IndexType("users", "user"); + public static final Index DESCRIPTOR = Index.simple("users"); + public static final IndexMainType TYPE_USER = IndexType.main(DESCRIPTOR, "user"); public static final String FIELD_UUID = "uuid"; public static final String FIELD_LOGIN = "login"; public static final String FIELD_NAME = "name"; @@ -48,15 +53,21 @@ public class UserIndexDefinition implements IndexDefinition { this.config = config; } + public static UserIndexDefinition createForTest() { + return new UserIndexDefinition(new MapSettings().asConfig()); + } + @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create(INDEX_TYPE_USER.getIndex(), + NewRegularIndex index = context.create( + DESCRIPTOR, newBuilder(config) .setDefaultNbOfShards(1) - .build()); + .build()) + // all information is retrieved from the index, not only IDs + .setEnableSource(true); - // type "user" - NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_USER.getType()); + TypeMapping mapping = index.createTypeMapping(TYPE_USER); mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_LOGIN).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build(); mapping.keywordFieldBuilder(FIELD_NAME).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build(); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexer.java index 2f94cd0de27..0dab27deed7 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -44,7 +44,7 @@ import org.sonar.server.es.ResilientIndexer; import static java.util.Collections.singletonList; import static org.sonar.core.util.stream.MoreCollectors.toHashSet; import static org.sonar.core.util.stream.MoreCollectors.toList; -import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER; +import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER; public class UserIndexer implements ResilientIndexer { @@ -58,7 +58,7 @@ public class UserIndexer implements ResilientIndexer { @Override public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(INDEX_TYPE_USER); + return ImmutableSet.of(TYPE_USER); } @Override @@ -84,7 +84,7 @@ public class UserIndexer implements ResilientIndexer { public void commitAndIndex(DbSession dbSession, Collection<UserDto> users) { List<String> uuids = users.stream().map(UserDto::getUuid).collect(toList()); List<EsQueueDto> items = uuids.stream() - .map(uuid -> EsQueueDto.create(INDEX_TYPE_USER.format(), uuid)) + .map(uuid -> EsQueueDto.create(TYPE_USER.format(), uuid)) .collect(MoreCollectors.toArrayList()); dbClient.esQueueDao().insert(dbSession, items); @@ -128,12 +128,12 @@ public class UserIndexer implements ResilientIndexer { // the remaining uuids reference rows that don't exist in db. They must // be deleted from index. - uuids.forEach(uuid -> bulkIndexer.addDeletion(INDEX_TYPE_USER, uuid)); + uuids.forEach(uuid -> bulkIndexer.addDeletion(TYPE_USER, uuid)); return bulkIndexer.stop(); } private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) { - return new BulkIndexer(esClient, INDEX_TYPE_USER, bulkSize, listener); + return new BulkIndexer(esClient, TYPE_USER, bulkSize, listener); } private static IndexRequest newIndexRequest(UserDto user, ListMultimap<String, String> organizationUuidsByUserUuid) { @@ -147,9 +147,6 @@ public class UserIndexer implements ResilientIndexer { doc.setScmAccounts(UserDto.decodeScmAccounts(user.getScmAccounts())); doc.setOrganizationUuids(organizationUuidsByUserUuid.get(user.getUuid())); - return new IndexRequest(INDEX_TYPE_USER.getIndex(), INDEX_TYPE_USER.getType()) - .id(doc.getId()) - .routing(doc.getRouting()) - .source(doc.getFields()); + return doc.toIndexRequest(); } } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewDoc.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewDoc.java index 60ba429e0d4..bff2c49ac2b 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewDoc.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewDoc.java @@ -24,14 +24,16 @@ import java.util.List; import java.util.Map; import org.sonar.server.es.BaseDoc; +import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW; + public class ViewDoc extends BaseDoc { public ViewDoc(Map<String, Object> fields) { - super(fields); + super(TYPE_VIEW, fields); } public ViewDoc() { - super(Maps.newHashMap()); + super(TYPE_VIEW, Maps.newHashMap()); } @Override @@ -39,16 +41,6 @@ public class ViewDoc extends BaseDoc { return uuid(); } - @Override - public String getRouting() { - return null; - } - - @Override - public String getParent() { - return null; - } - public String uuid() { return getField(ViewIndexDefinition.FIELD_UUID); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java index e226f9473fa..652e8fcbe97 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java @@ -45,7 +45,7 @@ public class ViewIndex { } public List<String> findAllViewUuids() { - SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.INDEX_TYPE_VIEW) + SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.TYPE_VIEW) .addSort("_doc", SortOrder.ASC) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .setFetchSource(false) diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java index 21ea4ba23f2..2abc9030b7b 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexDefinition.java @@ -20,18 +20,23 @@ package org.sonar.server.view.index; import org.sonar.api.config.Configuration; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.newindex.NewRegularIndex; +import org.sonar.server.es.newindex.TypeMapping; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; /** * Definition of ES index "views", including settings and fields. */ public class ViewIndexDefinition implements IndexDefinition { - public static final IndexType INDEX_TYPE_VIEW = new IndexType("views", "view"); + public static final Index DESCRIPTOR = Index.simple("views"); + public static final IndexMainType TYPE_VIEW = IndexType.main(DESCRIPTOR, "view"); public static final String FIELD_UUID = "uuid"; public static final String FIELD_PROJECTS = "projects"; @@ -41,16 +46,27 @@ public class ViewIndexDefinition implements IndexDefinition { this.config = config; } + /** + * Keep the document sources in index so that indexer tests can verify content + * of indexed documents. + */ + public static ViewIndexDefinition createForTest() { + return new ViewIndexDefinition(new MapSettings().asConfig()); + } + @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create( - INDEX_TYPE_VIEW.getIndex(), + NewRegularIndex index = context.create( + DESCRIPTOR, newBuilder(config) .setDefaultNbOfShards(5) - .build()); + .build()) + // storing source is required because some search queries on issue index use terms lookup query onto the view index + // and this requires source to be stored (https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html#query-dsl-terms-lookup) + .setEnableSource(true); // type "view" - NewIndex.NewIndexType mapping = index.createType(INDEX_TYPE_VIEW.getType()); + TypeMapping mapping = index.createTypeMapping(TYPE_VIEW); mapping.keywordFieldBuilder(FIELD_UUID).disableNorms().build(); mapping.keywordFieldBuilder(FIELD_PROJECTS).disableNorms().build(); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java index 0d5086ce2fa..5a7c581ac84 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -42,7 +42,7 @@ import org.sonar.server.es.ResilientIndexer; import static com.google.common.collect.Maps.newHashMap; import static org.sonar.core.util.stream.MoreCollectors.toHashSet; -import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW; +import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW; public class ViewIndexer implements ResilientIndexer { @@ -56,7 +56,7 @@ public class ViewIndexer implements ResilientIndexer { @Override public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(INDEX_TYPE_VIEW); + return ImmutableSet.of(TYPE_VIEW); } @Override @@ -92,14 +92,14 @@ public class ViewIndexer implements ResilientIndexer { * The views lookup cache will be cleared */ public void index(ViewDoc viewDoc) { - BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, Size.REGULAR); + BulkIndexer bulk = new BulkIndexer(esClient, TYPE_VIEW, Size.REGULAR); bulk.start(); doIndex(bulk, viewDoc, true); bulk.stop(); } private void index(DbSession dbSession, Map<String, String> viewAndProjectViewUuidMap, boolean needClearCache, Size bulkSize) { - BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, bulkSize); + BulkIndexer bulk = new BulkIndexer(esClient, TYPE_VIEW, bulkSize); bulk.start(); for (Map.Entry<String, String> entry : viewAndProjectViewUuidMap.entrySet()) { String viewUuid = entry.getKey(); @@ -119,7 +119,10 @@ public class ViewIndexer implements ResilientIndexer { } private static IndexRequest newIndexRequest(ViewDoc doc) { - return new IndexRequest(ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex(), ViewIndexDefinition.INDEX_TYPE_VIEW.getType(), doc.uuid()) + IndexType.IndexMainType mainType = TYPE_VIEW; + return new IndexRequest(mainType.getIndex().getName(), mainType.getType()) + .id(doc.getId()) + .routing(doc.getRouting().orElse(null)) .source(doc.getFields()); } @@ -156,13 +159,13 @@ public class ViewIndexer implements ResilientIndexer { // Safety check to remove all views that may not have been deleted views.removeAll(dbClient.componentDao().selectExistingUuids(dbSession, views)); - views.forEach(v -> bulkIndexer.addDeletion(INDEX_TYPE_VIEW, v)); + views.forEach(v -> bulkIndexer.addDeletion(TYPE_VIEW, v)); return bulkIndexer.stop(); } public void delete(DbSession dbSession, Collection<String> viewUuids) { List<EsQueueDto> items = viewUuids.stream() - .map(l -> EsQueueDto.create(INDEX_TYPE_VIEW.format(), l)) + .map(l -> EsQueueDto.create(TYPE_VIEW.format(), l)) .collect(MoreCollectors.toArrayList()); dbClient.esQueueDao().insert(dbSession, items); @@ -171,6 +174,6 @@ public class ViewIndexer implements ResilientIndexer { } private BulkIndexer newBulkIndexer(Size bulkSize, IndexingListener listener) { - return new BulkIndexer(esClient, INDEX_TYPE_VIEW, bulkSize, listener); + return new BulkIndexer(esClient, TYPE_VIEW, bulkSize, listener); } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java index 5a22fd76d9a..69a27cb0da2 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java @@ -42,8 +42,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.sonar.db.component.ComponentTesting.newFileDto; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME; -import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_TYPE_COMPONENT; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION; @@ -62,7 +62,7 @@ public class ComponentIndexerTest { @Test public void test_getIndexTypes() { - assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_COMPONENT); + assertThat(underTest.getIndexTypes()).containsExactly(TYPE_COMPONENT); } @Test @@ -90,7 +90,7 @@ public class ComponentIndexerTest { underTest.indexOnStartup(emptySet()); assertThatIndexContainsOnly(project); - ComponentDoc doc = es.getDocuments(INDEX_TYPE_COMPONENT, ComponentDoc.class).get(0); + ComponentDoc doc = es.getDocuments(TYPE_COMPONENT, ComponentDoc.class).get(0); assertThat(doc.getId()).isEqualTo(project.uuid()); assertThat(doc.getKey()).isEqualTo(project.getDbKey()); assertThat(doc.getProjectUuid()).isEqualTo(project.projectUuid()); @@ -234,7 +234,7 @@ public class ComponentIndexerTest { public void errors_during_indexing_are_recovered() { ComponentDto project = db.components().insertPrivateProject(); ComponentDto file = db.components().insertComponent(newFileDto(project)); - es.lockWrites(INDEX_TYPE_COMPONENT); + es.lockWrites(TYPE_COMPONENT); IndexingResult result = indexProject(project, PROJECT_CREATION); assertThat(result.getTotal()).isEqualTo(2L); @@ -244,9 +244,9 @@ public class ComponentIndexerTest { result = recover(); assertThat(result.getTotal()).isEqualTo(2L); assertThat(result.getFailures()).isEqualTo(2L); - assertThat(es.countDocuments(INDEX_TYPE_COMPONENT)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_COMPONENT)).isEqualTo(0); - es.unlockWrites(INDEX_TYPE_COMPONENT); + es.unlockWrites(TYPE_COMPONENT); result = recover(); assertThat(result.getTotal()).isEqualTo(2L); @@ -275,17 +275,17 @@ public class ComponentIndexerTest { } private void assertThatIndexHasSize(int expectedSize) { - assertThat(es.countDocuments(INDEX_TYPE_COMPONENT)).isEqualTo(expectedSize); + assertThat(es.countDocuments(TYPE_COMPONENT)).isEqualTo(expectedSize); } private void assertThatIndexContainsOnly(ComponentDto... expectedComponents) { - assertThat(es.getIds(INDEX_TYPE_COMPONENT)).containsExactlyInAnyOrder( + assertThat(es.getIds(TYPE_COMPONENT)).containsExactlyInAnyOrder( Arrays.stream(expectedComponents).map(ComponentDto::uuid).toArray(String[]::new)); } private void assertThatComponentHasName(ComponentDto component, String expectedName) { SearchHit[] hits = es.client() - .prepareSearch(INDEX_TYPE_COMPONENT) + .prepareSearch(TYPE_COMPONENT.getMainType()) .setQuery(matchQuery(SORTABLE_ANALYZER.subField(FIELD_NAME), expectedName)) .get() .getHits() diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java index 7890492b751..3a49dfdaf8a 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -34,11 +34,12 @@ import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.db.DbTester; import org.sonar.server.es.BulkIndexer.Size; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static java.util.Collections.emptyMap; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.es.FakeIndexDefinition.INDEX; -import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE; +import static org.sonar.server.es.newindex.FakeIndexDefinition.INDEX; +import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; public class BulkIndexerTest { @@ -53,7 +54,7 @@ public class BulkIndexerTest { @Test public void index_nothing() { - BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR); + BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR); indexer.start(); indexer.stop(); @@ -62,7 +63,7 @@ public class BulkIndexerTest { @Test public void index_documents() { - BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR); + BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR); indexer.start(); indexer.add(newIndexRequest(42)); indexer.add(newIndexRequest(78)); @@ -80,7 +81,7 @@ public class BulkIndexerTest { // index has one replica assertThat(replicas()).isEqualTo(1); - BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.LARGE); + BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.LARGE); indexer.start(); // replicas are temporarily disabled @@ -108,12 +109,12 @@ public class BulkIndexerTest { for (int i = 0; i < max; i++) { docs[i] = FakeIndexDefinition.newDoc(i); } - es.putDocuments(INDEX_TYPE_FAKE, docs); + es.putDocuments(TYPE_FAKE, docs); assertThat(count()).isEqualTo(max); - SearchRequestBuilder req = es.client().prepareSearch(INDEX_TYPE_FAKE) + SearchRequestBuilder req = es.client().prepareSearch(TYPE_FAKE) .setQuery(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom)); - BulkIndexer.delete(es.client(), INDEX_TYPE_FAKE, req); + BulkIndexer.delete(es.client(), TYPE_FAKE, req); assertThat(count()).isEqualTo(removeFrom); } @@ -121,12 +122,12 @@ public class BulkIndexerTest { @Test public void listener_is_called_on_successful_requests() { FakeListener listener = new FakeListener(); - BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); + BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener); indexer.start(); - indexer.addDeletion(INDEX_TYPE_FAKE, "foo"); + indexer.addDeletion(TYPE_FAKE, "foo"); indexer.stop(); assertThat(listener.calledDocIds) - .containsExactlyInAnyOrder(new DocId(INDEX_TYPE_FAKE, "foo")); + .containsExactlyInAnyOrder(newDocId(TYPE_FAKE, "foo")); assertThat(listener.calledResult.getSuccess()).isEqualTo(1); assertThat(listener.calledResult.getTotal()).isEqualTo(1); } @@ -134,13 +135,13 @@ public class BulkIndexerTest { @Test public void listener_is_called_even_if_deleting_a_doc_that_does_not_exist() { FakeListener listener = new FakeListener(); - BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); + BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.add(newIndexRequestWithDocId("bar")); indexer.stop(); assertThat(listener.calledDocIds) - .containsExactlyInAnyOrder(new DocId(INDEX_TYPE_FAKE, "foo"), new DocId(INDEX_TYPE_FAKE, "bar")); + .containsExactlyInAnyOrder(newDocId(TYPE_FAKE, "foo"), newDocId(TYPE_FAKE, "bar")); assertThat(listener.calledResult.getSuccess()).isEqualTo(2); assertThat(listener.calledResult.getTotal()).isEqualTo(2); } @@ -148,12 +149,12 @@ public class BulkIndexerTest { @Test public void listener_is_not_called_with_errors() { FakeListener listener = new FakeListener(); - BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); + BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.add(new IndexRequest("index_does_not_exist", "index_does_not_exist", "bar").source(emptyMap())); indexer.stop(); - assertThat(listener.calledDocIds).containsExactly(new DocId(INDEX_TYPE_FAKE, "foo")); + assertThat(listener.calledDocIds).containsExactly(newDocId(TYPE_FAKE, "foo")); assertThat(listener.calledResult.getSuccess()).isEqualTo(1); assertThat(listener.calledResult.getTotal()).isEqualTo(2); } @@ -162,10 +163,10 @@ public class BulkIndexerTest { public void log_requests_when_TRACE_level_is_enabled() { logTester.setLevel(LoggerLevel.TRACE); - BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, new FakeListener()); + BulkIndexer indexer = new BulkIndexer(es.client(), TYPE_FAKE, Size.REGULAR, new FakeListener()); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); - indexer.addDeletion(INDEX_TYPE_FAKE, "foo"); + indexer.addDeletion(TYPE_FAKE, "foo"); indexer.add(newIndexRequestWithDocId("bar")); indexer.stop(); @@ -192,7 +193,7 @@ public class BulkIndexerTest { } private long count() { - return es.countDocuments("fakes", "fake"); + return es.countDocuments(IndexType.main(Index.simple("fakes"), "fake")); } private int replicas() { @@ -202,13 +203,17 @@ public class BulkIndexerTest { } private IndexRequest newIndexRequest(int intField) { - return new IndexRequest(INDEX, INDEX_TYPE_FAKE.getType()) + return new IndexRequest(INDEX, TYPE_FAKE.getType()) .source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, intField)); } private IndexRequest newIndexRequestWithDocId(String id) { - return new IndexRequest(INDEX, INDEX_TYPE_FAKE.getType()) + return new IndexRequest(INDEX, TYPE_FAKE.getType()) .id(id) .source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, 42)); } + + private static DocId newDocId(IndexType.IndexMainType mainType, String id) { + return new DocId(TYPE_FAKE.getIndex().getName(), mainType.getType(), id); + } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java index bb82604b44f..3d3e91b6ae2 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/DefaultIndexSettingsTest.java @@ -21,10 +21,11 @@ package org.sonar.server.es; import java.util.Map; import org.junit.Test; +import org.sonar.server.es.newindex.DefaultIndexSettings; import org.sonar.test.TestUtils; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; public class DefaultIndexSettingsTest { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/DocIdTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/DocIdTest.java new file mode 100644 index 00000000000..eefae0cb4e2 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/DocIdTest.java @@ -0,0 +1,63 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import org.junit.Test; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; + +public class DocIdTest { + @Test + public void equals_is_based_on_index_type_and_id() { + String index = randomAlphabetic(5); + String type = randomAlphabetic(6); + String id = randomAlphabetic(7); + DocId underTest = new DocId(index, type, id); + + assertThat(underTest) + .isEqualTo(new DocId(index, type, id)) + .isNotEqualTo(new DocId(randomAlphabetic(7), type, id)) + .isNotEqualTo(new DocId(index, type, randomAlphabetic(7))) + .isNotEqualTo(new DocId(index, randomAlphabetic(7), id)) + .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), id)) + .isNotEqualTo(new DocId(randomAlphabetic(7), type, randomAlphabetic(8))) + .isNotEqualTo(new DocId(index, randomAlphabetic(7), randomAlphabetic(8))) + .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), randomAlphabetic(9))); + } + + @Test + public void hashcode_is_based_on_index_type_and_id() { + String index = randomAlphabetic(5); + String type = randomAlphabetic(6); + String id = randomAlphabetic(7); + DocId underTest = new DocId(index, type, id); + + assertThat(underTest.hashCode()) + .isEqualTo(new DocId(index, type, id).hashCode()) + .isNotEqualTo(new DocId(randomAlphabetic(7), type, id).hashCode()) + .isNotEqualTo(new DocId(index, type, randomAlphabetic(7)).hashCode()) + .isNotEqualTo(new DocId(index, randomAlphabetic(7), id).hashCode()) + .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), id).hashCode()) + .isNotEqualTo(new DocId(randomAlphabetic(7), type, randomAlphabetic(8)).hashCode()) + .isNotEqualTo(new DocId(index, randomAlphabetic(7), randomAlphabetic(8)).hashCode()) + .isNotEqualTo(new DocId(randomAlphabetic(7), randomAlphabetic(8), randomAlphabetic(9)).hashCode()); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java index cb78b45a587..82938da09bd 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java @@ -21,6 +21,7 @@ package org.sonar.server.es; import org.junit.Rule; import org.junit.Test; +import org.sonar.server.es.newindex.FakeIndexDefinition; import org.sonar.server.es.request.ProxyClusterHealthRequestBuilder; import org.sonar.server.es.request.ProxyClusterStateRequestBuilder; import org.sonar.server.es.request.ProxyClusterStatsRequestBuilder; @@ -44,21 +45,24 @@ public class EsClientTest { @Test public void proxify_requests() { + Index fakesIndex = Index.simple("fakes"); + IndexType.IndexMainType fakeMainType = IndexType.main(fakesIndex, "fake"); + EsClient underTest = es.client(); assertThat(underTest.nativeClient()).isNotNull(); assertThat(underTest.prepareClusterStats()).isInstanceOf(ProxyClusterStatsRequestBuilder.class); - assertThat(underTest.prepareCreate("fakes")).isInstanceOf(ProxyCreateIndexRequestBuilder.class); - assertThat(underTest.prepareDelete("fakes", "fake", "my_id")).isInstanceOf(ProxyDeleteRequestBuilder.class); - assertThat(underTest.prepareIndicesExist()).isInstanceOf(ProxyIndicesExistsRequestBuilder.class); - assertThat(underTest.prepareGet(new IndexType("fakes", "fake"), "1")).isInstanceOf(ProxyGetRequestBuilder.class); + assertThat(underTest.prepareCreate(fakesIndex)).isInstanceOf(ProxyCreateIndexRequestBuilder.class); + assertThat(underTest.prepareDelete(fakeMainType, "my_id")).isInstanceOf(ProxyDeleteRequestBuilder.class); + assertThat(underTest.prepareIndicesExist(fakesIndex)).isInstanceOf(ProxyIndicesExistsRequestBuilder.class); + assertThat(underTest.prepareGet(fakeMainType, "1")).isInstanceOf(ProxyGetRequestBuilder.class); assertThat(underTest.prepareHealth()).isInstanceOf(ProxyClusterHealthRequestBuilder.class); assertThat(underTest.prepareNodesStats()).isInstanceOf(ProxyNodesStatsRequestBuilder.class); - assertThat(underTest.preparePutMapping()).isInstanceOf(ProxyPutMappingRequestBuilder.class); - assertThat(underTest.prepareRefresh()).isInstanceOf(ProxyRefreshRequestBuilder.class); - assertThat(underTest.prepareSearch(new IndexType[0])).isInstanceOf(ProxySearchRequestBuilder.class); + assertThat(underTest.preparePutMapping(fakesIndex)).isInstanceOf(ProxyPutMappingRequestBuilder.class); + assertThat(underTest.prepareRefresh(fakesIndex)).isInstanceOf(ProxyRefreshRequestBuilder.class); + assertThat(underTest.prepareSearch(fakesIndex)).isInstanceOf(ProxySearchRequestBuilder.class); assertThat(underTest.prepareSearchScroll("1234")).isInstanceOf(ProxySearchScrollRequestBuilder.class); assertThat(underTest.prepareState()).isInstanceOf(ProxyClusterStateRequestBuilder.class); - assertThat(underTest.prepareStats()).isInstanceOf(ProxyIndicesStatsRequestBuilder.class); + assertThat(underTest.prepareStats(fakesIndex)).isInstanceOf(ProxyIndicesStatsRequestBuilder.class); underTest.close(); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsTester.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsTester.java index 0b3b266053d..a8253881b6d 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsTester.java @@ -21,6 +21,7 @@ package org.sonar.server.es; import com.google.common.base.Throwables; import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import java.nio.file.Files; @@ -53,13 +54,19 @@ import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.join.ParentJoinPlugin; +import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; import org.elasticsearch.search.SearchHit; import org.junit.rules.ExternalResource; -import org.sonar.api.config.Configuration; -import org.sonar.api.config.internal.MapSettings; import org.sonar.server.component.index.ComponentIndexDefinition; +import org.sonar.server.es.IndexDefinition.IndexDefinitionContext; +import org.sonar.server.es.IndexType.IndexRelationType; +import org.sonar.server.es.newindex.BuiltIndex; +import org.sonar.server.es.newindex.NewIndex; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition; import org.sonar.server.rule.index.RuleIndexDefinition; @@ -69,7 +76,9 @@ import org.sonar.server.view.index.ViewIndexDefinition; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Lists.newArrayList; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.sonar.server.es.DefaultIndexSettings.REFRESH_IMMEDIATE; +import static org.sonar.server.es.Index.ALL_INDICES; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.REFRESH_IMMEDIATE; public class EsTester extends ExternalResource { @@ -102,17 +111,16 @@ public class EsTester extends ExternalResource { */ public static EsTester create() { if (!CORE_INDICES_CREATED.get()) { - Configuration config = new MapSettings().asConfig(); - List<IndexDefinition.Index> createdIndices = createIndices( - new ComponentIndexDefinition(config), + List<BuiltIndex> createdIndices = createIndices( + ComponentIndexDefinition.createForTest(), IssueIndexDefinition.createForTest(), - new ProjectMeasuresIndexDefinition(config), + ProjectMeasuresIndexDefinition.createForTest(), RuleIndexDefinition.createForTest(), - new UserIndexDefinition(config), - new ViewIndexDefinition(config)); + UserIndexDefinition.createForTest(), + ViewIndexDefinition.createForTest()); CORE_INDICES_CREATED.set(true); - createdIndices.stream().map(IndexDefinition.Index::getName).forEach(CORE_INDICES_NAMES::add); + createdIndices.stream().map(t -> t.getMainType().getIndex().getName()).forEach(CORE_INDICES_NAMES::add); } return new EsTester(false); } @@ -135,26 +143,19 @@ public class EsTester extends ExternalResource { .filter(i -> !CORE_INDICES_NAMES.contains(i)) .forEach(EsTester::deleteIndexIfExists); } - BulkIndexer.delete(client(), new IndexType("_all", ""), client().prepareSearch("_all").setQuery(matchAllQuery())); + BulkIndexer.delete(client(), IndexType.main(ALL_INDICES, "dummy"), client().prepareSearch(ALL_INDICES).setQuery(matchAllQuery())); } public EsClient client() { return new EsClient(SHARED_NODE.client()); } - public void putDocuments(String index, String type, BaseDoc... docs) { - putDocuments(new IndexType(index, type), docs); - } - public void putDocuments(IndexType indexType, BaseDoc... docs) { try { BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk() .setRefreshPolicy(REFRESH_IMMEDIATE); for (BaseDoc doc : docs) { - bulk.add(new IndexRequest(indexType.getIndex(), indexType.getType(), doc.getId()) - .parent(doc.getParent()) - .routing(doc.getRouting()) - .source(doc.getFields())); + bulk.add(doc.toIndexRequest()); } BulkResponse bulkResponse = bulk.get(); if (bulkResponse.hasFailures()) { @@ -170,7 +171,8 @@ public class EsTester extends ExternalResource { BulkRequestBuilder bulk = SHARED_NODE.client().prepareBulk() .setRefreshPolicy(REFRESH_IMMEDIATE); for (Map<String, Object> doc : docs) { - bulk.add(new IndexRequest(indexType.getIndex(), indexType.getType()) + IndexType.IndexMainType mainType = indexType.getMainType(); + bulk.add(new IndexRequest(mainType.getIndex().getName(), mainType.getType()) .source(doc)); } BulkResponse bulkResponse = bulk.get(); @@ -182,12 +184,16 @@ public class EsTester extends ExternalResource { } } - public long countDocuments(String index, String type) { - return countDocuments(new IndexType(index, type)); + public long countDocuments(Index index) { + return client().prepareSearch(index) + .setQuery(matchAllQuery()) + .setSize(0).get().getHits().getTotalHits(); } public long countDocuments(IndexType indexType) { - return client().prepareSearch(indexType).setSize(0).get().getHits().getTotalHits(); + return client().prepareSearch(indexType.getMainType()) + .setQuery(getDocumentsQuery(indexType)) + .setSize(0).get().getHits().getTotalHits(); } /** @@ -206,10 +212,27 @@ public class EsTester extends ExternalResource { } /** - * Get all the indexed documents (no paginated results). Results are not sorted. + * Get all the indexed documents (no paginated results) in the specified index, whatever their type. Results are not sorted. + */ + public List<SearchHit> getDocuments(Index index) { + SearchRequestBuilder req = SHARED_NODE.client() + .prepareSearch(index.getName()) + .setQuery(matchAllQuery()); + return getDocuments(req); + } + + /** + * Get all the indexed documents (no paginated results) of the specified type. Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { - SearchRequestBuilder req = SHARED_NODE.client().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(matchAllQuery()); + IndexType.IndexMainType mainType = indexType.getMainType(); + SearchRequestBuilder req = SHARED_NODE.client() + .prepareSearch(mainType.getIndex().getName()) + .setQuery(getDocumentsQuery(indexType)); + return getDocuments(req); + } + + private List<SearchHit> getDocuments(SearchRequestBuilder req) { EsUtils.optimizeScrollRequest(req); req.setScroll(new TimeValue(60000)) .setSize(100); @@ -227,6 +250,20 @@ public class EsTester extends ExternalResource { return result; } + private QueryBuilder getDocumentsQuery(IndexType indexType) { + if (!indexType.getMainType().getIndex().acceptsRelations()) { + return matchAllQuery(); + } + + if (indexType instanceof IndexRelationType) { + return new TermQueryBuilder(FIELD_INDEX_TYPE, ((IndexRelationType) indexType).getName()); + } + if (indexType instanceof IndexType.IndexMainType) { + return new TermQueryBuilder(FIELD_INDEX_TYPE, ((IndexType.IndexMainType) indexType).getType()); + } + throw new IllegalArgumentException("Unsupported IndexType " + indexType.getClass()); + } + /** * Get a list of a specific field from all indexed documents. */ @@ -242,11 +279,11 @@ public class EsTester extends ExternalResource { } public void lockWrites(IndexType index) { - setIndexSettings(index.getIndex(), ImmutableMap.of("index.blocks.write", "true")); + setIndexSettings(index.getMainType().getIndex().getName(), ImmutableMap.of("index.blocks.write", "true")); } public void unlockWrites(IndexType index) { - setIndexSettings(index.getIndex(), ImmutableMap.of("index.blocks.write", "false")); + setIndexSettings(index.getMainType().getIndex().getName(), ImmutableMap.of("index.blocks.write", "false")); } private void setIndexSettings(String index, Map<String, Object> settings) { @@ -266,39 +303,39 @@ public class EsTester extends ExternalResource { } } - private static List<IndexDefinition.Index> createIndices(IndexDefinition... definitions) { - IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); + private static List<BuiltIndex> createIndices(IndexDefinition... definitions) { + IndexDefinitionContext context = new IndexDefinitionContext(); Stream.of(definitions).forEach(d -> d.define(context)); - List<IndexDefinition.Index> result = new ArrayList<>(); + List<BuiltIndex> result = new ArrayList<>(); for (NewIndex newIndex : context.getIndices().values()) { - IndexDefinition.Index index = new IndexDefinition.Index(newIndex); + BuiltIndex index = newIndex.build(); - deleteIndexIfExists(index.getName()); + String indexName = index.getMainType().getIndex().getName(); + deleteIndexIfExists(indexName); // create index Settings.Builder settings = Settings.builder(); settings.put(index.getSettings()); CreateIndexResponse indexResponse = SHARED_NODE.client().admin().indices() - .prepareCreate(index.getName()) + .prepareCreate(indexName) .setSettings(settings) .get(); if (!indexResponse.isAcknowledged()) { - throw new IllegalStateException("Failed to create index " + index.getName()); + throw new IllegalStateException("Failed to create index " + indexName); } - SHARED_NODE.client().admin().cluster().prepareHealth(index.getName()).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); + SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); // create types - for (Map.Entry<String, IndexDefinition.Type> entry : index.getTypes().entrySet()) { - PutMappingResponse mappingResponse = SHARED_NODE.client().admin().indices().preparePutMapping(index.getName()) - .setType(entry.getKey()) - .setSource(entry.getValue().getAttributes()) - .get(); - if (!mappingResponse.isAcknowledged()) { - throw new IllegalStateException("Failed to create type " + entry.getKey()); - } + String typeName = index.getMainType().getType(); + PutMappingResponse mappingResponse = SHARED_NODE.client().admin().indices().preparePutMapping(indexName) + .setType(typeName) + .setSource(index.getAttributes()) + .get(); + if (!mappingResponse.isAcknowledged()) { + throw new IllegalStateException("Failed to create type " + typeName); } - SHARED_NODE.client().admin().cluster().prepareHealth(index.getName()).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); + SHARED_NODE.client().admin().cluster().prepareHealth(indexName).setWaitForStatus(ClusterHealthStatus.YELLOW).get(); result.add(index); } return result; @@ -324,10 +361,20 @@ public class EsTester extends ExternalResource { .put(NetworkModule.HTTP_ENABLED.getKey(), false) .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "single-node") .build(); - Node node = new Node(settings); + Node node = new TesterNode(settings); return node.start(); } catch (Exception e) { throw new IllegalStateException("Fail to start embedded Elasticsearch", e); } } + + private static class TesterNode extends Node { + public TesterNode(Settings preparedSettings) { + super( + InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), + ImmutableList.of( + // install ParentJoin plugin required to create field of type "join" + ParentJoinPlugin.class)); + } + } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/FakeDoc.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/FakeDoc.java index 79d4ef24eea..d199ed9323a 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/FakeDoc.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/FakeDoc.java @@ -20,13 +20,13 @@ package org.sonar.server.es; import com.google.common.collect.Maps; -import java.util.Map; -import static org.sonar.server.es.FakeIndexDefinition.INT_FIELD; +import static org.sonar.server.es.newindex.FakeIndexDefinition.INT_FIELD; +import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; public class FakeDoc extends BaseDoc { - public FakeDoc(Map<String, Object> fields) { - super(fields); + public FakeDoc() { + super(TYPE_FAKE, Maps.newHashMap()); } @Override @@ -34,20 +34,6 @@ public class FakeDoc extends BaseDoc { return null; } - @Override - public String getRouting() { - return null; - } - - @Override - public String getParent() { - return null; - } - - public FakeDoc() { - super(Maps.newHashMap()); - } - public int getInt() { return getField(INT_FIELD); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionContextTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionContextTest.java index 83140b5a873..ed7adadab3e 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionContextTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionContextTest.java @@ -19,35 +19,59 @@ */ package org.sonar.server.es; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.Locale; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.newindex.SettingsConfiguration; +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; +@RunWith(DataProviderRunner.class) public class IndexDefinitionContextTest { - private NewIndex.SettingsConfiguration emptySettingsConfiguration = newBuilder(new MapSettings().asConfig()).build(); + private SettingsConfiguration emptySettingsConfiguration = newBuilder(new MapSettings().asConfig()).build(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); @Test public void create_indices() { IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); - context.create("issues", emptySettingsConfiguration); - context.create("measures", emptySettingsConfiguration); - assertThat(context.getIndices().keySet()).containsOnly("issues", "measures"); + context.create(Index.withRelations("issues"), emptySettingsConfiguration); + context.create(Index.simple("users"), emptySettingsConfiguration); + assertThat(context.getIndices().keySet()) + .containsOnly("issues", "users"); } @Test - public void fail_to_create_twice_the_same_index() { + @UseDataProvider("paarOfIndicesWithSameName") + public void fail_to_create_twice_index_with_given_name(Index index1, Index index2) { IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); - context.create("issues", emptySettingsConfiguration); - try { - context.create("issues", emptySettingsConfiguration); - fail(); - } catch (IllegalArgumentException ok) { - assertThat(ok).hasMessage("Index already exists: issues"); - } + context.create(index1, emptySettingsConfiguration); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index already exists: " + index1.getName()); + + context.create(index2, emptySettingsConfiguration); + } + + @DataProvider + public static Object[][] paarOfIndicesWithSameName() { + String indexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); + return new Object[][] { + {Index.simple(indexName), Index.simple(indexName)}, + {Index.withRelations(indexName), Index.withRelations(indexName)}, + {Index.simple(indexName), Index.withRelations(indexName)}, + {Index.withRelations(indexName), Index.simple(indexName)}, + }; } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java index 4c8fb8f74c8..c746a57f812 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexDefinitionHashTest.java @@ -19,37 +19,407 @@ */ package org.sonar.server.es; -import org.junit.Test; - import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import org.junit.Test; +import org.sonar.api.config.Configuration; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.newindex.SettingsConfiguration; +import org.sonar.server.es.newindex.TestNewIndex; +import org.sonar.server.es.newindex.TypeMapping; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; public class IndexDefinitionHashTest { + private final SettingsConfiguration settingsConfiguration = settingsConfigurationOf(new MapSettings()); + + @Test + public void hash_changes_if_mainType_is_different() { + Index simpleIndex = Index.simple("foo"); + Index withRelationsIndex = Index.withRelations("foo"); + IndexMainType mainTypeBar = IndexMainType.main(simpleIndex, "bar"); + TestNewIndex indexSimpleBar = new TestNewIndex(mainTypeBar, settingsConfiguration); + TestNewIndex indexSimpleDonut = new TestNewIndex(IndexMainType.main(simpleIndex, "donut"), settingsConfiguration); + TestNewIndex indexWithRelationsBar = new TestNewIndex(IndexMainType.main(withRelationsIndex, "bar"), settingsConfiguration); + + assertThat(hashOf(indexSimpleBar)) + .isEqualTo(hashOf(new TestNewIndex(mainTypeBar, settingsConfiguration))) + .isNotEqualTo(hashOf(indexSimpleDonut)) + .isNotEqualTo(hashOf(indexWithRelationsBar)); + assertThat(hashOf(indexSimpleDonut)) + .isNotEqualTo(hashOf(indexWithRelationsBar)); + } + + @Test + public void hash_changes_if_relations_are_different() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + TestNewIndex indexNoRelation = new TestNewIndex(mainType, settingsConfiguration); + TestNewIndex indexOneRelation = new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut1"); + TestNewIndex indexOneOtherRelation = new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut2"); + TestNewIndex indexTwoRelations = new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut1") + .addRelation("donut2"); + TestNewIndex indexTwoOtherRelations = new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut1") + .addRelation("donut3"); + + assertThat(hashOf(indexNoRelation)) + .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration))) + .isNotEqualTo(hashOf(indexOneRelation)) + .isNotEqualTo(hashOf(indexOneOtherRelation)) + .isNotEqualTo(hashOf(indexTwoRelations)) + .isNotEqualTo(hashOf(indexTwoOtherRelations)); + assertThat(hashOf(indexOneRelation)) + .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration).addRelation("donut1"))) + .isNotEqualTo(hashOf(indexOneOtherRelation)) + .isNotEqualTo(hashOf(indexTwoRelations)) + .isNotEqualTo(hashOf(indexTwoOtherRelations)); + assertThat(hashOf(indexTwoRelations)) + .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut1") + .addRelation("donut2"))) + .isNotEqualTo(hashOf(indexOneRelation)) + .isNotEqualTo(hashOf(indexOneOtherRelation)) + .isNotEqualTo(hashOf(indexTwoOtherRelations)); + } + + @Test + public void hash_is_the_same_if_only_relations_order_changes() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + TestNewIndex indexTwoRelations = new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut1") + .addRelation("donut2") + .addRelation("donut3"); + TestNewIndex indexTwoRelationsOtherOrder = new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut2") + .addRelation("donut1") + .addRelation("donut3"); + TestNewIndex indexTwoRelationsOtherOrder2 = new TestNewIndex(mainType, settingsConfiguration) + .addRelation("donut2") + .addRelation("donut3") + .addRelation("donut1"); + + assertThat(hashOf(indexTwoRelations)) + .isEqualTo(hashOf(indexTwoRelationsOtherOrder)) + .isEqualTo(hashOf(indexTwoRelationsOtherOrder2)); + } + + @Test + public void hash_changes_if_fields_on_main_type_mapping_are_different() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + TestNewIndex indexNoField = new TestNewIndex(mainType, settingsConfiguration); + TestNewIndex indexOneField = new TestNewIndex(mainType, settingsConfiguration); + indexOneField.getMainTypeMapping() + .createIntegerField("field1"); + TestNewIndex indexOneFieldAgain = new TestNewIndex(mainType, settingsConfiguration); + indexOneFieldAgain.getMainTypeMapping() + .createIntegerField("field1"); + TestNewIndex indexOneOtherField = new TestNewIndex(mainType, settingsConfiguration); + indexOneOtherField.getMainTypeMapping() + .createIntegerField("field2"); + TestNewIndex indexTwoFields = new TestNewIndex(mainType, settingsConfiguration); + indexTwoFields.getMainTypeMapping() + .createIntegerField("field1") + .createIntegerField("field2"); + TestNewIndex indexTwoFieldsAgain = new TestNewIndex(mainType, settingsConfiguration); + indexTwoFieldsAgain.getMainTypeMapping() + .createIntegerField("field1") + .createIntegerField("field2"); + TestNewIndex indexTwoOtherFields = new TestNewIndex(mainType, settingsConfiguration); + indexTwoOtherFields.getMainTypeMapping() + .createIntegerField("field1") + .createIntegerField("field3"); + + assertThat(hashOf(indexNoField)) + .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfiguration))) + .isNotEqualTo(hashOf(indexOneField)) + .isNotEqualTo(hashOf(indexOneOtherField)) + .isNotEqualTo(hashOf(indexTwoFields)) + .isNotEqualTo(hashOf(indexTwoOtherFields)); + assertThat(hashOf(indexOneField)) + .isEqualTo(hashOf(indexOneFieldAgain)) + .isNotEqualTo(hashOf(indexOneOtherField)) + .isNotEqualTo(hashOf(indexTwoFields)) + .isNotEqualTo(hashOf(indexTwoOtherFields)); + assertThat(hashOf(indexTwoFields)) + .isEqualTo(hashOf(indexTwoFieldsAgain)) + .isNotEqualTo(hashOf(indexOneField)) + .isNotEqualTo(hashOf(indexOneOtherField)) + .isNotEqualTo(hashOf(indexTwoOtherFields)); + } + + @Test + public void hash_is_the_same_if_only_fields_order_changes() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + TestNewIndex indexTwoFields = new TestNewIndex(mainType, settingsConfiguration); + indexTwoFields.getMainTypeMapping() + .createBooleanField("donut1") + .createBooleanField("donut2") + .createBooleanField("donut3"); + TestNewIndex indexTwoFieldsOtherOrder = new TestNewIndex(mainType, settingsConfiguration); + indexTwoFieldsOtherOrder.getMainTypeMapping() + .createBooleanField("donut2") + .createBooleanField("donut1") + .createBooleanField("donut3"); + TestNewIndex indexTwoFieldsOtherOrder2 = new TestNewIndex(mainType, settingsConfiguration); + indexTwoFieldsOtherOrder2.getMainTypeMapping() + .createBooleanField("donut2") + .createBooleanField("donut3") + .createBooleanField("donut1"); + + assertThat(hashOf(indexTwoFields)) + .isEqualTo(hashOf(indexTwoFieldsOtherOrder)) + .isEqualTo(hashOf(indexTwoFieldsOtherOrder2)); + } + + @Test + public void hash_changes_if_field_type_changes() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + String fieldName = "field1"; + + computeAndVerifyAllDifferentHashesOnMapping(mainType, + (mapping) -> mapping.createBooleanField(fieldName), + (mapping) -> mapping.createIntegerField(fieldName), + (mapping) -> mapping.createByteField(fieldName), + (mapping) -> mapping.createDateTimeField(fieldName), + (mapping) -> mapping.createDoubleField(fieldName), + (mapping) -> mapping.createLongField(fieldName), + (mapping) -> mapping.createShortField(fieldName), + (mapping) -> mapping.createUuidPathField(fieldName), + (mapping) -> mapping.keywordFieldBuilder(fieldName).build(), + (mapping) -> mapping.textFieldBuilder(fieldName).build(), + (mapping) -> mapping.nestedFieldBuilder(fieldName).addKeywordField("bar").build()); + } + + @Test + public void hash_changes_if_keyword_options_change() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + String fieldName = "field1"; + + computeAndVerifyAllDifferentHashesOnMapping(mainType, + (mapping) -> mapping.keywordFieldBuilder(fieldName).build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableNorms().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableSearch().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableNorms().disableSearch().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSearch().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().build()); + } + + @Test + public void hash_is_the_same_if_only_order_of_keyword_options_change() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + String fieldName = "field1"; + + computeAndVerifyAllSameHashesOnMapping(mainType, + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableNorms().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSortingAndAggregating().build()); + computeAndVerifyAllSameHashesOnMapping(mainType, + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableSearch().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableSortingAndAggregating().build()); + computeAndVerifyAllSameHashesOnMapping(mainType, + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableNorms().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSearch().build()); + computeAndVerifyAllSameHashesOnMapping(mainType, + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSortingAndAggregating().disableSearch().disableNorms().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableNorms().disableSortingAndAggregating().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSearch().disableSortingAndAggregating().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableNorms().disableSortingAndAggregating().disableSearch().build(), + (mapping) -> mapping.keywordFieldBuilder(fieldName).disableSearch().disableSortingAndAggregating().disableNorms().build()); + } + + @Test + public void hash_changes_if_textFieldBuilder_options_change() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + String fieldName = "field1"; + + computeAndVerifyAllDifferentHashesOnMapping(mainType, + (mapping) -> mapping.textFieldBuilder(fieldName).build(), + (mapping) -> mapping.textFieldBuilder(fieldName).disableSearch().build(), + (mapping) -> mapping.textFieldBuilder(fieldName).disableNorms().build(), + (mapping) -> mapping.textFieldBuilder(fieldName).disableNorms().disableSearch().build()); + } + + @Test + public void hash_is_the_same_if_only_order_of_textFieldBuilder_options_change() { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + String fieldName = "field1"; + + computeAndVerifyAllSameHashesOnMapping(mainType, + (mapping) -> mapping.textFieldBuilder(fieldName).disableSearch().disableNorms().build(), + (mapping) -> mapping.textFieldBuilder(fieldName).disableNorms().disableSearch().build()); + } + + @SafeVarargs + private final void computeAndVerifyAllSameHashesOnMapping(IndexMainType mainType, Consumer<TypeMapping>... fieldTypes) { + List<Consumer<TypeMapping>> fieldTypes1 = Arrays.asList(fieldTypes); + List<TestNewIndex> mainIndices = fieldTypes1.stream() + .map(consumer -> { + TestNewIndex mainTypeMapping = new TestNewIndex(mainType, settingsConfiguration); + consumer.accept(mainTypeMapping.getMainTypeMapping()); + return mainTypeMapping; + }) + .collect(toList()); + List<TestNewIndex> relationIndices = fieldTypes1.stream() + .map(consumer -> { + TestNewIndex relationTypeMapping = new TestNewIndex(mainType, settingsConfiguration); + consumer.accept(relationTypeMapping.createRelationMapping("donut")); + return relationTypeMapping; + }) + .collect(toList()); + + Set<String> mainHashes = mainIndices.stream() + .map(IndexDefinitionHashTest::hashOf) + .collect(toSet()); + Set<String> relationHashes = relationIndices.stream() + .map(IndexDefinitionHashTest::hashOf) + .collect(toSet()); + + assertThat(mainHashes) + // verify hashing is stable + .isEqualTo(mainIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet())) + .doesNotContainAnyElementsOf(relationHashes) + .hasSize(1); + assertThat(relationHashes) + // verify hashing is stable + .isEqualTo(relationIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet())) + .doesNotContainAnyElementsOf(mainHashes) + .hasSize(1); + } + + @SafeVarargs + private final void computeAndVerifyAllDifferentHashesOnMapping(IndexMainType mainType, Consumer<TypeMapping>... fieldTypes) { + List<TestNewIndex> mainIndices = Arrays.stream(fieldTypes) + .map(consumer -> { + TestNewIndex mainTypeMapping = new TestNewIndex(mainType, settingsConfiguration); + consumer.accept(mainTypeMapping.getMainTypeMapping()); + return mainTypeMapping; + }) + .collect(toList()); + List<TestNewIndex> relationIndices = Arrays.stream(fieldTypes) + .map(consumer -> { + TestNewIndex relationTypeMapping = new TestNewIndex(mainType, settingsConfiguration); + consumer.accept(relationTypeMapping.createRelationMapping("donut")); + return relationTypeMapping; + }) + .collect(toList()); + + Set<String> mainHashes = mainIndices.stream() + .map(IndexDefinitionHashTest::hashOf) + .collect(toSet()); + Set<String> relationHashes = relationIndices.stream() + .map(IndexDefinitionHashTest::hashOf) + .collect(toSet()); + + assertThat(mainHashes) + // verify hashing is stable + .isEqualTo(mainIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet())) + .doesNotContainAnyElementsOf(relationHashes) + .hasSize(fieldTypes.length); + assertThat(relationHashes) + // verify hashing is stable + .isEqualTo(relationIndices.stream().map(IndexDefinitionHashTest::hashOf).collect(toSet())) + .doesNotContainAnyElementsOf(mainHashes) + .hasSize(fieldTypes.length); + } @Test - public void of() { - IndexDefinition.Index indexV1 = new IndexDefinition.Index(createIndex()); - String hashV1 = IndexDefinitionHash.of(indexV1); - assertThat(hashV1).isNotEmpty(); - // always the same - assertThat(hashV1).isEqualTo(IndexDefinitionHash.of(indexV1)); + public void hash_changes_if_clustering_is_enabled_or_not() { + Index index = Index.simple("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + MapSettings empty = new MapSettings(); + MapSettings clusterDisabled = new MapSettings().setProperty(CLUSTER_ENABLED.getKey(), false); + MapSettings clusterEnabled = new MapSettings().setProperty(CLUSTER_ENABLED.getKey(), true); - NewIndex newIndexV2 = createIndex(); - newIndexV2.getTypes().get("fake").createIntegerField("max"); - String hashV2 = IndexDefinitionHash.of(new IndexDefinition.Index(newIndexV2)); - assertThat(hashV2).isNotEmpty().isNotEqualTo(hashV1); + assertThat(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(empty)))) + .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(empty)))) + .isEqualTo(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(clusterDisabled)))) + .isNotEqualTo(hashOf(new TestNewIndex(mainType, settingsConfigurationOf(clusterEnabled)))); } - private NewIndex createIndex() { - NewIndex newIndex = new NewIndex("fakes", newBuilder(new MapSettings().asConfig()).build()); - NewIndex.NewIndexType mapping = newIndex.createType("fake"); - mapping.setAttribute("list_attr", Arrays.asList("foo", "bar")); - mapping.keywordFieldBuilder("key").build(); - mapping.createDateTimeField("updatedAt"); - return newIndex; + @Test + public void hash_changes_if_number_of_shards_changes() { + Index index = Index.simple("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + Configuration emptySettings = new MapSettings().asConfig(); + SettingsConfiguration defaultNbOfShards = SettingsConfiguration.newBuilder(emptySettings) + .build(); + SettingsConfiguration specifiedDefaultNbOfShards = SettingsConfiguration.newBuilder(emptySettings) + .setDefaultNbOfShards(5) + .build(); + SettingsConfiguration specifyDefaultNbOfShards = SettingsConfiguration.newBuilder(new MapSettings() + .setProperty("sonar.search." + index.getName() + ".shards", 1) + .asConfig()) + .setDefaultNbOfShards(1) + .build(); + SettingsConfiguration specifiedNbOfShards = SettingsConfiguration.newBuilder(new MapSettings() + .setProperty("sonar.search." + index.getName() + ".shards", 10) + .asConfig()) + .setDefaultNbOfShards(5) + .build(); + + assertThat(hashOf(new TestNewIndex(mainType, defaultNbOfShards))) + // verify hash is stable + .isEqualTo(hashOf(new TestNewIndex(mainType, defaultNbOfShards))) + .isEqualTo(hashOf(new TestNewIndex(mainType, specifyDefaultNbOfShards))) + .isNotEqualTo(hashOf(new TestNewIndex(mainType, specifiedDefaultNbOfShards))) + .isNotEqualTo(hashOf(new TestNewIndex(mainType, specifiedNbOfShards))); + assertThat(hashOf(new TestNewIndex(mainType, specifiedDefaultNbOfShards))) + // verify hash is stable + .isEqualTo(hashOf(new TestNewIndex(mainType, specifiedDefaultNbOfShards))) + .isNotEqualTo(hashOf(new TestNewIndex(mainType, specifyDefaultNbOfShards))); } + @Test + public void hash_changes_if_refreshInterval_changes() { + Index index = Index.simple("foo"); + IndexMainType mainType = IndexMainType.main(index, "bar"); + Configuration emptySettings = new MapSettings().asConfig(); + SettingsConfiguration defaultRefreshInterval = SettingsConfiguration.newBuilder(emptySettings) + .build(); + SettingsConfiguration noRefreshInterval = SettingsConfiguration.newBuilder(emptySettings) + .setRefreshInterval(-1) + .build(); + SettingsConfiguration refreshInterval30 = SettingsConfiguration.newBuilder(emptySettings) + .setRefreshInterval(30) + .build(); + SettingsConfiguration someRefreshInterval = SettingsConfiguration.newBuilder(emptySettings) + .setRefreshInterval(56) + .build(); + + assertThat(hashOf(new TestNewIndex(mainType, defaultRefreshInterval))) + // verify hash is stable + .isEqualTo(hashOf(new TestNewIndex(mainType, defaultRefreshInterval))) + .isEqualTo(hashOf(new TestNewIndex(mainType, refreshInterval30))) + .isNotEqualTo(hashOf(new TestNewIndex(mainType, noRefreshInterval))) + .isNotEqualTo(hashOf(new TestNewIndex(mainType, someRefreshInterval))); + assertThat(hashOf(new TestNewIndex(mainType, noRefreshInterval))) + .isNotEqualTo(hashOf(new TestNewIndex(mainType, someRefreshInterval))); + } + + private static SettingsConfiguration settingsConfigurationOf(MapSettings settings) { + return SettingsConfiguration.newBuilder(settings.asConfig()).build(); + } + + private static String hashOf(TestNewIndex newIndex) { + return IndexDefinitionHash.of(newIndex.build()); + } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTest.java new file mode 100644 index 00000000000..65cf708c31b --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTest.java @@ -0,0 +1,161 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.Locale; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(DataProviderRunner.class) +public class IndexTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + @UseDataProvider("nullOrEmpty") + public void simple_index_constructor_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index name can't be null nor empty"); + + Index.simple(nullOrEmpty); + } + + @Test + @UseDataProvider("nullOrEmpty") + public void withRelations_index_constructor_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index name can't be null nor empty"); + + Index.withRelations(nullOrEmpty); + + } + + @DataProvider + public static Object[][] nullOrEmpty() { + return new Object[][] { + {null}, + {""} + }; + } + + @Test + public void simple_index_name_must_not_contain_upper_case_char() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index name must be lower-case letters or '_all': Issues"); + + Index.simple("Issues"); + } + + @Test + public void withRelations_index_name_must_not_contain_upper_case_char() { + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index name must be lower-case letters or '_all': Issues"); + + Index.withRelations("Issues"); + } + + @Test + public void simple_index_name_can_not_contain_underscore_except__all_keyword() { + // doesn't fail + Index.simple("_all"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index name must be lower-case letters or '_all': _"); + + Index.simple("_"); + } + + @Test + public void withRelations_index_name_can_not_contain_underscore_except__all_keyword() { + // doesn't fail + Index.withRelations("_all"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index name must be lower-case letters or '_all': _"); + + Index.withRelations("_"); + } + + @Test + public void simple_index_does_not_accept_relations() { + Index underTest = Index.simple("foo"); + + assertThat(underTest.acceptsRelations()).isFalse(); + } + + @Test + public void withRelations_index_does_not_accept_relations() { + Index underTest = Index.withRelations("foo"); + + assertThat(underTest.acceptsRelations()).isTrue(); + } + + @Test + public void getName_returns_constructor_parameter() { + String indexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); + + assertThat(Index.simple(indexName).getName()).isEqualTo(indexName); + assertThat(Index.withRelations(indexName).getName()).isEqualTo(indexName); + } + + @Test + public void getJoinField_throws_ISE_on_simple_index() { + Index underTest = Index.simple("foo"); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Only index accepting relations has a join field"); + + underTest.getJoinField(); + } + + @Test + public void getJoinField_returns_name_based_on_index_name() { + String indexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); + Index underTest = Index.withRelations(indexName); + + assertThat(underTest.getJoinField()).isEqualTo("join_" + indexName); + } + + @Test + public void equals_is_based_on_name_and_acceptRelations_flag() { + assertThat(Index.simple("foo")) + .isEqualTo(Index.simple("foo")) + .isNotEqualTo(Index.simple("bar")) + .isNotEqualTo(Index.withRelations("foo")); + } + + @Test + public void hashcode_is_based_on_name_and_acceptRelations_flag() { + assertThat(Index.simple("foo").hashCode()) + .isEqualTo(Index.simple("foo").hashCode()) + .isNotEqualTo(Index.simple("bar").hashCode()) + .isNotEqualTo(Index.withRelations("foo").hashCode()); + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java index a6df6010ff1..dbcf45eaec8 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/IndexTypeTest.java @@ -17,26 +17,59 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ - package org.sonar.server.es; +package org.sonar.server.es; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.IndexType.IndexRelationType; +import org.sonar.server.es.IndexType.SimpleIndexMainType; import static org.assertj.core.api.Assertions.assertThat; +@RunWith(DataProviderRunner.class) public class IndexTypeTest { @Rule public ExpectedException expectedException = ExpectedException.none(); @Test - public void format_and_parse() { - IndexType type1 = new IndexType("foo", "bar"); + public void parseMainType_from_main_type_without_relations() { + IndexMainType type1 = IndexType.main(Index.simple("foo"), "bar"); assertThat(type1.format()).isEqualTo("foo/bar"); - IndexType type2 = IndexType.parse(type1.format()); - assertThat(type2.equals(type1)).isTrue(); + SimpleIndexMainType type2 = IndexType.parseMainType(type1.format()); + assertThat(type2) + .extracting(SimpleIndexMainType::getIndex, SimpleIndexMainType::getType) + .containsExactly("foo", "bar"); + } + + @Test + public void parseMainType_from_maintype_with_relations() { + IndexMainType type1 = IndexType.main(Index.withRelations("foo"), "bar"); + assertThat(type1.format()).isEqualTo("foo/bar"); + + SimpleIndexMainType type2 = IndexType.parseMainType(type1.format()); + assertThat(type2) + .extracting(SimpleIndexMainType::getIndex, SimpleIndexMainType::getType) + .containsExactly("foo", "bar"); + } + + @Test + public void parseMainType_from_relationtype() { + IndexMainType mainType = IndexType.main(Index.withRelations("foo"), "bar"); + IndexRelationType type1 = IndexType.relation(mainType, "donut"); + assertThat(type1.format()).isEqualTo("foo/bar/donut"); + + SimpleIndexMainType type2 = IndexType.parseMainType(type1.format()); + assertThat(type2) + .extracting(SimpleIndexMainType::getIndex, SimpleIndexMainType::getType) + .containsExactly("foo", "bar"); } @Test @@ -44,20 +77,79 @@ public class IndexTypeTest { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Unsupported IndexType value: foo"); - IndexType.parse("foo"); + IndexType.parseMainType("foo"); + } + + @Test + @UseDataProvider("nullOrEmpty") + public void main_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) { + Index index = Index.simple("foo"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("type name can't be null nor empty"); + + IndexType.main(index, nullOrEmpty); + } + + @Test + @UseDataProvider("nullOrEmpty") + public void relation_fails_with_IAE_if_index_name_is_null_or_empty(String nullOrEmpty) { + Index index = Index.withRelations("foo"); + IndexMainType mainType = IndexType.main(index, "foobar"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("type name can't be null nor empty"); + + IndexType.relation(mainType, nullOrEmpty); + } + + @DataProvider + public static Object[][] nullOrEmpty() { + return new Object[][] { + {null}, + {""} + }; + } + + @Test + public void IndexMainType_equals_and_hashCode() { + IndexMainType type1 = IndexType.main(Index.simple("foo"), "bar"); + IndexMainType type1b = IndexType.main(Index.simple("foo"), "bar"); + IndexMainType type1c = IndexType.main(Index.withRelations("foo"), "bar"); + IndexMainType type2 = IndexType.main(Index.simple("foo"), "baz"); + + assertThat(type1) + .isEqualTo(type1) + .isEqualTo(type1b) + .isNotEqualTo(type1c) + .isNotEqualTo(type2); + + assertThat(type1.hashCode()).isEqualTo(type1.hashCode()); + assertThat(type1.hashCode()).isEqualTo(type1b.hashCode()); + assertThat(type1.hashCode()).isNotEqualTo(type1c.hashCode()); + assertThat(type2.hashCode()).isNotEqualTo(type1.hashCode()); } @Test - public void equals_and_hashCode() { - IndexType type1 = new IndexType("foo", "bar"); - IndexType type1b = new IndexType("foo", "bar"); - IndexType type2 = new IndexType("foo", "baz"); + public void IndexRelationType_equals_and_hashCode() { + IndexMainType mainType1 = IndexType.main(Index.withRelations("foo"), "bar"); + IndexMainType mainType2 = IndexType.main(Index.withRelations("foo"), "baz"); + IndexRelationType type1 = IndexType.relation(mainType1, "donut"); + IndexRelationType type1b = IndexType.relation(mainType1, "donut"); + IndexRelationType type2 = IndexType.relation(mainType1, "donuz"); + IndexRelationType type3 = IndexType.relation(mainType2, "donut"); - assertThat(type1.equals(type1)).isTrue(); - assertThat(type1.equals(type1b)).isTrue(); - assertThat(type1.equals(type2)).isFalse(); + assertThat(type1) + .isEqualTo(type1) + .isEqualTo(type1b) + .isNotEqualTo(type2) + .isNotEqualTo(type3); assertThat(type1.hashCode()).isEqualTo(type1.hashCode()); assertThat(type1.hashCode()).isEqualTo(type1b.hashCode()); + assertThat(type2.hashCode()).isNotEqualTo(type1.hashCode()); + assertThat(type3.hashCode()) + .isNotEqualTo(type2.hashCode()) + .isNotEqualTo(type1.hashCode()); } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/NewIndexSettingsConfigurationTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/NewIndexSettingsConfigurationTest.java index 014727d6df3..17ba47fd6e4 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/NewIndexSettingsConfigurationTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/NewIndexSettingsConfigurationTest.java @@ -24,10 +24,11 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.sonar.api.config.Configuration; +import org.sonar.server.es.newindex.SettingsConfiguration; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class NewIndexSettingsConfigurationTest { @Rule @@ -45,7 +46,7 @@ public class NewIndexSettingsConfigurationTest { @Test public void setDefaultNbOfShards_fails_with_IAE_if_argument_is_zero() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("defaultNbOfShards must be >= 1"); @@ -55,7 +56,7 @@ public class NewIndexSettingsConfigurationTest { @Test public void setDefaultNbOfShards_fails_with_IAE_if_argument_is_less_than_zero() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("defaultNbOfShards must be >= 1"); @@ -65,14 +66,14 @@ public class NewIndexSettingsConfigurationTest { @Test public void setDefaultNbOfShards_accepts_1() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); assertThat(underTest.setDefaultNbOfShards(1).build().getDefaultNbOfShards()).isEqualTo(1); } @Test public void setDefaultNbOfShards_accepts_any_int_greater_than_1() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); int value = 1 + new Random().nextInt(200); @@ -86,7 +87,7 @@ public class NewIndexSettingsConfigurationTest { @Test public void setRefreshInterval_fails_with_IAE_if_argument_is_zero() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("refreshInterval must be either -1 or strictly positive"); @@ -96,7 +97,7 @@ public class NewIndexSettingsConfigurationTest { @Test public void setRefreshInterval_fails_with_IAE_if_argument_is_less_than_minus_1() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("refreshInterval must be either -1 or strictly positive"); @@ -106,14 +107,14 @@ public class NewIndexSettingsConfigurationTest { @Test public void setRefreshInterval_accepts_minus_1() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); assertThat(underTest.setRefreshInterval(-1).build().getRefreshInterval()).isEqualTo(-1); } @Test public void setRefreshInterval_accepts_any_int_greater_than_1() { - NewIndex.SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); + SettingsConfiguration.Builder underTest = newBuilder(mockConfiguration); int value = 1 + new Random().nextInt(200); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/NewIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/NewIndexTest.java deleted file mode 100644 index fab2c8c7c80..00000000000 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/NewIndexTest.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2019 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.collect.ImmutableMap; -import java.util.Map; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.settings.Settings; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.sonar.api.config.internal.MapSettings; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.data.MapEntry.entry; -import static org.junit.Assert.fail; -import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; -import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; - -public class NewIndexTest { - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - private MapSettings settings = new MapSettings(); - private NewIndex.SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build(); - - @Test - public void getName_returns_constructor_argument() { - assertThat(new NewIndex("foo", defaultSettingsConfiguration).getName()).isEqualTo("foo"); - } - - @Test - public void no_types_of_none_are_specified() { - assertThat(new NewIndex("foo", defaultSettingsConfiguration).getTypes()).isEmpty(); - } - - @Test - public void verify_default_index_settings_in_standalone() { - Settings underTest = new NewIndex("issues", defaultSettingsConfiguration).getSettings().build(); - - assertThat(underTest.get("index.number_of_shards")).isNotEmpty(); - assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false"); - assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s"); - assertThat(underTest.get("index.number_of_shards")).isEqualTo("1"); - assertThat(underTest.get("index.number_of_replicas")).isEqualTo("0"); - } - - @Test - public void verify_default_index_settings_in_cluster() { - settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); - Settings underTest = new NewIndex("issues", defaultSettingsConfiguration).getSettings().build(); - - assertThat(underTest.get("index.number_of_shards")).isNotEmpty(); - assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false"); - assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s"); - assertThat(underTest.get("index.number_of_shards")).isEqualTo("1"); - assertThat(underTest.get("index.number_of_replicas")).isEqualTo("1"); - } - - @Test - public void index_name_is_lower_case() { - try { - new NewIndex("Issues", defaultSettingsConfiguration); - fail(); - } catch (IllegalArgumentException e) { - assertThat(e).hasMessage("Index name must be lower-case: Issues"); - } - } - - @Test - public void define_fields() { - NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); - NewIndex.NewIndexType mapping = index.createType("issue"); - mapping.setAttribute("dynamic", "true"); - mapping.setProperty("foo_field", ImmutableMap.of("type", "keyword")); - mapping.createBooleanField("boolean_field"); - mapping.createByteField("byte_field"); - mapping.createDateTimeField("dt_field"); - mapping.createDoubleField("double_field"); - mapping.createIntegerField("int_field"); - mapping.createLongField("long_field"); - mapping.createShortField("short_field"); - mapping.createUuidPathField("uuid_path_field"); - - mapping = index.getTypes().get("issue"); - assertThat(mapping).isNotNull(); - assertThat(mapping.getAttributes().get("dynamic")).isEqualTo("true"); - assertThat(mapping.getProperty("foo_field")).isInstanceOf(Map.class); - assertThat((Map) mapping.getProperty("foo_field")).containsEntry("type", "keyword"); - assertThat((Map) mapping.getProperty("byte_field")).isNotEmpty(); - assertThat((Map) mapping.getProperty("double_field")).isNotEmpty(); - assertThat((Map) mapping.getProperty("dt_field")).isNotEmpty(); - assertThat((Map) mapping.getProperty("int_field")).containsEntry("type", "integer"); - assertThat((Map) mapping.getProperty("long_field")).isNotEmpty(); - assertThat((Map) mapping.getProperty("short_field")).isNotEmpty(); - assertThat((Map) mapping.getProperty("uuid_path_field")).isNotEmpty(); - assertThat((Map) mapping.getProperty("unknown")).isNull(); - } - - @Test - public void define_string_field() { - NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); - NewIndex.NewIndexType mapping = index.createType("issue"); - mapping.keywordFieldBuilder("basic_field").build(); - mapping.keywordFieldBuilder("not_searchable_field").disableSearch().build(); - mapping.keywordFieldBuilder("all_capabilities_field") - .addSubFields( - DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER, - DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER, - DefaultIndexSettingsElement.SORTABLE_ANALYZER) - .build(); - mapping.keywordFieldBuilder("dumb_text_storage") - .disableSearch() - .disableNorms() - .disableSortingAndAggregating() - .build(); - - Map<String, Object> props = (Map) mapping.getProperty("basic_field"); - assertThat(props.get("type")).isEqualTo("keyword"); - assertThat(props.get("index")).isEqualTo("true"); - assertThat(props.get("fields")).isNull(); - - props = (Map) mapping.getProperty("not_searchable_field"); - assertThat(props.get("type")).isEqualTo("keyword"); - assertThat(props.get("index")).isEqualTo("false"); - assertThat(props.get("norms")).isEqualTo("true"); - assertThat(props.get("store")).isEqualTo("false"); - assertThat(props.get("doc_values")).isEqualTo("true"); - assertThat(props.get("fields")).isNull(); - - props = (Map) mapping.getProperty("all_capabilities_field"); - assertThat(props.get("type")).isEqualTo("keyword"); - // no need to test values, it's not the scope of this test - assertThat((Map) props.get("fields")).isNotEmpty(); - - props = (Map) mapping.getProperty("dumb_text_storage"); - assertThat(props.get("type")).isEqualTo("keyword"); - assertThat(props.get("index")).isEqualTo("false"); - assertThat(props.get("norms")).isEqualTo("false"); - assertThat(props.get("store")).isEqualTo("false"); - assertThat(props.get("doc_values")).isEqualTo("false"); - assertThat(props.get("fields")).isNull(); - } - - @Test - public void define_nested_field() { - NewIndex index = new NewIndex("projectmeasures", defaultSettingsConfiguration); - NewIndex.NewIndexType mapping = index.createType("projectmeasures"); - - mapping.nestedFieldBuilder("measures") - .addKeywordField("key") - .addDoubleField("value") - .build(); - Map<String, Object> result = (Map) mapping.getProperty("measures"); - - assertThat(result.get("type")).isEqualTo("nested"); - Map<String, Map<String, Object>> subProperties = (Map) result.get("properties"); - assertThat(subProperties.get("key").get("type")).isEqualTo("keyword"); - assertThat(subProperties.get("value").get("type")).isEqualTo("double"); - } - - @Test - public void fail_when_nested_with_no_field() { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("At least one sub-field must be declared in nested property 'measures'"); - - NewIndex index = new NewIndex("projectmeasures", defaultSettingsConfiguration); - NewIndex.NewIndexType mapping = index.createType("project_measures"); - - mapping.nestedFieldBuilder("measures").build(); - } - - @Test - public void use_doc_values_by_default() { - NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); - NewIndex.NewIndexType mapping = index.createType("issue"); - mapping.keywordFieldBuilder("the_doc_value").build(); - - Map<String, Object> props = (Map) mapping.getProperty("the_doc_value"); - assertThat(props.get("type")).isEqualTo("keyword"); - assertThat(props.get("doc_values")).isEqualTo("true"); - } - - @Test - public void default_shards_and_replicas() { - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); - } - - @Test - public void five_shards_and_one_replica_by_default_on_cluster() { - settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); - } - - @Test - public void customize_number_of_shards() { - settings.setProperty("sonar.search.issues.shards", "3"); - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3"); - // keep default value - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); - } - - @Test - public void default_number_of_replicas_on_standalone_instance_must_be_0() { - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); - } - - @Test - public void default_number_of_replicas_on_non_enabled_cluster_must_be_0() { - settings.setProperty(CLUSTER_ENABLED.getKey(), "false"); - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); - } - - @Test - public void default_number_of_replicas_on_cluster_instance_must_be_1() { - settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); - } - - @Test - public void when_number_of_replicas_on_cluster_is_specified_to_zero_default_value_must_not_be_used() { - settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); - settings.setProperty(SEARCH_REPLICAS.getKey(), "0"); - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); - } - - @Test - public void index_defined_with_specified_number_of_replicas_when_cluster_enabled() { - settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); - settings.setProperty(SEARCH_REPLICAS.getKey(), "3"); - NewIndex index = new NewIndex("issues", newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); - - assertThat(index.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3"); - } - - @Test - public void fail_when_replica_customization_cant_be_parsed() { - settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); - settings.setProperty(SEARCH_REPLICAS.getKey(), "ꝱꝲꝳପ"); - NewIndex.SettingsConfiguration settingsConfiguration = newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build(); - - expectedException.expect(IllegalStateException.class); - expectedException.expectMessage("The property 'sonar.search.replicas' is not an int value: For input string: \"ꝱꝲꝳପ\""); - - new NewIndex("issues", settingsConfiguration); - } - - @Test - public void in_standalone_searchReplicas_is_not_overridable() { - settings.setProperty(SEARCH_REPLICAS.getKey(), "5"); - NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); - - assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0"); - } - - @Test - public void index_with_source() { - NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); - NewIndex.NewIndexType mapping = index.createType("issue"); - mapping.setEnableSource(true); - - mapping = index.getTypes().get("issue"); - assertThat(mapping).isNotNull(); - assertThat(getAttributeAsMap(mapping, "_source")).containsExactly(entry("enabled", true)); - } - - @Test - public void index_without_source() { - NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); - NewIndex.NewIndexType mapping = index.createType("issue"); - mapping.setEnableSource(false); - - mapping = index.getTypes().get("issue"); - assertThat(mapping).isNotNull(); - assertThat(getAttributeAsMap(mapping, "_source")).containsExactly(entry("enabled", false)); - } - - @Test - public void index_requires_project_authorization() { - NewIndex index = new NewIndex("issues", defaultSettingsConfiguration); - index.createType("issue") - // creates a second type "authorization" and configures _parent and _routing fields - .requireProjectAuthorization(); - - // issue type - NewIndex.NewIndexType issueType = index.getTypes().get("issue"); - assertThat(getAttributeAsMap(issueType, "_parent")).containsExactly(entry("type", "authorization")); - assertThat(getAttributeAsMap(issueType, "_routing")).containsExactly(entry("required", true)); - - // authorization type - NewIndex.NewIndexType authorizationType = index.getTypes().get("authorization"); - assertThat(getAttributeAsMap(authorizationType, "_parent")).isNull(); - assertThat(getAttributeAsMap(authorizationType, "_routing")).containsExactly(entry("required", true)); - } - - private static Map<String, Object> getAttributeAsMap(NewIndex.NewIndexType type, String attributeKey) { - return (Map<String, Object>) type.getAttributes().get(attributeKey); - } -} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToManyResilientIndexingListenerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToManyResilientIndexingListenerTest.java index 92e8f8c1f68..8bceb9261e4 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToManyResilientIndexingListenerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToManyResilientIndexingListenerTest.java @@ -34,7 +34,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; public class OneToManyResilientIndexingListenerTest { @@ -47,16 +47,16 @@ public class OneToManyResilientIndexingListenerTest { @Test public void ES_QUEUE_rows_are_deleted_when_all_docs_are_successfully_indexed() { - EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "P1"); - EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, "P2"); - EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, "P1"); + EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "P1"); + EsQueueDto item2 = insertInQueue(TYPE_ISSUE, "P2"); + EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.TYPE_COMPONENT, "P1"); db.commit(); // does not contain outOfScopeItem IndexingListener underTest = newListener(asList(item1, item2)); - DocId issue1 = newDocId(INDEX_TYPE_ISSUE, "I1"); - DocId issue2 = newDocId(INDEX_TYPE_ISSUE, "I2"); + DocId issue1 = newDocId(TYPE_ISSUE, "I1"); + DocId issue2 = newDocId(TYPE_ISSUE, "I2"); underTest.onSuccess(asList(issue1, issue2)); assertThatEsTableContainsOnly(item1, item2, outOfScopeItem); @@ -71,16 +71,16 @@ public class OneToManyResilientIndexingListenerTest { @Test public void ES_QUEUE_rows_are_not_deleted_on_partial_error() { - EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "P1"); - EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, "P2"); - EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, "P1"); + EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "P1"); + EsQueueDto item2 = insertInQueue(TYPE_ISSUE, "P2"); + EsQueueDto outOfScopeItem = insertInQueue(ComponentIndexDefinition.TYPE_COMPONENT, "P1"); db.commit(); // does not contain outOfScopeItem IndexingListener underTest = newListener(asList(item1, item2)); - DocId issue1 = newDocId(INDEX_TYPE_ISSUE, "I1"); - DocId issue2 = newDocId(INDEX_TYPE_ISSUE, "I2"); + DocId issue1 = newDocId(TYPE_ISSUE, "I1"); + DocId issue2 = newDocId(TYPE_ISSUE, "I2"); underTest.onSuccess(asList(issue1, issue2)); assertThatEsTableContainsOnly(item1, item2, outOfScopeItem); @@ -93,8 +93,9 @@ public class OneToManyResilientIndexingListenerTest { assertThatEsTableContainsOnly(item1, item2, outOfScopeItem); } - private static DocId newDocId(IndexType indexType, String id) { - return new DocId(indexType, id); + private static DocId newDocId(IndexType.IndexRelationType indexType, String id) { + IndexType.IndexMainType mainType = indexType.getMainType(); + return new DocId(mainType.getIndex().getName(), mainType.getType(), id); } private IndexingListener newListener(Collection<EsQueueDto> items) { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToOneResilientIndexingListenerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToOneResilientIndexingListenerTest.java index ce5ecdb6df1..c46a0ad3732 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToOneResilientIndexingListenerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/OneToOneResilientIndexingListenerTest.java @@ -34,7 +34,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; public class OneToOneResilientIndexingListenerTest { @@ -47,9 +47,9 @@ public class OneToOneResilientIndexingListenerTest { @Test public void onSuccess_deletes_rows_from_ES_QUEUE_table() { - EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "foo"); - EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, "bar"); - EsQueueDto item3 = insertInQueue(INDEX_TYPE_ISSUE, "baz"); + EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "foo"); + EsQueueDto item2 = insertInQueue(TYPE_ISSUE, "bar"); + EsQueueDto item3 = insertInQueue(TYPE_ISSUE, "baz"); db.commit(); IndexingListener underTest = newListener(asList(item1, item2, item3)); @@ -76,10 +76,10 @@ public class OneToOneResilientIndexingListenerTest { */ @Test public void onSuccess_deletes_all_the_rows_with_same_doc_id() { - EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "foo"); + EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "foo"); // same id as item1 - EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, item1.getDocId()); - EsQueueDto item3 = insertInQueue(INDEX_TYPE_ISSUE, "bar"); + EsQueueDto item2 = insertInQueue(TYPE_ISSUE, item1.getDocId()); + EsQueueDto item3 = insertInQueue(TYPE_ISSUE, "bar"); db.commit(); IndexingListener underTest = newListener(asList(item1, item2, item3)); @@ -89,7 +89,8 @@ public class OneToOneResilientIndexingListenerTest { } private static DocId toDocId(EsQueueDto dto) { - return new DocId(IndexType.parse(dto.getDocType()), dto.getDocId()); + IndexType.SimpleIndexMainType mainType = IndexType.parseMainType(dto.getDocType()); + return new DocId(mainType.getIndex(), mainType.getType(), dto.getDocId()); } private IndexingListener newListener(Collection<EsQueueDto> items) { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java index f0c5b4d796b..a9ed154f9fc 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java @@ -19,35 +19,56 @@ */ package org.sonar.server.es.metadata; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.Locale; +import java.util.Random; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.newindex.FakeIndexDefinition; +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; +@RunWith(DataProviderRunner.class) public class MetadataIndexTest { @Rule public EsTester es = EsTester.createCustom(new MetadataIndexDefinitionBridge(), new FakeIndexDefinition()); private final MetadataIndex underTest = new MetadataIndex(es.client()); - private final String index = randomAlphanumeric(20); + private final String indexName = randomAlphabetic(20).toLowerCase(Locale.ENGLISH); + private final Index index = new Random().nextBoolean() ? Index.simple(indexName) : Index.withRelations(indexName); @Test - public void type_should_be_not_initialized_by_default() { - IndexType indexType = new IndexType("examples", "example"); + @UseDataProvider("mainOrRelationType") + public void type_should_be_not_initialized_by_default(IndexType indexType) { assertThat(underTest.getInitialized(indexType)).isFalse(); } @Test - public void type_should_be_initialized_after_explicitly_set_to_initialized() { - IndexType indexType = new IndexType("examples", "example"); + @UseDataProvider("mainOrRelationType") + public void type_should_be_initialized_after_explicitly_set_to_initialized(IndexType indexType) { + underTest.setInitialized(indexType, true); assertThat(underTest.getInitialized(indexType)).isTrue(); } + @DataProvider + public static Object[][] mainOrRelationType() { + IndexMainType mainType = IndexType.main(Index.withRelations("examples"), "example"); + return new Object[][] { + {mainType}, + {IndexType.relation(mainType, "doo")} + }; + } + @Test public void hash_should_be_empty_by_default() { assertThat(underTest.getHash(index)).isEmpty(); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/FakeIndexDefinition.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/FakeIndexDefinition.java index 5bb78cee4a1..d6dfd760540 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/FakeIndexDefinition.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/FakeIndexDefinition.java @@ -17,18 +17,24 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package org.sonar.server.es; +package org.sonar.server.es.newindex; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.FakeDoc; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexDefinition; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class FakeIndexDefinition implements IndexDefinition { public static final String INDEX = "fakes"; public static final String TYPE = "fake"; - public static final IndexType INDEX_TYPE_FAKE = new IndexType("fakes", "fake"); + public static final Index DESCRIPTOR = Index.simple(INDEX); + public static final IndexMainType TYPE_FAKE = IndexType.main(DESCRIPTOR, TYPE); public static final String INT_FIELD = "intField"; private int replicas = 0; @@ -40,11 +46,11 @@ public class FakeIndexDefinition implements IndexDefinition { @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create(INDEX, newBuilder(new MapSettings().asConfig()).build()); + NewIndex index = context.create(DESCRIPTOR, newBuilder(new MapSettings().asConfig()).build()); index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas); index.getSettings().put("index.refresh_interval", "-1"); - NewIndex.NewIndexType type = index.createType(INDEX_TYPE_FAKE.getType()); - type.createIntegerField(INT_FIELD); + index.createTypeMapping(TYPE_FAKE) + .createIntegerField(INT_FIELD); } public static FakeDoc newDoc(int value) { diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/FieldAwareTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/FieldAwareTest.java new file mode 100644 index 00000000000..99af8144e95 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/FieldAwareTest.java @@ -0,0 +1,75 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import java.util.Random; +import java.util.function.BiConsumer; +import java.util.stream.Stream; +import org.junit.Test; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.fail; + +public class FieldAwareTest { + + @Test + public void indexType_is_a_reserved_field_name_whatever_the_case() { + Stream<BiConsumer<TestFieldAware, String>> fieldSetters = Stream.of( + (testFieldAware, fieldName) -> testFieldAware.createBooleanField(fieldName), + (testFieldAware, fieldName) -> testFieldAware.createByteField(fieldName), + (testFieldAware, fieldName) -> testFieldAware.createDateTimeField(fieldName), + (testFieldAware, fieldName) -> testFieldAware.createDoubleField(fieldName), + (testFieldAware, fieldName) -> testFieldAware.createIntegerField(fieldName), + (testFieldAware, fieldName) -> testFieldAware.createLongField(fieldName), + (testFieldAware, fieldName) -> testFieldAware.keywordFieldBuilder(fieldName).build(), + (testFieldAware, fieldName) -> testFieldAware.textFieldBuilder(fieldName).build(), + (testFieldAware, fieldName) -> testFieldAware.nestedFieldBuilder(fieldName).addKeywordField("foo").build() + ); + + fieldSetters.forEach(c -> { + TestFieldAware underTest = new TestFieldAware(); + // should not fail for other field name + c.accept(underTest, randomAlphabetic(1 + new Random().nextInt(10))); + // fails whatever the case + Stream.of("indexType", "indextype", "InDexType", "INDEXTYPE") + .forEach(illegalFieldName -> { + try { + c.accept(underTest, illegalFieldName); + fail("should have thrown a IllegalArgumentException"); + } catch (IllegalArgumentException e) { + assertThat(e).hasMessage("indexType is a reserved field name"); + } + }); + }); + } + + private static class TestFieldAware extends FieldAware<TestFieldAware> { + private String fieldName; + private Object attributes; + + @Override + TestFieldAware setFieldImpl(String fieldName, Object attributes) { + this.fieldName = fieldName; + this.attributes = attributes; + return this; + } + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewAuthorizedIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewAuthorizedIndexTest.java new file mode 100644 index 00000000000..4628a01025f --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewAuthorizedIndexTest.java @@ -0,0 +1,129 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.ImmutableMap; +import java.util.Locale; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; + +public class NewAuthorizedIndexTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private String someIndexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); + private Index someIndex = Index.withRelations(someIndexName); + private MapSettings settings = new MapSettings(); + private SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build(); + + @Test + public void constructor_fails_with_IAE_if_index_does_not_support_relations() { + Index simpleIndex = Index.simple(someIndexName); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Index must accept relations"); + + new NewAuthorizedIndex(simpleIndex, defaultSettingsConfiguration); + } + + @Test + public void getMainType_returns_main_type_of_authorization_for_index_of_constructor() { + NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration); + + assertThat(underTest.getMainType()).isEqualTo(IndexType.main(someIndex, "auth")); + } + + @Test + public void build_fails_if_no_relation_mapping_has_been_created() { + NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("At least one relation mapping must be defined"); + + underTest.build(); + } + + @Test + public void build_enforces_routing() { + NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "donut")); + + BuiltIndex<NewAuthorizedIndex> builtIndex = underTest.build(); + + assertThat(getAttributeAsMap(builtIndex, "_routing")) + .containsOnly(entry("required", true)); + } + + @Test + public void build_defines_type_field() { + NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "donut")); + + BuiltIndex<NewAuthorizedIndex> builtIndex = underTest.build(); + + Map<String, Object> properties = getProperties(builtIndex); + assertThat(getFieldAsMap(properties, "indexType")) + .isEqualTo(ImmutableMap.of( + "type", "keyword", + "norms", false, + "store", false, + "doc_values", false)); + } + + @Test + public void constructor_creates_mapping_for_authorization_type() { + NewAuthorizedIndex underTest = new NewAuthorizedIndex(someIndex, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "donut")); + + BuiltIndex<NewAuthorizedIndex> builtIndex = underTest.build(); + + Map<String, Object> properties = getProperties(builtIndex); + assertThat(getFieldAsMap(properties, "auth_groupIds")) + .containsOnly(entry("type", "long")); + assertThat(getFieldAsMap(properties, "auth_userIds")) + .containsOnly(entry("type", "long")); + assertThat(getFieldAsMap(properties, "auth_allowAnyone")) + .containsOnly(entry("type", "boolean")); + } + + private static Map<String, Object> getProperties(BuiltIndex<?> index) { + return getAttributeAsMap(index, "properties"); + } + + @SuppressWarnings("unchecked") + private static Map<String, Object> getAttributeAsMap(BuiltIndex<?> index, String attributeKey) { + return (Map<String, Object>) index.getAttributes().get(attributeKey); + } + + @SuppressWarnings("unchecked") + private Map<String, Object> getFieldAsMap(Map<String, Object> properties, String fieldName) { + return (Map<String, Object>) properties.get(fieldName); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java new file mode 100644 index 00000000000..e11262c2e52 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java @@ -0,0 +1,383 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.ImmutableMap; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.Map; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.data.MapEntry.entry; +import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; +import static org.sonar.process.ProcessProperties.Property.SEARCH_REPLICAS; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; + +@RunWith(DataProviderRunner.class) +public class NewIndexTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private static final String someIndexName = randomAlphabetic(5).toLowerCase(); + private MapSettings settings = new MapSettings(); + private SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build(); + + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void getRelations_returns_empty_if_no_relation_added(Index index) { + NewIndex<?> newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration); + + assertThat(newIndex.getRelations()).isEmpty(); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void verify_default_index_settings_in_standalone(Index index) { + Settings underTest = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration) + .getSettings().build(); + + assertThat(underTest.get("index.number_of_shards")).isNotEmpty(); + assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false"); + assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s"); + assertThat(underTest.get("index.number_of_shards")).isEqualTo("1"); + assertThat(underTest.get("index.number_of_replicas")).isEqualTo("0"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void verify_default_index_settings_in_cluster(Index index) { + settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); + Settings underTest = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration).getSettings().build(); + + assertThat(underTest.get("index.number_of_shards")).isNotEmpty(); + assertThat(underTest.get("index.mapper.dynamic")).isEqualTo("false"); + assertThat(underTest.get("index.refresh_interval")).isEqualTo("30s"); + assertThat(underTest.get("index.number_of_shards")).isEqualTo("1"); + assertThat(underTest.get("index.number_of_replicas")).isEqualTo("1"); + } + + @Test + @UseDataProvider("indexAndTypeMappings") + public void define_fields(NewIndex<?> newIndex, TypeMapping typeMapping) { + typeMapping.setField("foo_field", ImmutableMap.of("type", "keyword")); + typeMapping.createBooleanField("boolean_field"); + typeMapping.createByteField("byte_field"); + typeMapping.createDateTimeField("dt_field"); + typeMapping.createDoubleField("double_field"); + typeMapping.createIntegerField("int_field"); + typeMapping.createLongField("long_field"); + typeMapping.createShortField("short_field"); + typeMapping.createUuidPathField("uuid_path_field"); + + assertThat(newIndex.getProperty("foo_field")).isInstanceOf(Map.class); + assertThat((Map) newIndex.getProperty("foo_field")).containsEntry("type", "keyword"); + assertThat((Map) newIndex.getProperty("byte_field")).isNotEmpty(); + assertThat((Map) newIndex.getProperty("double_field")).isNotEmpty(); + assertThat((Map) newIndex.getProperty("dt_field")).isNotEmpty(); + assertThat((Map) newIndex.getProperty("int_field")).containsEntry("type", "integer"); + assertThat((Map) newIndex.getProperty("long_field")).isNotEmpty(); + assertThat((Map) newIndex.getProperty("short_field")).isNotEmpty(); + assertThat((Map) newIndex.getProperty("uuid_path_field")).isNotEmpty(); + assertThat((Map) newIndex.getProperty("unknown")).isNull(); + } + + @Test + @UseDataProvider("indexAndTypeMappings") + public void define_string_field(NewIndex<?> newIndex, TypeMapping typeMapping) { + typeMapping.keywordFieldBuilder("basic_field").build(); + typeMapping.keywordFieldBuilder("not_searchable_field").disableSearch().build(); + typeMapping.keywordFieldBuilder("all_capabilities_field") + .addSubFields( + DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER, + DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER, + DefaultIndexSettingsElement.SORTABLE_ANALYZER) + .build(); + typeMapping.keywordFieldBuilder("dumb_text_storage") + .disableSearch() + .disableNorms() + .disableSortingAndAggregating() + .build(); + + Map<String, Object> props = (Map) newIndex.getProperty("basic_field"); + assertThat(props.get("type")).isEqualTo("keyword"); + assertThat(props.get("index")).isEqualTo("true"); + assertThat(props.get("fields")).isNull(); + + props = (Map) newIndex.getProperty("not_searchable_field"); + assertThat(props.get("type")).isEqualTo("keyword"); + assertThat(props.get("index")).isEqualTo("false"); + assertThat(props.get("norms")).isEqualTo("true"); + assertThat(props.get("store")).isEqualTo("false"); + assertThat(props.get("doc_values")).isEqualTo("true"); + assertThat(props.get("fields")).isNull(); + + props = (Map) newIndex.getProperty("all_capabilities_field"); + assertThat(props.get("type")).isEqualTo("keyword"); + // no need to test values, it's not the scope of this test + assertThat((Map) props.get("fields")).isNotEmpty(); + + props = (Map) newIndex.getProperty("dumb_text_storage"); + assertThat(props.get("type")).isEqualTo("keyword"); + assertThat(props.get("index")).isEqualTo("false"); + assertThat(props.get("norms")).isEqualTo("false"); + assertThat(props.get("store")).isEqualTo("false"); + assertThat(props.get("doc_values")).isEqualTo("false"); + assertThat(props.get("fields")).isNull(); + } + + @Test + @UseDataProvider("indexAndTypeMappings") + public void define_nested_field(NewIndex<?> newIndex, TypeMapping typeMapping) { + typeMapping.nestedFieldBuilder("measures") + .addKeywordField("key") + .addDoubleField("value") + .build(); + Map<String, Object> result = (Map) newIndex.getProperty("measures"); + + assertThat(result.get("type")).isEqualTo("nested"); + Map<String, Map<String, Object>> subProperties = (Map) result.get("properties"); + assertThat(subProperties.get("key").get("type")).isEqualTo("keyword"); + assertThat(subProperties.get("value").get("type")).isEqualTo("double"); + } + + @Test + @UseDataProvider("indexAndTypeMappings") + public void fail_when_nested_with_no_field(NewIndex<?> newIndex, TypeMapping typeMapping) { + NestedFieldBuilder<TypeMapping> nestedFieldBuilder = typeMapping.nestedFieldBuilder("measures"); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("At least one sub-field must be declared in nested property 'measures'"); + + nestedFieldBuilder.build(); + } + + @Test + @UseDataProvider("indexAndTypeMappings") + public void use_doc_values_by_default(NewIndex<?> newIndex, TypeMapping typeMapping) { + typeMapping.keywordFieldBuilder("the_doc_value").build(); + + Map<String, Object> props = (Map) newIndex.getProperty("the_doc_value"); + assertThat(props.get("type")).isEqualTo("keyword"); + assertThat(props.get("doc_values")).isEqualTo("true"); + } + + @DataProvider + public static Object[][] indexAndTypeMappings() { + String indexName = randomAlphabetic(5).toLowerCase(); + MapSettings settings = new MapSettings(); + SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build(); + Index index = Index.withRelations(indexName); + IndexMainType mainType = IndexType.main(index, "foo"); + SimplestNewIndex newIndex = new SimplestNewIndex(mainType, defaultSettingsConfiguration); + TypeMapping mainMapping = newIndex.createTypeMapping(mainType); + TypeMapping relationMapping = newIndex.createTypeMapping(IndexType.relation(mainType, "bar")); + return new Object[][] { + {newIndex, mainMapping}, + {newIndex, relationMapping}, + }; + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void default_shards_and_replicas(Index index) { + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void five_shards_and_one_replica_by_default_on_cluster(Index index) { + settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("5"); + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void customize_number_of_shards(Index index) { + settings.setProperty("sonar.search." + index.getName() + ".shards", "3"); + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_SHARDS)).isEqualTo("3"); + // keep default value + assertThat(newIndex.getSetting(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void default_number_of_replicas_on_standalone_instance_must_be_0(Index index) { + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void default_number_of_replicas_on_non_enabled_cluster_must_be_0(Index index) { + settings.setProperty(CLUSTER_ENABLED.getKey(), "false"); + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void default_number_of_replicas_on_cluster_instance_must_be_1(Index index) { + settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("1"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void when_number_of_replicas_on_cluster_is_specified_to_zero_default_value_must_not_be_used(Index index) { + settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); + settings.setProperty(SEARCH_REPLICAS.getKey(), "0"); + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("0"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void index_defined_with_specified_number_of_replicas_when_cluster_enabled(Index index) { + settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); + settings.setProperty(SEARCH_REPLICAS.getKey(), "3"); + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build()); + + assertThat(newIndex.getSettings().get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS)).isEqualTo("3"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void fail_when_replica_customization_cant_be_parsed(Index index) { + settings.setProperty(CLUSTER_ENABLED.getKey(), "true"); + settings.setProperty(SEARCH_REPLICAS.getKey(), "ꝱꝲꝳପ"); + SettingsConfiguration settingsConfiguration = newBuilder(settings.asConfig()).setDefaultNbOfShards(5).build(); + IndexMainType mainType = IndexType.main(index, "foo"); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("The property 'sonar.search.replicas' is not an int value: For input string: \"ꝱꝲꝳପ\""); + + new SimplestNewIndex(mainType, settingsConfiguration); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void in_standalone_searchReplicas_is_not_overridable(Index index) { + settings.setProperty(SEARCH_REPLICAS.getKey(), "5"); + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration); + + assertThat(newIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void index_with_source(Index index) { + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration); + newIndex.setEnableSource(true); + + assertThat(newIndex).isNotNull(); + assertThat(getAttributeAsMap(newIndex, "_source")).containsExactly(entry("enabled", true)); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void index_without_source(Index index) { + NewIndex newIndex = new SimplestNewIndex(IndexType.main(index, "foo"), defaultSettingsConfiguration); + newIndex.setEnableSource(false); + + assertThat(getAttributeAsMap(newIndex, "_source")).containsExactly(entry("enabled", false)); + } + + @Test + public void createTypeMapping_with_IndexRelationType_fails_with_ISE_if_index_does_not_allow_relations() { + IndexType.IndexRelationType indexRelationType = IndexType.relation(IndexType.main(Index.withRelations(someIndexName), "bar"), "bar"); + + Index index = Index.simple(someIndexName); + IndexMainType mainType = IndexType.main(index, "foo"); + NewIndex underTest = new NewIndex(index, defaultSettingsConfiguration) { + @Override + public IndexMainType getMainType() { + return mainType; + } + + @Override + public BuiltIndex build() { + throw new UnsupportedOperationException("build not implemented"); + } + }; + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Index is not configured to accept relations. Update IndexDefinition.Descriptor instance for this index"); + + underTest.createTypeMapping(indexRelationType); + } + + @DataProvider + public static Object[][] indexWithAndWithoutRelations() { + return new Object[][] { + {Index.simple(someIndexName)}, + {Index.withRelations(someIndexName)} + }; + } + + private static Map<String, Object> getAttributeAsMap(NewIndex newIndex, String attributeKey) { + return (Map<String, Object>) newIndex.getAttributes().get(attributeKey); + } + + private static final class SimplestNewIndex extends NewIndex<SimplestNewIndex> { + private final IndexMainType mainType; + + public SimplestNewIndex(IndexMainType mainType, SettingsConfiguration settingsConfiguration) { + super(mainType.getIndex(), settingsConfiguration); + this.mainType = mainType; + } + + @Override + public IndexMainType getMainType() { + return mainType; + } + + @Override + public BuiltIndex<SimplestNewIndex> build() { + return new BuiltIndex<>(this); + } + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewRegularIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewRegularIndexTest.java new file mode 100644 index 00000000000..92894309005 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewRegularIndexTest.java @@ -0,0 +1,197 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import com.google.common.collect.ImmutableMap; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.Locale; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.sonar.server.es.newindex.DefaultIndexSettings.NORMS; +import static org.sonar.server.es.newindex.DefaultIndexSettings.STORE; +import static org.sonar.server.es.newindex.DefaultIndexSettings.TYPE; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; + +@RunWith(DataProviderRunner.class) +public class NewRegularIndexTest { + private static final String SOME_INDEX_NAME = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private MapSettings settings = new MapSettings(); + private SettingsConfiguration defaultSettingsConfiguration = newBuilder(settings.asConfig()).build(); + + @Test + @UseDataProvider("indexes") + public void getMainType_fails_with_ISE_if_createTypeMapping_with_IndexMainType_has_not_been_called(Index index) { + NewRegularIndex newIndex = new NewRegularIndex(index, defaultSettingsConfiguration); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Main type has not been defined"); + + newIndex.getMainType(); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void createTypeMapping_with_IndexMainType_fails_with_ISE_if_called_twice(Index index) { + NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.main(index, "foo")); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Main type can only be defined once"); + + underTest.createTypeMapping(IndexType.main(index, "foo")); + } + + @Test + public void createTypeMapping_with_IndexRelationType_fails_with_ISE_if_called_before_createType_with_IndexMainType() { + Index index = Index.withRelations(SOME_INDEX_NAME); + NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Mapping for main type must be created first"); + + underTest.createTypeMapping(IndexType.relation(IndexType.main(index, "foo"), "bar")); + } + + @Test + public void createTypeMapping_with_IndexRelationType_fails_with_IAE_if_mainType_does_not_match_defined_one() { + Index index = Index.withRelations(SOME_INDEX_NAME); + IndexType.IndexMainType mainType = IndexType.main(index, "foo"); + NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration); + underTest.createTypeMapping(mainType); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("main type of relation must be "+ mainType); + + underTest.createTypeMapping(IndexType.relation(IndexType.main(index, "donut"), "bar")); + } + + @Test + @UseDataProvider("indexWithAndWithoutRelations") + public void build_fails_with_ISE_if_no_mainType_is_defined(Index index) { + NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("Mapping for main type must be defined"); + + underTest.build(); + } + + @DataProvider + public static Object[][] indexWithAndWithoutRelations() { + return new Object[][] { + {Index.simple(SOME_INDEX_NAME)}, + {Index.withRelations(SOME_INDEX_NAME)} + }; + } + + @Test + public void build_fails_with_ISE_if_index_accepts_relations_and_none_is_defined() { + Index index = Index.withRelations(SOME_INDEX_NAME); + NewRegularIndex underTest = new NewRegularIndex(index, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.main(index, "foo")); + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("At least one relation must be defined when index accepts relations"); + + underTest.build(); + } + + @Test + public void build_does_not_enforce_routing_if_mainType_does_not_accepts_relations() { + Index someIndex = Index.simple(SOME_INDEX_NAME); + NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.main(someIndex, "foo")); + + BuiltIndex<NewRegularIndex> builtIndex = underTest.build(); + + assertThat(builtIndex.getAttributes().get("_routing")) + .isNull(); + } + + @Test + public void build_enforces_routing_if_mainType_accepts_relations() { + Index someIndex = Index.withRelations(SOME_INDEX_NAME); + NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.main(someIndex, "foo")); + underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "bar")); + + BuiltIndex<NewRegularIndex> builtIndex = underTest.build(); + + assertThat((Map<String, Object>) builtIndex.getAttributes().get("_routing")) + .contains(entry("required", true)); + } + + @Test + public void build_does_not_define_type_field_if_index_does_not_accept_relations() { + Index someIndex = Index.simple(SOME_INDEX_NAME); + NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.main(someIndex, "foo")); + + BuiltIndex<NewRegularIndex> builtIndex = underTest.build(); + + Map<String, Object> properties = (Map<String, Object>) builtIndex.getAttributes().get("properties"); + assertThat(properties.get("indexType")) + .isNull(); + } + + @Test + public void build_defines_type_field_if_index_accepts_relations() { + Index someIndex = Index.withRelations(SOME_INDEX_NAME); + NewRegularIndex underTest = new NewRegularIndex(someIndex, defaultSettingsConfiguration); + underTest.createTypeMapping(IndexType.main(someIndex, "foo")); + underTest.createTypeMapping(IndexType.relation(underTest.getMainType(), "bar")); + + BuiltIndex<NewRegularIndex> builtIndex = underTest.build(); + + Map<String, Object> properties = (Map<String, Object>) builtIndex.getAttributes().get("properties"); + assertThat((Map) properties.get("indexType")) + .isEqualTo(ImmutableMap.of( + TYPE, "keyword", + NORMS, false, + STORE, false, + "doc_values", false)); + } + + @DataProvider + public static Object[][] indexes() { + String someIndexName = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); + return new Object[][] { + {Index.simple(someIndexName)}, + {Index.withRelations(someIndexName)} + }; + } + +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/TestNewIndex.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/TestNewIndex.java new file mode 100644 index 00000000000..01cd8e8c63b --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/TestNewIndex.java @@ -0,0 +1,56 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.newindex; + +import org.sonar.server.es.IndexType; + +public final class TestNewIndex extends NewIndex<TestNewIndex> { + private final IndexType.IndexMainType mainType; + private final TypeMapping mainTypeMapping; + + public TestNewIndex(IndexType.IndexMainType mainType, SettingsConfiguration settingsConfiguration) { + super(mainType.getIndex(), settingsConfiguration); + this.mainType = mainType; + mainTypeMapping = super.createTypeMapping(mainType); + } + + @Override + public IndexType.IndexMainType getMainType() { + return mainType; + } + + public TypeMapping getMainTypeMapping() { + return mainTypeMapping; + } + + @Override + public BuiltIndex<TestNewIndex> build() { + return new BuiltIndex<>(this); + } + + public TestNewIndex addRelation(String name) { + super.createTypeMapping(IndexType.relation(mainType, name)); + return this; + } + + public TypeMapping createRelationMapping(String name) { + return super.createTypeMapping(IndexType.relation(mainType, name)); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java index 1b2dd6ca360..88ea79f5b40 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java @@ -19,15 +19,18 @@ */ package org.sonar.server.es.request; +import java.util.Locale; +import java.util.Random; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.common.unit.TimeValue; import org.junit.Rule; import org.junit.Test; -import org.sonar.api.utils.System2; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; +import org.sonar.server.es.Index; +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -41,21 +44,21 @@ public class ProxyCreateIndexRequestBuilderTest { @Test public void create_index() { - CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndexName()); + CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex()); requestBuilder.get(); } @Test public void to_string() { - String indexName = generateNewIndexName(); - assertThat(es.client().prepareCreate(indexName).toString()).contains("ES create index '" + indexName + "'"); + Index index = generateNewIndex(); + assertThat(es.client().prepareCreate(index).toString()).contains("ES create index '" + index.getName() + "'"); } @Test public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndexName()); + CreateIndexRequestBuilder requestBuilder = es.client().prepareCreate(generateNewIndex()); requestBuilder.get(); assertThat(logTester.logs()).hasSize(1); } @@ -63,7 +66,7 @@ public class ProxyCreateIndexRequestBuilderTest { @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().prepareCreate(generateNewIndexName()).get("1"); + es.client().prepareCreate(generateNewIndex()).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -73,7 +76,7 @@ public class ProxyCreateIndexRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareCreate(generateNewIndexName()).get(TimeValue.timeValueMinutes(1)); + es.client().prepareCreate(generateNewIndex()).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -83,15 +86,16 @@ public class ProxyCreateIndexRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().prepareCreate(generateNewIndexName()).execute(); + es.client().prepareCreate(generateNewIndex()).execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); } } - private static String generateNewIndexName(){ - return "index_" + Long.toString(System2.INSTANCE.now()); + private static Index generateNewIndex(){ + String name = randomAlphabetic(10).toLowerCase(Locale.ENGLISH); + return new Random().nextBoolean() ? Index.simple(name) : Index.withRelations(name); } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java index ed3c07391df..e5303f75c59 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java @@ -25,7 +25,9 @@ import org.junit.Test; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -34,24 +36,26 @@ public class ProxyDeleteRequestBuilderTest { @Rule public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); - @Rule public LogTester logTester = new LogTester(); + private final Index index = Index.simple("fakes"); + private final IndexType.IndexMainType mainType = IndexType.main(index, "fake"); + @Test public void delete() { - es.client().prepareDelete("fakes", "fake", "the_id").get(); + es.client().prepareDelete(mainType, "the_id").get(); } @Test public void to_string() { - assertThat(es.client().prepareDelete("fakes", "fake", "the_id").toString()).isEqualTo("ES delete request of doc the_id in index fakes/fake"); + assertThat(es.client().prepareDelete(mainType, "the_id").toString()).isEqualTo("ES delete request of doc the_id in index fakes/fake"); } @Test public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - es.client().prepareDelete("fakes", "fake", "the_id").get(); + es.client().prepareDelete(mainType, "the_id").get(); assertThat(logTester.logs()).hasSize(1); } @@ -59,7 +63,7 @@ public class ProxyDeleteRequestBuilderTest { @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().prepareDelete("fakes", "fake", "the_id").get("1"); + es.client().prepareDelete(mainType, "the_id").get("1"); fail(); } catch (UnsupportedOperationException e) { assertThat(e).hasMessage("Not yet implemented"); @@ -69,7 +73,7 @@ public class ProxyDeleteRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareDelete("fakes", "fake", "the_id").get(TimeValue.timeValueMinutes(1)); + es.client().prepareDelete(mainType, "the_id").get(TimeValue.timeValueMinutes(1)); fail(); } catch (UnsupportedOperationException e) { assertThat(e).hasMessage("Not yet implemented"); @@ -79,7 +83,7 @@ public class ProxyDeleteRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().prepareDelete("fakes", "fake", "the_id").execute(); + es.client().prepareDelete(mainType, "the_id").execute(); fail(); } catch (UnsupportedOperationException e) { assertThat(e).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java index ed72c7f428f..2c30ddd9931 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java @@ -19,20 +19,26 @@ */ package org.sonar.server.es.request; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.common.unit.TimeValue; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.Index; +import org.sonar.server.es.newindex.FakeIndexDefinition; import org.sonar.server.es.IndexType; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; -import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE; +import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; +@RunWith(DataProviderRunner.class) public class ProxyGetRequestBuilderTest { @Rule @@ -45,14 +51,15 @@ public class ProxyGetRequestBuilderTest { public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey") + es.client().prepareGet(TYPE_FAKE, "ruleKey") .get(); assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); } @Test - public void fail_to_get_bad_query() { - GetRequestBuilder requestBuilder = es.client().prepareGet(new IndexType("unknown", "test"), "rule1"); + @UseDataProvider("mainAndRelationWithUnknownIndex") + public void prepareGet_fails_if_index_unknown(IndexType indexType) { + GetRequestBuilder requestBuilder = es.client().prepareGet(indexType, "rule1"); try { requestBuilder.get(); fail(); @@ -62,10 +69,19 @@ public class ProxyGetRequestBuilderTest { } } + @DataProvider + public static Object[][] mainAndRelationWithUnknownIndex() { + IndexType.IndexMainType mainType = IndexType.main(Index.withRelations("unknown"), "test"); + return new Object[][] { + {mainType}, + {IndexType.relation(mainType, "donut")} + }; + } + @Test public void get_with_string_timeout_is_not_implemented() { try { - es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey").get("1"); + es.client().prepareGet(TYPE_FAKE, "ruleKey").get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -75,7 +91,7 @@ public class ProxyGetRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey").get(TimeValue.timeValueMinutes(1)); + es.client().prepareGet(TYPE_FAKE, "ruleKey").get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -85,7 +101,7 @@ public class ProxyGetRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().prepareGet(INDEX_TYPE_FAKE, "ruleKey").execute(); + es.client().prepareGet(TYPE_FAKE, "ruleKey").execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java index 084095536d1..cab9b711576 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java @@ -19,21 +19,29 @@ */ package org.sonar.server.es.request; +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; import org.elasticsearch.action.DocWriteResponse.Result; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.common.unit.TimeValue; import org.junit.Rule; import org.junit.Test; +import org.junit.runner.RunWith; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.Index; +import org.sonar.server.es.newindex.FakeIndexDefinition; import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; +import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; +@RunWith(DataProviderRunner.class) public class ProxyIndexRequestBuilderTest { @Rule @@ -44,7 +52,7 @@ public class ProxyIndexRequestBuilderTest { @Test public void index_with_index_type_and_id() { - IndexResponse response = es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE) + IndexResponse response = es.client().prepareIndex(TYPE_FAKE) .setSource(FakeIndexDefinition.newDoc(42).getFields()) .get(); assertThat(response.getResult()).isSameAs(Result.CREATED); @@ -53,7 +61,7 @@ public class ProxyIndexRequestBuilderTest { @Test public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - IndexResponse response = es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE) + IndexResponse response = es.client().prepareIndex(TYPE_FAKE) .setSource(FakeIndexDefinition.newDoc(42).getFields()) .get(); assertThat(response.getResult()).isSameAs(Result.CREATED); @@ -61,33 +69,31 @@ public class ProxyIndexRequestBuilderTest { } @Test - public void fail_if_bad_query() { - IndexRequestBuilder requestBuilder = es.client().prepareIndex(new IndexType("unknownIndex", "unknownType")); + @UseDataProvider("mainOrRelationType") + public void fail_if_bad_query(IndexType indexType) { + IndexRequestBuilder requestBuilder = es.client().prepareIndex(indexType); try { requestBuilder.get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'unknownIndex' on type 'unknownType'"); + assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'foo' on type 'bar'"); } } - @Test - public void fail_if_bad_query_with_basic_profiling() { - IndexRequestBuilder requestBuilder = es.client().prepareIndex(new IndexType("unknownIndex", "unknownType")); - try { - requestBuilder.get(); - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'unknownIndex' on type 'unknownType'"); - } + @DataProvider + public static Object[][] mainOrRelationType() { + IndexMainType mainType = IndexType.main(Index.withRelations("foo"), "bar"); + return new Object[][] { + {mainType}, + {IndexType.relation(mainType, "donut")} + }; } @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).get("1"); + es.client().prepareIndex(TYPE_FAKE).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -97,7 +103,7 @@ public class ProxyIndexRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).get(TimeValue.timeValueMinutes(1)); + es.client().prepareIndex(TYPE_FAKE).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -107,7 +113,7 @@ public class ProxyIndexRequestBuilderTest { @Test public void do_not_support_execute_method() { try { - es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).execute(); + es.client().prepareIndex(TYPE_FAKE).execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java index 96db5c90eff..312585d5d00 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java @@ -25,7 +25,8 @@ import org.junit.Test; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.Index; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -40,41 +41,28 @@ public class ProxyIndicesExistsRequestBuilderTest { @Test public void exists() { - assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.INDEX).get().isExists()).isTrue(); - assertThat(es.client().prepareIndicesExist("unknown").get().isExists()).isFalse(); + assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get().isExists()).isTrue(); + assertThat(es.client().prepareIndicesExist(Index.simple("unknown")).get().isExists()).isFalse(); } @Test public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - es.client().prepareIndicesExist(FakeIndexDefinition.INDEX).get(); + es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get(); assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); } @Test - public void fail_to_exists() { - try { - es.client().prepareIndicesExist().get(); - - // expected to fail because elasticsearch is not correctly configured, but that does not matter - fail(); - } catch (Exception e) { - assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES indices exists request"); - } - } - - @Test public void to_string() { - assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.INDEX).toString()).isEqualTo("ES indices exists request on indices 'fakes'"); + assertThat(es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES indices exists request on indices 'fakes'"); } @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().prepareIndicesExist().get("1"); + es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -84,7 +72,7 @@ public class ProxyIndicesExistsRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareIndicesExist().get(TimeValue.timeValueMinutes(1)); + es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -94,7 +82,7 @@ public class ProxyIndicesExistsRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().prepareIndicesExist().execute(); + es.client().prepareIndicesExist(FakeIndexDefinition.DESCRIPTOR).execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java index e12b3b7f84f..d159cfe0a45 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java @@ -25,7 +25,8 @@ import org.junit.Test; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.Index; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -40,12 +41,12 @@ public class ProxyIndicesStatsRequestBuilderTest { @Test public void stats() { - es.client().prepareStats(FakeIndexDefinition.INDEX).get(); + es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(); } @Test public void to_string() { - assertThat(es.client().prepareStats(FakeIndexDefinition.INDEX).setIndices("rules").toString()).isEqualTo("ES indices stats request on indices 'rules'"); + assertThat(es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).setIndices("rules").toString()).isEqualTo("ES indices stats request on indices 'rules'"); assertThat(es.client().prepareStats().toString()).isEqualTo("ES indices stats request"); } @@ -53,7 +54,7 @@ public class ProxyIndicesStatsRequestBuilderTest { public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - es.client().prepareStats(FakeIndexDefinition.INDEX).get(); + es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(); assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); } @@ -61,7 +62,7 @@ public class ProxyIndicesStatsRequestBuilderTest { @Test public void fail_to_stats() { try { - es.client().prepareStats("unknown").get(); + es.client().prepareStats(Index.simple("unknown")).get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); @@ -72,7 +73,7 @@ public class ProxyIndicesStatsRequestBuilderTest { @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().prepareStats(FakeIndexDefinition.INDEX).get("1"); + es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -82,7 +83,7 @@ public class ProxyIndicesStatsRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareStats(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1)); + es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -92,7 +93,7 @@ public class ProxyIndicesStatsRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().prepareStats(FakeIndexDefinition.INDEX).execute(); + es.client().prepareStats(FakeIndexDefinition.DESCRIPTOR).execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java index df01d50a0a6..68b028ecfe5 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java @@ -26,7 +26,7 @@ import org.junit.rules.ExpectedException; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java index 00c7e1b19be..36d8ba25e11 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java @@ -29,7 +29,7 @@ import org.junit.Test; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -44,7 +44,7 @@ public class ProxyPutMappingRequestBuilderTest { @Test public void put_mapping() { - PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.INDEX) + PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR) .setType(FakeIndexDefinition.TYPE) .setSource(mapDomain()); requestBuilder.get(); @@ -52,9 +52,9 @@ public class ProxyPutMappingRequestBuilderTest { @Test public void to_string() { - assertThat(es.client().preparePutMapping(FakeIndexDefinition.INDEX).setSource(mapDomain()).toString()) + assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setSource(mapDomain()).toString()) .isEqualTo("ES put mapping request on indices 'fakes' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'"); - assertThat(es.client().preparePutMapping(FakeIndexDefinition.INDEX).setType(FakeIndexDefinition.TYPE).setSource(mapDomain()).toString()) + assertThat(es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).setType(FakeIndexDefinition.TYPE).setSource(mapDomain()).toString()) .isEqualTo("ES put mapping request on indices 'fakes' on type 'fake' with source '{\"dynamic\":false,\"_all\":{\"enabled\":false}}'"); } @@ -62,7 +62,7 @@ public class ProxyPutMappingRequestBuilderTest { public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.INDEX) + PutMappingRequestBuilder requestBuilder = es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR) .setType(FakeIndexDefinition.TYPE) .setSource(mapDomain()); requestBuilder.get(); @@ -73,7 +73,7 @@ public class ProxyPutMappingRequestBuilderTest { @Test public void fail_on_bad_query() { try { - es.client().preparePutMapping().get(); + es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); @@ -84,7 +84,7 @@ public class ProxyPutMappingRequestBuilderTest { @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().preparePutMapping().get("1"); + es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -94,7 +94,7 @@ public class ProxyPutMappingRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().preparePutMapping().get(TimeValue.timeValueMinutes(1)); + es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -104,7 +104,7 @@ public class ProxyPutMappingRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().preparePutMapping().execute(); + es.client().preparePutMapping(FakeIndexDefinition.DESCRIPTOR).execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java index 19f8fb29e99..12b68a7af9e 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java @@ -26,7 +26,8 @@ import org.junit.Test; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.Index; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -41,21 +42,20 @@ public class ProxyRefreshRequestBuilderTest { @Test public void refresh() { - RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.INDEX); + RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR); requestBuilder.get(); } @Test public void to_string() { - assertThat(es.client().prepareRefresh(FakeIndexDefinition.INDEX).toString()).isEqualTo("ES refresh request on indices 'fakes'"); - assertThat(es.client().prepareRefresh().toString()).isEqualTo("ES refresh request"); + assertThat(es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).toString()).isEqualTo("ES refresh request on indices 'fakes'"); } @Test public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.INDEX); + RefreshRequestBuilder requestBuilder = es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR); requestBuilder.get(); assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); } @@ -63,7 +63,7 @@ public class ProxyRefreshRequestBuilderTest { @Test public void fail_to_refresh() { try { - es.client().prepareRefresh("unknown").get(); + es.client().prepareRefresh(Index.simple("unknown")).get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); @@ -74,7 +74,7 @@ public class ProxyRefreshRequestBuilderTest { @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().prepareRefresh(FakeIndexDefinition.INDEX).get("1"); + es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -84,7 +84,7 @@ public class ProxyRefreshRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareRefresh(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1)); + es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -94,7 +94,7 @@ public class ProxyRefreshRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().prepareRefresh(FakeIndexDefinition.INDEX).execute(); + es.client().prepareRefresh(FakeIndexDefinition.DESCRIPTOR).execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java index 03ff5c1318f..7edb2970d31 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java @@ -25,8 +25,8 @@ import org.junit.Test; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; -import org.sonar.server.es.IndexType; +import org.sonar.server.es.Index; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -41,40 +41,43 @@ public class ProxySearchRequestBuilderTest { @Test public void search() { - es.client().prepareSearch(FakeIndexDefinition.INDEX).get(); + es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(); } @Test public void to_string() { - assertThat(es.client().prepareSearch(FakeIndexDefinition.INDEX).setTypes(FakeIndexDefinition.TYPE).toString()).contains("ES search request '").contains( - "' on indices '[fakes]' on types '[fake]'"); - assertThat(es.client().prepareSearch(FakeIndexDefinition.INDEX).toString()).contains("ES search request '").contains("' on indices '[fakes]'"); - assertThat(es.client().prepareSearch(new IndexType[0]).toString()).contains("ES search request"); + assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).setTypes(FakeIndexDefinition.TYPE).toString()).contains("ES search request '") + .contains("' on indices '[fakes]' on types '[fake]'"); + assertThat(es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).toString()) + .contains("ES search request '") + .contains("' on indices '[fakes]'"); } @Test public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - es.client().prepareSearch(FakeIndexDefinition.INDEX).get(); + es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(); assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); } @Test public void fail_to_search_bad_query() { try { - es.client().prepareSearch("non-existing-index").get(); + es.client().prepareSearch(Index.simple("unknown")).get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); - assertThat(e.getMessage()).contains("Fail to execute ES search request 'SearchRequest{").contains("}' on indices '[non-existing-index]'"); + assertThat(e.getMessage()) + .contains("Fail to execute ES search request 'SearchRequest{") + .contains("}' on indices '[unknown]'"); } } @Test public void get_with_string_timeout_is_not_yet_implemented() { try { - es.client().prepareSearch(FakeIndexDefinition.INDEX).get("1"); + es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -84,7 +87,7 @@ public class ProxySearchRequestBuilderTest { @Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { - es.client().prepareSearch(FakeIndexDefinition.INDEX).get(TimeValue.timeValueMinutes(1)); + es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); @@ -94,7 +97,7 @@ public class ProxySearchRequestBuilderTest { @Test public void execute_should_throw_an_unsupported_operation_exception() { try { - es.client().prepareSearch(FakeIndexDefinition.INDEX).execute(); + es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR).execute(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(UnsupportedOperationException.class).hasMessage("execute() should not be called as it's used for asynchronous"); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java index 5a4961b7dd6..859f5a643f2 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java @@ -26,7 +26,7 @@ import org.junit.Test; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; @@ -43,7 +43,7 @@ public class ProxySearchScrollRequestBuilderTest { public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); - SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.INDEX) + SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR) .setScroll(TimeValue.timeValueMinutes(1)) .get(); logTester.clear(); @@ -55,7 +55,7 @@ public class ProxySearchScrollRequestBuilderTest { public void no_trace_logs() { logTester.setLevel(LoggerLevel.DEBUG); - SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.INDEX) + SearchResponse response = es.client().prepareSearch(FakeIndexDefinition.DESCRIPTOR) .setScroll(TimeValue.timeValueMinutes(1)) .get(); logTester.clear(); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java index 12412646e35..109c1e6452d 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java @@ -49,7 +49,6 @@ public class ProxyWebServerHealthRequestBuilderTest { @Test public void to_string() { - assertThat(es.client().prepareHealth("rules").toString()).isEqualTo("ES cluster health request on indices 'rules'"); assertThat(es.client().prepareHealth().toString()).isEqualTo("ES cluster health request"); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexDefinitionTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexDefinitionTest.java index a9dd93ec486..43bd47022fb 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexDefinitionTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexDefinitionTest.java @@ -21,10 +21,13 @@ package org.sonar.server.issue.index; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.newindex.NewIndex; import static org.assertj.core.api.Assertions.assertThat; +import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; public class IssueIndexDefinitionTest { @@ -36,12 +39,14 @@ public class IssueIndexDefinitionTest { def.define(underTest); assertThat(underTest.getIndices()).hasSize(1); - NewIndex issuesIndex = underTest.getIndices().get("issues"); - assertThat(issuesIndex).isNotNull(); - assertThat(issuesIndex.getTypes().keySet()).containsOnly("issue", "authorization"); + NewIndex<?> issuesIndex = underTest.getIndices().get("issues"); + IndexType.IndexMainType mainType = IndexType.main(Index.withRelations("issues"), TYPE_AUTHORIZATION); + assertThat(issuesIndex.getMainType()).isEqualTo(mainType); + assertThat(issuesIndex.getRelationsStream()) + .containsOnly(IndexType.relation(mainType, "issue")); // no cluster by default - assertThat(issuesIndex.getSettings().get("index.number_of_shards")).isEqualTo("5"); - assertThat(issuesIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + assertThat(issuesIndex.getSetting("index.number_of_shards")).isEqualTo("5"); + assertThat(issuesIndex.getSetting("index.number_of_replicas")).isEqualTo("0"); } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java index 4354c530108..67264571c84 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java @@ -56,7 +56,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.rules.ExpectedException.none; import static org.sonar.db.component.ComponentTesting.newFileDto; import static org.sonar.server.issue.IssueDocTesting.newDoc; -import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_POROUS_DEFENSES; import static org.sonar.server.issue.index.SecurityStandardHelper.UNKNOWN_STANDARD; import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE_AUTHORIZATION; @@ -82,13 +82,13 @@ public class IssueIndexerTest { @Test public void test_getIndexTypes() { - assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_ISSUE); + assertThat(underTest.getIndexTypes()).containsExactly(TYPE_ISSUE); } @Test public void test_getAuthorizationScope() { AuthorizationScope scope = underTest.getAuthorizationScope(); - assertThat(scope.getIndexType().getIndex()).isEqualTo(INDEX_TYPE_ISSUE.getIndex()); + assertThat(scope.getIndexType().getIndex()).isEqualTo(IssueIndexDefinition.DESCRIPTOR); assertThat(scope.getIndexType().getType()).isEqualTo(TYPE_AUTHORIZATION); Predicate<IndexPermissions> projectPredicate = scope.getProjectPredicate(); @@ -118,7 +118,7 @@ public class IssueIndexerTest { underTest.indexOnStartup(emptySet()); - IssueDoc doc = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class).get(0); + IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0); assertThat(doc.getId()).isEqualTo(issue.getKey()); assertThat(doc.organizationUuid()).isEqualTo(organization.getUuid()); assertThat(doc.assigneeUuid()).isEqualTo(issue.getAssigneeUuid()); @@ -131,8 +131,6 @@ public class IssueIndexerTest { assertThat(doc.creationDate()).isEqualToIgnoringMillis(issue.getIssueCreationDate()); assertThat(doc.directoryPath()).isEqualTo(dir.path()); assertThat(doc.filePath()).isEqualTo(file.path()); - assertThat(doc.getParent()).isEqualTo(project.uuid()); - assertThat(doc.getRouting()).isEqualTo(project.uuid()); assertThat(doc.language()).isEqualTo(issue.getLanguage()); assertThat(doc.line()).isEqualTo(issue.getLine()); // functional date @@ -152,7 +150,7 @@ public class IssueIndexerTest { underTest.indexOnStartup(emptySet()); - IssueDoc doc = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class).get(0); + IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0); assertThat(doc.getCwe()).containsExactlyInAnyOrder("123", "863"); assertThat(doc.getOwaspTop10()).containsExactlyInAnyOrder("a3"); assertThat(doc.getSansTop25()).containsExactlyInAnyOrder(SANS_TOP_25_POROUS_DEFENSES); @@ -160,7 +158,7 @@ public class IssueIndexerTest { @Test public void indexOnStartup_does_not_fail_on_errors_and_does_enable_recovery_mode() { - es.lockWrites(INDEX_TYPE_ISSUE); + es.lockWrites(TYPE_ISSUE); db.issues().insertIssue(organization); try { @@ -170,7 +168,7 @@ public class IssueIndexerTest { } finally { assertThatIndexHasSize(0); assertThatEsQueueTableHasSize(0); - es.unlockWrites(INDEX_TYPE_ISSUE); + es.unlockWrites(TYPE_ISSUE); } } @@ -200,7 +198,7 @@ public class IssueIndexerTest { underTest.indexOnAnalysis(project.uuid()); - assertThat(es.getDocuments(INDEX_TYPE_ISSUE)) + assertThat(es.getDocuments(TYPE_ISSUE)) .extracting(SearchHit::getId) .containsExactlyInAnyOrder(issue.getKey(), "orphan"); } @@ -211,7 +209,7 @@ public class IssueIndexerTest { */ @Test public void indexOnAnalysis_does_not_fail_on_errors_and_does_not_enable_recovery_mode() { - es.lockWrites(INDEX_TYPE_ISSUE); + es.lockWrites(TYPE_ISSUE); IssueDto issue = db.issues().insertIssue(organization); try { @@ -221,7 +219,7 @@ public class IssueIndexerTest { } finally { assertThatIndexHasSize(0); assertThatEsQueueTableHasSize(0); - es.unlockWrites(INDEX_TYPE_ISSUE); + es.unlockWrites(TYPE_ISSUE); } } @@ -273,7 +271,7 @@ public class IssueIndexerTest { public void errors_during_project_deletion_are_recovered() { addIssueToIndex("P1", "I1"); assertThatIndexHasSize(1); - es.lockWrites(INDEX_TYPE_ISSUE); + es.lockWrites(TYPE_ISSUE); IndexingResult result = indexProject("P1", ProjectIndexer.Cause.PROJECT_DELETION); assertThat(result.getTotal()).isEqualTo(1L); @@ -285,7 +283,7 @@ public class IssueIndexerTest { assertThat(result.getFailures()).isEqualTo(1L); assertThatIndexHasSize(1); - es.unlockWrites(INDEX_TYPE_ISSUE); + es.unlockWrites(TYPE_ISSUE); result = recover(); assertThat(result.getTotal()).isEqualTo(1L); @@ -338,7 +336,7 @@ public class IssueIndexerTest { db.getDbClient().issueDao().insert(db.getSession(), issue1, issue2); // index is read-only - es.lockWrites(INDEX_TYPE_ISSUE); + es.lockWrites(TYPE_ISSUE); underTest.commitAndIndexIssues(db.getSession(), asList(issue1, issue2)); @@ -348,7 +346,7 @@ public class IssueIndexerTest { assertThatEsQueueTableHasSize(2); // re-enable write on index - es.unlockWrites(INDEX_TYPE_ISSUE); + es.unlockWrites(TYPE_ISSUE); // emulate the recovery daemon IndexingResult result = recover(); @@ -361,7 +359,7 @@ public class IssueIndexerTest { @Test public void recovery_does_not_fail_if_unsupported_docIdType() { - EsQueueDto item = EsQueueDto.create(INDEX_TYPE_ISSUE.format(), "I1", "unknown", "P1"); + EsQueueDto item = EsQueueDto.create(TYPE_ISSUE.format(), "I1", "unknown", "P1"); db.getDbClient().esQueueDao().insert(db.getSession(), item); db.commit(); @@ -375,7 +373,7 @@ public class IssueIndexerTest { @Test public void indexing_recovers_multiple_errors_on_the_same_issue() { - es.lockWrites(INDEX_TYPE_ISSUE); + es.lockWrites(TYPE_ISSUE); IssueDto issue = db.issues().insertIssue(organization); // three changes on the same issue @@ -387,7 +385,7 @@ public class IssueIndexerTest { // three attempts of indexing are stored in es_queue recovery table assertThatEsQueueTableHasSize(3); - es.unlockWrites(INDEX_TYPE_ISSUE); + es.unlockWrites(TYPE_ISSUE); recover(); assertThatIndexHasOnly(issue); @@ -402,7 +400,7 @@ public class IssueIndexerTest { IssueDto issue1 = db.issues().insertIssue(IssueTesting.newIssue(rule, project, file)); IssueDto issue2 = db.issues().insertIssue(IssueTesting.newIssue(rule, project, file)); - es.lockWrites(INDEX_TYPE_ISSUE); + es.lockWrites(TYPE_ISSUE); IndexingResult result = indexProject(project.uuid(), ProjectIndexer.Cause.PROJECT_DELETION); assertThat(result.getTotal()).isEqualTo(2L); @@ -414,7 +412,7 @@ public class IssueIndexerTest { assertThat(result.getFailures()).isEqualTo(2L); assertThatIndexHasSize(0); - es.unlockWrites(INDEX_TYPE_ISSUE); + es.unlockWrites(TYPE_ISSUE); result = recover(); assertThat(result.getTotal()).isEqualTo(2L); @@ -446,7 +444,7 @@ public class IssueIndexerTest { @Test public void deleteByKeys_does_not_recover_from_errors() { addIssueToIndex("P1", "Issue1"); - es.lockWrites(INDEX_TYPE_ISSUE); + es.lockWrites(TYPE_ISSUE); try { // FIXME : test also message @@ -455,7 +453,7 @@ public class IssueIndexerTest { } finally { assertThatIndexHasOnly("Issue1"); assertThatEsQueueTableHasSize(0); - es.unlockWrites(INDEX_TYPE_ISSUE); + es.unlockWrites(TYPE_ISSUE); } } @@ -481,7 +479,7 @@ public class IssueIndexerTest { new IssueIndexer(es.client(), db.getDbClient(), new IssueIteratorFactory(db.getDbClient())) .index(asList(issueDoc).iterator()); - assertThat(es.countDocuments(INDEX_TYPE_ISSUE)).isEqualTo(1L); + assertThat(es.countDocuments(TYPE_ISSUE)).isEqualTo(1L); } @Test @@ -495,7 +493,7 @@ public class IssueIndexerTest { underTest.indexOnStartup(emptySet()); - IssueDoc doc = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class).get(0); + IssueDoc doc = es.getDocuments(TYPE_ISSUE, IssueDoc.class).get(0); assertThat(doc.getId()).isEqualTo(issue.getKey()); assertThat(doc.organizationUuid()).isEqualTo(organization.getUuid()); assertThat(doc.componentUuid()).isEqualTo(file.uuid()); @@ -505,22 +503,22 @@ public class IssueIndexerTest { } private void addIssueToIndex(String projectUuid, String issueKey) { - es.putDocuments(INDEX_TYPE_ISSUE, + es.putDocuments(TYPE_ISSUE, newDoc().setKey(issueKey).setProjectUuid(projectUuid)); } private void assertThatIndexHasSize(long expectedSize) { - assertThat(es.countDocuments(INDEX_TYPE_ISSUE)).isEqualTo(expectedSize); + assertThat(es.countDocuments(TYPE_ISSUE)).isEqualTo(expectedSize); } private void assertThatIndexHasOnly(IssueDto... expectedIssues) { - assertThat(es.getDocuments(INDEX_TYPE_ISSUE)) + assertThat(es.getDocuments(TYPE_ISSUE)) .extracting(SearchHit::getId) .containsExactlyInAnyOrder(Arrays.stream(expectedIssues).map(IssueDto::getKey).toArray(String[]::new)); } private void assertThatIndexHasOnly(String... expectedKeys) { - List<IssueDoc> issues = es.getDocuments(INDEX_TYPE_ISSUE, IssueDoc.class); + List<IssueDoc> issues = es.getDocuments(TYPE_ISSUE, IssueDoc.class); assertThat(issues).extracting(IssueDoc::key).containsOnly(expectedKeys); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java index 8136c1e9c7a..11e543901f5 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java @@ -43,12 +43,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_CREATION; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_DELETION; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_KEY_UPDATE; import static org.sonar.server.es.ProjectIndexer.Cause.PROJECT_TAGS_UPDATE; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; public class ProjectMeasuresIndexerTest { @@ -65,7 +66,7 @@ public class ProjectMeasuresIndexerTest { public void index_nothing() { underTest.indexOnStartup(emptySet()); - assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isZero(); + assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isZero(); } @Test @@ -158,7 +159,7 @@ public class ProjectMeasuresIndexerTest { db.getDbClient().componentDao().delete(db.getSession(), project.getId()); IndexingResult result = indexProject(project, PROJECT_DELETION); - assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getSuccess()).isEqualTo(1L); } @@ -170,13 +171,13 @@ public class ProjectMeasuresIndexerTest { underTest.index(db.getSession(), emptyList()); - assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0); } @Test public void errors_during_indexing_are_recovered() { ComponentDto project = db.components().insertPrivateProject(); - es.lockWrites(INDEX_TYPE_PROJECT_MEASURES); + es.lockWrites(TYPE_PROJECT_MEASURES); IndexingResult result = indexProject(project, PROJECT_CREATION); assertThat(result.getTotal()).isEqualTo(1L); @@ -186,10 +187,10 @@ public class ProjectMeasuresIndexerTest { result = recover(); assertThat(result.getTotal()).isEqualTo(1L); assertThat(result.getFailures()).isEqualTo(1L); - assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0); assertThatEsQueueTableHasSize(1); - es.unlockWrites(INDEX_TYPE_PROJECT_MEASURES); + es.unlockWrites(TYPE_PROJECT_MEASURES); result = recover(); assertThat(result.getTotal()).isEqualTo(1L); @@ -205,7 +206,7 @@ public class ProjectMeasuresIndexerTest { underTest.indexOnAnalysis(branch.uuid()); - assertThat(es.countDocuments(INDEX_TYPE_PROJECT_MEASURES)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_PROJECT_MEASURES)).isEqualTo(0); } private IndexingResult indexProject(ComponentDto project, ProjectIndexer.Cause cause) { @@ -217,8 +218,10 @@ public class ProjectMeasuresIndexerTest { private void assertThatProjectHasTag(ComponentDto project, String expectedTag) { SearchRequestBuilder request = es.client() - .prepareSearch(INDEX_TYPE_PROJECT_MEASURES) - .setQuery(boolQuery().filter(termQuery(FIELD_TAGS, expectedTag))); + .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) + .setQuery(boolQuery() + .filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())) + .filter(termQuery(FIELD_TAGS, expectedTag))); assertThat(request.get().getHits().getHits()) .extracting(SearchHit::getId) .contains(project.uuid()); @@ -229,12 +232,12 @@ public class ProjectMeasuresIndexerTest { } private void assertThatIndexContainsOnly(SnapshotDto... expectedProjects) { - assertThat(es.getIds(INDEX_TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder( + assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder( Arrays.stream(expectedProjects).map(SnapshotDto::getComponentUuid).toArray(String[]::new)); } private void assertThatIndexContainsOnly(ComponentDto... expectedProjects) { - assertThat(es.getIds(INDEX_TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder( + assertThat(es.getIds(TYPE_PROJECT_MEASURES)).containsExactlyInAnyOrder( Arrays.stream(expectedProjects).map(ComponentDto::uuid).toArray(String[]::new)); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/permission/index/AuthorizationDocTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/permission/index/AuthorizationDocTest.java new file mode 100644 index 00000000000..0d4e5368139 --- /dev/null +++ b/server/sonar-server-common/src/test/java/org/sonar/server/permission/index/AuthorizationDocTest.java @@ -0,0 +1,182 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.permission.index; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import java.util.List; +import java.util.Random; +import java.util.stream.IntStream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.fail; + +@RunWith(DataProviderRunner.class) +public class AuthorizationDocTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void idOf_returns_argument_with_a_prefix() { + String s = randomAlphabetic(12); + + assertThat(AuthorizationDoc.idOf(s)).isEqualTo("auth_" + s); + } + + @Test + public void idOf_fails_with_NPE_if_argument_is_null() { + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("projectUuid can't be null"); + + AuthorizationDoc.idOf(null); + } + + @Test + public void projectUuidOf_fails_with_NPE_if_argument_is_null() { + expectedException.expect(NullPointerException.class); + + AuthorizationDoc.projectUuidOf(null); + } + + @Test + public void projectUuidOf_returns_substring_if_starts_with_id_prefix() { + assertThat(AuthorizationDoc.projectUuidOf("auth_")).isEmpty(); + + String id = randomAlphabetic(1 + new Random().nextInt(10)); + assertThat(AuthorizationDoc.projectUuidOf("auth_" + id)).isEqualTo(id); + } + + @Test + public void projectUuidOf_returns_argument_if_does_not_starts_with_id_prefix() { + String id = randomAlphabetic(1 + new Random().nextInt(10)); + assertThat(AuthorizationDoc.projectUuidOf(id)).isEqualTo(id); + assertThat(AuthorizationDoc.projectUuidOf("")).isEqualTo(""); + } + + @Test + public void getId_fails_with_NPE_if_IndexPermissions_has_null_projectUuid() { + IndexPermissions dto = new IndexPermissions(null, null); + IndexType.IndexMainType mainType = IndexType.main(Index.simple("foo"), "bar"); + AuthorizationDoc underTest = AuthorizationDoc.fromDto(mainType, dto); + + expectedException.expect(NullPointerException.class); + expectedException.expectMessage("projectUuid can't be null"); + + underTest.getId(); + } + + @Test + @UseDataProvider("dtos") + public void getId_returns_projectUuid_with_a_prefix(IndexPermissions dto) { + AuthorizationDoc underTest = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), dto); + + assertThat(underTest.getId()).isEqualTo("auth_" + dto.getProjectUuid()); + } + + @Test + @UseDataProvider("dtos") + public void getRouting_returns_projectUuid(IndexPermissions dto) { + AuthorizationDoc underTest = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), dto); + + assertThat(underTest.getRouting()).contains(dto.getProjectUuid()); + } + + @Test + public void fromDto_of_allowAnyone_is_false_and_no_user_nor_group() { + IndexPermissions underTest = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4)); + + AuthorizationDoc doc = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), underTest); + + boolean auth_allowAnyone = doc.getField("auth_allowAnyone"); + assertThat(auth_allowAnyone).isFalse(); + List<Integer> userIds = doc.getField("auth_userIds"); + assertThat(userIds).isEmpty(); + List<Integer> groupIds = doc.getField("auth_groupIds"); + assertThat(groupIds).isEmpty(); + } + + @Test + public void fromDto_defines_userIds_and_groupIds_if_allowAnyone_is_false() { + IndexPermissions underTest = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4)); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addUserId); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addGroupId); + + AuthorizationDoc doc = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), underTest); + + boolean auth_allowAnyone = doc.getField("auth_allowAnyone"); + assertThat(auth_allowAnyone).isFalse(); + List<Integer> userIds = doc.getField("auth_userIds"); + assertThat(userIds).isEqualTo(underTest.getUserIds()); + List<Integer> groupIds = doc.getField("auth_groupIds"); + assertThat(groupIds).isEqualTo(underTest.getGroupIds()); + } + + @Test + public void fromDto_ignores_userIds_and_groupIds_if_allowAnyone_is_true() { + IndexPermissions underTest = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4)); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addUserId); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(underTest::addGroupId); + underTest.allowAnyone(); + + AuthorizationDoc doc = AuthorizationDoc.fromDto(IndexType.main(Index.simple("foo"), "bar"), underTest); + + boolean auth_allowAnyone = doc.getField("auth_allowAnyone"); + assertThat(auth_allowAnyone).isTrue(); + try { + doc.getField("auth_userIds"); + fail("should have thrown IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e).hasMessage("Field auth_userIds not specified in query options"); + } + try { + doc.getField("auth_groupIds"); + fail("should have thrown IllegalStateException"); + } catch (IllegalStateException e) { + assertThat(e).hasMessage("Field auth_groupIds not specified in query options"); + } + } + + @DataProvider + public static Object[][] dtos() { + IndexPermissions allowAnyone = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4)); + allowAnyone.allowAnyone(); + IndexPermissions someUserIds = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4)); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someUserIds::addUserId); + IndexPermissions someGroupIds = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4)); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someGroupIds::addGroupId); + IndexPermissions someGroupIdAndUserIs = new IndexPermissions(randomAlphabetic(3), randomAlphabetic(4)); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someGroupIdAndUserIs::addUserId); + IntStream.range(0, 1 + new Random().nextInt(5)).forEach(someGroupIdAndUserIs::addGroupId); + return new Object[][] { + {allowAnyone}, + {someUserIds}, + {someGroupIds}, + {someGroupIdAndUserIs} + }; + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexerTest.java index 9d61119d2a0..060f0e290e3 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexerTest.java @@ -43,7 +43,7 @@ import static java.util.Arrays.stream; import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE; public class ActiveRuleIndexerTest { @@ -73,13 +73,13 @@ public class ActiveRuleIndexerTest { @Test public void getIndexTypes() { - assertThat(underTest.getIndexTypes()).containsExactly(INDEX_TYPE_ACTIVE_RULE); + assertThat(underTest.getIndexTypes()).containsExactly(TYPE_ACTIVE_RULE); } @Test public void indexOnStartup_does_nothing_if_no_data() { underTest.indexOnStartup(emptySet()); - assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isZero(); + assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isZero(); } @Test @@ -88,7 +88,7 @@ public class ActiveRuleIndexerTest { underTest.indexOnStartup(emptySet()); - List<ActiveRuleDoc> docs = es.getDocuments(INDEX_TYPE_ACTIVE_RULE, ActiveRuleDoc.class); + List<ActiveRuleDoc> docs = es.getDocuments(TYPE_ACTIVE_RULE, ActiveRuleDoc.class); assertThat(docs).hasSize(1); verify(docs.get(0), profile1, activeRule); assertThatEsQueueTableIsEmpty(); @@ -112,44 +112,44 @@ public class ActiveRuleIndexerTest { underTest.commitAndIndex(db.getSession(), Collections.emptyList()); - assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(0); assertThatEsQueueTableIsEmpty(); } @Test public void commitAndIndex_keeps_elements_to_recover_in_ES_QUEUE_on_errors() { ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1); - es.lockWrites(INDEX_TYPE_ACTIVE_RULE); + es.lockWrites(TYPE_ACTIVE_RULE); commitAndIndex(rule1, ar); - EsQueueDto expectedItem = EsQueueDto.create(INDEX_TYPE_ACTIVE_RULE.format(), "" + ar.getId(), "activeRuleId", valueOf(ar.getRuleId())); + EsQueueDto expectedItem = EsQueueDto.create(TYPE_ACTIVE_RULE.format(), "ar_" + ar.getId(), "activeRuleId", valueOf(ar.getRuleId())); assertThatEsQueueContainsExactly(expectedItem); - es.unlockWrites(INDEX_TYPE_ACTIVE_RULE); + es.unlockWrites(TYPE_ACTIVE_RULE); } @Test public void commitAndIndex_deletes_the_documents_that_dont_exist_in_database() { ActiveRuleDto ar = db.qualityProfiles().activateRule(profile1, rule1); indexAll(); - assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(1); db.getDbClient().activeRuleDao().delete(db.getSession(), ar.getKey()); commitAndIndex(rule1, ar); - assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(0); assertThatEsQueueTableIsEmpty(); } @Test public void index_fails_and_deletes_doc_if_docIdType_is_unsupported() { - EsQueueDto item = EsQueueDto.create(INDEX_TYPE_ACTIVE_RULE.format(), "the_id", "unsupported", "the_routing"); + EsQueueDto item = EsQueueDto.create(TYPE_ACTIVE_RULE.format(), "the_id", "unsupported", "the_routing"); db.getDbClient().esQueueDao().insert(db.getSession(), item); underTest.index(db.getSession(), asList(item)); assertThatEsQueueTableIsEmpty(); - assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(0); + assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(0); } @Test @@ -169,11 +169,11 @@ public class ActiveRuleIndexerTest { public void commitDeletionOfProfiles_does_nothing_if_profiles_are_not_indexed() { db.qualityProfiles().activateRule(profile1, rule1); indexAll(); - assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(1); underTest.commitDeletionOfProfiles(db.getSession(), singletonList(profile2)); - assertThat(es.countDocuments(INDEX_TYPE_ACTIVE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(TYPE_ACTIVE_RULE)).isEqualTo(1); } private void assertThatEsQueueTableIsEmpty() { @@ -194,16 +194,16 @@ public class ActiveRuleIndexerTest { } private void verifyOnlyIndexed(ActiveRuleDto... expected) { - List<String> docs = es.getIds(INDEX_TYPE_ACTIVE_RULE); + List<String> docs = es.getIds(TYPE_ACTIVE_RULE); assertThat(docs).hasSize(expected.length); for (ActiveRuleDto activeRuleDto : expected) { - assertThat(docs).contains(activeRuleDto.getId().toString()); + assertThat(docs).contains("ar_" + activeRuleDto.getId()); } } private void verify(ActiveRuleDoc doc1, QProfileDto profile, ActiveRuleDto activeRule) { assertThat(doc1) - .matches(doc -> doc.getId().equals("" + activeRule.getId())) + .matches(doc -> doc.getId().equals("ar_" + activeRule.getId())) .matches(doc -> doc.getRuleProfileUuid().equals(profile.getRulesProfileUuid())) .matches(doc -> doc.getSeverity().equals(activeRule.getSeverityString())); } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java index f71eec9c264..81bd2172c74 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java @@ -27,18 +27,20 @@ import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; import org.sonar.server.es.EsTester; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.newindex.NewIndex; import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.sonar.process.ProcessProperties.Property.CLUSTER_ENABLED; -import static org.sonar.server.es.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_ID; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_KEY; import static org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_REPOSITORY; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE; public class RuleIndexDefinitionTest { @@ -54,13 +56,16 @@ public class RuleIndexDefinitionTest { underTest.define(context); assertThat(context.getIndices()).hasSize(1); - NewIndex ruleIndex = context.getIndices().get("rules"); - assertThat(ruleIndex).isNotNull(); - assertThat(ruleIndex.getTypes().keySet()).containsOnly("activeRule", "ruleExtension", "rule"); + NewIndex<?> ruleIndex = context.getIndices().get("rules"); + assertThat(ruleIndex.getMainType()) + .isEqualTo(IndexType.main(Index.withRelations("rules"), "rule")); + assertThat(ruleIndex.getRelationsStream()) + .extracting(IndexType.IndexRelationType::getName) + .containsOnly("activeRule", "ruleExtension"); // no cluster by default - assertThat(ruleIndex.getSettings().get("index.number_of_shards")).isEqualTo("2"); - assertThat(ruleIndex.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + assertThat(ruleIndex.getSetting("index.number_of_shards")).isEqualTo("2"); + assertThat(ruleIndex.getSetting("index.number_of_replicas")).isEqualTo("0"); } @Test @@ -70,7 +75,7 @@ public class RuleIndexDefinitionTest { underTest.define(context); NewIndex ruleIndex = context.getIndices().get("rules"); - assertThat(ruleIndex.getSettings().get("index.number_of_replicas")).isEqualTo("1"); + assertThat(ruleIndex.getSetting("index.number_of_replicas")).isEqualTo("1"); } @Test @@ -82,13 +87,13 @@ public class RuleIndexDefinitionTest { "quick", "brown", "fox", "jump", "over", "lazi", "dog"); // the following method fails if PUT fails - tester.putDocuments(INDEX_TYPE_RULE, new RuleDoc(ImmutableMap.of( + tester.putDocuments(TYPE_RULE, new RuleDoc(ImmutableMap.of( FIELD_RULE_ID, "123", FIELD_RULE_HTML_DESCRIPTION, longText, FIELD_RULE_REPOSITORY, "squid", FIELD_RULE_KEY, "squid:S001"))); - assertThat(tester.countDocuments(INDEX_TYPE_RULE)).isEqualTo(1); - assertThat(tester.client().prepareSearch(INDEX_TYPE_RULE.getIndex()).setQuery(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy")) + assertThat(tester.countDocuments(TYPE_RULE)).isEqualTo(1); + assertThat(tester.client().prepareSearch(TYPE_RULE.getIndex()).setQuery(matchQuery(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION), "brown fox jumps lazy")) .get().getHits().getTotalHits()).isEqualTo(1); } @@ -110,7 +115,7 @@ public class RuleIndexDefinitionTest { } private List<AnalyzeResponse.AnalyzeToken> analyzeIndexedTokens(String text) { - return tester.client().nativeClient().admin().indices().prepareAnalyze(INDEX_TYPE_RULE.getIndex(), + return tester.client().nativeClient().admin().indices().prepareAnalyze(TYPE_RULE.getIndex().getName(), text) .setField(ENGLISH_HTML_ANALYZER.subField(FIELD_RULE_HTML_DESCRIPTION)) .execute().actionGet().getTokens(); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java index 3068233aaa5..e69c0042c08 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexTest.java @@ -82,9 +82,9 @@ import static org.sonar.server.rule.index.RuleIndex.FACET_LANGUAGES; import static org.sonar.server.rule.index.RuleIndex.FACET_REPOSITORIES; import static org.sonar.server.rule.index.RuleIndex.FACET_TAGS; import static org.sonar.server.rule.index.RuleIndex.FACET_TYPES; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE; -import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION; public class RuleIndexTest { @@ -296,7 +296,7 @@ public class RuleIndexTest { createRuleMetadata(rule2, organization, setTags("tag2")); index(); - assertThat(es.countDocuments(INDEX_TYPE_RULE_EXTENSION)).isEqualTo(4); + assertThat(es.countDocuments(TYPE_RULE_EXTENSION)).isEqualTo(4); // tag2s in filter RuleQuery query = new RuleQuery().setOrganization(organization).setTags(of("tag2s")); verifySearch(query, rule2); @@ -588,8 +588,8 @@ public class RuleIndexTest { } private void index() { - ruleIndexer.indexOnStartup(Sets.newHashSet(INDEX_TYPE_RULE, INDEX_TYPE_RULE_EXTENSION)); - activeRuleIndexer.indexOnStartup(Sets.newHashSet(INDEX_TYPE_ACTIVE_RULE)); + ruleIndexer.indexOnStartup(Sets.newHashSet(TYPE_RULE, TYPE_RULE_EXTENSION)); + activeRuleIndexer.indexOnStartup(Sets.newHashSet(TYPE_ACTIVE_RULE)); } private RuleQuery newRuleQuery() { @@ -1102,8 +1102,8 @@ public class RuleIndexTest { index(); // inactive rules on profile - List<SearchHit> ruleDocs = es.getDocuments(INDEX_TYPE_RULE); - List<SearchHit> activeRuleDocs = es.getDocuments(INDEX_TYPE_ACTIVE_RULE); + List<SearchHit> ruleDocs = es.getDocuments(TYPE_RULE); + List<SearchHit> activeRuleDocs = es.getDocuments(TYPE_ACTIVE_RULE); assertThat(underTest.searchAll(new RuleQuery().setActivation(false).setQProfile(profile2))) .containsOnly(rule2.getId(), rule3.getId()); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java index 83123f4c57f..6161448d091 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java @@ -43,6 +43,8 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptySet; import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE; +import static org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE_EXTENSION; public class RuleIndexerTest { @@ -75,7 +77,7 @@ public class RuleIndexerTest { @Test public void index_nothing() { underTest.index(dbSession, emptyList()); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(0L); + assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(0L); } @Test @@ -83,7 +85,7 @@ public class RuleIndexerTest { dbClient.ruleDao().insert(dbSession, rule); underTest.commitAndIndex(dbSession, rule.getId()); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1); } @Test @@ -92,13 +94,13 @@ public class RuleIndexerTest { dbClient.ruleDao().insert(dbSession, rule.setStatus(RuleStatus.READY)); dbSession.commit(); underTest.commitAndIndex(dbTester.getSession(), rule.getId()); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1); // Remove rule dbTester.getDbClient().ruleDao().update(dbTester.getSession(), rule.setStatus(RuleStatus.READY).setUpdatedAt(2000000000000L)); underTest.commitAndIndex(dbTester.getSession(), rule.getId()); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1); } @Test @@ -115,7 +117,7 @@ public class RuleIndexerTest { .setScope(RuleExtensionScope.organization(organization.getUuid())); assertThat( es.client() - .prepareSearch(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION) + .prepareSearch(TYPE_RULE_EXTENSION.getMainType()) .setQuery(termQuery("_id", doc.getId())) .get() .getHits() @@ -136,13 +138,13 @@ public class RuleIndexerTest { RuleExtensionDoc doc = new RuleExtensionDoc() .setRuleId(rule.getId()) .setScope(RuleExtensionScope.organization(organization.getUuid())); - assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION)).contains(doc.getId()); + assertThat(es.getIds(TYPE_RULE_EXTENSION)).contains(doc.getId()); // update db table "rules_metadata" with empty tags and delete tags from index metadata = RuleTesting.newRuleMetadata(rule, organization).setTags(emptySet()); dbTester.getDbClient().ruleDao().insertOrUpdate(dbTester.getSession(), metadata); underTest.commitAndIndex(dbTester.getSession(), rule.getId(), organization); - assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION)).doesNotContain(doc.getId()); + assertThat(es.getIds(TYPE_RULE_EXTENSION)).doesNotContain(doc.getId()); } @Test @@ -151,6 +153,6 @@ public class RuleIndexerTest { RuleDefinitionDto rule = dbTester.rules().insert(r -> r.setDescription(description)); underTest.commitAndIndex(dbTester.getSession(), rule.getId()); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(TYPE_RULE)).isEqualTo(1); } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java index df336a60359..4e7e8fedc09 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexDefinitionTest.java @@ -21,8 +21,10 @@ package org.sonar.server.user.index; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.newindex.NewIndex; import static org.assertj.core.api.Assertions.assertThat; @@ -37,11 +39,12 @@ public class UserIndexDefinitionTest { assertThat(underTest.getIndices()).hasSize(1); NewIndex index = underTest.getIndices().get("users"); - assertThat(index).isNotNull(); - assertThat(index.getTypes().keySet()).containsOnly("user"); + assertThat(index.getMainType()) + .isEqualTo(IndexType.main(Index.simple("users"), "user")); + assertThat(index.getRelationsStream()).isEmpty(); // no cluster by default - assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("1"); - assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + assertThat(index.getSetting("index.number_of_shards")).isEqualTo("1"); + assertThat(index.getSetting("index.number_of_replicas")).isEqualTo("0"); } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexTest.java index 11939f9aa28..1775f3de25b 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexTest.java @@ -34,7 +34,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER; +import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER; public class UserIndexTest { @@ -53,9 +53,9 @@ public class UserIndexTest { UserDoc user1 = newUser("user1", asList("user_1", "u1")); UserDoc user2 = newUser("user_with_same_email_as_user1", asList("user_2")).setEmail(user1.email()); UserDoc user3 = newUser("inactive_user_with_same_scm_as_user1", user1.scmAccounts()).setActive(false); - es.putDocuments(INDEX_TYPE_USER, user1); - es.putDocuments(INDEX_TYPE_USER, user2); - es.putDocuments(INDEX_TYPE_USER, user3); + es.putDocuments(TYPE_USER, user1); + es.putDocuments(TYPE_USER, user2); + es.putDocuments(TYPE_USER, user3); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(user1.scmAccounts().get(0), ORGANIZATION_UUID)).extractingResultOf("login").containsOnly(user1.login()); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(user1.login(), ORGANIZATION_UUID)).extractingResultOf("login").containsOnly(user1.login()); @@ -71,7 +71,7 @@ public class UserIndexTest { public void getAtMostThreeActiveUsersForScmAccount_ignores_inactive_user() { String scmAccount = "scmA"; UserDoc user = newUser(USER1_LOGIN, singletonList(scmAccount)).setActive(false); - es.putDocuments(INDEX_TYPE_USER, user); + es.putDocuments(TYPE_USER, user); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(user.login(), ORGANIZATION_UUID)).isEmpty(); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(scmAccount, ORGANIZATION_UUID)).isEmpty(); @@ -84,10 +84,10 @@ public class UserIndexTest { UserDoc user2 = newUser("user2", Collections.emptyList()).setEmail(email); UserDoc user3 = newUser("user3", Collections.emptyList()).setEmail(email); UserDoc user4 = newUser("user4", Collections.emptyList()).setEmail(email); - es.putDocuments(INDEX_TYPE_USER, user1); - es.putDocuments(INDEX_TYPE_USER, user2); - es.putDocuments(INDEX_TYPE_USER, user3); - es.putDocuments(INDEX_TYPE_USER, user4); + es.putDocuments(TYPE_USER, user1); + es.putDocuments(TYPE_USER, user2); + es.putDocuments(TYPE_USER, user3); + es.putDocuments(TYPE_USER, user4); // restrict results to 3 users assertThat(underTest.getAtMostThreeActiveUsersForScmAccount(email, ORGANIZATION_UUID)).hasSize(3); @@ -96,7 +96,7 @@ public class UserIndexTest { @Test public void getAtMostThreeActiveUsersForScmAccount_is_case_sensitive_for_login() { UserDoc user = newUser("the_login", singletonList("John.Smith")); - es.putDocuments(INDEX_TYPE_USER, user); + es.putDocuments(TYPE_USER, user); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_login", ORGANIZATION_UUID)).hasSize(1); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_Login", ORGANIZATION_UUID)).isEmpty(); @@ -105,7 +105,7 @@ public class UserIndexTest { @Test public void getAtMostThreeActiveUsersForScmAccount_is_case_insensitive_for_email() { UserDoc user = newUser("the_login", "the_EMAIL@corp.com", singletonList("John.Smith")); - es.putDocuments(INDEX_TYPE_USER, user); + es.putDocuments(TYPE_USER, user); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_EMAIL@corp.com", ORGANIZATION_UUID)).hasSize(1); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("the_email@corp.com", ORGANIZATION_UUID)).hasSize(1); @@ -115,7 +115,7 @@ public class UserIndexTest { @Test public void getAtMostThreeActiveUsersForScmAccount_is_case_insensitive_for_scm_account() { UserDoc user = newUser("the_login", singletonList("John.Smith")); - es.putDocuments(INDEX_TYPE_USER, user); + es.putDocuments(TYPE_USER, user); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("John.Smith", ORGANIZATION_UUID)).hasSize(1); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("JOHN.SMIth", ORGANIZATION_UUID)).hasSize(1); @@ -127,16 +127,16 @@ public class UserIndexTest { public void getAtMostThreeActiveUsersForScmAccount_search_only_user_within_given_organization() { UserDoc user1 = newUser("user1", singletonList("same_scm")).setOrganizationUuids(singletonList(ORGANIZATION_UUID)); UserDoc user2 = newUser("user2", singletonList("same_scm")).setOrganizationUuids(singletonList("another_organization")); - es.putDocuments(INDEX_TYPE_USER, user1); - es.putDocuments(INDEX_TYPE_USER, user2); + es.putDocuments(TYPE_USER, user1); + es.putDocuments(TYPE_USER, user2); assertThat(underTest.getAtMostThreeActiveUsersForScmAccount("same_scm", ORGANIZATION_UUID)).extractingResultOf("login").containsOnly(user1.login()); } @Test public void searchUsers() { - es.putDocuments(INDEX_TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1")); - es.putDocuments(INDEX_TYPE_USER, newUser(USER2_LOGIN, Collections.emptyList()).setEmail("email2")); + es.putDocuments(TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1")); + es.putDocuments(TYPE_USER, newUser(USER2_LOGIN, Collections.emptyList()).setEmail("email2")); assertThat(underTest.search(userQuery.build(), new SearchOptions()).getDocs()).hasSize(2); assertThat(underTest.search(userQuery.setTextQuery("user").build(), new SearchOptions()).getDocs()).hasSize(2); @@ -149,9 +149,9 @@ public class UserIndexTest { @Test public void search_users_filter_by_organization_uuid() { - es.putDocuments(INDEX_TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1") + es.putDocuments(TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1") .setOrganizationUuids(newArrayList("O1", "O2"))); - es.putDocuments(INDEX_TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2") + es.putDocuments(TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2") .setOrganizationUuids(newArrayList("O2"))); assertThat(underTest.search(userQuery.setOrganizationUuid("O42").build(), new SearchOptions()).getDocs()).isEmpty(); @@ -161,9 +161,9 @@ public class UserIndexTest { @Test public void search_users_filter_by_excluded_organization_uuid() { - es.putDocuments(INDEX_TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1") + es.putDocuments(TYPE_USER, newUser(USER1_LOGIN, asList("user_1", "u1")).setEmail("email1") .setOrganizationUuids(newArrayList("O1", "O2"))); - es.putDocuments(INDEX_TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2") + es.putDocuments(TYPE_USER, newUser(USER2_LOGIN, emptyList()).setEmail("email2") .setOrganizationUuids(newArrayList("O2"))); assertThat(underTest.search(userQuery.setExcludedOrganizationUuid("O42").build(), new SearchOptions()).getDocs()).hasSize(2); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexerTest.java index a269a9b0326..8f4dcc30198 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/user/index/UserIndexerTest.java @@ -33,6 +33,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; +import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER; public class UserIndexerTest { @@ -50,7 +51,7 @@ public class UserIndexerTest { public void index_nothing_on_startup() { underTest.indexOnStartup(new HashSet<>()); - assertThat(es.countDocuments(UserIndexDefinition.INDEX_TYPE_USER)).isEqualTo(0L); + assertThat(es.countDocuments(TYPE_USER)).isEqualTo(0L); } @Test @@ -59,7 +60,7 @@ public class UserIndexerTest { underTest.indexOnStartup(new HashSet<>()); - List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); + List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class); assertThat(docs).hasSize(1); UserDoc doc = docs.get(0); assertThat(doc.uuid()).isEqualTo(user.getUuid()); @@ -81,7 +82,7 @@ public class UserIndexerTest { underTest.indexOnStartup(new HashSet<>()); - List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); + List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class); assertThat(docs).hasSize(1); UserDoc doc = docs.get(0); assertThat(doc.uuid()).isEqualTo(user.getUuid()); @@ -96,7 +97,7 @@ public class UserIndexerTest { underTest.commitAndIndex(db.getSession(), user); - List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); + List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class); assertThat(docs) .extracting(UserDoc::uuid) .containsExactlyInAnyOrder(user.getUuid()) @@ -115,7 +116,7 @@ public class UserIndexerTest { underTest.commitAndIndex(db.getSession(), user); - List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); + List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class); assertThat(docs) .extracting(UserDoc::uuid, UserDoc::organizationUuids) .containsExactlyInAnyOrder(tuple(user.getUuid(), asList(organization1.getUuid(), organization2.getUuid()))); @@ -132,7 +133,7 @@ public class UserIndexerTest { underTest.commitAndIndex(db.getSession(), asList(user1, user2)); - List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); + List<UserDoc> docs = es.getDocuments(TYPE_USER, UserDoc.class); assertThat(docs) .extracting(UserDoc::login, UserDoc::organizationUuids) .containsExactlyInAnyOrder( diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexDefinitionTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexDefinitionTest.java index 3bed1317750..e18c49b422f 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexDefinitionTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexDefinitionTest.java @@ -21,8 +21,10 @@ package org.sonar.server.view.index; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.newindex.NewIndex; import static org.assertj.core.api.Assertions.assertThat; @@ -37,10 +39,11 @@ public class ViewIndexDefinitionTest { assertThat(underTest.getIndices()).hasSize(1); NewIndex index = underTest.getIndices().get("views"); - assertThat(index).isNotNull(); - assertThat(index.getTypes().keySet()).containsOnly("view"); + assertThat(index.getMainType()) + .isEqualTo(IndexType.main(Index.simple("views"), "view")); + assertThat(index.getRelationsStream()).isEmpty(); - assertThat(index.getSettings().get("index.number_of_shards")).isEqualTo("5"); - assertThat(index.getSettings().get("index.number_of_replicas")).isEqualTo("0"); + assertThat(index.getSetting("index.number_of_shards")).isEqualTo("5"); + assertThat(index.getSetting("index.number_of_replicas")).isEqualTo("0"); } } diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexTest.java index f82551ac90b..e1ac8c49094 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexTest.java @@ -27,7 +27,7 @@ import org.sonar.server.es.EsTester; import static com.google.common.collect.Lists.newArrayList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW; +import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW; public class ViewIndexTest { @@ -40,8 +40,8 @@ public class ViewIndexTest { public void find_all_view_uuids() { ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(singletonList("P1")); ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(singletonList("P2")); - es.putDocuments(INDEX_TYPE_VIEW, view1); - es.putDocuments(INDEX_TYPE_VIEW, view2); + es.putDocuments(TYPE_VIEW, view1); + es.putDocuments(TYPE_VIEW, view2); List<String> result = newArrayList(index.findAllViewUuids()); diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java index 5d814b909b7..096b9fb6611 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/view/index/ViewIndexerTest.java @@ -40,7 +40,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; import static org.sonar.api.resources.Qualifiers.APP; import static org.sonar.db.component.ComponentTesting.newProjectCopy; -import static org.sonar.server.view.index.ViewIndexDefinition.INDEX_TYPE_VIEW; +import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW; public class ViewIndexerTest { @@ -58,7 +58,7 @@ public class ViewIndexerTest { @Test public void index_nothing() { underTest.indexOnStartup(emptySet()); - assertThat(es.countDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW)).isEqualTo(0L); + assertThat(es.countDocuments(TYPE_VIEW)).isEqualTo(0L); } @Test @@ -67,7 +67,7 @@ public class ViewIndexerTest { underTest.indexOnStartup(emptySet()); - List<ViewDoc> docs = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class); + List<ViewDoc> docs = es.getDocuments(TYPE_VIEW, ViewDoc.class); assertThat(docs).hasSize(4); Map<String, ViewDoc> viewsByUuid = Maps.uniqueIndex(docs, ViewDoc::uuid); @@ -84,7 +84,7 @@ public class ViewIndexerTest { underTest.index("EFGH"); - List<ViewDoc> docs = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class); + List<ViewDoc> docs = es.getDocuments(TYPE_VIEW, ViewDoc.class); assertThat(docs).hasSize(2); Map<String, ViewDoc> viewsByUuid = Maps.uniqueIndex(docs, ViewDoc::uuid); @@ -97,7 +97,7 @@ public class ViewIndexerTest { public void index_view_doc() { underTest.index(new ViewDoc().setUuid("EFGH").setProjects(newArrayList("KLMN", "JKLM"))); - List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class); + List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class); assertThat(result).hasSize(1); ViewDoc view = result.get(0); @@ -112,7 +112,7 @@ public class ViewIndexerTest { db.components().insertComponent(newProjectCopy("PC1", project, application)); underTest.index(application.uuid()); - List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class); + List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class); assertThat(result).hasSize(1); ViewDoc resultApp = result.get(0); @@ -127,7 +127,7 @@ public class ViewIndexerTest { db.components().insertComponent(newProjectCopy("PC1", project, application)); underTest.indexOnStartup(emptySet()); - List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class); + List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class); assertThat(result).hasSize(1); ViewDoc resultApp = result.get(0); @@ -153,7 +153,7 @@ public class ViewIndexerTest { underTest.index(applicationBranch1.uuid()); - List<ViewDoc> result = es.getDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, ViewDoc.class); + List<ViewDoc> result = es.getDocuments(TYPE_VIEW, ViewDoc.class); assertThat(result) .extracting(ViewDoc::uuid, ViewDoc::projects) .containsExactlyInAnyOrder( @@ -165,16 +165,16 @@ public class ViewIndexerTest { ViewDoc view1 = new ViewDoc().setUuid("UUID1").setProjects(asList("P1")); ViewDoc view2 = new ViewDoc().setUuid("UUID2").setProjects(asList("P2", "P3", "P4")); ViewDoc view3 = new ViewDoc().setUuid("UUID3").setProjects(asList("P2", "P3", "P4")); - es.putDocuments(INDEX_TYPE_VIEW, view1); - es.putDocuments(INDEX_TYPE_VIEW, view2); - es.putDocuments(INDEX_TYPE_VIEW, view3); + es.putDocuments(TYPE_VIEW, view1); + es.putDocuments(TYPE_VIEW, view2); + es.putDocuments(TYPE_VIEW, view3); - assertThat(es.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID)) + assertThat(es.getDocumentFieldValues(TYPE_VIEW, ViewIndexDefinition.FIELD_UUID)) .containsOnly(view1.uuid(), view2.uuid(), view3.uuid()); underTest.delete(dbSession, asList(view1.uuid(), view2.uuid())); - assertThat(es.getDocumentFieldValues(INDEX_TYPE_VIEW, ViewIndexDefinition.FIELD_UUID)) + assertThat(es.getDocumentFieldValues(TYPE_VIEW, ViewIndexDefinition.FIELD_UUID)) .containsOnly(view3.uuid()); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java index 72d1a041793..6b382ebe452 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndex.java @@ -61,9 +61,10 @@ import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_LA import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_NAME; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_ORGANIZATION_UUID; import static org.sonar.server.component.index.ComponentIndexDefinition.FIELD_QUALIFIER; -import static org.sonar.server.component.index.ComponentIndexDefinition.INDEX_TYPE_COMPONENT; import static org.sonar.server.component.index.ComponentIndexDefinition.NAME_ANALYZERS; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.component.index.ComponentIndexDefinition.TYPE_COMPONENT; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; public class ComponentIndex { @@ -82,7 +83,7 @@ public class ComponentIndex { public SearchIdResult<String> search(ComponentQuery query, SearchOptions searchOptions) { SearchRequestBuilder requestBuilder = client - .prepareSearch(INDEX_TYPE_COMPONENT) + .prepareSearch(TYPE_COMPONENT.getMainType()) .setFetchSource(false) .setFrom(searchOptions.getOffset()) .setSize(searchOptions.getLimit()); @@ -118,7 +119,7 @@ public class ComponentIndex { } SearchRequestBuilder request = client - .prepareSearch(INDEX_TYPE_COMPONENT) + .prepareSearch(TYPE_COMPONENT.getMainType()) .setQuery(createQuery(query, features)) .addAggregation(createAggregation(query)) @@ -166,6 +167,7 @@ public class ComponentIndex { private QueryBuilder createQuery(SuggestionQuery query, ComponentTextSearchFeature... features) { BoolQueryBuilder esQuery = boolQuery(); + esQuery.filter(termQuery(FIELD_INDEX_TYPE, TYPE_COMPONENT.getName())); esQuery.filter(authorizationTypeSupport.createQueryFilter()); ComponentTextSearchQuery componentTextSearchQuery = ComponentTextSearchQuery.builder() .setQueryText(query.getQuery()) diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java index 20f15f1c903..c7f8dbbddd5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/ws/SuggestionsAction.java @@ -53,7 +53,7 @@ import org.sonar.server.component.index.ComponentHitsPerQualifier; import org.sonar.server.component.index.ComponentIndex; import org.sonar.server.component.index.ComponentIndexResults; import org.sonar.server.component.index.SuggestionQuery; -import org.sonar.server.es.DefaultIndexSettings; +import org.sonar.server.es.newindex.DefaultIndexSettings; import org.sonar.server.favorite.FavoriteFinder; import org.sonar.server.user.UserSession; import org.sonarqube.ws.Components.SuggestionsWsResponse; @@ -70,7 +70,7 @@ import static org.sonar.api.web.UserRole.USER; import static org.sonar.core.util.stream.MoreCollectors.toList; import static org.sonar.core.util.stream.MoreCollectors.toSet; import static org.sonar.server.component.index.SuggestionQuery.DEFAULT_LIMIT; -import static org.sonar.server.es.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH; +import static org.sonar.server.es.newindex.DefaultIndexSettings.MINIMUM_NGRAM_LENGTH; import static org.sonar.server.ws.WsUtils.writeProtobuf; import static org.sonarqube.ws.Common.Organization; import static org.sonarqube.ws.Components.SuggestionsWsResponse.newBuilder; diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java index c589c186186..7112f77b28a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java @@ -22,7 +22,6 @@ package org.sonar.server.es; import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; @@ -36,10 +35,14 @@ import org.sonar.api.server.ServerSide; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.process.ProcessProperties; -import org.sonar.server.es.IndexDefinition.Index; import org.sonar.server.es.metadata.EsDbCompatibility; import org.sonar.server.es.metadata.MetadataIndex; import org.sonar.server.es.metadata.MetadataIndexDefinition; +import org.sonar.server.es.newindex.BuiltIndex; +import org.sonar.server.es.newindex.NewIndex; + +import static org.sonar.server.es.metadata.MetadataIndexDefinition.DESCRIPTOR; +import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA; /** * Creates/deletes all indices in Elasticsearch during server startup. @@ -69,26 +72,29 @@ public class IndexCreator implements Startable { @Override public void start() { // create the "metadata" index first - if (!client.prepareIndicesExist(MetadataIndexDefinition.INDEX_TYPE_METADATA.getIndex()).get().isExists()) { + IndexType.IndexMainType metadataMainType = TYPE_METADATA; + if (!client.prepareIndicesExist(metadataMainType.getIndex()).get().isExists()) { IndexDefinition.IndexDefinitionContext context = new IndexDefinition.IndexDefinitionContext(); metadataIndexDefinition.define(context); NewIndex index = context.getIndices().values().iterator().next(); - createIndex(new Index(index), false); + createIndex(index.build(), false); } checkDbCompatibility(definitions.getIndices().values()); // create indices that do not exist or that have a new definition (different mapping, cluster enabled, ...) - for (Index index : definitions.getIndices().values()) { - boolean exists = client.prepareIndicesExist(index.getName()).get().isExists(); - if (exists && !index.getName().equals(MetadataIndexDefinition.INDEX_TYPE_METADATA.getIndex()) && hasDefinitionChange(index)) { - verifyNotBlueGreenDeployment(index.getName()); - LOGGER.info("Delete Elasticsearch index {} (structure changed)", index.getName()); - deleteIndex(index.getName()); + for (BuiltIndex<?> builtIndex : definitions.getIndices().values()) { + Index index = builtIndex.getMainType().getIndex(); + String indexName = index.getName(); + boolean exists = client.prepareIndicesExist(index).get().isExists(); + if (exists && !builtIndex.getMainType().equals(metadataMainType) && hasDefinitionChange(builtIndex)) { + verifyNotBlueGreenDeployment(indexName); + LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName); + deleteIndex(indexName); exists = false; } if (!exists) { - createIndex(index, true); + createIndex(builtIndex, true); } } } @@ -104,18 +110,18 @@ public class IndexCreator implements Startable { // nothing to do } - private void createIndex(Index index, boolean useMetadata) { + private void createIndex(BuiltIndex<?> builtIndex, boolean useMetadata) { + Index index = builtIndex.getMainType().getIndex(); LOGGER.info(String.format("Create index %s", index.getName())); Settings.Builder settings = Settings.builder(); - settings.put(index.getSettings()); + settings.put(builtIndex.getSettings()); if (useMetadata) { - metadataIndex.setHash(index.getName(), IndexDefinitionHash.of(index)); - for (IndexDefinition.Type type : index.getTypes().values()) { - metadataIndex.setInitialized(new IndexType(index.getName(), type.getName()), false); - } + metadataIndex.setHash(index, IndexDefinitionHash.of(builtIndex)); + metadataIndex.setInitialized(builtIndex.getMainType(), false); + builtIndex.getRelationTypes().forEach(relationType -> metadataIndex.setInitialized(relationType, false)); } CreateIndexResponse indexResponse = client - .prepareCreate(index.getName()) + .prepareCreate(index) .setSettings(settings) .get(); if (!indexResponse.isAcknowledged()) { @@ -124,15 +130,13 @@ public class IndexCreator implements Startable { client.waitForStatus(ClusterHealthStatus.YELLOW); // create types - for (Map.Entry<String, IndexDefinition.Type> entry : index.getTypes().entrySet()) { - LOGGER.info(String.format("Create type %s/%s", index.getName(), entry.getKey())); - PutMappingResponse mappingResponse = client.preparePutMapping(index.getName()) - .setType(entry.getKey()) - .setSource(entry.getValue().getAttributes()) - .get(); - if (!mappingResponse.isAcknowledged()) { - throw new IllegalStateException("Failed to create type " + entry.getKey()); - } + LOGGER.info("Create type {}", builtIndex.getMainType().format()); + PutMappingResponse mappingResponse = client.preparePutMapping(index) + .setType(builtIndex.getMainType().getType()) + .setSource(builtIndex.getAttributes()) + .get(); + if (!mappingResponse.isAcknowledged()) { + throw new IllegalStateException("Failed to create type " + builtIndex.getMainType().getType()); } client.waitForStatus(ClusterHealthStatus.YELLOW); } @@ -141,15 +145,15 @@ public class IndexCreator implements Startable { client.nativeClient().admin().indices().prepareDelete(indexName).get(); } - private boolean hasDefinitionChange(Index index) { - return metadataIndex.getHash(index.getName()) + private boolean hasDefinitionChange(BuiltIndex<?> index) { + return metadataIndex.getHash(index.getMainType().getIndex()) .map(hash -> { String defHash = IndexDefinitionHash.of(index); return !StringUtils.equals(hash, defHash); }).orElse(true); } - private void checkDbCompatibility(Collection<Index> definitions) { + private void checkDbCompatibility(Collection<BuiltIndex> definitions) { List<String> existingIndices = loadExistingIndicesExceptMetadata(definitions); if (!existingIndices.isEmpty()) { boolean delete = false; @@ -164,11 +168,13 @@ public class IndexCreator implements Startable { esDbCompatibility.markAsCompatible(); } - private List<String> loadExistingIndicesExceptMetadata(Collection<Index> definitions) { - Set<String> definedNames = definitions.stream().map(IndexDefinition.Index::getName).collect(Collectors.toSet()); + private List<String> loadExistingIndicesExceptMetadata(Collection<BuiltIndex> definitions) { + Set<String> definedNames = definitions.stream() + .map(t -> t.getMainType().getIndex().getName()) + .collect(Collectors.toSet()); return Arrays.stream(client.nativeClient().admin().indices().prepareGetIndex().get().getIndices()) .filter(definedNames::contains) - .filter(index -> !MetadataIndexDefinition.INDEX_TYPE_METADATA.getIndex().equals(index)) + .filter(index -> !DESCRIPTOR.getName().equals(index)) .collect(Collectors.toList()); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java index 4439ca5d494..22b8b59b0c3 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexDefinitions.java @@ -24,6 +24,8 @@ import java.util.Map; import org.picocontainer.Startable; import org.sonar.api.config.Configuration; import org.sonar.api.server.ServerSide; +import org.sonar.server.es.newindex.BuiltIndex; +import org.sonar.server.es.newindex.NewIndex; /** * This class collects definitions of all Elasticsearch indices during server startup @@ -31,7 +33,7 @@ import org.sonar.api.server.ServerSide; @ServerSide public class IndexDefinitions implements Startable { - private final Map<String, IndexDefinition.Index> byKey = Maps.newHashMap(); + private final Map<String, BuiltIndex> byKey = Maps.newHashMap(); private final IndexDefinition[] defs; private final Configuration config; @@ -40,7 +42,7 @@ public class IndexDefinitions implements Startable { this.config = config; } - public Map<String, IndexDefinition.Index> getIndices() { + public Map<String, BuiltIndex> getIndices() { return byKey; } @@ -55,7 +57,7 @@ public class IndexDefinitions implements Startable { } for (Map.Entry<String, NewIndex> entry : context.getIndices().entrySet()) { - byKey.put(entry.getKey(), new IndexDefinition.Index(entry.getValue())); + byKey.put(entry.getKey(), entry.getValue().build()); } } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java index cea201376af..be64ce0a057 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexerStartupTask.java @@ -72,12 +72,13 @@ public class IndexerStartupTask { } private Set<IndexType> getUninitializedTypes(StartupIndexer indexer) { - return indexer.getIndexTypes().stream().filter(indexType -> !metadataIndex.getInitialized(indexType)).collect(toSet()); + return indexer.getIndexTypes().stream() + .filter(indexType -> !metadataIndex.getInitialized(indexType)) + .collect(toSet()); } private void setInitialized(IndexType indexType) { - String index = indexType.getIndex(); - waitForIndexYellow(index); + waitForIndexYellow(indexType.getMainType().getIndex().getName()); metadataIndex.setInitialized(indexType, true); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java index 9b38225af18..f700fc635f5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java @@ -64,7 +64,7 @@ public class RecoveryIndexer implements Startable { private final System2 system2; private final Configuration config; private final DbClient dbClient; - private final Map<IndexType, ResilientIndexer> indexersByType; + private final Map<String, Indexer> indexersByType; private final long minAgeInMs; private final long loopLimit; @@ -73,11 +73,29 @@ public class RecoveryIndexer implements Startable { this.config = config; this.dbClient = dbClient; this.indexersByType = new HashMap<>(); - Arrays.stream(indexers).forEach(i -> i.getIndexTypes().forEach(indexType -> indexersByType.put(indexType, i))); + Arrays.stream(indexers).forEach(i -> i.getIndexTypes().forEach(indexType -> indexersByType.put(indexType.format(), new Indexer(indexType, i)))); this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS); this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT); } + private static final class Indexer { + private final IndexType indexType; + private final ResilientIndexer delegate; + + private Indexer(IndexType indexType, ResilientIndexer delegate) { + this.indexType = indexType; + this.delegate = delegate; + } + + public IndexType getIndexType() { + return indexType; + } + + public ResilientIndexer getDelegate() { + return delegate; + } + } + @Override public void start() { long delayInMs = getSetting(PROPERTY_DELAY, DEFAULT_DELAY_IN_MS); @@ -116,7 +134,7 @@ public class RecoveryIndexer implements Startable { while (!items.isEmpty()) { IndexingResult loopResult = new IndexingResult(); - groupItemsByType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems))); + groupItemsByDocType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems))); result.add(loopResult); if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) { @@ -138,19 +156,19 @@ public class RecoveryIndexer implements Startable { } } - private IndexingResult doIndex(DbSession dbSession, IndexType type, Collection<EsQueueDto> typeItems) { - LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type); + private IndexingResult doIndex(DbSession dbSession, String docType, Collection<EsQueueDto> typeItems) { + LOGGER.trace(LOG_PREFIX + "processing {} [{}]", typeItems.size(), docType); - ResilientIndexer indexer = indexersByType.get(type); + Indexer indexer = indexersByType.get(docType); if (indexer == null) { - LOGGER.error(LOG_PREFIX + "ignore {} items with unsupported type {}", typeItems.size(), type); + LOGGER.error(LOG_PREFIX + "ignore {} items with unsupported type [{}]", typeItems.size(), docType); return new IndexingResult(); } - return indexer.index(dbSession, typeItems); + return indexer.delegate.index(dbSession, typeItems); } - private static ListMultimap<IndexType, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) { - return items.stream().collect(MoreCollectors.index(i -> IndexType.parse(i.getDocType()))); + private static ListMultimap<String, EsQueueDto> groupItemsByDocType(Collection<EsQueueDto> items) { + return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType)); } private long getSetting(String key, long defaultValue) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java index 631bc02d998..8e5976d5f4d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndex.java @@ -75,10 +75,12 @@ import org.sonar.db.rule.RuleDefinitionDto; import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsClient; import org.sonar.server.es.EsUtils; +import org.sonar.server.es.IndexType; import org.sonar.server.es.SearchOptions; import org.sonar.server.es.Sorting; import org.sonar.server.es.StickyFacetBuilder; import org.sonar.server.issue.index.IssueQuery.PeriodStart; +import org.sonar.server.permission.index.AuthorizationDoc; import org.sonar.server.permission.index.WebAuthorizationTypeSupport; import org.sonar.server.user.UserSession; import org.sonar.server.view.index.ViewIndexDefinition; @@ -96,6 +98,7 @@ import static org.elasticsearch.index.query.QueryBuilders.termsQuery; import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; import static org.sonar.server.es.BaseDoc.epochMillisToEpochSeconds; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNED_TO_ME; import static org.sonar.server.issue.index.IssueIndex.Facet.ASSIGNEES; import static org.sonar.server.issue.index.IssueIndex.Facet.AUTHOR; @@ -143,11 +146,12 @@ import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_SEVE import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_STATUS; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TAGS; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_TYPE; -import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; +import static org.sonar.server.issue.index.IssueIndexDefinition.TYPE_ISSUE; import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_INSECURE_INTERACTION; import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_POROUS_DEFENSES; import static org.sonar.server.issue.index.SecurityStandardHelper.SANS_TOP_25_RISKY_RESOURCE; import static org.sonar.server.issue.index.SecurityStandardHelper.UNKNOWN_STANDARD; +import static org.sonar.server.view.index.ViewIndexDefinition.TYPE_VIEW; import static org.sonarqube.ws.client.issue.IssuesWsParameters.DEPRECATED_PARAM_AUTHORS; import static org.sonarqube.ws.client.issue.IssuesWsParameters.FACET_MODE_EFFORT; import static org.sonarqube.ws.client.issue.IssuesWsParameters.PARAM_ASSIGNEES; @@ -277,7 +281,7 @@ public class IssueIndex { } public SearchResponse search(IssueQuery query, SearchOptions options) { - SearchRequestBuilder requestBuilder = client.prepareSearch(INDEX_TYPE_ISSUE); + SearchRequestBuilder requestBuilder = client.prepareSearch(TYPE_ISSUE.getMainType()); configureSorting(query, requestBuilder); configurePagination(options, requestBuilder); @@ -313,7 +317,7 @@ public class IssueIndex { private static void configureRouting(IssueQuery query, SearchOptions options, SearchRequestBuilder requestBuilder) { Collection<String> uuids = query.projectUuids(); if (!uuids.isEmpty() && options.getFacets().isEmpty()) { - requestBuilder.setRouting(uuids.toArray(new String[uuids.size()])); + requestBuilder.setRouting(uuids.stream().map(AuthorizationDoc::idOf).toArray(String[]::new)); } } @@ -323,6 +327,7 @@ public class IssueIndex { private Map<String, QueryBuilder> createFilters(IssueQuery query) { Map<String, QueryBuilder> filters = new HashMap<>(); + filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_ISSUE.getName())); filters.put("__authorization", createAuthorizationFilter()); // Issue is assigned Filter @@ -425,10 +430,11 @@ public class IssueIndex { BoolQueryBuilder viewsFilter = boolQuery(); for (String viewUuid : viewUuids) { + IndexType.IndexMainType mainType = TYPE_VIEW; viewsFilter.should(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID, new TermsLookup( - ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex(), - ViewIndexDefinition.INDEX_TYPE_VIEW.getType(), + mainType.getIndex().getName(), + mainType.getType(), viewUuid, ViewIndexDefinition.FIELD_PROJECTS))); } @@ -635,7 +641,7 @@ public class IssueIndex { private OptionalLong getMinCreatedAt(Map<String, QueryBuilder> filters, QueryBuilder esQuery) { String facetNameAndField = CREATED_AT.getFieldName(); SearchRequestBuilder esRequest = client - .prepareSearch(INDEX_TYPE_ISSUE) + .prepareSearch(TYPE_ISSUE.getMainType()) .setSize(0); BoolQueryBuilder esFilter = boolQuery(); filters.values().stream().filter(Objects::nonNull).forEach(esFilter::must); @@ -710,7 +716,7 @@ public class IssueIndex { private Terms listTermsMatching(String fieldName, IssueQuery query, @Nullable String textQuery, Terms.Order termsOrder, int size) { SearchRequestBuilder requestBuilder = client - .prepareSearch(INDEX_TYPE_ISSUE) + .prepareSearch(TYPE_ISSUE.getMainType()) // Avoids returning search hits .setSize(0); @@ -745,7 +751,7 @@ public class IssueIndex { if (projectUuids.isEmpty()) { return Collections.emptyList(); } - SearchRequestBuilder request = client.prepareSearch(INDEX_TYPE_ISSUE) + SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) .setQuery( boolQuery() .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) @@ -787,8 +793,8 @@ public class IssueIndex { return Collections.emptyList(); } - SearchRequestBuilder request = client.prepareSearch(INDEX_TYPE_ISSUE) - .setRouting(projectUuid) + SearchRequestBuilder request = client.prepareSearch(TYPE_ISSUE.getMainType()) + .setRouting(AuthorizationDoc.idOf(projectUuid)) .setQuery( boolQuery() .must(termsQuery(FIELD_ISSUE_BRANCH_UUID, branchUuids)) @@ -912,16 +918,17 @@ public class IssueIndex { private SearchRequestBuilder prepareNonClosedVulnerabilitiesAndHotspotSearch(String projectUuid, boolean isViewOrApp) { BoolQueryBuilder componentFilter = boolQuery(); if (isViewOrApp) { + IndexType.IndexMainType mainType = TYPE_VIEW; componentFilter.filter(QueryBuilders.termsLookupQuery(FIELD_ISSUE_BRANCH_UUID, new TermsLookup( - ViewIndexDefinition.INDEX_TYPE_VIEW.getIndex(), - ViewIndexDefinition.INDEX_TYPE_VIEW.getType(), + mainType.getIndex().getName(), + mainType.getType(), projectUuid, ViewIndexDefinition.FIELD_PROJECTS))); } else { componentFilter.filter(termQuery(FIELD_ISSUE_BRANCH_UUID, projectUuid)); } - return client.prepareSearch(INDEX_TYPE_ISSUE) + return client.prepareSearch(TYPE_ISSUE.getMainType()) .setQuery( componentFilter .filter(termsQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name(), RuleType.VULNERABILITY.name())) diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java index 4c87f95d646..2839c4acdf2 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java @@ -51,7 +51,7 @@ import org.elasticsearch.search.sort.FieldSortBuilder; import org.sonar.api.server.ServerSide; import org.sonar.api.utils.System2; import org.sonar.core.util.stream.MoreCollectors; -import org.sonar.server.es.DefaultIndexSettingsElement; +import org.sonar.server.es.newindex.DefaultIndexSettingsElement; import org.sonar.server.es.EsClient; import org.sonar.server.es.SearchIdResult; import org.sonar.server.es.SearchOptions; @@ -86,6 +86,7 @@ import static org.sonar.api.measures.CoreMetrics.SECURITY_RATING_KEY; import static org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY; import static org.sonar.server.es.EsUtils.escapeSpecialRegexChars; import static org.sonar.server.es.EsUtils.termsToMap; +import static org.sonar.server.es.IndexType.FIELD_INDEX_TYPE; import static org.sonar.server.measure.index.ProjectMeasuresDoc.QUALITY_GATE_STATUS; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY; @@ -96,7 +97,7 @@ import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIEL import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ORGANIZATION_UUID; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_LAST_ANALYSIS_DATE; import static org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_NAME; import static org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_LANGUAGES; @@ -164,7 +165,7 @@ public class ProjectMeasuresIndex { public SearchIdResult<String> search(ProjectMeasuresQuery query, SearchOptions searchOptions) { SearchRequestBuilder requestBuilder = client - .prepareSearch(INDEX_TYPE_PROJECT_MEASURES) + .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) .setFetchSource(false) .setFrom(searchOptions.getOffset()) .setSize(searchOptions.getLimit()); @@ -181,7 +182,7 @@ public class ProjectMeasuresIndex { public ProjectMeasuresStatistics searchTelemetryStatistics() { SearchRequestBuilder request = client - .prepareSearch(INDEX_TYPE_PROJECT_MEASURES) + .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) .setFetchSource(false) .setSize(0); @@ -344,6 +345,7 @@ public class ProjectMeasuresIndex { private Map<String, QueryBuilder> createFilters(ProjectMeasuresQuery query) { Map<String, QueryBuilder> filters = new HashMap<>(); + filters.put("__indexType", termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())); filters.put("__authorization", authorizationTypeSupport.createQueryFilter()); Multimap<String, MetricCriterion> metricCriterionMultimap = ArrayListMultimap.create(); query.getMetricCriteria().forEach(metricCriterion -> metricCriterionMultimap.put(metricCriterion.getMetricKey(), metricCriterion)); @@ -429,7 +431,7 @@ public class ProjectMeasuresIndex { } SearchRequestBuilder searchQuery = client - .prepareSearch(INDEX_TYPE_PROJECT_MEASURES) + .prepareSearch(TYPE_PROJECT_MEASURES.getMainType()) .setQuery(authorizationTypeSupport.createQueryFilter()) .setFetchSource(false) .setSize(0) diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsTextSearchQueryFactory.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsTextSearchQueryFactory.java index 022bc73247a..63a487a0a37 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsTextSearchQueryFactory.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectsTextSearchQueryFactory.java @@ -27,13 +27,13 @@ import org.apache.commons.lang.StringUtils; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.sonar.server.es.DefaultIndexSettings; +import org.sonar.server.es.newindex.DefaultIndexSettings; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.prefixQuery; -import static org.sonar.server.es.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; -import static org.sonar.server.es.DefaultIndexSettingsElement.SORTABLE_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER; +import static org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME; diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java index f58cf64be92..f509a15b099 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java @@ -20,15 +20,15 @@ package org.sonar.server.permission.index; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.elasticsearch.action.index.IndexRequest; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -43,7 +43,6 @@ import org.sonar.server.es.ProjectIndexer; import static java.util.Collections.emptyList; import static org.sonar.core.util.stream.MoreCollectors.toArrayList; -import static org.sonar.core.util.stream.MoreCollectors.toSet; /** * Populates the types "authorization" of each index requiring project @@ -54,7 +53,7 @@ public class PermissionIndexer implements ProjectIndexer { private final DbClient dbClient; private final EsClient esClient; private final Collection<AuthorizationScope> authorizationScopes; - private final Set<IndexType> indexTypes; + private final Map<String, IndexType> indexTypeByFormat; public PermissionIndexer(DbClient dbClient, EsClient esClient, NeedAuthorizationIndexer... needAuthorizationIndexers) { this(dbClient, esClient, Arrays.stream(needAuthorizationIndexers) @@ -67,14 +66,14 @@ public class PermissionIndexer implements ProjectIndexer { this.dbClient = dbClient; this.esClient = esClient; this.authorizationScopes = authorizationScopes; - this.indexTypes = authorizationScopes.stream() + this.indexTypeByFormat = authorizationScopes.stream() .map(AuthorizationScope::getIndexType) - .collect(toSet(authorizationScopes.size())); + .collect(MoreCollectors.uniqueIndex(IndexType.IndexMainType::format, t -> t, authorizationScopes.size())); } @Override public Set<IndexType> getIndexTypes() { - return indexTypes; + return ImmutableSet.copyOf(indexTypeByFormat.values()); } @Override @@ -116,8 +115,8 @@ public class PermissionIndexer implements ProjectIndexer { } private Collection<EsQueueDto> insertIntoEsQueue(DbSession dbSession, Collection<String> projectUuids) { - List<EsQueueDto> items = indexTypes.stream() - .flatMap(indexType -> projectUuids.stream().map(projectUuid -> EsQueueDto.create(indexType.format(), projectUuid, null, projectUuid))) + List<EsQueueDto> items = indexTypeByFormat.values().stream() + .flatMap(indexType -> projectUuids.stream().map(projectUuid -> EsQueueDto.create(indexType.format(), AuthorizationDoc.idOf(projectUuid), null, projectUuid))) .collect(toArrayList()); dbClient.esQueueDao().insert(dbSession, items); @@ -138,7 +137,7 @@ public class PermissionIndexer implements ProjectIndexer { authorizations.stream() .filter(scope.getProjectPredicate()) - .map(dto -> newIndexRequest(dto, indexType)) + .map(dto -> AuthorizationDoc.fromDto(indexType, dto).toIndexRequest()) .forEach(bulkIndexer::add); bulkIndexer.stop(); @@ -152,8 +151,8 @@ public class PermissionIndexer implements ProjectIndexer { List<BulkIndexer> bulkIndexers = items.stream() .map(EsQueueDto::getDocType) .distinct() - .map(IndexType::parse) - .filter(indexTypes::contains) + .map(indexTypeByFormat::get) + .filter(Objects::nonNull) .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) .collect(Collectors.toList()); @@ -164,37 +163,26 @@ public class PermissionIndexer implements ProjectIndexer { bulkIndexers.forEach(BulkIndexer::start); PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); - Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet()); + Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId) + .map(AuthorizationDoc::projectUuidOf) + .collect(MoreCollectors.toHashSet()); permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> { remainingProjectUuids.remove(p.getProjectUuid()); - bulkIndexers.forEach(bi -> bi.add(newIndexRequest(p, bi.getIndexType()))); + bulkIndexers.forEach(bi -> bi.add(AuthorizationDoc.fromDto(bi.getIndexType(), p).toIndexRequest())); }); // the remaining references on projects that don't exist in db. They must // be deleted from index. - remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> bi.addDeletion(bi.getIndexType(), projectUuid, projectUuid))); + remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> { + String authorizationDocId = AuthorizationDoc.idOf(projectUuid); + bi.addDeletion(bi.getIndexType(), authorizationDocId, authorizationDocId); + })); bulkIndexers.forEach(b -> result.add(b.stop())); return result; } - private static IndexRequest newIndexRequest(IndexPermissions dto, IndexType indexType) { - Map<String, Object> doc = new HashMap<>(); - if (dto.isAllowAnyone()) { - doc.put(IndexAuthorizationConstants.FIELD_ALLOW_ANYONE, true); - // no need to feed users and groups - } else { - doc.put(IndexAuthorizationConstants.FIELD_ALLOW_ANYONE, false); - doc.put(IndexAuthorizationConstants.FIELD_GROUP_IDS, dto.getGroupIds()); - doc.put(IndexAuthorizationConstants.FIELD_USER_IDS, dto.getUserIds()); - } - return new IndexRequest(indexType.getIndex(), indexType.getType()) - .id(dto.getProjectUuid()) - .routing(dto.getProjectUuid()) - .source(doc); - } - private Stream<AuthorizationScope> getScopes(Set<IndexType> indexTypes) { return authorizationScopes.stream() .filter(scope -> indexTypes.contains(scope.getIndexType())); diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java index 0a9024956a8..e4fc7c2bce6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/BackendCleanup.java @@ -36,6 +36,7 @@ import org.sonar.db.version.SqTables; import org.sonar.server.component.index.ComponentIndexDefinition; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.EsClient; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexType; import org.sonar.server.issue.index.IssueIndexDefinition; import org.sonar.server.measure.index.ProjectMeasuresIndexDefinition; @@ -96,7 +97,8 @@ public class BackendCleanup { esClient.prepareClearCache().get(); for (String index : esClient.prepareState().get().getState().getMetaData().getConcreteAllIndices()) { - clearIndex(new IndexType(index, index)); + /*under the hood, type is not used to perform clearIndex, so it's ok it does not match any existing index*/ + clearIndex(Index.simple(index)); } } catch (Exception e) { throw new IllegalStateException("Unable to clear indexes", e); @@ -121,10 +123,10 @@ public class BackendCleanup { throw new IllegalStateException("Fail to reset data", e); } - clearIndex(IssueIndexDefinition.INDEX_TYPE_ISSUE); - clearIndex(ViewIndexDefinition.INDEX_TYPE_VIEW); - clearIndex(ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES); - clearIndex(ComponentIndexDefinition.INDEX_TYPE_COMPONENT); + clearIndex(IssueIndexDefinition.DESCRIPTOR); + clearIndex(ViewIndexDefinition.DESCRIPTOR); + clearIndex(ProjectMeasuresIndexDefinition.DESCRIPTOR); + clearIndex(ComponentIndexDefinition.DESCRIPTOR); } private void truncateAnalysisTables(Connection connection) throws SQLException { @@ -166,8 +168,8 @@ public class BackendCleanup { /** * Completely remove a index with all types */ - public void clearIndex(IndexType indexType) { - BulkIndexer.delete(esClient, indexType, esClient.prepareSearch(indexType.getIndex()).setQuery(matchAllQuery())); + public void clearIndex(Index index) { + BulkIndexer.delete(esClient, IndexType.main(index, index.getName()), esClient.prepareSearch(index).setQuery(matchAllQuery())); } @FunctionalInterface diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java index 07509022422..1575ac315fb 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -32,16 +32,20 @@ import org.junit.rules.ExpectedException; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.utils.log.LogTester; import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.server.es.IndexType.IndexMainType; import org.sonar.server.es.metadata.MetadataIndex; import org.sonar.server.es.metadata.MetadataIndexDefinition; +import org.sonar.server.es.newindex.NewRegularIndex; +import org.sonar.server.es.newindex.SettingsConfiguration; import static org.assertj.core.api.Assertions.assertThat; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.IndexType.main; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class IndexCreatorTest { - private static final NewIndex.SettingsConfiguration SETTINGS_CONFIGURATION = newBuilder(new MapSettings().asConfig()).build(); + private static final SettingsConfiguration SETTINGS_CONFIGURATION = newBuilder(new MapSettings().asConfig()).build(); private static final String LOG_DB_VENDOR_CHANGED = "Delete Elasticsearch indices (DB vendor changed)"; private static final String LOG_DB_SCHEMA_CHANGED = "Delete Elasticsearch indices (DB schema changed)"; @@ -80,15 +84,19 @@ public class IndexCreatorTest { @Test public void mark_all_non_existing_index_types_as_uninitialized() { + Index fakesIndex = Index.simple("fakes"); + Index fakersIndex = Index.simple("fakers"); startNewCreator(context -> { - NewIndex i = context.create("fakes", SETTINGS_CONFIGURATION); - i.createType("t1"); - i.createType("t2"); + context.create(fakesIndex, SETTINGS_CONFIGURATION) + .createTypeMapping(IndexType.main(fakesIndex, "fake")); + context.create(fakersIndex, SETTINGS_CONFIGURATION) + .createTypeMapping(IndexType.main(fakersIndex, "faker")); }); - assertThat(metadataIndex.getHash("fakes")).isNotEmpty(); - assertThat(metadataIndex.getInitialized(new IndexType("fakes", "t1"))).isFalse(); - assertThat(metadataIndex.getInitialized(new IndexType("fakes", "t2"))).isFalse(); + assertThat(metadataIndex.getHash(fakesIndex)).isNotEmpty(); + assertThat(metadataIndex.getHash(fakersIndex)).isNotEmpty(); + assertThat(metadataIndex.getInitialized(main(fakesIndex, "fake"))).isFalse(); + assertThat(metadataIndex.getInitialized(main(fakersIndex, "faker"))).isFalse(); } @Test @@ -96,7 +104,7 @@ public class IndexCreatorTest { // v1 startNewCreator(new FakeIndexDefinition()); - IndexType fakeIndexType = new IndexType("fakes", "fake"); + IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake"); String id = "1"; es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); @@ -135,7 +143,7 @@ public class IndexCreatorTest { public void do_not_recreate_index_on_unchanged_definition() { // v1 startNewCreator(new FakeIndexDefinition()); - IndexType fakeIndexType = new IndexType("fakes", "fake"); + IndexMainType fakeIndexType = main(Index.simple("fakes"), "fake"); String id = "1"; es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(IMMEDIATE).get(); assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); @@ -225,25 +233,27 @@ public class IndexCreatorTest { private static class FakeIndexDefinition implements IndexDefinition { - private static final IndexType INDEX_TYPE = new IndexType("fakes", "fake"); + private static final IndexMainType INDEX_TYPE = main(Index.simple("fakes"), "fake"); @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create("fakes", SETTINGS_CONFIGURATION); - NewIndex.NewIndexType mapping = index.createType("fake"); - mapping.keywordFieldBuilder("key").build(); - mapping.createDateTimeField("updatedAt"); + Index index = Index.simple("fakes"); + NewRegularIndex newIndex = context.create(index, SETTINGS_CONFIGURATION); + newIndex.createTypeMapping(IndexType.main(index, "fake")) + .keywordFieldBuilder("key").build() + .createDateTimeField("updatedAt"); } } private static class FakeIndexDefinitionV2 implements IndexDefinition { @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create("fakes", SETTINGS_CONFIGURATION); - NewIndex.NewIndexType mapping = index.createType("fake"); - mapping.keywordFieldBuilder("key").build(); - mapping.createDateTimeField("updatedAt"); - mapping.createIntegerField("newField"); + Index index = Index.simple("fakes"); + NewRegularIndex newIndex = context.create(index, SETTINGS_CONFIGURATION); + newIndex.createTypeMapping(IndexType.main(index, "fake")) + .keywordFieldBuilder("key").build() + .createDateTimeField("updatedAt") + .createIntegerField("newField"); } } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java index 129cfd003ee..b9b17c31ee2 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java @@ -26,13 +26,14 @@ import org.junit.Test; import org.mockito.Mockito; import org.sonar.api.config.internal.MapSettings; import org.sonar.server.es.metadata.MetadataIndex; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE; +import static org.sonar.server.es.newindex.FakeIndexDefinition.TYPE_FAKE; public class IndexerStartupTaskTest { @@ -46,31 +47,31 @@ public class IndexerStartupTaskTest { @Before public void setUp() throws Exception { - doReturn(ImmutableSet.of(INDEX_TYPE_FAKE)).when(indexer).getIndexTypes(); + doReturn(ImmutableSet.of(TYPE_FAKE)).when(indexer).getIndexTypes(); } @Test public void index_if_not_initialized() { - doReturn(false).when(metadataIndex).getInitialized(INDEX_TYPE_FAKE); + doReturn(false).when(metadataIndex).getInitialized(TYPE_FAKE); underTest.execute(); verify(indexer).getIndexTypes(); - verify(indexer).indexOnStartup(Mockito.eq(ImmutableSet.of(INDEX_TYPE_FAKE))); + verify(indexer).indexOnStartup(Mockito.eq(ImmutableSet.of(TYPE_FAKE))); } @Test public void set_initialized_after_indexation() { - doReturn(false).when(metadataIndex).getInitialized(INDEX_TYPE_FAKE); + doReturn(false).when(metadataIndex).getInitialized(TYPE_FAKE); underTest.execute(); - verify(metadataIndex).setInitialized(eq(INDEX_TYPE_FAKE), eq(true)); + verify(metadataIndex).setInitialized(eq(TYPE_FAKE), eq(true)); } @Test public void do_not_index_if_already_initialized() { - doReturn(true).when(metadataIndex).getInitialized(INDEX_TYPE_FAKE); + doReturn(true).when(metadataIndex).getInitialized(TYPE_FAKE); underTest.execute(); @@ -81,7 +82,7 @@ public class IndexerStartupTaskTest { @Test public void do_not_index_if_indexes_are_disabled() { settings.setProperty("sonar.internal.es.disableIndexes", "true"); - es.putDocuments(INDEX_TYPE_FAKE, new FakeDoc()); + es.putDocuments(TYPE_FAKE, new FakeDoc()); underTest.execute(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java index bc84c56724a..8d82b2c5bd7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java @@ -20,7 +20,6 @@ package org.sonar.server.es; import java.util.Iterator; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.junit.Rule; import org.junit.Test; import org.sonar.api.config.internal.MapSettings; @@ -29,39 +28,39 @@ import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.platform.db.migration.es.MigrationEsClient; import static org.assertj.core.api.Assertions.assertThat; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class MigrationEsClientImplTest { @Rule public LogTester logTester = new LogTester(); @Rule public EsTester es = EsTester.createCustom( - new SimpleIndexDefinition("a"), - new SimpleIndexDefinition("b"), - new SimpleIndexDefinition("c")); + new SimpleIndexDefinition("as"), + new SimpleIndexDefinition("bs"), + new SimpleIndexDefinition("cs")); private MigrationEsClient underTest = new MigrationEsClientImpl(es.client()); @Test public void delete_existing_index() { - underTest.deleteIndexes("a"); + underTest.deleteIndexes("as"); assertThat(loadExistingIndices()) - .doesNotContain("a") - .contains("b", "c"); + .doesNotContain("as") + .contains("bs", "cs"); assertThat(logTester.logs(LoggerLevel.INFO)) - .contains("Drop Elasticsearch index [a]"); + .contains("Drop Elasticsearch index [as]"); } @Test public void ignore_indices_that_do_not_exist() { - underTest.deleteIndexes("a", "xxx", "c"); + underTest.deleteIndexes("as", "xxx", "cs"); assertThat(loadExistingIndices()) - .doesNotContain("a", "c") - .contains("b"); + .doesNotContain("as", "cs") + .contains("bs"); assertThat(logTester.logs(LoggerLevel.INFO)) - .contains("Drop Elasticsearch index [a]", "Drop Elasticsearch index [c]") + .contains("Drop Elasticsearch index [as]", "Drop Elasticsearch index [cs]") .doesNotContain("Drop Elasticsearch index [xxx]"); } @@ -78,9 +77,11 @@ public class MigrationEsClientImplTest { @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create(indexName, newBuilder(new MapSettings().asConfig()).build()); - index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0); - index.getSettings().put("index.refresh_interval", "-1"); + IndexType.IndexMainType mainType = IndexType.main(Index.simple(indexName), indexName.substring(1)); + context.create( + mainType.getIndex(), + newBuilder(new MapSettings().asConfig()).build()) + .createTypeMapping(mainType); } } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java index b0cdd01d7c4..cad24375750 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java @@ -43,6 +43,7 @@ import org.sonar.api.utils.log.LoggerLevel; import org.sonar.db.DbSession; import org.sonar.db.DbTester; import org.sonar.db.es.EsQueueDto; +import org.sonar.server.es.IndexType.IndexMainType; import org.sonar.server.rule.index.RuleIndexer; import org.sonar.server.user.index.UserIndexer; @@ -57,7 +58,7 @@ import static org.sonar.api.utils.log.LoggerLevel.TRACE; public class RecoveryIndexerTest { private static final long PAST = 1_000L; - private static final IndexType FOO_TYPE = new IndexType("foos", "foo"); + private static final IndexMainType FOO_TYPE = IndexType.main(Index.simple("foos"), "foo"); private TestSystem2 system2 = new TestSystem2().setNow(PAST); private MapSettings emptySettings = new MapSettings(); @@ -114,13 +115,12 @@ public class RecoveryIndexerTest { @Test public void successfully_recover_indexing_requests() { - IndexType type1 = new IndexType("foos", "foo"); - EsQueueDto item1a = insertItem(type1, "f1"); - EsQueueDto item1b = insertItem(type1, "f2"); - IndexType type2 = new IndexType("bars", "bar"); + EsQueueDto item1a = insertItem(FOO_TYPE, "f1"); + EsQueueDto item1b = insertItem(FOO_TYPE, "f2"); + IndexMainType type2 = IndexType.main(Index.simple("bars"), "bar"); EsQueueDto item2 = insertItem(type2, "b1"); - SuccessfulFakeIndexer indexer1 = new SuccessfulFakeIndexer(type1); + SuccessfulFakeIndexer indexer1 = new SuccessfulFakeIndexer(FOO_TYPE); SuccessfulFakeIndexer indexer2 = new SuccessfulFakeIndexer(type2); advanceInTime(); @@ -221,7 +221,7 @@ public class RecoveryIndexerTest { public void unsupported_types_are_kept_in_queue_for_manual_fix_operation() { insertItem(FOO_TYPE, "f1"); - ResilientIndexer indexer = new SuccessfulFakeIndexer(new IndexType("bars", "bar")); + ResilientIndexer indexer = new SuccessfulFakeIndexer(IndexType.main(Index.simple("bars"), "bar")); advanceInTime(); underTest = newRecoveryIndexer(indexer); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java index 78dcca23b21..108c89475a6 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java @@ -25,7 +25,7 @@ import org.junit.rules.ExpectedException; import org.mockito.Mockito; import org.sonar.db.DbClient; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/WebIssueStorageTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/WebIssueStorageTest.java index 55c4aa462e1..b326fd398f7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/WebIssueStorageTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/WebIssueStorageTest.java @@ -61,7 +61,8 @@ public class WebIssueStorageTest { private TestDefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); - private WebIssueStorage underTest = new WebIssueStorage(system2, dbClient, new DefaultRuleFinder(db.getDbClient(), defaultOrganizationProvider), mock(IssueIndexer.class)); + private IssueIndexer issueIndexer = mock(IssueIndexer.class); + private WebIssueStorage underTest = new WebIssueStorage(system2, dbClient, new DefaultRuleFinder(db.getDbClient(), defaultOrganizationProvider), issueIndexer); @Test public void load_component_id_from_db() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java index cb5c4ef44ca..184daed2564 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java @@ -149,7 +149,7 @@ public class IssueIndexTest { // project2 can be seen by group2 indexIssue(newDoc("I2", file2)); authorizationIndexer.allowOnlyGroup(project2, group2); - // project3 can be seen by nobody + // project3 can be seen by nobody but root indexIssue(newDoc("I3", file3)); userSessionRule.logIn().setGroups(group1); @@ -167,6 +167,9 @@ public class IssueIndexTest { userSessionRule.logIn().setGroups(group1, group2); assertThatSearchReturnsEmpty(IssueQuery.builder().projectUuids(singletonList(project3.uuid()))); + + userSessionRule.setRoot(); + assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2", "I3"); } @Test @@ -198,6 +201,9 @@ public class IssueIndexTest { // another user userSessionRule.logIn(newUserDto()); assertThatSearchReturnsEmpty(IssueQuery.builder()); + + userSessionRule.setRoot(); + assertThatSearchReturnsOnly(IssueQuery.builder(), "I1", "I2", "I3"); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java index e6d4616cdc2..601b84b441d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTest.java @@ -68,7 +68,7 @@ import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; import static org.sonar.db.user.GroupTesting.newGroupDto; import static org.sonar.db.user.UserTesting.newUserDto; import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; import static org.sonar.server.measure.index.ProjectMeasuresQuery.Operator; @RunWith(DataProviderRunner.class) @@ -1383,7 +1383,7 @@ public class ProjectMeasuresIndexTest { @Test public void search_statistics() { - es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, + es.putDocuments(TYPE_PROJECT_MEASURES, newDoc("lines", 10, "coverage", 80) .setLanguages(Arrays.asList("java", "cs", "js")) .setNclocLanguageDistributionFromMap(ImmutableMap.of("java", 200, "cs", 250, "js", 50)), @@ -1409,17 +1409,17 @@ public class ProjectMeasuresIndexTest { } private void index(ProjectMeasuresDoc... docs) { - es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs); + es.putDocuments(TYPE_PROJECT_MEASURES, docs); authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).allowAnyone()).collect(toList())); } private void indexForUser(UserDto user, ProjectMeasuresDoc... docs) { - es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs); + es.putDocuments(TYPE_PROJECT_MEASURES, docs); authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).addUserId(user.getId())).collect(toList())); } private void indexForGroup(GroupDto group, ProjectMeasuresDoc... docs) { - es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs); + es.putDocuments(TYPE_PROJECT_MEASURES, docs); authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).addGroupId(group.getId())).collect(toList())); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTextSearchTest.java b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTextSearchTest.java index 918a889312b..344b598c5c7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTextSearchTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexTextSearchTest.java @@ -46,7 +46,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.entry; import static org.sonar.api.resources.Qualifiers.PROJECT; import static org.sonar.db.component.ComponentTesting.newPrivateProjectDto; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; import static org.sonar.server.measure.index.ProjectMeasuresQuery.Operator.GT; import static org.sonar.server.measure.index.ProjectMeasuresQuery.Operator.LT; @@ -287,7 +287,7 @@ public class ProjectMeasuresIndexTextSearchTest { } private void index(ProjectMeasuresDoc... docs) { - es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs); + es.putDocuments(TYPE_PROJECT_MEASURES, docs); authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).allowAnyone()).collect(toList())); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/MemberUpdaterTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/MemberUpdaterTest.java index 157981454a4..309698e3c49 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/MemberUpdaterTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/MemberUpdaterTest.java @@ -489,7 +489,7 @@ public class MemberUpdaterTest { private void assertUserIsNotMember(OrganizationDto organization, UserDto user) { db.organizations().assertUserIsNotMemberOfOrganization(organization, user); - SearchRequestBuilder request = es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) + SearchRequestBuilder request = es.client().prepareSearch(UserIndexDefinition.TYPE_USER) .setQuery(boolQuery() .must(termQuery(FIELD_ORGANIZATION_UUIDS, organization.getUuid())) .must(termQuery(FIELD_UUID, user.getUuid()))); diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java index 84bd85739c7..d6f3fbc5114 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java @@ -60,7 +60,6 @@ import org.sonar.server.permission.PermissionService; import org.sonar.server.permission.PermissionServiceImpl; import org.sonar.server.qualityprofile.BuiltInQProfileRepository; import org.sonar.server.tester.UserSessionRule; -import org.sonar.server.user.index.UserIndexDefinition; import org.sonar.server.user.index.UserIndexer; import org.sonar.server.usergroups.DefaultGroupCreatorImpl; import org.sonar.server.ws.TestRequest; @@ -82,6 +81,7 @@ import static org.sonar.core.config.CorePropertyDefinitions.ORGANIZATIONS_ANYONE import static org.sonar.server.organization.ws.OrganizationsWsSupport.PARAM_NAME; import static org.sonar.server.organization.ws.OrganizationsWsTestSupport.STRING_257_CHARS_LONG; import static org.sonar.server.organization.ws.OrganizationsWsTestSupport.STRING_65_CHARS_LONG; +import static org.sonar.server.user.index.UserIndexDefinition.TYPE_USER; import static org.sonar.server.user.index.UserIndexDefinition.FIELD_ORGANIZATION_UUIDS; import static org.sonar.server.user.index.UserIndexDefinition.FIELD_UUID; import static org.sonar.test.JsonAssert.assertJson; @@ -236,7 +236,7 @@ public class CreateActionTest { OrganizationDto organization = dbClient.organizationDao().selectByKey(dbSession, "bar").get(); assertThat(dbClient.organizationMemberDao().select(dbSession, organization.getUuid(), user.getId())).isPresent(); - assertThat(es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) + assertThat(es.client().prepareSearch(TYPE_USER) .setQuery(boolQuery() .must(termQuery(FIELD_ORGANIZATION_UUIDS, organization.getUuid())) .must(termQuery(FIELD_UUID, user.getUuid()))) diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndex.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndex.java index 8c5fd471ed2..0deceacf733 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndex.java +++ b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndex.java @@ -26,8 +26,8 @@ import org.elasticsearch.search.SearchHits; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.server.es.EsClient; -import static org.sonar.server.permission.index.FooIndexDefinition.FOO_INDEX; -import static org.sonar.server.permission.index.FooIndexDefinition.FOO_TYPE; +import static org.sonar.server.permission.index.FooIndexDefinition.DESCRIPTOR; +import static org.sonar.server.permission.index.FooIndexDefinition.TYPE_AUTHORIZATION; public class FooIndex { @@ -40,8 +40,8 @@ public class FooIndex { } public boolean hasAccessToProject(String projectUuid) { - SearchHits hits = esClient.prepareSearch(FOO_INDEX) - .setTypes(FOO_TYPE) + SearchHits hits = esClient.prepareSearch(DESCRIPTOR) + .setTypes(TYPE_AUTHORIZATION.getType()) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) .filter(authorizationTypeSupport.createQueryFilter())) diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexDefinition.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexDefinition.java index c34f2153898..2dfe93c4c29 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexDefinition.java +++ b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexDefinition.java @@ -20,29 +20,33 @@ package org.sonar.server.permission.index; import org.sonar.api.config.internal.MapSettings; +import org.sonar.server.es.Index; import org.sonar.server.es.IndexDefinition; import org.sonar.server.es.IndexType; -import org.sonar.server.es.NewIndex; +import org.sonar.server.es.newindex.NewAuthorizedIndex; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; -import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder; +import static org.sonar.server.es.newindex.SettingsConfiguration.MANUAL_REFRESH_INTERVAL; +import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; public class FooIndexDefinition implements IndexDefinition { - public static final String FOO_INDEX = "foos"; + public static final Index DESCRIPTOR = Index.withRelations("foos"); public static final String FOO_TYPE = "foo"; - public static final IndexType INDEX_TYPE_FOO = new IndexType(FOO_INDEX, FOO_TYPE); + public static final IndexType.IndexMainType TYPE_AUTHORIZATION = IndexType.main(DESCRIPTOR, IndexAuthorizationConstants.TYPE_AUTHORIZATION); + public static final IndexType.IndexRelationType TYPE_FOO = IndexType.relation(TYPE_AUTHORIZATION, FOO_TYPE); public static final String FIELD_NAME = "name"; public static final String FIELD_PROJECT_UUID = "projectUuid"; @Override public void define(IndexDefinitionContext context) { - NewIndex index = context.create(FOO_INDEX, newBuilder(new MapSettings().asConfig()).setRefreshInterval(MANUAL_REFRESH_INTERVAL).build()); + NewAuthorizedIndex newIndex = context.createWithAuthorization( + DESCRIPTOR, + newBuilder(new MapSettings().asConfig()) + .setRefreshInterval(MANUAL_REFRESH_INTERVAL) + .build()); - NewIndex.NewIndexType type = index.createType(FOO_TYPE) - .requireProjectAuthorization(); - - type.keywordFieldBuilder(FIELD_NAME).build(); - type.keywordFieldBuilder(FIELD_PROJECT_UUID).build(); + newIndex.createTypeMapping(TYPE_FOO) + .keywordFieldBuilder(FIELD_NAME).build() + .keywordFieldBuilder(FIELD_PROJECT_UUID).build(); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java index 8572418fd44..fd9e79f31bf 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java +++ b/server/sonar-server/src/test/java/org/sonar/server/permission/index/FooIndexer.java @@ -19,22 +19,22 @@ */ package org.sonar.server.permission.index; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.Collection; import java.util.Set; import org.sonar.db.DbSession; import org.sonar.db.es.EsQueueDto; +import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.IndexingResult; import org.sonar.server.es.ProjectIndexer; -import static org.sonar.server.permission.index.FooIndexDefinition.INDEX_TYPE_FOO; +import static org.sonar.server.permission.index.FooIndexDefinition.TYPE_FOO; public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer { - private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(INDEX_TYPE_FOO, p -> true); + private static final AuthorizationScope AUTHORIZATION_SCOPE = new AuthorizationScope(TYPE_FOO, p -> true); private final EsClient esClient; @@ -59,15 +59,34 @@ public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer { } private void addToIndex(String projectUuid, String name) { - esClient.prepareIndex(INDEX_TYPE_FOO) - .setRouting(projectUuid) - .setParent(projectUuid) - .setSource(ImmutableMap.of( - FooIndexDefinition.FIELD_NAME, name, - FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) + FooDoc fooDoc = new FooDoc(projectUuid, name); + esClient.prepareIndex(TYPE_FOO) + .setId(fooDoc.getId()) + .setRouting(fooDoc.getRouting().orElse(null)) + .setSource(fooDoc.getFields()) .get(); } + private static final class FooDoc extends BaseDoc { + private final String projectUuid; + private final String name; + + private FooDoc(String projectUuid, String name) { + super(TYPE_FOO); + this.projectUuid = projectUuid; + this.name = name; + setField(FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid); + setField(FooIndexDefinition.FIELD_NAME, name); + setParent(AuthorizationDoc.idOf(projectUuid)); + } + + @Override + public String getId() { + return projectUuid + "_" + name; + } + + } + @Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { throw new UnsupportedOperationException(); @@ -75,7 +94,7 @@ public class FooIndexer implements ProjectIndexer, NeedAuthorizationIndexer { @Override public Set<IndexType> getIndexTypes() { - return ImmutableSet.of(INDEX_TYPE_FOO); + return ImmutableSet.of(TYPE_FOO); } @Override diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java index 37708c1da8b..1dda0b4e89a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/permission/index/PermissionIndexerTest.java @@ -33,6 +33,7 @@ import org.sonar.db.user.GroupDto; import org.sonar.db.user.UserDto; import org.sonar.server.es.EsTester; import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; import org.sonar.server.es.IndexingResult; import org.sonar.server.es.ProjectIndexer; import org.sonar.server.tester.UserSessionRule; @@ -47,7 +48,7 @@ import static org.sonar.server.permission.index.IndexAuthorizationConstants.TYPE public class PermissionIndexerTest { - private static final IndexType INDEX_TYPE_FOO_AUTH = new IndexType(FooIndexDefinition.INDEX_TYPE_FOO.getIndex(), TYPE_AUTHORIZATION); + private static final IndexMainType INDEX_TYPE_FOO_AUTH = IndexType.main(FooIndexDefinition.DESCRIPTOR, TYPE_AUTHORIZATION); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -336,8 +337,7 @@ public class PermissionIndexerTest { } private void assertThatAuthIndexHasSize(int expectedSize) { - IndexType authIndexType = underTest.getIndexTypes().iterator().next(); - assertThat(es.countDocuments(authIndexType)).isEqualTo(expectedSize); + assertThat(es.countDocuments(FooIndexDefinition.TYPE_AUTHORIZATION)).isEqualTo(expectedSize); } private void indexOnStartup() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/permission/index/WebAuthorizationTypeSupportTest.java b/server/sonar-server/src/test/java/org/sonar/server/permission/index/WebAuthorizationTypeSupportTest.java index 808befb7bac..9a94ba36f93 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/permission/index/WebAuthorizationTypeSupportTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/permission/index/WebAuthorizationTypeSupportTest.java @@ -61,14 +61,14 @@ public class WebAuthorizationTypeSupportTest { " \"bool\" : {" + " \"should\" : [{" + " \"term\" : {" + - " \"allowAnyone\" : {\"value\": true}" + + " \"auth_allowAnyone\" : {\"value\": true}" + " }" + " }]" + " }" + " }]" + " }" + " }," + - " \"parent_type\" : \"authorization\"" + + " \"parent_type\" : \"auth\"" + " }" + "}"); } @@ -88,12 +88,12 @@ public class WebAuthorizationTypeSupportTest { " \"should\": [" + " {" + " \"term\": {" + - " \"allowAnyone\": {\"value\": true}" + + " \"auth_allowAnyone\": {\"value\": true}" + " }" + " }," + " {" + " \"term\": {" + - " \"userIds\": {\"value\": 1234}" + + " \"auth_userIds\": {\"value\": 1234}" + " }" + " }" + " ]" + @@ -101,7 +101,7 @@ public class WebAuthorizationTypeSupportTest { " }]" + " }" + " }," + - " \"parent_type\": \"authorization\"" + + " \"parent_type\": \"auth\"" + " }" + "}"); } @@ -123,22 +123,22 @@ public class WebAuthorizationTypeSupportTest { " \"should\": [" + " {" + " \"term\": {" + - " \"allowAnyone\": {\"value\": true}" + + " \"auth_allowAnyone\": {\"value\": true}" + " }" + " }," + " {" + " \"term\": {" + - " \"userIds\": {\"value\": 1234}" + + " \"auth_userIds\": {\"value\": 1234}" + " }" + " }," + " {" + " \"term\": {" + - " \"groupIds\": {\"value\": 10}" + + " \"auth_groupIds\": {\"value\": 10}" + " }" + " }," + " {" + " \"term\": {" + - " \"groupIds\": {\"value\": 11}" + + " \"auth_groupIds\": {\"value\": 11}" + " }" + " }" + " ]" + @@ -146,7 +146,7 @@ public class WebAuthorizationTypeSupportTest { " }]" + " }" + " }," + - " \"parent_type\": \"authorization\"" + + " \"parent_type\": \"auth\"" + " }" + "}"); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupTest.java index 90c92c9d56b..abf5408ad55 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/BackendCleanupTest.java @@ -75,28 +75,28 @@ public class BackendCleanupTest { @Test public void clear_indexes() { - es.putDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE, IssueDocTesting.newDoc()); - es.putDocuments(RuleIndexDefinition.INDEX_TYPE_RULE, newRuleDoc()); - es.putDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, newComponentDoc()); + es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); + es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); + es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); underTest.clearIndexes(); - assertThat(es.countDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE)).isEqualTo(0); - assertThat(es.countDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT)).isEqualTo(0); + assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0); + assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0); } @Test public void clear_all() { dbTester.prepareDbUnit(getClass(), "shared.xml"); - es.putDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE, IssueDocTesting.newDoc()); - es.putDocuments(RuleIndexDefinition.INDEX_TYPE_RULE, newRuleDoc()); - es.putDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, newComponentDoc()); + es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); + es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); + es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); underTest.clearAll(); - assertThat(es.countDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE)).isEqualTo(0); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(0); - assertThat(es.countDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT)).isEqualTo(0); + assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isEqualTo(0); + assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(0); + assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isEqualTo(0); assertThat(dbTester.countRowsOfTable("projects")).isEqualTo(0); assertThat(dbTester.countRowsOfTable("snapshots")).isEqualTo(0); @@ -107,28 +107,28 @@ public class BackendCleanupTest { @Test public void reset_data() { dbTester.prepareDbUnit(getClass(), "shared.xml"); - es.putDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE, IssueDocTesting.newDoc()); - es.putDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG"))); - es.putDocuments(RuleIndexDefinition.INDEX_TYPE_RULE, newRuleDoc()); - es.putDocuments(ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc() + es.putDocuments(IssueIndexDefinition.TYPE_ISSUE, IssueDocTesting.newDoc()); + es.putDocuments(ViewIndexDefinition.TYPE_VIEW, new ViewDoc().setUuid("CDEF").setProjects(newArrayList("DEFG"))); + es.putDocuments(RuleIndexDefinition.TYPE_RULE, newRuleDoc()); + es.putDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES, new ProjectMeasuresDoc() .setId("PROJECT") .setKey("Key") .setName("Name")); - es.putDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT, newComponentDoc()); + es.putDocuments(ComponentIndexDefinition.TYPE_COMPONENT, newComponentDoc()); underTest.resetData(); assertThat(dbTester.countRowsOfTable("projects")).isZero(); assertThat(dbTester.countRowsOfTable("snapshots")).isZero(); assertThat(dbTester.countRowsOfTable("properties")).isZero(); - assertThat(es.countDocuments(IssueIndexDefinition.INDEX_TYPE_ISSUE)).isZero(); - assertThat(es.countDocuments(ViewIndexDefinition.INDEX_TYPE_VIEW)).isZero(); - assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES)).isZero(); - assertThat(es.countDocuments(ComponentIndexDefinition.INDEX_TYPE_COMPONENT)).isZero(); + assertThat(es.countDocuments(IssueIndexDefinition.TYPE_ISSUE)).isZero(); + assertThat(es.countDocuments(ViewIndexDefinition.TYPE_VIEW)).isZero(); + assertThat(es.countDocuments(ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES)).isZero(); + assertThat(es.countDocuments(ComponentIndexDefinition.TYPE_COMPONENT)).isZero(); // Rules should not be removed assertThat(dbTester.countRowsOfTable("rules")).isEqualTo(1); - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(1); + assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(1); } private static RuleDoc newRuleDoc() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImplTest.java index c5283f37921..37a712ecc47 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImplTest.java @@ -24,7 +24,7 @@ import org.junit.Rule; import org.junit.Test; import org.sonar.process.systeminfo.protobuf.ProtobufSystemInfo; import org.sonar.server.es.EsTester; -import org.sonar.server.es.FakeIndexDefinition; +import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.assertj.core.api.Assertions.assertThat; diff --git a/server/sonar-server/src/test/java/org/sonar/server/projecttag/ws/SearchActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/projecttag/ws/SearchActionTest.java index 0fe9304e70a..cd4d9ad0b45 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/projecttag/ws/SearchActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/projecttag/ws/SearchActionTest.java @@ -47,7 +47,7 @@ import static java.util.Optional.ofNullable; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; import static org.sonar.api.resources.Qualifiers.PROJECT; -import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.INDEX_TYPE_PROJECT_MEASURES; +import static org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES; import static org.sonar.test.JsonAssert.assertJson; public class SearchActionTest { @@ -110,7 +110,7 @@ public class SearchActionTest { } private void index(ProjectMeasuresDoc... docs) { - es.putDocuments(INDEX_TYPE_PROJECT_MEASURES, docs); + es.putDocuments(TYPE_PROJECT_MEASURES, docs); authorizationIndexer.allow(stream(docs).map(doc -> new IndexPermissions(doc.getId(), PROJECT).allowAnyone()).collect(toList())); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/RegisterRulesTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/RegisterRulesTest.java index ab2da8a585e..281e15aab70 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/RegisterRulesTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/RegisterRulesTest.java @@ -277,7 +277,7 @@ public class RegisterRulesTest { .containsOnly(READY); // verify index - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(numberOfRules); + assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(numberOfRules); assertThat(ruleIndex.search(new RuleQuery(), new SearchOptions()).getIds()) .isNotEmpty(); @@ -291,7 +291,7 @@ public class RegisterRulesTest { .containsOnly(REMOVED); // verify index (documents are still in the index, but all are removed) - assertThat(es.countDocuments(RuleIndexDefinition.INDEX_TYPE_RULE)).isEqualTo(numberOfRules); + assertThat(es.countDocuments(RuleIndexDefinition.TYPE_RULE)).isEqualTo(numberOfRules); assertThat(ruleIndex.search(new RuleQuery(), new SearchOptions()).getIds()) .isEmpty(); } @@ -315,7 +315,7 @@ public class RegisterRulesTest { RuleDto rule1 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY1); RuleDto rule2 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY2); RuleDto hotspotRule = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, HOTSPOT_RULE_KEY); - assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId())); + assertThat(es.getIds(RuleIndexDefinition.TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId())); // user adds tags and sets markdown note rule1.setTags(newHashSet("usertag1", "usertag2")); @@ -739,7 +739,7 @@ public class RegisterRulesTest { RuleDto rule1 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY1); RuleDto rule2 = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, RULE_KEY2); RuleDto hotspotRule = dbClient.ruleDao().selectOrFailByKey(db.getSession(), defaultOrganization, HOTSPOT_RULE_KEY); - assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId())); + assertThat(es.getIds(RuleIndexDefinition.TYPE_RULE)).containsOnly(valueOf(rule1.getId()), valueOf(rule2.getId()), valueOf(hotspotRule.getId())); assertThat(rule2.getStatus()).isEqualTo(READY); @@ -774,7 +774,7 @@ public class RegisterRulesTest { execute(new BigRepository()); assertThat(db.countRowsOfTable("rules")).isEqualTo(BigRepository.SIZE); assertThat(db.countRowsOfTable("rules_parameters")).isEqualTo(BigRepository.SIZE * 20); - assertThat(es.getIds(RuleIndexDefinition.INDEX_TYPE_RULE)).hasSize(BigRepository.SIZE); + assertThat(es.getIds(RuleIndexDefinition.TYPE_RULE)).hasSize(BigRepository.SIZE); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java index 315b5a03d2f..450bf1b96d2 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/search/BaseDocTest.java @@ -19,18 +19,27 @@ */ package org.sonar.server.search; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import java.util.Collections; import java.util.Date; import java.util.Map; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.sonar.server.es.BaseDoc; import org.sonar.server.es.EsUtils; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.fail; public class BaseDocTest { + private final IndexType.IndexMainType someType = IndexType.main(Index.simple("bar"), "donut"); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); @Test public void getField() { @@ -38,21 +47,12 @@ public class BaseDocTest { fields.put("a_string", "foo"); fields.put("a_int", 42); fields.put("a_null", null); - BaseDoc doc = new BaseDoc(fields) { + BaseDoc doc = new BaseDoc(someType, fields) { @Override public String getId() { return null; } - @Override - public String getRouting() { - return null; - } - - @Override - public String getParent() { - return null; - } }; assertThat((String) doc.getNullableField("a_string")).isEqualTo("foo"); @@ -63,21 +63,12 @@ public class BaseDocTest { @Test public void getField_fails_if_missing_field() { Map<String, Object> fields = Collections.emptyMap(); - BaseDoc doc = new BaseDoc(fields) { + BaseDoc doc = new BaseDoc(someType, fields) { @Override public String getId() { return null; } - @Override - public String getRouting() { - return null; - } - - @Override - public String getParent() { - return null; - } }; try { @@ -90,21 +81,12 @@ public class BaseDocTest { @Test public void getFieldAsDate() { - BaseDoc doc = new BaseDoc(Maps.newHashMap()) { + BaseDoc doc = new BaseDoc(someType, Maps.newHashMap()) { @Override public String getId() { return null; } - @Override - public String getRouting() { - return null; - } - - @Override - public String getParent() { - return null; - } }; Date now = new Date(); doc.setField("javaDate", now); @@ -116,21 +98,12 @@ public class BaseDocTest { @Test public void getNullableFieldAsDate() { - BaseDoc doc = new BaseDoc(Maps.newHashMap()) { + BaseDoc doc = new BaseDoc(someType, Maps.newHashMap()) { @Override public String getId() { return null; } - @Override - public String getRouting() { - return null; - } - - @Override - public String getParent() { - return null; - } }; Date now = new Date(); doc.setField("javaDate", now); @@ -142,4 +115,45 @@ public class BaseDocTest { doc.setField("noValue", null); assertThat(doc.getNullableFieldAsDate("noValue")).isNull(); } + + @Test + public void getFields_fails_with_ISE_if_setParent_has_not_been_called_on_IndexRelationType() { + IndexType.IndexRelationType relationType = IndexType.relation(IndexType.main(Index.withRelations("foo"), "bar"), "donut"); + BaseDoc doc = new BaseDoc(relationType) { + + @Override + public String getId() { + throw new UnsupportedOperationException("getId not implemented"); + } + + }; + + expectedException.expect(IllegalStateException.class); + expectedException.expectMessage("parent must be set on a doc associated to a IndexRelationType (see BaseDoc#setParent(String))"); + + doc.getFields(); + } + + @Test + public void getFields_contains_join_field_and_indexType_field_when_setParent_has_been_called_on_IndexRelationType() { + Index index = Index.withRelations("foo"); + IndexType.IndexRelationType relationType = IndexType.relation(IndexType.main(index, "bar"), "donut"); + BaseDoc doc = new BaseDoc(relationType) { + { + setParent("miam"); + } + + @Override + public String getId() { + throw new UnsupportedOperationException("getId not implemented"); + } + + }; + + Map<String, Object> fields = doc.getFields(); + + assertThat((Map) fields.get(index.getJoinField())) + .isEqualTo(ImmutableMap.of("name", relationType.getName(), "parent", "miam")); + assertThat(fields.get("indexType")).isEqualTo(relationType.getName()); + } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterCreateTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterCreateTest.java index 697defe8b1c..56bf5052cc7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterCreateTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterCreateTest.java @@ -121,7 +121,7 @@ public class UserUpdaterCreateTest { .isEqualTo(dto.getUpdatedAt()); assertThat(dbClient.userDao().selectByLogin(session, "user").getId()).isEqualTo(dto.getId()); - List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER); + List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.TYPE_USER); assertThat(indexUsers).hasSize(1); assertThat(indexUsers.get(0).getSource()) .contains( @@ -351,7 +351,7 @@ public class UserUpdaterCreateTest { .build(), u -> { }, otherUser); - assertThat(es.getIds(UserIndexDefinition.INDEX_TYPE_USER)).containsExactlyInAnyOrder(created.getUuid(), otherUser.getUuid()); + assertThat(es.getIds(UserIndexDefinition.TYPE_USER)).containsExactlyInAnyOrder(created.getUuid(), otherUser.getUuid()); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterUpdateTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterUpdateTest.java index 9b03952c851..dd76ec4f824 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterUpdateTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterUpdateTest.java @@ -106,7 +106,7 @@ public class UserUpdaterUpdateTest { assertThat(updatedUser.getCreatedAt()).isEqualTo(user.getCreatedAt()); assertThat(updatedUser.getUpdatedAt()).isGreaterThan(user.getCreatedAt()); - List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER); + List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.TYPE_USER); assertThat(indexUsers).hasSize(1); assertThat(indexUsers.get(0).getSource()) .contains( @@ -228,7 +228,7 @@ public class UserUpdaterUpdateTest { .setLogin("new_login"), u -> { }); - List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER); + List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.TYPE_USER); assertThat(indexUsers).hasSize(1); assertThat(indexUsers.get(0).getSource()) .contains(entry("login", "new_login")); @@ -473,7 +473,7 @@ public class UserUpdaterUpdateTest { .setScmAccounts(asList("ma2")), u -> { }, otherUser); - assertThat(es.getIds(UserIndexDefinition.INDEX_TYPE_USER)).containsExactlyInAnyOrder(user.getUuid(), otherUser.getUuid()); + assertThat(es.getIds(UserIndexDefinition.TYPE_USER)).containsExactlyInAnyOrder(user.getUuid(), otherUser.getUuid()); } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java index 0a6f6f5fb80..e01fb412b8c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java @@ -117,7 +117,7 @@ public class CreateActionTest { .containsOnly("john", "John", "john@email.com", singletonList("jn"), true); // exists in index - assertThat(es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) + assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER) .setQuery(boolQuery() .must(termQuery(FIELD_LOGIN, "john")) .must(termQuery(FIELD_NAME, "John")) diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java index 73e4b958061..54aac993829 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java @@ -103,7 +103,7 @@ public class DeactivateActionTest { deactivate(user.getLogin()); verifyThatUserIsDeactivated(user.getLogin()); - assertThat(es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) + assertThat(es.client().prepareSearch(UserIndexDefinition.TYPE_USER) .setQuery(boolQuery() .must(termQuery(FIELD_UUID, user.getUuid())) .must(termQuery(FIELD_ACTIVE, "false"))) |