]> source.dussan.org Git - sonarqube.git/commit
SONAR-12686 upgrade es client to 7.9.3 and move to HTTP
authorJacek <jacek.poreda@sonarsource.com>
Thu, 9 Jan 2020 08:26:53 +0000 (09:26 +0100)
committersonartech <sonartech@sonarsource.com>
Thu, 5 Nov 2020 20:06:21 +0000 (20:06 +0000)
commitf4751bd13509f8d325d17cb4cf4ed9d85025f65f
tree369137df20a5df287bf77cdb7dcf49888f282f31
parent8cdee7d30f96e87b8bb7ec55fdfd8101ab717dfd
SONAR-12686 upgrade es client to 7.9.3 and move to HTTP

- add should minimum match eq 1 to user index queries
ES 7.X changed behaviour in case filter query with bool it defaults to '0'
https://www.elastic.co/guide/en/elasticsearch/reference/7.x/breaking-changes-7.0.html#_the_filter_context_has_been_removed

- fix issue index routing param
ES 7.X helped discover this bug as new setting has been auto configured which is 'index.number_of_routing_shards'.
This has changed how documents are distributed across shards depending on how many shards the index has.

Without that change issues docs has been incorrectly routed to the same shard hash as projects and it worked no matter what routing key you used projectUuid or auth_projectUuid.

- update ngram and edge_ngram names to match with es 7.x
nGram and edgeNgram has been deprecated in favour of ngram and edge_ngram
https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#deprecated-ngram-edgengram-token-filter-cannot-be-used

- remove `_all : enabled` usage from UT
This field was already deprecated in 6.X, now it has been removed.
https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#all-meta-field-removed

- add Elasticsearch High Level REST client dependency

- use sonar.search.port for ES HTTP

- main process use ES Rest client to check ES status

- sonar.cluster.search.hosts has HTTP ports on APP nodes
also sonar.search.port and sonar.search.host MUST be configured on each Search node with the host and HTTP port of the current node

- use Elasticsearch high level rest client

- use in EsTester

- use as primary es client

- use indices api to get all indices name instead of cluster api

- use cluster health api to check cluster state

- support raw requests for 'nodes/_stats' and '_cluster/stats'

- support raw requests for 'indices/_stats'

- leave netty4plugin as testCompile dependency it is used in UTs

- all ES non-test calls go through EsClient class

- add rest client ES profiling
124 files changed:
build.gradle
server/sonar-main/build.gradle
server/sonar-main/src/main/java/org/sonar/application/AppStateFactory.java
server/sonar-main/src/main/java/org/sonar/application/ProcessLauncherImpl.java
server/sonar-main/src/main/java/org/sonar/application/cluster/ClusterAppStateImpl.java
server/sonar-main/src/main/java/org/sonar/application/command/CommandFactoryImpl.java
server/sonar-main/src/main/java/org/sonar/application/config/ClusterSettings.java
server/sonar-main/src/main/java/org/sonar/application/es/EsConnector.java
server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java
server/sonar-main/src/main/java/org/sonar/application/es/EsInstallation.java
server/sonar-main/src/main/java/org/sonar/application/es/EsSettings.java
server/sonar-main/src/main/java/org/sonar/application/process/EsManagedProcess.java
server/sonar-main/src/test/java/org/sonar/application/AppStateFactoryTest.java
server/sonar-main/src/test/java/org/sonar/application/ProcessLauncherImplTest.java
server/sonar-main/src/test/java/org/sonar/application/command/CommandFactoryImplTest.java
server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java [new file with mode: 0644]
server/sonar-main/src/test/java/org/sonar/application/es/EsSettingsTest.java
server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java
server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java
server/sonar-server-common/build.gradle
server/sonar-server-common/src/main/java/org/sonar/server/component/index/ComponentIndexer.java
server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java
server/sonar-server-common/src/main/java/org/sonar/server/es/ElasticsearchException.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/EsClient.java
server/sonar-server-common/src/main/java/org/sonar/server/es/EsClientProvider.java
server/sonar-server-common/src/main/java/org/sonar/server/es/EsRequestDetails.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java
server/sonar-server-common/src/main/java/org/sonar/server/es/Facets.java
server/sonar-server-common/src/main/java/org/sonar/server/es/SearchIdResult.java
server/sonar-server-common/src/main/java/org/sonar/server/es/SearchResult.java
server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java
server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettings.java
server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/DefaultIndexSettingsElement.java
server/sonar-server-common/src/main/java/org/sonar/server/es/newindex/NewIndex.java
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterHealthRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStateRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyClusterStatsRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyDeleteRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyGetRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndexRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxyRefreshRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilder.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/request/package-info.java [deleted file]
server/sonar-server-common/src/main/java/org/sonar/server/es/response/ClusterStatsResponse.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndexStats.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStats.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/response/IndicesStatsResponse.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/response/JvmStats.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStats.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/es/response/NodeStatsResponse.java [new file with mode: 0644]
server/sonar-server-common/src/main/java/org/sonar/server/issue/index/IssueIndexer.java
server/sonar-server-common/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java
server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java
server/sonar-server-common/src/main/java/org/sonar/server/user/index/UserIndex.java
server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java
server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndexer.java
server/sonar-server-common/src/test/java/org/sonar/server/component/index/ComponentIndexerTest.java
server/sonar-server-common/src/test/java/org/sonar/server/es/BulkIndexerTest.java
server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientProviderTest.java
server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientStopperTest.java
server/sonar-server-common/src/test/java/org/sonar/server/es/EsClientTest.java
server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java [new file with mode: 0644]
server/sonar-server-common/src/test/java/org/sonar/server/es/EsUtilsTest.java
server/sonar-server-common/src/test/java/org/sonar/server/es/newindex/NewIndexTest.java
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyClearCacheRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyCreateIndexRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyDeleteRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyGetRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndexRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesExistsRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyIndicesStatsRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyNodesStatsRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyPutMappingRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyRefreshRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxySearchScrollRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerHealthRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStateRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/request/ProxyWebServerStatsRequestBuilderTest.java [deleted file]
server/sonar-server-common/src/test/java/org/sonar/server/es/response/ClusterStatsResponseTest.java [new file with mode: 0644]
server/sonar-server-common/src/test/java/org/sonar/server/es/response/IndicesStatsResponseTest.java [new file with mode: 0644]
server/sonar-server-common/src/test/java/org/sonar/server/es/response/NodeStatsResponseTest.java [new file with mode: 0644]
server/sonar-server-common/src/test/java/org/sonar/server/es/searchrequest/TopAggregationHelperTest.java
server/sonar-server-common/src/test/java/org/sonar/server/measure/index/ProjectMeasuresIndexerTest.java
server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexDefinitionTest.java
server/sonar-server-common/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java
server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java
server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/newindex/FakeIndexDefinition.java
server/sonar-webserver-core/src/main/java/org/sonar/server/es/IndexerStartupTask.java
server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java
server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsIndexesSection.java
server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/EsStateSection.java
server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/EsClusterStateSection.java
server/sonar-webserver-core/src/main/java/org/sonar/server/platform/monitoring/cluster/SearchNodesInfoLoaderImpl.java
server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java
server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsIndexesSectionTest.java
server/sonar-webserver-core/src/test/java/org/sonar/server/platform/monitoring/EsStateSectionTest.java
server/sonar-webserver-es/src/main/java/org/sonar/server/component/index/ComponentIndex.java
server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java
server/sonar-webserver-es/src/main/java/org/sonar/server/issue/index/IssueIndex.java
server/sonar-webserver-es/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndex.java
server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java
server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexDebtTest.java
server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexSecurityHotspotsTest.java
server/sonar-webserver-es/src/test/java/org/sonar/server/issue/index/IssueIndexTest.java
server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndex.java
server/sonar-webserver-es/src/test/java/org/sonar/server/permission/index/FooIndexer.java
server/sonar-webserver-webapi/src/main/java/org/sonar/server/health/EsStatusCheck.java
server/sonar-webserver-webapi/src/main/java/org/sonar/server/hotspot/ws/SearchAction.java
server/sonar-webserver-webapi/src/main/java/org/sonar/server/issue/ws/SearchAction.java
server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusClusterCheckTest.java
server/sonar-webserver-webapi/src/test/java/org/sonar/server/health/EsStatusNodeCheckTest.java
server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/CreateActionTest.java
server/sonar-webserver-webapi/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java
sonar-application/build.gradle
sonar-application/src/main/assembly/conf/sonar.properties
sonar-application/src/main/java/org/sonar/application/App.java