From ad08d96448ed6e4c80643f6f82add2c38644e153 Mon Sep 17 00:00:00 2001 From: Jacek Date: Wed, 8 Sep 2021 11:14:57 +0200 Subject: [PATCH] SONAR-15375 Fix SSF-191 (cherry picked from commit e4ee455c8b5bada664cce96f00146621366c8561) --- build.gradle | 7 ++--- gradle.properties | 4 +-- .../sonar/application/es/EsConnectorImpl.java | 2 +- .../application/es/EsConnectorImplTest.java | 26 +++++++++++++++- .../process/EsManagedProcessTest.java | 30 +++++++++---------- .../java/org/sonar/server/es/BulkIndexer.java | 2 +- .../java/org/sonar/server/es/EsUtils.java | 2 +- .../sonar/server/rule/index/RuleIndex.java | 30 +++++++++---------- .../sonar/server/view/index/ViewIndex.java | 2 +- .../sonar/server/es/EsRequestDetailsTest.java | 2 +- .../java/org/sonar/server/es/EsTester.java | 2 +- sonar-application/build.gradle | 4 +-- 12 files changed, 68 insertions(+), 45 deletions(-) diff --git a/build.gradle b/build.gradle index aef1dfd1e45..c2208a2d941 100644 --- a/build.gradle +++ b/build.gradle @@ -339,13 +339,12 @@ subprojects { entry 'jetty-server' entry 'jetty-servlet' } - dependency('org.elasticsearch.client:elasticsearch-rest-high-level-client:7.13.4') { + dependency('org.elasticsearch.client:elasticsearch-rest-high-level-client:7.14.1') { exclude 'org.apache.logging.log4j:log4j-core' } - dependency 'org.elasticsearch.plugin:transport-netty4-client:7.13.4' + dependency 'org.elasticsearch.plugin:transport-netty4-client:7.14.1' dependency 'org.elasticsearch:mocksocket:1.0' - //analysis-common need to stay at 7.10.2 as it hasn't been published to mvn central - dependency 'org.codelibs.elasticsearch.module:analysis-common:7.10.2' + dependency 'org.codelibs.elasticsearch.module:analysis-common:7.14.1' dependency 'org.eclipse.jgit:org.eclipse.jgit:5.9.0.202009080501-r' dependency 'org.tmatesoft.svnkit:svnkit:1.10.1' dependency 'org.hamcrest:hamcrest-all:1.3' diff --git a/gradle.properties b/gradle.properties index 5c5ce1ecb65..4afeda34ef6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,5 +11,5 @@ org.gradle.vfs.watch=true # https://www.elastic.co/downloads/elasticsearch-no-jdk elasticsearchDownloadUrlPath=https://artifacts.elastic.co/downloads/elasticsearch/ elasticsearchDownloadRepoxUrlPath=https://repox.jfrog.io/artifactory/sonarsource-bucket/sonarqube/elasticsearch/ -elasticsearchDownloadUrlFile=elasticsearch-7.12.1-no-jdk-linux-x86_64.tar.gz -elasticsearchDownloadSha512=c91fd66668977280d7339caf6b4fef9fce5640492f0e2a65bc537c6da8397c801a3727f1ff56b7a1591b11ea47195720c588a4eba473043f829a87cd3dcc0af4 +elasticsearchDownloadUrlFile=elasticsearch-7.14.1-no-jdk-linux-x86_64.tar.gz +elasticsearchDownloadSha512=77dca78ba865ae74863b3b2a3cd61e8a8e4478cd02eb020184dbf89fa32cf145a6bbd1d11a1cb88c2236a3b8cdb8b0047e3c0f1a40f609f31b898c905b2c211d diff --git a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java index 1c5a3263df1..59bc2f9b75d 100644 --- a/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java +++ b/server/sonar-main/src/main/java/org/sonar/application/es/EsConnectorImpl.java @@ -41,7 +41,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; +import static org.elasticsearch.core.TimeValue.timeValueSeconds; public class EsConnectorImpl implements EsConnector { private static final String ES_USERNAME = "elastic"; diff --git a/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java index 9c5e3d0d914..a9d473d9cc8 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/es/EsConnectorImplTest.java @@ -56,6 +56,24 @@ public class EsConnectorImplTest { " \"error\" : \"i-have-a-bad-feelings-about-this\"" + "}"; + private static final String ES_INFO_RESPONSE = "{" + + " \"name\" : \"sonarqube\"," + + " \"cluster_name\" : \"sonarqube\"," + + " \"cluster_uuid\" : \"6Oj9lFIyQVa_d5HgQWqQpA\"," + + " \"version\" : {" + + " \"number\" : \"7.14.1\"," + + " \"build_flavor\" : \"default\"," + + " \"build_type\" : \"tar\"," + + " \"build_hash\" : \"66b55ebfa59c92c15db3f69a335d500018b3331e\"," + + " \"build_date\" : \"2021-08-26T09:01:05.390870785Z\"," + + " \"build_snapshot\" : false," + + " \"lucene_version\" : \"8.9.0\"," + + " \"minimum_wire_compatibility_version\" : \"6.8.0\"," + + " \"minimum_index_compatibility_version\" : \"6.0.0-beta1\"" + + " }," + + " \"tagline\" : \"You Know, for Search\"" + + "}"; + @Rule public MockWebServer mockWebServer = new MockWebServer(); @@ -95,10 +113,16 @@ public class EsConnectorImplTest { } private void mockServerResponse(int httpCode, String jsonResponse) { + mockWebServer.enqueue(new MockResponse() + .setResponseCode(200) + .setBody(ES_INFO_RESPONSE) + .setHeader("Content-Type", "application/json") + .setHeader("X-elastic-product", "Elasticsearch")); mockWebServer.enqueue(new MockResponse() .setResponseCode(httpCode) .setBody(jsonResponse) - .setHeader("Content-Type", "application/json")); + .setHeader("Content-Type", "application/json") + .setHeader("X-elastic-product", "Elasticsearch")); } } diff --git a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java index 4bb1b5f92b3..2b1dc45bb15 100644 --- a/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java +++ b/server/sonar-main/src/test/java/org/sonar/application/process/EsManagedProcessTest.java @@ -94,8 +94,8 @@ public class EsManagedProcessTest { public void isOperational_should_retry_if_Elasticsearch_is_unreachable() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenReturn(Optional.empty()) - .thenReturn(Optional.of(ClusterHealthStatus.GREEN)); + .thenReturn(Optional.empty()) + .thenReturn(Optional.of(ClusterHealthStatus.GREEN)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isTrue(); } @@ -104,7 +104,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_false_if_Elasticsearch_status_cannot_be_evaluated() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new RuntimeException("test")); + .thenThrow(new RuntimeException("test")); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); } @@ -113,7 +113,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_false_if_ElasticsearchException_with_connection_refused_thrown() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new ElasticsearchException("Connection refused")); + .thenThrow(new ElasticsearchException("Connection refused")); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); } @@ -122,7 +122,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_false_if_ElasticsearchException_with_connection_timeout_thrown() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new ElasticsearchException(new ExecutionException(new ConnectException("Timeout connecting to [/127.0.0.1:9001]")))); + .thenThrow(new ElasticsearchException(new ExecutionException(new ConnectException("Timeout connecting to [/127.0.0.1:9001]")))); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT_LONG); assertThat(underTest.isOperational()).isFalse(); } @@ -131,7 +131,7 @@ public class EsManagedProcessTest { public void isOperational_should_return_false_if_ElasticsearchException_thrown() { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new ElasticsearchException("test")); + .thenThrow(new ElasticsearchException("test")); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); } @@ -147,24 +147,24 @@ public class EsManagedProcessTest { EsConnector esConnector = mock(EsConnector.class); when(esConnector.getClusterHealthStatus()) - .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE)); + .thenThrow(new ElasticsearchStatusException("foobar[type=master_not_discovered_exception,acme]...", RestStatus.SERVICE_UNAVAILABLE)); EsManagedProcess underTest = new EsManagedProcess(mock(Process.class), ProcessId.ELASTICSEARCH, esConnector, WAIT_FOR_UP_TIMEOUT); assertThat(underTest.isOperational()).isFalse(); assertThat(memoryAppender.events).isNotEmpty(); assertThat(memoryAppender.events) - .extracting(ILoggingEvent::getLevel, ILoggingEvent::getMessage) - .containsOnlyOnce( - tuple(Level.INFO, "Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?") - ); + .extracting(ILoggingEvent::getLevel, ILoggingEvent::getMessage) + .containsOnlyOnce( + tuple(Level.INFO, "Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?") + ); // Second call must not log another message assertThat(underTest.isOperational()).isFalse(); assertThat(memoryAppender.events) - .extracting(ILoggingEvent::getLevel, ILoggingEvent::getMessage) - .containsOnlyOnce( - tuple(Level.INFO, "Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?") - ); + .extracting(ILoggingEvent::getLevel, ILoggingEvent::getMessage) + .containsOnlyOnce( + tuple(Level.INFO, "Elasticsearch is waiting for a master to be elected. Did you start all the search nodes ?") + ); } private static class MemoryAppender extends AppenderBase { diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java index 063d35cbfb1..b73ce9d0e4b 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -52,7 +52,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortOrder; import org.sonar.api.utils.log.Logger; diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java index e5966a90c6a..204d7966a58 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/EsUtils.java @@ -35,7 +35,7 @@ import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.bucket.terms.Terms; diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java index d618bcc6923..e8ba164fa4d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/rule/index/RuleIndex.java @@ -33,7 +33,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilder; @@ -250,7 +250,7 @@ public class RuleIndex { /* Add enforced filter on main type Rule */ filters.put( FIELD_INDEX_TYPE, - boolQuery().must(QueryBuilders.termsQuery(FIELD_INDEX_TYPE, TYPE_RULE.getType()))); + boolQuery().must(termsQuery(FIELD_INDEX_TYPE, TYPE_RULE.getType()))); /* Add enforced filter on rules that are REMOVED */ filters.put(FIELD_RULE_STATUS, @@ -270,45 +270,45 @@ public class RuleIndex { if (isNotEmpty(query.getLanguages())) { filters.put(FIELD_RULE_LANGUAGE, - QueryBuilders.termsQuery(FIELD_RULE_LANGUAGE, query.getLanguages())); + termsQuery(FIELD_RULE_LANGUAGE, query.getLanguages())); } if (isNotEmpty(query.getRepositories())) { filters.put(FIELD_RULE_REPOSITORY, - QueryBuilders.termsQuery(FIELD_RULE_REPOSITORY, query.getRepositories())); + termsQuery(FIELD_RULE_REPOSITORY, query.getRepositories())); } if (isNotEmpty(query.getSeverities())) { filters.put(FIELD_RULE_SEVERITY, - QueryBuilders.termsQuery(FIELD_RULE_SEVERITY, query.getSeverities())); + termsQuery(FIELD_RULE_SEVERITY, query.getSeverities())); } if (isNotEmpty(query.getCwe())) { filters.put(FIELD_RULE_CWE, boolQuery() - .must(QueryBuilders.termsQuery(FIELD_RULE_CWE, query.getCwe())) - .must(QueryBuilders.termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); + .must(termsQuery(FIELD_RULE_CWE, query.getCwe())) + .must(termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); } if (isNotEmpty(query.getOwaspTop10())) { filters.put(FIELD_RULE_OWASP_TOP_10, boolQuery() - .must(QueryBuilders.termsQuery(FIELD_RULE_OWASP_TOP_10, query.getOwaspTop10())) - .must(QueryBuilders.termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); + .must(termsQuery(FIELD_RULE_OWASP_TOP_10, query.getOwaspTop10())) + .must(termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); } if (isNotEmpty(query.getSansTop25())) { filters.put(FIELD_RULE_SANS_TOP_25, boolQuery() - .must(QueryBuilders.termsQuery(FIELD_RULE_SANS_TOP_25, query.getSansTop25())) - .must(QueryBuilders.termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); + .must(termsQuery(FIELD_RULE_SANS_TOP_25, query.getSansTop25())) + .must(termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); } if (isNotEmpty(query.getSonarsourceSecurity())) { filters.put(FIELD_RULE_SONARSOURCE_SECURITY, boolQuery() - .must(QueryBuilders.termsQuery(FIELD_RULE_SONARSOURCE_SECURITY, query.getSonarsourceSecurity())) - .must(QueryBuilders.termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); + .must(termsQuery(FIELD_RULE_SONARSOURCE_SECURITY, query.getSonarsourceSecurity())) + .must(termsQuery(FIELD_RULE_TYPE, VULNERABILITY.name(), SECURITY_HOTSPOT.name()))); } if (StringUtils.isNotEmpty(query.getKey())) { @@ -324,7 +324,7 @@ public class RuleIndex { if (isNotEmpty(types)) { List typeNames = types.stream().map(RuleType::toString).collect(MoreCollectors.toList()); filters.put(FIELD_RULE_TYPE, - QueryBuilders.termsQuery(FIELD_RULE_TYPE, typeNames)); + termsQuery(FIELD_RULE_TYPE, typeNames)); } if (query.getAvailableSinceLong() != null) { @@ -338,7 +338,7 @@ public class RuleIndex { stringStatus.add(status.name()); } filters.put(FIELD_RULE_STATUS, - QueryBuilders.termsQuery(FIELD_RULE_STATUS, stringStatus)); + termsQuery(FIELD_RULE_STATUS, stringStatus)); } Boolean isTemplate = query.isTemplate(); diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java index bff8fb5665f..f7de7d66704 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/view/index/ViewIndex.java @@ -24,7 +24,7 @@ import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java index b0987394f89..8cb3ca582f3 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/EsRequestDetailsTest.java @@ -31,7 +31,7 @@ import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.client.Requests; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetIndexRequest; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java index 611a89c3c47..006254c7c4e 100644 --- a/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java +++ b/server/sonar-server-common/src/testFixtures/java/org/sonar/server/es/EsTester.java @@ -65,7 +65,7 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; diff --git a/sonar-application/build.gradle b/sonar-application/build.gradle index f5f3e5012c9..efdd644b223 100644 --- a/sonar-application/build.gradle +++ b/sonar-application/build.gradle @@ -247,8 +247,8 @@ zip.doFirst { } // Check the size of the archive zip.doLast { - def minLength = 270000000 - def maxLength = 290000000 + def minLength = 290000000 + def maxLength = 315000000 def length = archiveFile.get().asFile.length() if (length < minLength) -- 2.39.5