From 94b72ea61ba5dd9a16c90f5674df234f1131baf4 Mon Sep 17 00:00:00 2001 From: =?utf8?q?S=C3=A9bastien=20Lesaint?= Date: Wed, 12 Jun 2019 14:05:21 +0200 Subject: [PATCH] SONAR-12187 do not fail at startup if ES indices are read-only unless the DB vendor is changed --- .../server/es/metadata/MetadataIndex.java | 80 ++----------- .../server/es/metadata/MetadataIndexImpl.java | 110 ++++++++++++++++++ .../server/es/metadata/MetadataIndexTest.java | 2 +- .../es/metadata/EsDbCompatibilityImpl.java | 4 +- .../platformlevel/PlatformLevel4.java | 4 +- .../org/sonar/server/es/IndexCreatorTest.java | 3 +- .../server/es/IndexerStartupTaskTest.java | 3 +- .../metadata/EsDbCompatibilityImplTest.java | 77 ++++++++++-- 8 files changed, 196 insertions(+), 87 deletions(-) create mode 100644 server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java index 2254b368911..db7d162670d 100644 --- a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java @@ -20,85 +20,19 @@ package org.sonar.server.es.metadata; import java.util.Optional; -import org.elasticsearch.action.get.GetRequestBuilder; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.common.document.DocumentField; -import org.sonar.server.es.EsClient; import org.sonar.server.es.Index; import org.sonar.server.es.IndexType; -import org.sonar.server.es.IndexType.IndexMainType; -import org.sonar.server.es.IndexType.IndexRelationType; -import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA; -import static org.sonar.server.es.newindex.DefaultIndexSettings.REFRESH_IMMEDIATE; +public interface MetadataIndex { + Optional getHash(Index index); -public class MetadataIndex { + void setHash(Index index, String hash); - private static final String DB_VENDOR_KEY = "dbVendor"; + boolean getInitialized(IndexType indexType); - private final EsClient esClient; + void setInitialized(IndexType indexType, boolean initialized); - public MetadataIndex(EsClient esClient) { - this.esClient = esClient; - } + Optional getDbVendor(); - public Optional getHash(Index index) { - return getMetadata(hashId(index)); - } - - public void setHash(Index index, String hash) { - setMetadata(hashId(index), hash); - } - - private static String hashId(Index index) { - return index.getName() + ".indexStructure"; - } - - public boolean getInitialized(IndexType indexType) { - return getMetadata(initializedId(indexType)).map(Boolean::parseBoolean).orElse(false); - } - - public void setInitialized(IndexType indexType, boolean initialized) { - setMetadata(initializedId(indexType), String.valueOf(initialized)); - } - - private static String initializedId(IndexType indexType) { - if (indexType instanceof IndexMainType) { - IndexMainType mainType = (IndexMainType) indexType; - return mainType.getIndex().getName() + "." + mainType.getType() + ".initialized"; - } - if (indexType instanceof IndexRelationType) { - IndexRelationType relationType = (IndexRelationType) indexType; - IndexMainType mainType = relationType.getMainType(); - return mainType.getIndex().getName() + "." + mainType.getType() + "." + relationType.getName() + ".initialized"; - } - throw new IllegalArgumentException("Unsupported IndexType " + indexType.getClass()); - } - - public Optional getDbVendor() { - return getMetadata(DB_VENDOR_KEY); - } - - public void setDbMetadata(String vendor) { - setMetadata(DB_VENDOR_KEY, vendor); - } - - private Optional getMetadata(String id) { - GetRequestBuilder request = esClient.prepareGet(TYPE_METADATA, id) - .setStoredFields(MetadataIndexDefinition.FIELD_VALUE); - GetResponse response = request.get(); - if (response.isExists()) { - DocumentField field = response.getField(MetadataIndexDefinition.FIELD_VALUE); - return Optional.of(field.getValue()); - } - return Optional.empty(); - } - - private void setMetadata(String id, String value) { - esClient.prepareIndex(TYPE_METADATA) - .setId(id) - .setSource(MetadataIndexDefinition.FIELD_VALUE, value) - .setRefreshPolicy(REFRESH_IMMEDIATE) - .get(); - } + void setDbMetadata(String vendor); } diff --git a/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java new file mode 100644 index 00000000000..ad028e08bcd --- /dev/null +++ b/server/sonar-server-common/src/main/java/org/sonar/server/es/metadata/MetadataIndexImpl.java @@ -0,0 +1,110 @@ +/* + * SonarQube + * Copyright (C) 2009-2019 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es.metadata; + +import java.util.Optional; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.common.document.DocumentField; +import org.sonar.server.es.EsClient; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; +import org.sonar.server.es.IndexType.IndexMainType; +import org.sonar.server.es.IndexType.IndexRelationType; + +import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA; +import static org.sonar.server.es.newindex.DefaultIndexSettings.REFRESH_IMMEDIATE; + +public class MetadataIndexImpl implements MetadataIndex { + + private static final String DB_VENDOR_KEY = "dbVendor"; + + private final EsClient esClient; + + public MetadataIndexImpl(EsClient esClient) { + this.esClient = esClient; + } + + @Override + public Optional getHash(Index index) { + return getMetadata(hashId(index)); + } + + @Override + public void setHash(Index index, String hash) { + setMetadata(hashId(index), hash); + } + + private static String hashId(Index index) { + return index.getName() + ".indexStructure"; + } + + @Override + public boolean getInitialized(IndexType indexType) { + return getMetadata(initializedId(indexType)).map(Boolean::parseBoolean).orElse(false); + } + + @Override + public void setInitialized(IndexType indexType, boolean initialized) { + setMetadata(initializedId(indexType), String.valueOf(initialized)); + } + + private static String initializedId(IndexType indexType) { + if (indexType instanceof IndexMainType) { + IndexMainType mainType = (IndexMainType) indexType; + return mainType.getIndex().getName() + "." + mainType.getType() + ".initialized"; + } + if (indexType instanceof IndexRelationType) { + IndexRelationType relationType = (IndexRelationType) indexType; + IndexMainType mainType = relationType.getMainType(); + return mainType.getIndex().getName() + "." + mainType.getType() + "." + relationType.getName() + ".initialized"; + } + throw new IllegalArgumentException("Unsupported IndexType " + indexType.getClass()); + } + + @Override + public Optional getDbVendor() { + return getMetadata(DB_VENDOR_KEY); + } + + @Override + public void setDbMetadata(String vendor) { + setMetadata(DB_VENDOR_KEY, vendor); + } + + private Optional getMetadata(String id) { + GetRequestBuilder request = esClient.prepareGet(TYPE_METADATA, id) + .setStoredFields(MetadataIndexDefinition.FIELD_VALUE); + GetResponse response = request.get(); + if (response.isExists()) { + DocumentField field = response.getField(MetadataIndexDefinition.FIELD_VALUE); + return Optional.of(field.getValue()); + } + return Optional.empty(); + } + + private void setMetadata(String id, String value) { + esClient.prepareIndex(TYPE_METADATA) + .setId(id) + .setSource(MetadataIndexDefinition.FIELD_VALUE, value) + .setRefreshPolicy(REFRESH_IMMEDIATE) + .get(); + } +} diff --git a/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java b/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java index a9ed154f9fc..1be4bce6837 100644 --- a/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java +++ b/server/sonar-server-common/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java @@ -42,7 +42,7 @@ public class MetadataIndexTest { @Rule public EsTester es = EsTester.createCustom(new MetadataIndexDefinitionBridge(), new FakeIndexDefinition()); - private final MetadataIndex underTest = new MetadataIndex(es.client()); + private final MetadataIndex underTest = new MetadataIndexImpl(es.client()); private final String indexName = randomAlphabetic(20).toLowerCase(Locale.ENGLISH); private final Index index = new Random().nextBoolean() ? Index.simple(indexName) : Index.withRelations(indexName); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java b/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java index b846d1fc450..c535e81d92d 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java @@ -40,7 +40,9 @@ public class EsDbCompatibilityImpl implements EsDbCompatibility { @Override public void markAsCompatible() { - metadataIndex.setDbMetadata(getDbVendor()); + if (!hasSameDbVendor()) { + metadataIndex.setDbMetadata(getDbVendor()); + } } private String getDbVendor() { diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index e47bbbe1d28..f7081969ecf 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -63,8 +63,8 @@ import org.sonar.server.es.IndexDefinitions; import org.sonar.server.es.ProjectIndexersImpl; import org.sonar.server.es.RecoveryIndexer; import org.sonar.server.es.metadata.EsDbCompatibilityImpl; -import org.sonar.server.es.metadata.MetadataIndex; import org.sonar.server.es.metadata.MetadataIndexDefinition; +import org.sonar.server.es.metadata.MetadataIndexImpl; import org.sonar.server.extension.CoreExtensionBootstraper; import org.sonar.server.extension.CoreExtensionStopper; import org.sonar.server.favorite.FavoriteModule; @@ -244,7 +244,7 @@ public class PlatformLevel4 extends PlatformLevel { addIfStartupLeader( IndexCreator.class, MetadataIndexDefinition.class, - MetadataIndex.class, + MetadataIndexImpl.class, EsDbCompatibilityImpl.class); addIfCluster(NodeHealthModule.class); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java index ba7f0138ef1..d3d6797a4d3 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java @@ -36,6 +36,7 @@ import org.sonar.api.utils.log.LoggerLevel; import org.sonar.server.es.IndexType.IndexMainType; import org.sonar.server.es.metadata.MetadataIndex; import org.sonar.server.es.metadata.MetadataIndexDefinition; +import org.sonar.server.es.metadata.MetadataIndexImpl; import org.sonar.server.es.newindex.NewRegularIndex; import org.sonar.server.es.newindex.SettingsConfiguration; import org.sonar.server.platform.db.migration.es.MigrationEsClient; @@ -60,7 +61,7 @@ public class IndexCreatorTest { public EsTester es = EsTester.createCustom(); private MetadataIndexDefinition metadataIndexDefinition = new MetadataIndexDefinition(new MapSettings().asConfig()); - private MetadataIndex metadataIndex = new MetadataIndex(es.client()); + private MetadataIndex metadataIndex = new MetadataIndexImpl(es.client()); private TestEsDbCompatibility esDbCompatibility = new TestEsDbCompatibility(); private MapSettings settings = new MapSettings(); private MigrationEsClient migrationEsClient = mock(MigrationEsClient.class); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java index b9b17c31ee2..de10b8e1905 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/IndexerStartupTaskTest.java @@ -26,6 +26,7 @@ import org.junit.Test; import org.mockito.Mockito; import org.sonar.api.config.internal.MapSettings; import org.sonar.server.es.metadata.MetadataIndex; +import org.sonar.server.es.metadata.MetadataIndexImpl; import org.sonar.server.es.newindex.FakeIndexDefinition; import static org.mockito.ArgumentMatchers.eq; @@ -41,7 +42,7 @@ public class IndexerStartupTaskTest { public EsTester es = EsTester.createCustom(new FakeIndexDefinition()); private final MapSettings settings = new MapSettings(); - private final MetadataIndex metadataIndex = mock(MetadataIndex.class); + private final MetadataIndex metadataIndex = mock(MetadataIndexImpl.class); private final StartupIndexer indexer = mock(StartupIndexer.class); private final IndexerStartupTask underTest = new IndexerStartupTask(es.client(), settings.asConfig(), metadataIndex, indexer); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java index 108c89475a6..66001567d87 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java @@ -19,24 +19,34 @@ */ package org.sonar.server.es.metadata; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import javax.annotation.CheckForNull; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; import org.sonar.db.DbClient; -import org.sonar.server.es.EsTester; -import org.sonar.server.es.newindex.FakeIndexDefinition; +import org.sonar.server.es.Index; +import org.sonar.server.es.IndexType; + +import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class EsDbCompatibilityImplTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - @Rule - public EsTester es = EsTester.createCustom(new MetadataIndexDefinitionBridge(), new FakeIndexDefinition()); + private DbClient dbClient = mock(DbClient.class, Mockito.RETURNS_DEEP_STUBS); - private MetadataIndex metadataIndex = new MetadataIndex(es.client()); + private MetadataIndex metadataIndex = spy(new TestMetadataIndex()); private EsDbCompatibilityImpl underTest = new EsDbCompatibilityImpl(dbClient, metadataIndex); @Test @@ -63,7 +73,7 @@ public class EsDbCompatibilityImplTest { } @Test - public void store_db_metadata_in_es() { + public void markAsCompatible_db_metadata_in_es() { prepareDb("mysql"); underTest.markAsCompatible(); @@ -72,7 +82,7 @@ public class EsDbCompatibilityImplTest { } @Test - public void store_updates_db_metadata_in_es() { + public void markAsCompatible_updates_db_metadata_in_es() { prepareEs("mysql"); prepareDb("postgres"); @@ -82,7 +92,7 @@ public class EsDbCompatibilityImplTest { } @Test - public void store_marks_es_as_compatible_with_db() { + public void markAsCompatible_marks_es_as_compatible_with_db() { prepareDb("postgres"); underTest.markAsCompatible(); @@ -90,11 +100,62 @@ public class EsDbCompatibilityImplTest { assertThat(underTest.hasSameDbVendor()).isTrue(); } + @Test + public void markAsCompatible_has_no_effect_if_vendor_is_the_same() { + String vendor = randomAlphabetic(12); + prepareEs(vendor); + prepareDb(vendor); + + underTest.markAsCompatible(); + + assertThat(underTest.hasSameDbVendor()).isTrue(); + verify(metadataIndex, times(0)).setDbMetadata(anyString()); + } + private void prepareDb(String dbVendor) { when(dbClient.getDatabase().getDialect().getId()).thenReturn(dbVendor); } private void prepareEs(String dbVendor) { metadataIndex.setDbMetadata(dbVendor); + // reset spy to not perturbate assertions on spy from verified code + reset(metadataIndex); + } + + private static class TestMetadataIndex implements MetadataIndex { + private final Map hashes = new HashMap<>(); + private final Map initializeds = new HashMap<>(); + @CheckForNull + private String dbVendor = null; + + @Override + public Optional getHash(Index index) { + return Optional.ofNullable(hashes.get(index)); + } + + @Override + public void setHash(Index index, String hash) { + hashes.put(index, hash); + } + + @Override + public boolean getInitialized(IndexType indexType) { + return initializeds.getOrDefault(indexType, false); + } + + @Override + public void setInitialized(IndexType indexType, boolean initialized) { + initializeds.put(indexType, initialized); + } + + @Override + public Optional getDbVendor() { + return Optional.ofNullable(dbVendor); + } + + @Override + public void setDbMetadata(String vendor) { + this.dbVendor = vendor; + } } } -- 2.39.5