Browse Source

SONAR-10691 Introduce ES drop in DB migration

tags/7.5
Eric Hartmann 6 years ago
parent
commit
aadcb25039

+ 29
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/ElasticsearchClient.java View File

@@ -0,0 +1,29 @@
/*
* SonarQube
* Copyright (C) 2009-2018 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

package org.sonar.server.platform.db.migration.es;

public interface ElasticsearchClient {

/**
* This method is reentrant and does not fail if indexName or otherIndexNames does not exist
*/
void deleteIndexes(String... otherIndexNames);
}

+ 24
- 0
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java View File

@@ -0,0 +1,24 @@
/*
* SonarQube
* Copyright (C) 2009-2018 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
@ParametersAreNonnullByDefault
package org.sonar.server.platform.db.migration.es;

import javax.annotation.ParametersAreNonnullByDefault;


+ 1
- 1
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/step/SelectImpl.java View File

@@ -28,7 +28,7 @@ import java.util.List;
import org.apache.commons.dbutils.DbUtils;
import org.sonar.db.Database;

public class SelectImpl extends BaseSqlStatement<Select>implements Select {
public class SelectImpl extends BaseSqlStatement<Select> implements Select {

private SelectImpl(PreparedStatement pstmt) {
super(pstmt);

+ 51
- 0
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/step/ElasticsearchChangeTest.java View File

@@ -0,0 +1,51 @@
/*
* SonarQube
* Copyright (C) 2009-2018 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

package org.sonar.server.platform.db.migration.step;

import java.sql.SQLException;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.sonar.server.platform.db.migration.es.ElasticsearchClient;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;

public class ElasticsearchChangeTest {

private ElasticsearchClient client = mock(ElasticsearchClient.class);

@Test
public void deleteIndice_call() throws SQLException {
ArgumentCaptor<String> indices = ArgumentCaptor.forClass(String.class);

new ElasticsearchChange(client) {
@Override
protected void execute(Context context) throws SQLException {
context.deleteIndice("a", "b", "c");
}
}.execute();

verify(client, times(1)).deleteIndice(indices.capture());
assertThat(indices.getAllValues()).containsExactly("a", "b", "c");
}
}

+ 1
- 10
server/sonar-server/src/main/java/org/sonar/server/es/IndexCreator.java View File

@@ -48,7 +48,6 @@ import org.sonar.server.es.metadata.MetadataIndexDefinition;
public class IndexCreator implements Startable {

private static final Logger LOGGER = Loggers.get(IndexCreator.class);
private static final String PROPERY_DISABLE_CHECK = "sonar.search.disableDropOnDbMigration";

private final MetadataIndexDefinition metadataIndexDefinition;
private final MetadataIndex metadataIndex;
@@ -151,20 +150,12 @@ public class IndexCreator implements Startable {
}

private void checkDbCompatibility(Collection<Index> definitions) {
boolean disabledCheck = configuration.getBoolean(PROPERY_DISABLE_CHECK).orElse(false);
if (disabledCheck) {
LOGGER.warn("Automatic drop of search indices in turned off (see property " + PROPERY_DISABLE_CHECK + ")");
}

List<String> existingIndices = loadExistingIndicesExceptMetadata(definitions);
if (!disabledCheck && !existingIndices.isEmpty()) {
if (!existingIndices.isEmpty()) {
boolean delete = false;
if (!esDbCompatibility.hasSameDbVendor()) {
LOGGER.info("Delete Elasticsearch indices (DB vendor changed)");
delete = true;
} else if (!esDbCompatibility.hasSameDbSchemaVersion()) {
LOGGER.info("Delete Elasticsearch indices (DB schema changed)");
delete = true;
}
if (delete) {
existingIndices.forEach(this::deleteIndex);

+ 57
- 0
server/sonar-server/src/main/java/org/sonar/server/es/SimpleEsClientImpl.java View File

@@ -0,0 +1,57 @@
/*
* SonarQube
* Copyright (C) 2009-2018 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

package org.sonar.server.es;

import java.util.Arrays;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.client.Client;
import org.sonar.server.platform.db.migration.es.ElasticsearchClient;

import static java.util.Objects.requireNonNull;

public class SimpleEsClientImpl implements ElasticsearchClient {
private final Client nativeClient;

public SimpleEsClientImpl(Client nativeClient) {
this.nativeClient = requireNonNull(nativeClient);
}

/**
* This method is reentrant and does not fail if indexName or otherIndexNames does not exist
*/
public void deleteIndexes(String... indexNames) {
if (indexNames.length == 0) {
return;
}

GetMappingsResponse getMappingsResponse = nativeClient.admin().indices().prepareGetMappings("_all").get();
String[] allIndexes = getMappingsResponse.mappings().keys().toArray(String.class);
String[] intersection = intersection(indexNames, allIndexes);
nativeClient.admin().indices().prepareDelete(intersection).get();
}

private String[] intersection(String[] a, String[] b) {
return Arrays.stream(a)
.distinct()
.filter(x -> Arrays.stream(b).anyMatch(y -> y != null && y.equals(x)))
.toArray(String[]::new);
}
}

+ 0
- 7
server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibility.java View File

@@ -32,13 +32,6 @@ public interface EsDbCompatibility {
*/
boolean hasSameDbVendor();

/**
* Whether the effective DB schema version equals the version
* registered in Elasticsearch metadata.
* Return {@code false} if at least one of the values is absent
*/
boolean hasSameDbSchemaVersion();

/**
* Stores in Elasticsearch the metadata about database
*/

+ 2
- 22
server/sonar-server/src/main/java/org/sonar/server/es/metadata/EsDbCompatibilityImpl.java View File

@@ -19,21 +19,17 @@
*/
package org.sonar.server.es.metadata;

import java.util.Objects;
import java.util.Optional;
import org.sonar.db.DbClient;
import org.sonar.server.platform.db.migration.history.MigrationHistory;

public class EsDbCompatibilityImpl implements EsDbCompatibility {

private final DbClient dbClient;
private final MetadataIndex metadataIndex;
private final MigrationHistory dbMigrationHistory;

public EsDbCompatibilityImpl(DbClient dbClient, MetadataIndex metadataIndex, MigrationHistory dbMigrationHistory) {
public EsDbCompatibilityImpl(DbClient dbClient, MetadataIndex metadataIndex) {
this.dbClient = dbClient;
this.metadataIndex = metadataIndex;
this.dbMigrationHistory = dbMigrationHistory;
}

@Override
@@ -42,28 +38,12 @@ public class EsDbCompatibilityImpl implements EsDbCompatibility {
return registeredDbVendor.isPresent() && registeredDbVendor.get().equals(getDbVendor());
}

@Override
public boolean hasSameDbSchemaVersion() {
Optional<Long> registeredVersion = metadataIndex.getDbSchemaVersion();
if (!registeredVersion.isPresent()) {
return false;
}
return getDbSchemaVersion()
.filter(effectiveVersion -> Objects.equals(registeredVersion.get(), effectiveVersion))
.isPresent();
}

@Override
public void markAsCompatible() {
metadataIndex.setDbMetadata(getDbVendor(), getDbSchemaVersion()
.orElseThrow(() -> new IllegalStateException("DB schema version is not present in database")));
metadataIndex.setDbMetadata(getDbVendor());
}

private String getDbVendor() {
return dbClient.getDatabase().getDialect().getId();
}

private Optional<Long> getDbSchemaVersion() {
return dbMigrationHistory.getLastMigrationNumber();
}
}

+ 1
- 7
server/sonar-server/src/main/java/org/sonar/server/es/metadata/MetadataIndex.java View File

@@ -31,7 +31,6 @@ import static org.sonar.server.es.DefaultIndexSettings.REFRESH_IMMEDIATE;
public class MetadataIndex {

private static final String DB_VENDOR_KEY = "dbVendor";
private static final String DB_SCHEMA_VERSION_KEY = "dbSchemaVersion";

private final EsClient esClient;

@@ -67,13 +66,8 @@ public class MetadataIndex {
return getMetadata(DB_VENDOR_KEY);
}

public Optional<Long> getDbSchemaVersion() {
return getMetadata(DB_SCHEMA_VERSION_KEY).map(Long::parseLong);
}

public void setDbMetadata(String vendor, long schemaVersion) {
public void setDbMetadata(String vendor) {
setMetadata(DB_VENDOR_KEY, vendor);
setMetadata(DB_SCHEMA_VERSION_KEY, String.valueOf(schemaVersion));
}

private Optional<String> getMetadata(String id) {

+ 0
- 29
server/sonar-server/src/test/java/org/sonar/server/es/IndexCreatorTest.java View File

@@ -151,17 +151,10 @@ public class IndexCreatorTest {
c -> c.setHasSameDbVendor(false));
}

@Test
public void delete_existing_indices_if_db_schema_changed() {
testDeleteOnDbChange(LOG_DB_SCHEMA_CHANGED,
c -> c.setHasSameDbSchemaVersion(false));
}

@Test
public void do_not_check_db_compatibility_on_fresh_es() {
// supposed to be ignored
esDbCompatibility.setHasSameDbVendor(false);
esDbCompatibility.setHasSameDbSchemaVersion(false);

startNewCreator(new FakeIndexDefinition());

@@ -172,28 +165,6 @@ public class IndexCreatorTest {
.contains("Create type metadatas/metadata");
}

@Test
public void do_not_check_db_compatibility_if_disabled_by_configuration() {
settings.setProperty("sonar.search.disableDropOnDbMigration", true);

// initial startup, automatic drop may be ignored because indices do not exist
startNewCreator(new FakeIndexDefinition());
logTester.clear();

// second startup, automatic drop can be disabled only by configuration

// supposed to be ignored
esDbCompatibility.setHasSameDbVendor(false);
esDbCompatibility.setHasSameDbSchemaVersion(false);

startNewCreator(new FakeIndexDefinition());

assertThat(logTester.logs(LoggerLevel.INFO))
.doesNotContain(LOG_DB_VENDOR_CHANGED)
.doesNotContain(LOG_DB_SCHEMA_CHANGED);
assertThat(logTester.logs(LoggerLevel.WARN)).contains("Automatic drop of search indices in turned off (see property sonar.search.disableDropOnDbMigration)");
}

private void testDeleteOnDbChange(String expectedLog, Consumer<TestEsDbCompatibility> afterFirstStart) {
startNewCreator(new FakeIndexDefinition());
assertThat(logTester.logs(LoggerLevel.INFO))

+ 82
- 0
server/sonar-server/src/test/java/org/sonar/server/es/SimpleEsClientImplTest.java View File

@@ -0,0 +1,82 @@
/*
* SonarQube
* Copyright (C) 2009-2018 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.es;

import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;

public class SimpleEsClientImplTest {
@Rule
public EsTester es = EsTester.createCustom(new FakeIndexDefinition(),
new SimpleIndexDefinition("a"),
new SimpleIndexDefinition("b"),
new SimpleIndexDefinition("c"));

private Client client = es.client().nativeClient();
private SimpleEsClientImpl underTest = new SimpleEsClientImpl(client);

@Test
public void call_without_arguments_does_not_generate_an_elasticsearch_call() {
Client client = mock(Client.class);
SimpleEsClientImpl underTest = new SimpleEsClientImpl(client);
underTest.deleteIndexes();

verify(client, never()).admin();
}

@Test
public void delete_known_indice_must_delete_the_index() {
underTest.deleteIndexes("fakes");

assertThat(es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().get("fakes")).isNull();
}

@Test
public void delete_unknown_indice_must_delete_all_existing_indexes() {
underTest.deleteIndexes("a", "xxx", "c");

assertThat(es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().get("a")).isNull();
assertThat(es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().get("c")).isNull();
}

public class SimpleIndexDefinition implements IndexDefinition {
private final String indexName;

public SimpleIndexDefinition(String indexName) {
this.indexName = indexName;
}

@Override
public void define(IndexDefinitionContext context) {
NewIndex index = context.create(indexName, newBuilder(new MapSettings().asConfig()).build());
index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
index.getSettings().put("index.refresh_interval", "-1");
}
}
}

+ 0
- 11
server/sonar-server/src/test/java/org/sonar/server/es/TestEsDbCompatibility.java View File

@@ -24,28 +24,17 @@ import org.sonar.server.es.metadata.EsDbCompatibility;
public class TestEsDbCompatibility implements EsDbCompatibility {

private boolean hasSameDbVendor = true;
private boolean hasSameDbSchemaVersion = true;

public TestEsDbCompatibility setHasSameDbVendor(boolean b) {
this.hasSameDbVendor = b;
return this;
}

public TestEsDbCompatibility setHasSameDbSchemaVersion(boolean b) {
this.hasSameDbSchemaVersion = b;
return this;
}

@Override
public boolean hasSameDbVendor() {
return hasSameDbVendor;
}

@Override
public boolean hasSameDbSchemaVersion() {
return hasSameDbSchemaVersion;
}

@Override
public void markAsCompatible() {


+ 13
- 63
server/sonar-server/src/test/java/org/sonar/server/es/metadata/EsDbCompatibilityImplTest.java View File

@@ -19,8 +19,6 @@
*/
package org.sonar.server.es.metadata;

import java.util.Optional;
import javax.annotation.Nullable;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@@ -28,8 +26,6 @@ import org.mockito.Mockito;
import org.sonar.db.DbClient;
import org.sonar.server.es.EsTester;
import org.sonar.server.es.FakeIndexDefinition;
import org.sonar.server.platform.db.migration.history.MigrationHistory;

import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -40,111 +36,65 @@ public class EsDbCompatibilityImplTest {
@Rule
public EsTester es = EsTester.createCustom(new MetadataIndexDefinitionBridge(), new FakeIndexDefinition());
private DbClient dbClient = mock(DbClient.class, Mockito.RETURNS_DEEP_STUBS);
private MigrationHistory migrationHistory = mock(MigrationHistory.class);
private MetadataIndex metadataIndex = new MetadataIndex(es.client());
private EsDbCompatibilityImpl underTest = new EsDbCompatibilityImpl(dbClient, metadataIndex, migrationHistory);
private EsDbCompatibilityImpl underTest = new EsDbCompatibilityImpl(dbClient, metadataIndex);

@Test
public void hasSameDbVendor_is_true_if_values_match() {
prepareDb("mysql", 1_800L);
prepareEs("mysql", 2_000L);
prepareDb("mysql");
prepareEs("mysql");

assertThat(underTest.hasSameDbVendor()).isTrue();
}

@Test
public void hasSameDbVendor_is_false_if_values_dont_match() {
prepareDb("mysql", 1_800L);
prepareEs("postgres", 1_800L);
prepareDb("mysql");
prepareEs("postgres");

assertThat(underTest.hasSameDbVendor()).isFalse();
}

@Test
public void hasSameDbVendor_is_false_if_value_is_absent_from_es() {
prepareDb("mysql", 1_800L);
prepareDb("mysql");

assertThat(underTest.hasSameDbVendor()).isFalse();
}

@Test
public void hasSameDbSchemaVersion_is_true_if_values_match() {
prepareDb("mysql", 1_800L);
prepareEs("postgres", 1_800L);

assertThat(underTest.hasSameDbSchemaVersion()).isTrue();
}

@Test
public void hasSameDbSchemaVersion_is_false_if_values_dont_match() {
prepareDb("mysql", 1_800L);
prepareEs("postgres", 2_000L);

assertThat(underTest.hasSameDbSchemaVersion()).isFalse();
}

@Test
public void hasSameDbSchemaVersion_is_false_if_value_is_absent_from_db() {
prepareDb("mysql", null);
prepareEs("postgres", 1_800L);

assertThat(underTest.hasSameDbSchemaVersion()).isFalse();
}

@Test
public void hasSameDbSchemaVersion_is_false_if_value_is_absent_from_es() {
prepareDb("mysql", 1_800L);

assertThat(underTest.hasSameDbSchemaVersion()).isFalse();
}

@Test
public void store_db_metadata_in_es() {
prepareDb("mysql", 1_800L);
prepareDb("mysql");

underTest.markAsCompatible();

assertThat(metadataIndex.getDbVendor()).hasValue("mysql");
assertThat(metadataIndex.getDbSchemaVersion()).hasValue(1_800L);
}

@Test
public void store_updates_db_metadata_in_es() {
prepareEs("mysql", 1_800L);
prepareDb("postgres", 2_000L);
prepareEs("mysql");
prepareDb("postgres");

underTest.markAsCompatible();

assertThat(metadataIndex.getDbVendor()).hasValue("postgres");
assertThat(metadataIndex.getDbSchemaVersion()).hasValue(2_000L);
}

@Test
public void store_throws_ISE_if_metadata_cant_be_loaded_from_db() {
prepareDb("postgres", null);

expectedException.expect(IllegalStateException.class);
expectedException.expectMessage("DB schema version is not present in database");

underTest.markAsCompatible();
}

@Test
public void store_marks_es_as_compatible_with_db() {
prepareDb("postgres", 1_800L);
prepareDb("postgres");

underTest.markAsCompatible();

assertThat(underTest.hasSameDbSchemaVersion()).isTrue();
assertThat(underTest.hasSameDbVendor()).isTrue();
}

private void prepareDb(String dbVendor, @Nullable Long dbSchemaVersion) {
private void prepareDb(String dbVendor) {
when(dbClient.getDatabase().getDialect().getId()).thenReturn(dbVendor);
when(migrationHistory.getLastMigrationNumber()).thenReturn(Optional.ofNullable(dbSchemaVersion));
}

private void prepareEs(String dbVendor, long dbSchemaVersion) {
metadataIndex.setDbMetadata(dbVendor, dbSchemaVersion);
private void prepareEs(String dbVendor) {
metadataIndex.setDbMetadata(dbVendor);
}
}

+ 1
- 3
server/sonar-server/src/test/java/org/sonar/server/es/metadata/MetadataIndexTest.java View File

@@ -63,14 +63,12 @@ public class MetadataIndexTest {
@Test
public void database_metadata_are_empty_if_absent_from_index() {
assertThat(underTest.getDbVendor()).isNotPresent();
assertThat(underTest.getDbSchemaVersion()).isNotPresent();
}

@Test
public void database_metadata_are_present_from_index() {
underTest.setDbMetadata("postgres", 1_800);
underTest.setDbMetadata("postgres");

assertThat(underTest.getDbVendor()).hasValue("postgres");
assertThat(underTest.getDbSchemaVersion()).hasValue(1_800L);
}
}

Loading…
Cancel
Save