--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.platform.db.migration.es;
+
+public interface ElasticsearchClient {
+
+ /**
+ * This method is reentrant and does not fail if indexName or otherIndexNames does not exist
+ */
+ void deleteIndexes(String... otherIndexNames);
+}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+@ParametersAreNonnullByDefault
+package org.sonar.server.platform.db.migration.es;
+
+import javax.annotation.ParametersAreNonnullByDefault;
+
import org.apache.commons.dbutils.DbUtils;
import org.sonar.db.Database;
-public class SelectImpl extends BaseSqlStatement<Select>implements Select {
+public class SelectImpl extends BaseSqlStatement<Select> implements Select {
private SelectImpl(PreparedStatement pstmt) {
super(pstmt);
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.platform.db.migration.step;
+
+import java.sql.SQLException;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.sonar.server.platform.db.migration.es.ElasticsearchClient;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class ElasticsearchChangeTest {
+
+ private ElasticsearchClient client = mock(ElasticsearchClient.class);
+
+ @Test
+ public void deleteIndice_call() throws SQLException {
+ ArgumentCaptor<String> indices = ArgumentCaptor.forClass(String.class);
+
+ new ElasticsearchChange(client) {
+ @Override
+ protected void execute(Context context) throws SQLException {
+ context.deleteIndice("a", "b", "c");
+ }
+ }.execute();
+
+ verify(client, times(1)).deleteIndice(indices.capture());
+ assertThat(indices.getAllValues()).containsExactly("a", "b", "c");
+ }
+}
public class IndexCreator implements Startable {
private static final Logger LOGGER = Loggers.get(IndexCreator.class);
- private static final String PROPERY_DISABLE_CHECK = "sonar.search.disableDropOnDbMigration";
private final MetadataIndexDefinition metadataIndexDefinition;
private final MetadataIndex metadataIndex;
}
private void checkDbCompatibility(Collection<Index> definitions) {
- boolean disabledCheck = configuration.getBoolean(PROPERY_DISABLE_CHECK).orElse(false);
- if (disabledCheck) {
- LOGGER.warn("Automatic drop of search indices in turned off (see property " + PROPERY_DISABLE_CHECK + ")");
- }
-
List<String> existingIndices = loadExistingIndicesExceptMetadata(definitions);
- if (!disabledCheck && !existingIndices.isEmpty()) {
+ if (!existingIndices.isEmpty()) {
boolean delete = false;
if (!esDbCompatibility.hasSameDbVendor()) {
LOGGER.info("Delete Elasticsearch indices (DB vendor changed)");
delete = true;
- } else if (!esDbCompatibility.hasSameDbSchemaVersion()) {
- LOGGER.info("Delete Elasticsearch indices (DB schema changed)");
- delete = true;
}
if (delete) {
existingIndices.forEach(this::deleteIndex);
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+package org.sonar.server.es;
+
+import java.util.Arrays;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.client.Client;
+import org.sonar.server.platform.db.migration.es.ElasticsearchClient;
+
+import static java.util.Objects.requireNonNull;
+
+public class SimpleEsClientImpl implements ElasticsearchClient {
+ private final Client nativeClient;
+
+ public SimpleEsClientImpl(Client nativeClient) {
+ this.nativeClient = requireNonNull(nativeClient);
+ }
+
+ /**
+ * This method is reentrant and does not fail if indexName or otherIndexNames does not exist
+ */
+ public void deleteIndexes(String... indexNames) {
+ if (indexNames.length == 0) {
+ return;
+ }
+
+ GetMappingsResponse getMappingsResponse = nativeClient.admin().indices().prepareGetMappings("_all").get();
+ String[] allIndexes = getMappingsResponse.mappings().keys().toArray(String.class);
+ String[] intersection = intersection(indexNames, allIndexes);
+ nativeClient.admin().indices().prepareDelete(intersection).get();
+ }
+
+ private String[] intersection(String[] a, String[] b) {
+ return Arrays.stream(a)
+ .distinct()
+ .filter(x -> Arrays.stream(b).anyMatch(y -> y != null && y.equals(x)))
+ .toArray(String[]::new);
+ }
+}
*/
boolean hasSameDbVendor();
- /**
- * Whether the effective DB schema version equals the version
- * registered in Elasticsearch metadata.
- * Return {@code false} if at least one of the values is absent
- */
- boolean hasSameDbSchemaVersion();
-
/**
* Stores in Elasticsearch the metadata about database
*/
*/
package org.sonar.server.es.metadata;
-import java.util.Objects;
import java.util.Optional;
import org.sonar.db.DbClient;
-import org.sonar.server.platform.db.migration.history.MigrationHistory;
public class EsDbCompatibilityImpl implements EsDbCompatibility {
private final DbClient dbClient;
private final MetadataIndex metadataIndex;
- private final MigrationHistory dbMigrationHistory;
- public EsDbCompatibilityImpl(DbClient dbClient, MetadataIndex metadataIndex, MigrationHistory dbMigrationHistory) {
+ public EsDbCompatibilityImpl(DbClient dbClient, MetadataIndex metadataIndex) {
this.dbClient = dbClient;
this.metadataIndex = metadataIndex;
- this.dbMigrationHistory = dbMigrationHistory;
}
@Override
return registeredDbVendor.isPresent() && registeredDbVendor.get().equals(getDbVendor());
}
- @Override
- public boolean hasSameDbSchemaVersion() {
- Optional<Long> registeredVersion = metadataIndex.getDbSchemaVersion();
- if (!registeredVersion.isPresent()) {
- return false;
- }
- return getDbSchemaVersion()
- .filter(effectiveVersion -> Objects.equals(registeredVersion.get(), effectiveVersion))
- .isPresent();
- }
-
@Override
public void markAsCompatible() {
- metadataIndex.setDbMetadata(getDbVendor(), getDbSchemaVersion()
- .orElseThrow(() -> new IllegalStateException("DB schema version is not present in database")));
+ metadataIndex.setDbMetadata(getDbVendor());
}
private String getDbVendor() {
return dbClient.getDatabase().getDialect().getId();
}
-
- private Optional<Long> getDbSchemaVersion() {
- return dbMigrationHistory.getLastMigrationNumber();
- }
}
public class MetadataIndex {
private static final String DB_VENDOR_KEY = "dbVendor";
- private static final String DB_SCHEMA_VERSION_KEY = "dbSchemaVersion";
private final EsClient esClient;
return getMetadata(DB_VENDOR_KEY);
}
- public Optional<Long> getDbSchemaVersion() {
- return getMetadata(DB_SCHEMA_VERSION_KEY).map(Long::parseLong);
- }
-
- public void setDbMetadata(String vendor, long schemaVersion) {
+ public void setDbMetadata(String vendor) {
setMetadata(DB_VENDOR_KEY, vendor);
- setMetadata(DB_SCHEMA_VERSION_KEY, String.valueOf(schemaVersion));
}
private Optional<String> getMetadata(String id) {
c -> c.setHasSameDbVendor(false));
}
- @Test
- public void delete_existing_indices_if_db_schema_changed() {
- testDeleteOnDbChange(LOG_DB_SCHEMA_CHANGED,
- c -> c.setHasSameDbSchemaVersion(false));
- }
-
@Test
public void do_not_check_db_compatibility_on_fresh_es() {
// supposed to be ignored
esDbCompatibility.setHasSameDbVendor(false);
- esDbCompatibility.setHasSameDbSchemaVersion(false);
startNewCreator(new FakeIndexDefinition());
.contains("Create type metadatas/metadata");
}
- @Test
- public void do_not_check_db_compatibility_if_disabled_by_configuration() {
- settings.setProperty("sonar.search.disableDropOnDbMigration", true);
-
- // initial startup, automatic drop may be ignored because indices do not exist
- startNewCreator(new FakeIndexDefinition());
- logTester.clear();
-
- // second startup, automatic drop can be disabled only by configuration
-
- // supposed to be ignored
- esDbCompatibility.setHasSameDbVendor(false);
- esDbCompatibility.setHasSameDbSchemaVersion(false);
-
- startNewCreator(new FakeIndexDefinition());
-
- assertThat(logTester.logs(LoggerLevel.INFO))
- .doesNotContain(LOG_DB_VENDOR_CHANGED)
- .doesNotContain(LOG_DB_SCHEMA_CHANGED);
- assertThat(logTester.logs(LoggerLevel.WARN)).contains("Automatic drop of search indices in turned off (see property sonar.search.disableDropOnDbMigration)");
- }
-
private void testDeleteOnDbChange(String expectedLog, Consumer<TestEsDbCompatibility> afterFirstStart) {
startNewCreator(new FakeIndexDefinition());
assertThat(logTester.logs(LoggerLevel.INFO))
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.server.es;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.junit.Rule;
+import org.junit.Test;
+import org.sonar.api.config.internal.MapSettings;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.sonar.server.es.NewIndex.SettingsConfiguration.newBuilder;
+
+public class SimpleEsClientImplTest {
+ @Rule
+ public EsTester es = EsTester.createCustom(new FakeIndexDefinition(),
+ new SimpleIndexDefinition("a"),
+ new SimpleIndexDefinition("b"),
+ new SimpleIndexDefinition("c"));
+
+ private Client client = es.client().nativeClient();
+ private SimpleEsClientImpl underTest = new SimpleEsClientImpl(client);
+
+ @Test
+ public void call_without_arguments_does_not_generate_an_elasticsearch_call() {
+ Client client = mock(Client.class);
+ SimpleEsClientImpl underTest = new SimpleEsClientImpl(client);
+ underTest.deleteIndexes();
+
+ verify(client, never()).admin();
+ }
+
+ @Test
+ public void delete_known_indice_must_delete_the_index() {
+ underTest.deleteIndexes("fakes");
+
+ assertThat(es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().get("fakes")).isNull();
+ }
+
+ @Test
+ public void delete_unknown_indice_must_delete_all_existing_indexes() {
+ underTest.deleteIndexes("a", "xxx", "c");
+
+ assertThat(es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().get("a")).isNull();
+ assertThat(es.client().nativeClient().admin().indices().prepareGetMappings().get().mappings().get("c")).isNull();
+ }
+
+ public class SimpleIndexDefinition implements IndexDefinition {
+ private final String indexName;
+
+ public SimpleIndexDefinition(String indexName) {
+ this.indexName = indexName;
+ }
+
+ @Override
+ public void define(IndexDefinitionContext context) {
+ NewIndex index = context.create(indexName, newBuilder(new MapSettings().asConfig()).build());
+ index.getSettings().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
+ index.getSettings().put("index.refresh_interval", "-1");
+ }
+ }
+}
public class TestEsDbCompatibility implements EsDbCompatibility {
private boolean hasSameDbVendor = true;
- private boolean hasSameDbSchemaVersion = true;
public TestEsDbCompatibility setHasSameDbVendor(boolean b) {
this.hasSameDbVendor = b;
return this;
}
- public TestEsDbCompatibility setHasSameDbSchemaVersion(boolean b) {
- this.hasSameDbSchemaVersion = b;
- return this;
- }
-
@Override
public boolean hasSameDbVendor() {
return hasSameDbVendor;
}
- @Override
- public boolean hasSameDbSchemaVersion() {
- return hasSameDbSchemaVersion;
- }
-
@Override
public void markAsCompatible() {
*/
package org.sonar.server.es.metadata;
-import java.util.Optional;
-import javax.annotation.Nullable;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.sonar.db.DbClient;
import org.sonar.server.es.EsTester;
import org.sonar.server.es.FakeIndexDefinition;
-import org.sonar.server.platform.db.migration.history.MigrationHistory;
-
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Rule
public EsTester es = EsTester.createCustom(new MetadataIndexDefinitionBridge(), new FakeIndexDefinition());
private DbClient dbClient = mock(DbClient.class, Mockito.RETURNS_DEEP_STUBS);
- private MigrationHistory migrationHistory = mock(MigrationHistory.class);
private MetadataIndex metadataIndex = new MetadataIndex(es.client());
- private EsDbCompatibilityImpl underTest = new EsDbCompatibilityImpl(dbClient, metadataIndex, migrationHistory);
+ private EsDbCompatibilityImpl underTest = new EsDbCompatibilityImpl(dbClient, metadataIndex);
@Test
public void hasSameDbVendor_is_true_if_values_match() {
- prepareDb("mysql", 1_800L);
- prepareEs("mysql", 2_000L);
+ prepareDb("mysql");
+ prepareEs("mysql");
assertThat(underTest.hasSameDbVendor()).isTrue();
}
@Test
public void hasSameDbVendor_is_false_if_values_dont_match() {
- prepareDb("mysql", 1_800L);
- prepareEs("postgres", 1_800L);
+ prepareDb("mysql");
+ prepareEs("postgres");
assertThat(underTest.hasSameDbVendor()).isFalse();
}
@Test
public void hasSameDbVendor_is_false_if_value_is_absent_from_es() {
- prepareDb("mysql", 1_800L);
+ prepareDb("mysql");
assertThat(underTest.hasSameDbVendor()).isFalse();
}
- @Test
- public void hasSameDbSchemaVersion_is_true_if_values_match() {
- prepareDb("mysql", 1_800L);
- prepareEs("postgres", 1_800L);
-
- assertThat(underTest.hasSameDbSchemaVersion()).isTrue();
- }
-
- @Test
- public void hasSameDbSchemaVersion_is_false_if_values_dont_match() {
- prepareDb("mysql", 1_800L);
- prepareEs("postgres", 2_000L);
-
- assertThat(underTest.hasSameDbSchemaVersion()).isFalse();
- }
-
- @Test
- public void hasSameDbSchemaVersion_is_false_if_value_is_absent_from_db() {
- prepareDb("mysql", null);
- prepareEs("postgres", 1_800L);
-
- assertThat(underTest.hasSameDbSchemaVersion()).isFalse();
- }
-
- @Test
- public void hasSameDbSchemaVersion_is_false_if_value_is_absent_from_es() {
- prepareDb("mysql", 1_800L);
-
- assertThat(underTest.hasSameDbSchemaVersion()).isFalse();
- }
-
@Test
public void store_db_metadata_in_es() {
- prepareDb("mysql", 1_800L);
+ prepareDb("mysql");
underTest.markAsCompatible();
assertThat(metadataIndex.getDbVendor()).hasValue("mysql");
- assertThat(metadataIndex.getDbSchemaVersion()).hasValue(1_800L);
}
@Test
public void store_updates_db_metadata_in_es() {
- prepareEs("mysql", 1_800L);
- prepareDb("postgres", 2_000L);
+ prepareEs("mysql");
+ prepareDb("postgres");
underTest.markAsCompatible();
assertThat(metadataIndex.getDbVendor()).hasValue("postgres");
- assertThat(metadataIndex.getDbSchemaVersion()).hasValue(2_000L);
- }
-
- @Test
- public void store_throws_ISE_if_metadata_cant_be_loaded_from_db() {
- prepareDb("postgres", null);
-
- expectedException.expect(IllegalStateException.class);
- expectedException.expectMessage("DB schema version is not present in database");
-
- underTest.markAsCompatible();
}
@Test
public void store_marks_es_as_compatible_with_db() {
- prepareDb("postgres", 1_800L);
+ prepareDb("postgres");
underTest.markAsCompatible();
- assertThat(underTest.hasSameDbSchemaVersion()).isTrue();
assertThat(underTest.hasSameDbVendor()).isTrue();
}
- private void prepareDb(String dbVendor, @Nullable Long dbSchemaVersion) {
+ private void prepareDb(String dbVendor) {
when(dbClient.getDatabase().getDialect().getId()).thenReturn(dbVendor);
- when(migrationHistory.getLastMigrationNumber()).thenReturn(Optional.ofNullable(dbSchemaVersion));
}
- private void prepareEs(String dbVendor, long dbSchemaVersion) {
- metadataIndex.setDbMetadata(dbVendor, dbSchemaVersion);
+ private void prepareEs(String dbVendor) {
+ metadataIndex.setDbMetadata(dbVendor);
}
}
@Test
public void database_metadata_are_empty_if_absent_from_index() {
assertThat(underTest.getDbVendor()).isNotPresent();
- assertThat(underTest.getDbSchemaVersion()).isNotPresent();
}
@Test
public void database_metadata_are_present_from_index() {
- underTest.setDbMetadata("postgres", 1_800);
+ underTest.setDbMetadata("postgres");
assertThat(underTest.getDbVendor()).hasValue("postgres");
- assertThat(underTest.getDbSchemaVersion()).hasValue(1_800L);
}
}
projects / sonarqube.git / commitdiff