@@ -22,10 +22,7 @@ package org.sonar.db; | |||
import java.io.PrintWriter; | |||
import java.sql.Connection; | |||
import java.sql.SQLException; | |||
import java.util.Collections; | |||
import java.util.List; | |||
import java.util.Map; | |||
import java.util.Set; | |||
import javax.sql.DataSource; | |||
import org.apache.commons.io.output.NullWriter; | |||
import org.apache.ibatis.io.Resources; | |||
@@ -45,7 +42,6 @@ import org.sonar.process.logging.LogbackHelper; | |||
import org.sonar.server.platform.db.migration.MigrationConfigurationModule; | |||
import org.sonar.server.platform.db.migration.engine.MigrationContainer; | |||
import org.sonar.server.platform.db.migration.engine.MigrationContainerImpl; | |||
import org.sonar.server.platform.db.migration.es.MigrationEsClient; | |||
import org.sonar.server.platform.db.migration.history.MigrationHistoryTableImpl; | |||
import org.sonar.server.platform.db.migration.step.MigrationStep; | |||
import org.sonar.server.platform.db.migration.step.MigrationStepExecutionException; | |||
@@ -153,7 +149,6 @@ public class SQDatabase extends DefaultDatabase { | |||
container.add(UuidFactoryFast.getInstance()); | |||
container.add(System2.INSTANCE); | |||
container.add(MapSettings.class); | |||
container.add(createMockMigrationEsClient()); | |||
container.startComponents(); | |||
MigrationContainer migrationContainer = new MigrationContainerImpl(container, H2StepExecutor.class); | |||
@@ -163,25 +158,6 @@ public class SQDatabase extends DefaultDatabase { | |||
executor.execute(migrationSteps.readAll()); | |||
} | |||
private static MigrationEsClient createMockMigrationEsClient() { | |||
return new MigrationEsClient() { | |||
@Override | |||
public void deleteIndexes(String name, String... otherNames) { | |||
//No ES operation required for database tests | |||
} | |||
@Override | |||
public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options) { | |||
//No ES operation required for database tests | |||
} | |||
@Override | |||
public Set<String> getUpdatedIndices() { | |||
return Collections.emptySet(); | |||
} | |||
}; | |||
} | |||
private void createMigrationHistoryTable(NoopDatabase noopDatabase) { | |||
new MigrationHistoryTableImpl(noopDatabase).start(); | |||
} |
@@ -21,36 +21,27 @@ package org.sonar.server.platform.db.migration.engine; | |||
import java.util.List; | |||
import java.util.Optional; | |||
import org.sonar.api.config.Configuration; | |||
import org.sonar.api.utils.AnnotationUtils; | |||
import org.sonar.core.platform.SpringComponentContainer; | |||
import org.sonar.process.ProcessProperties; | |||
import org.sonar.server.platform.db.migration.SupportsBlueGreen; | |||
import org.sonar.server.platform.db.migration.history.MigrationHistory; | |||
import org.sonar.server.platform.db.migration.step.MigrationSteps; | |||
import org.sonar.server.platform.db.migration.step.MigrationStepsExecutor; | |||
import org.sonar.server.platform.db.migration.step.MigrationStepsExecutorImpl; | |||
import org.sonar.server.platform.db.migration.step.RegisteredMigrationStep; | |||
import static java.lang.String.format; | |||
public class MigrationEngineImpl implements MigrationEngine { | |||
private final MigrationHistory migrationHistory; | |||
private final SpringComponentContainer serverContainer; | |||
private final MigrationSteps migrationSteps; | |||
private final Configuration configuration; | |||
public MigrationEngineImpl(MigrationHistory migrationHistory, SpringComponentContainer serverContainer, MigrationSteps migrationSteps, Configuration configuration) { | |||
public MigrationEngineImpl(MigrationHistory migrationHistory, SpringComponentContainer serverContainer, MigrationSteps migrationSteps) { | |||
this.migrationHistory = migrationHistory; | |||
this.serverContainer = serverContainer; | |||
this.migrationSteps = migrationSteps; | |||
this.configuration = configuration; | |||
} | |||
@Override | |||
public void execute() { | |||
MigrationContainer migrationContainer = new MigrationContainerImpl(serverContainer, MigrationStepsExecutorImpl.class); | |||
boolean blueGreen = configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false); | |||
try { | |||
MigrationStepsExecutor stepsExecutor = migrationContainer.getComponentByType(MigrationStepsExecutor.class); | |||
Optional<Long> lastMigrationNumber = migrationHistory.getLastMigrationNumber(); | |||
@@ -59,9 +50,6 @@ public class MigrationEngineImpl implements MigrationEngine { | |||
.map(i -> migrationSteps.readFrom(i + 1)) | |||
.orElse(migrationSteps.readAll()); | |||
if (blueGreen) { | |||
ensureSupportBlueGreen(steps); | |||
} | |||
stepsExecutor.execute(steps); | |||
@@ -69,13 +57,4 @@ public class MigrationEngineImpl implements MigrationEngine { | |||
migrationContainer.cleanup(); | |||
} | |||
} | |||
private static void ensureSupportBlueGreen(List<RegisteredMigrationStep> steps) { | |||
for (RegisteredMigrationStep step : steps) { | |||
if (AnnotationUtils.getAnnotation(step.getStepClass(), SupportsBlueGreen.class) == null) { | |||
throw new IllegalStateException(format("All migrations canceled. #%d does not support blue/green deployment: %s", | |||
step.getMigrationNumber(), step.getDescription())); | |||
} | |||
} | |||
} | |||
} |
@@ -1,47 +0,0 @@ | |||
/* | |||
* SonarQube | |||
* Copyright (C) 2009-2023 SonarSource SA | |||
* mailto:info AT sonarsource DOT com | |||
* | |||
* This program is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* This program is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.platform.db.migration.es; | |||
import java.util.Map; | |||
import java.util.Set; | |||
public interface MigrationEsClient { | |||
/** | |||
* This method is re-entrant and does not fail if indexName or otherIndexNames do not exist | |||
*/ | |||
void deleteIndexes(String name, String... otherNames); | |||
/** | |||
* Adds a new mapping to an existing Elasticsearch index. Does nothing if index does not exist. | |||
* | |||
* @param index name of the index that the mapping is added to | |||
* @param type document type in the index | |||
* @param mappingName name of the new mapping | |||
* @param mappingType type of the new mapping | |||
* @param options additional options to be applied to the mapping | |||
*/ | |||
void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options); | |||
/** | |||
* Returns the indices that have been touched by {@link #addMappingToExistingIndex(String, String, String, String, Map)} | |||
*/ | |||
Set<String> getUpdatedIndices(); | |||
} |
@@ -1,24 +0,0 @@ | |||
/* | |||
* SonarQube | |||
* Copyright (C) 2009-2023 SonarSource SA | |||
* mailto:info AT sonarsource DOT com | |||
* | |||
* This program is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* This program is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
@ParametersAreNonnullByDefault | |||
package org.sonar.server.platform.db.migration.es; | |||
import javax.annotation.ParametersAreNonnullByDefault; | |||
@@ -24,10 +24,8 @@ import java.util.List; | |||
import java.util.Optional; | |||
import org.junit.Before; | |||
import org.junit.Test; | |||
import org.sonar.api.config.internal.ConfigurationBridge; | |||
import org.sonar.api.config.internal.MapSettings; | |||
import org.sonar.core.platform.SpringComponentContainer; | |||
import org.sonar.process.ProcessProperties; | |||
import org.sonar.server.platform.db.migration.SupportsBlueGreen; | |||
import org.sonar.server.platform.db.migration.history.MigrationHistory; | |||
import org.sonar.server.platform.db.migration.step.MigrationStep; | |||
@@ -35,10 +33,8 @@ import org.sonar.server.platform.db.migration.step.MigrationSteps; | |||
import org.sonar.server.platform.db.migration.step.MigrationStepsExecutor; | |||
import org.sonar.server.platform.db.migration.step.RegisteredMigrationStep; | |||
import static java.util.Arrays.asList; | |||
import static java.util.Collections.singletonList; | |||
import static org.assertj.core.api.Assertions.assertThat; | |||
import static org.junit.Assert.fail; | |||
import static org.mockito.Mockito.mock; | |||
import static org.mockito.Mockito.times; | |||
import static org.mockito.Mockito.verify; | |||
@@ -50,7 +46,7 @@ public class MigrationEngineImplTest { | |||
private final MigrationSteps migrationSteps = mock(MigrationSteps.class); | |||
private final StepRegistry stepRegistry = new StepRegistry(); | |||
private final MapSettings settings = new MapSettings(); | |||
private final MigrationEngineImpl underTest = new MigrationEngineImpl(migrationHistory, serverContainer, migrationSteps, new ConfigurationBridge(settings)); | |||
private final MigrationEngineImpl underTest = new MigrationEngineImpl(migrationHistory, serverContainer, migrationSteps); | |||
@Before | |||
public void before() { | |||
@@ -85,38 +81,6 @@ public class MigrationEngineImplTest { | |||
assertThat(stepRegistry.stepRan).isTrue(); | |||
} | |||
@Test | |||
public void execute_steps_in_blue_green_mode() { | |||
settings.setProperty(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey(), true); | |||
when(migrationHistory.getLastMigrationNumber()).thenReturn(Optional.of(50L)); | |||
List<RegisteredMigrationStep> steps = singletonList(new RegisteredMigrationStep(1, "doo", TestBlueGreenMigrationStep.class)); | |||
when(migrationSteps.readFrom(51)).thenReturn(steps); | |||
when(migrationSteps.readAll()).thenReturn(steps); | |||
underTest.execute(); | |||
verify(migrationSteps).readFrom(51); | |||
assertThat(stepRegistry.stepRan).isTrue(); | |||
} | |||
@Test | |||
public void fail_blue_green_execution_if_some_migrations_are_not_compatible() { | |||
settings.setProperty(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey(), true); | |||
when(migrationHistory.getLastMigrationNumber()).thenReturn(Optional.of(50L)); | |||
List<RegisteredMigrationStep> steps = asList( | |||
new RegisteredMigrationStep(1, "foo", TestBlueGreenMigrationStep.class), | |||
new RegisteredMigrationStep(2, "bar", TestMigrationStep.class)); | |||
when(migrationSteps.readFrom(51)).thenReturn(steps); | |||
try { | |||
underTest.execute(); | |||
fail(); | |||
} catch (IllegalStateException e) { | |||
assertThat(e).hasMessage("All migrations canceled. #2 does not support blue/green deployment: bar"); | |||
assertThat(stepRegistry.stepRan).isFalse(); | |||
} | |||
} | |||
private static class NoOpExecutor implements MigrationStepsExecutor { | |||
@Override | |||
public void execute(List<RegisteredMigrationStep> steps) { |
@@ -182,9 +182,6 @@ public class ProcessProperties { | |||
*/ | |||
ENABLE_STOP_COMMAND("sonar.enableStopCommand"), | |||
// whether the blue/green deployment of server is enabled | |||
BLUE_GREEN_ENABLED("sonar.blueGreenEnabled", DEFAULT_FALSE), | |||
AUTO_DATABASE_UPGRADE("sonar.autoDatabaseUpgrade", DEFAULT_FALSE); | |||
/** |
@@ -1,77 +0,0 @@ | |||
/* | |||
* SonarQube | |||
* Copyright (C) 2009-2023 SonarSource SA | |||
* mailto:info AT sonarsource DOT com | |||
* | |||
* This program is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* This program is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.es; | |||
import java.util.Arrays; | |||
import java.util.HashSet; | |||
import java.util.Map; | |||
import java.util.Set; | |||
import java.util.stream.Stream; | |||
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; | |||
import org.elasticsearch.client.indices.GetIndexRequest; | |||
import org.sonar.api.utils.log.Loggers; | |||
import org.sonar.core.util.stream.MoreCollectors; | |||
import org.sonar.server.platform.db.migration.es.MigrationEsClient; | |||
public class MigrationEsClientImpl implements MigrationEsClient { | |||
private final EsClient client; | |||
private final Set<String> updatedIndices = new HashSet<>(); | |||
public MigrationEsClientImpl(EsClient client) { | |||
this.client = client; | |||
} | |||
@Override | |||
public void deleteIndexes(String name, String... otherNames) { | |||
String[] indices = client.getIndex(new GetIndexRequest("_all")).getIndices(); | |||
Set<String> existingIndices = Arrays.stream(indices).collect(MoreCollectors.toSet()); | |||
Stream.concat(Stream.of(name), Arrays.stream(otherNames)) | |||
.distinct() | |||
.filter(existingIndices::contains) | |||
.forEach(this::deleteIndex); | |||
} | |||
@Override | |||
public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options) { | |||
//TODO:: remove? | |||
// String[] indices = client.getIndex(new GetIndexRequest(index)).getIndices(); | |||
// if (indices != null && indices.length == 1) { | |||
// Loggers.get(getClass()).info("Add mapping [{}] to Elasticsearch index [{}]", mappingName, index); | |||
// String mappingOptions = Stream.concat(Stream.of(Maps.immutableEntry("type", mappingType)), options.entrySet().stream()) | |||
// .map(e -> e.getKey() + "=" + e.getValue()) | |||
// .collect(Collectors.joining(",")); | |||
// client.putMapping(new PutMappingRequest(index) | |||
// .source(mappingName, mappingOptions)); | |||
// updatedIndices.add(index); | |||
// } else { | |||
// throw new IllegalStateException("Expected only one index to be found, actual [" + String.join(",", indices) + "]"); | |||
// } | |||
} | |||
@Override | |||
public Set<String> getUpdatedIndices() { | |||
return updatedIndices; | |||
} | |||
private void deleteIndex(String index) { | |||
Loggers.get(getClass()).info("Drop Elasticsearch index [{}]", index); | |||
client.deleteIndex(new DeleteIndexRequest(index)); | |||
} | |||
} |
@@ -21,11 +21,9 @@ package org.sonar.server.platform; | |||
import java.util.Optional; | |||
import org.sonar.api.Startable; | |||
import org.sonar.api.config.Configuration; | |||
import org.sonar.api.utils.MessageException; | |||
import org.sonar.api.utils.log.Logger; | |||
import org.sonar.api.utils.log.Loggers; | |||
import org.sonar.process.ProcessProperties; | |||
import org.sonar.server.platform.db.migration.version.DatabaseVersion; | |||
import static org.sonar.server.log.ServerProcessLogging.STARTUP_LOGGER_NAME; | |||
@@ -35,11 +33,9 @@ public class DatabaseServerCompatibility implements Startable { | |||
private static final String HIGHLIGHTER = "################################################################################"; | |||
private final DatabaseVersion version; | |||
private final Configuration configuration; | |||
public DatabaseServerCompatibility(DatabaseVersion version, Configuration configuration) { | |||
public DatabaseServerCompatibility(DatabaseVersion version) { | |||
this.version = version; | |||
this.configuration = configuration; | |||
} | |||
@Override | |||
@@ -54,16 +50,14 @@ public class DatabaseServerCompatibility implements Startable { | |||
if (currentVersion.isPresent() && currentVersion.get() < DatabaseVersion.MIN_UPGRADE_VERSION) { | |||
throw MessageException.of("The version of SonarQube is too old. Please upgrade to the Long Term Support version first."); | |||
} | |||
boolean blueGreen = configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false); | |||
if (!blueGreen) { | |||
String msg = "The database must be manually upgraded. Please backup the database and browse /setup. " | |||
+ "For more information: https://docs.sonarqube.org/latest/setup/upgrading"; | |||
Loggers.get(DatabaseServerCompatibility.class).warn(msg); | |||
Logger logger = Loggers.get(STARTUP_LOGGER_NAME); | |||
logger.warn(HIGHLIGHTER); | |||
logger.warn(msg); | |||
logger.warn(HIGHLIGHTER); | |||
} | |||
String msg = "The database must be manually upgraded. Please backup the database and browse /setup. " | |||
+ "For more information: https://docs.sonarqube.org/latest/setup/upgrading"; | |||
Loggers.get(DatabaseServerCompatibility.class).warn(msg); | |||
Logger logger = Loggers.get(STARTUP_LOGGER_NAME); | |||
logger.warn(HIGHLIGHTER); | |||
logger.warn(msg); | |||
logger.warn(HIGHLIGHTER); | |||
} | |||
} | |||
@@ -70,10 +70,6 @@ public class DefaultServerUpgradeStatus implements ServerUpgradeStatus, Startabl | |||
return (int) initialDbVersion; | |||
} | |||
public boolean isBlueGreen() { | |||
return configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false); | |||
} | |||
public boolean isAutoDbUpgrade() { | |||
return configuration.getBoolean(ProcessProperties.Property.AUTO_DATABASE_UPGRADE.getKey()).orElse(false); | |||
} |
@@ -41,9 +41,6 @@ public class AutoDbMigration implements Startable { | |||
} else if (serverUpgradeStatus.isUpgraded() && serverUpgradeStatus.isAutoDbUpgrade()) { | |||
Loggers.get(getClass()).info("Automatically perform DB migration, as automatic database upgrade is enabled"); | |||
migrationEngine.execute(); | |||
} else if (serverUpgradeStatus.isUpgraded() && serverUpgradeStatus.isBlueGreen()) { | |||
Loggers.get(getClass()).info("Automatically perform DB migration on blue/green deployment"); | |||
migrationEngine.execute(); | |||
} | |||
} | |||
@@ -1,137 +0,0 @@ | |||
/* | |||
* SonarQube | |||
* Copyright (C) 2009-2023 SonarSource SA | |||
* mailto:info AT sonarsource DOT com | |||
* | |||
* This program is free software; you can redistribute it and/or | |||
* modify it under the terms of the GNU Lesser General Public | |||
* License as published by the Free Software Foundation; either | |||
* version 3 of the License, or (at your option) any later version. | |||
* | |||
* This program is distributed in the hope that it will be useful, | |||
* but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |||
* Lesser General Public License for more details. | |||
* | |||
* You should have received a copy of the GNU Lesser General Public License | |||
* along with this program; if not, write to the Free Software Foundation, | |||
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. | |||
*/ | |||
package org.sonar.server.es; | |||
import com.google.common.collect.ImmutableMap; | |||
import java.util.Iterator; | |||
import java.util.Map; | |||
import javax.annotation.CheckForNull; | |||
import org.elasticsearch.client.indices.GetMappingsRequest; | |||
import org.elasticsearch.cluster.metadata.MappingMetadata; | |||
import org.junit.Ignore; | |||
import org.junit.Rule; | |||
import org.junit.Test; | |||
import org.sonar.api.config.internal.MapSettings; | |||
import org.sonar.api.utils.log.LogTester; | |||
import org.sonar.api.utils.log.LoggerLevel; | |||
import org.sonar.server.platform.db.migration.es.MigrationEsClient; | |||
import static org.assertj.core.api.Assertions.assertThat; | |||
import static org.assertj.core.api.Assertions.assertThatThrownBy; | |||
import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; | |||
@Ignore | |||
public class MigrationEsClientImplTest { | |||
@Rule | |||
public LogTester logTester = new LogTester(); | |||
@Rule | |||
public EsTester es = EsTester.createCustom( | |||
new SimpleIndexDefinition("as"), | |||
new SimpleIndexDefinition("bs"), | |||
new SimpleIndexDefinition("cs")); | |||
private MigrationEsClient underTest = new MigrationEsClientImpl(es.client()); | |||
@Test | |||
public void delete_existing_index() { | |||
underTest.deleteIndexes("as"); | |||
assertThat(loadExistingIndices()) | |||
.toIterable() | |||
.doesNotContain("as") | |||
.contains("bs", "cs"); | |||
assertThat(logTester.logs(LoggerLevel.INFO)) | |||
.contains("Drop Elasticsearch index [as]"); | |||
} | |||
@Test | |||
public void delete_index_that_does_not_exist() { | |||
underTest.deleteIndexes("as", "xxx", "cs"); | |||
assertThat(loadExistingIndices()) | |||
.toIterable() | |||
.doesNotContain("as", "cs") | |||
.contains("bs"); | |||
assertThat(logTester.logs(LoggerLevel.INFO)) | |||
.contains("Drop Elasticsearch index [as]", "Drop Elasticsearch index [cs]") | |||
.doesNotContain("Drop Elasticsearch index [xxx]"); | |||
} | |||
@Test | |||
public void addMappingToExistingIndex() { | |||
Map<String, String> mappingOptions = ImmutableMap.of("norms", "false"); | |||
underTest.addMappingToExistingIndex("as", "s", "new_field", "keyword", mappingOptions); | |||
assertThat(loadExistingIndices()).toIterable().contains("as"); | |||
Map<String, MappingMetadata> mappings = mappings(); | |||
MappingMetadata mapping = mappings.get("as"); | |||
assertThat(countMappingFields(mapping)).isOne(); | |||
assertThat(field(mapping, "new_field")).isNotNull(); | |||
assertThat(logTester.logs(LoggerLevel.INFO)) | |||
.contains("Add mapping [new_field] to Elasticsearch index [as]"); | |||
assertThat(underTest.getUpdatedIndices()).containsExactly("as"); | |||
} | |||
@Test | |||
public void shouldFailIfMoreThanOneIndexReturned() { | |||
String indexPattern = "*s"; | |||
Map<String, String> mappingOptions = ImmutableMap.of("norms", "false"); | |||
assertThatThrownBy(() -> underTest.addMappingToExistingIndex(indexPattern, "s", "new_field", "keyword", mappingOptions)) | |||
.isInstanceOf(IllegalStateException.class) | |||
.hasMessageContaining("Expected only one index to be found, actual"); | |||
} | |||
private Iterator<String> loadExistingIndices() { | |||
return es.client().getMapping(new GetMappingsRequest()).mappings().keySet().iterator(); | |||
} | |||
private Map<String, MappingMetadata> mappings() { | |||
return es.client().getMapping(new GetMappingsRequest()).mappings(); | |||
} | |||
@CheckForNull | |||
@SuppressWarnings("unchecked") | |||
private Map<String, Object> field(MappingMetadata mapping, String field) { | |||
Map<String, Object> props = (Map<String, Object>) mapping.getSourceAsMap().get("properties"); | |||
return (Map<String, Object>) props.get(field); | |||
} | |||
private int countMappingFields(MappingMetadata mapping) { | |||
return ((Map) mapping.getSourceAsMap().get("properties")).size(); | |||
} | |||
private static class SimpleIndexDefinition implements IndexDefinition { | |||
private final String indexName; | |||
public SimpleIndexDefinition(String indexName) { | |||
this.indexName = indexName; | |||
} | |||
@Override | |||
public void define(IndexDefinitionContext context) { | |||
IndexType.IndexMainType mainType = IndexType.main(Index.simple(indexName), indexName.substring(1)); | |||
context.create( | |||
mainType.getIndex(), | |||
newBuilder(new MapSettings().asConfig()).build()) | |||
.createTypeMapping(mainType); | |||
} | |||
} | |||
} |
@@ -37,14 +37,12 @@ public class DatabaseServerCompatibilityTest { | |||
@Rule | |||
public LogTester logTester = new LogTester(); | |||
private MapSettings settings = new MapSettings(); | |||
@Test | |||
public void fail_if_requires_downgrade() { | |||
DatabaseVersion version = mock(DatabaseVersion.class); | |||
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_DOWNGRADE); | |||
var config = settings.asConfig(); | |||
var compatibility = new DatabaseServerCompatibility(version, config); | |||
var compatibility = new DatabaseServerCompatibility(version); | |||
assertThatThrownBy(compatibility::start) | |||
.isInstanceOf(MessageException.class) | |||
.hasMessage("Database was upgraded to a more recent version of SonarQube. " | |||
@@ -56,8 +54,7 @@ public class DatabaseServerCompatibilityTest { | |||
DatabaseVersion version = mock(DatabaseVersion.class); | |||
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_UPGRADE); | |||
when(version.getVersion()).thenReturn(Optional.of(12L)); | |||
var config = settings.asConfig(); | |||
var compatibility = new DatabaseServerCompatibility(version, config); | |||
var compatibility = new DatabaseServerCompatibility(version); | |||
assertThatThrownBy(compatibility::start) | |||
.isInstanceOf(MessageException.class) | |||
.hasMessage("The version of SonarQube is too old. Please upgrade to the Long Term Support version first."); | |||
@@ -68,7 +65,7 @@ public class DatabaseServerCompatibilityTest { | |||
DatabaseVersion version = mock(DatabaseVersion.class); | |||
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_UPGRADE); | |||
when(version.getVersion()).thenReturn(Optional.of(DatabaseVersion.MIN_UPGRADE_VERSION)); | |||
new DatabaseServerCompatibility(version, settings.asConfig()).start(); | |||
new DatabaseServerCompatibility(version).start(); | |||
assertThat(logTester.logs()).hasSize(4); | |||
assertThat(logTester.logs(LoggerLevel.WARN)).contains( | |||
@@ -84,19 +81,7 @@ public class DatabaseServerCompatibilityTest { | |||
public void do_nothing_if_up_to_date() { | |||
DatabaseVersion version = mock(DatabaseVersion.class); | |||
when(version.getStatus()).thenReturn(DatabaseVersion.Status.UP_TO_DATE); | |||
new DatabaseServerCompatibility(version, settings.asConfig()).start(); | |||
new DatabaseServerCompatibility(version).start(); | |||
// no error | |||
} | |||
@Test | |||
public void upgrade_automatically_if_blue_green_deployment() { | |||
settings.setProperty("sonar.blueGreenEnabled", "true"); | |||
DatabaseVersion version = mock(DatabaseVersion.class); | |||
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_UPGRADE); | |||
when(version.getVersion()).thenReturn(Optional.of(DatabaseVersion.MIN_UPGRADE_VERSION)); | |||
new DatabaseServerCompatibility(version, settings.asConfig()).start(); | |||
assertThat(logTester.logs(LoggerLevel.WARN)).isEmpty(); | |||
} | |||
} |
@@ -77,19 +77,6 @@ public class DefaultServerUpgradeStatusTest { | |||
assertThat(underTest.getInitialDbVersion()).isEqualTo((int) LAST_VERSION); | |||
} | |||
@Test | |||
public void isBlueGreen() { | |||
settings.clear(); | |||
assertThat(underTest.isBlueGreen()).isFalse(); | |||
settings.setProperty("sonar.blueGreenEnabled", true); | |||
assertThat(underTest.isBlueGreen()).isTrue(); | |||
settings.setProperty("sonar.blueGreenEnabled", false); | |||
assertThat(underTest.isBlueGreen()).isFalse(); | |||
} | |||
@Test | |||
public void isAutoDbUpgrade() { | |||
settings.clear(); |
@@ -88,30 +88,6 @@ public class AutoDbMigrationTest { | |||
assertThat(logTester.logs(LoggerLevel.INFO)).isEmpty(); | |||
} | |||
@Test | |||
public void start_runs_MigrationEngine_if_blue_green_upgrade() { | |||
mockFreshInstall(false); | |||
when(serverUpgradeStatus.isUpgraded()).thenReturn(true); | |||
when(serverUpgradeStatus.isBlueGreen()).thenReturn(true); | |||
underTest.start(); | |||
verify(migrationEngine).execute(); | |||
assertThat(logTester.logs(LoggerLevel.INFO)).contains("Automatically perform DB migration on blue/green deployment"); | |||
} | |||
@Test | |||
public void start_does_nothing_if_blue_green_but_no_upgrade() { | |||
mockFreshInstall(false); | |||
when(serverUpgradeStatus.isUpgraded()).thenReturn(false); | |||
when(serverUpgradeStatus.isBlueGreen()).thenReturn(true); | |||
underTest.start(); | |||
verifyNoInteractions(migrationEngine); | |||
assertThat(logTester.logs(LoggerLevel.INFO)).isEmpty(); | |||
} | |||
@Test | |||
public void start_runs_MigrationEngine_if_autoDbMigration_enabled() { | |||
mockFreshInstall(false); |
@@ -36,17 +36,14 @@ import org.elasticsearch.client.indices.PutMappingRequest; | |||
import org.elasticsearch.cluster.health.ClusterHealthStatus; | |||
import org.elasticsearch.common.settings.Settings; | |||
import org.sonar.api.Startable; | |||
import org.sonar.api.config.Configuration; | |||
import org.sonar.api.server.ServerSide; | |||
import org.sonar.api.utils.log.Logger; | |||
import org.sonar.api.utils.log.Loggers; | |||
import org.sonar.process.ProcessProperties; | |||
import org.sonar.server.es.metadata.EsDbCompatibility; | |||
import org.sonar.server.es.metadata.MetadataIndex; | |||
import org.sonar.server.es.metadata.MetadataIndexDefinition; | |||
import org.sonar.server.es.newindex.BuiltIndex; | |||
import org.sonar.server.es.newindex.NewIndex; | |||
import org.sonar.server.platform.db.migration.es.MigrationEsClient; | |||
import static org.sonar.server.es.metadata.MetadataIndexDefinition.DESCRIPTOR; | |||
import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA; | |||
@@ -62,20 +59,16 @@ public class IndexCreator implements Startable { | |||
private final MetadataIndexDefinition metadataIndexDefinition; | |||
private final MetadataIndex metadataIndex; | |||
private final EsClient client; | |||
private final MigrationEsClient migrationEsClient; | |||
private final IndexDefinitions definitions; | |||
private final EsDbCompatibility esDbCompatibility; | |||
private final Configuration configuration; | |||
public IndexCreator(EsClient client, IndexDefinitions definitions, MetadataIndexDefinition metadataIndexDefinition, | |||
MetadataIndex metadataIndex, MigrationEsClient migrationEsClient, EsDbCompatibility esDbCompatibility, Configuration configuration) { | |||
MetadataIndex metadataIndex, EsDbCompatibility esDbCompatibility) { | |||
this.client = client; | |||
this.definitions = definitions; | |||
this.metadataIndexDefinition = metadataIndexDefinition; | |||
this.metadataIndex = metadataIndex; | |||
this.migrationEsClient = migrationEsClient; | |||
this.esDbCompatibility = esDbCompatibility; | |||
this.configuration = configuration; | |||
} | |||
@Override | |||
@@ -168,23 +161,11 @@ public class IndexCreator implements Startable { | |||
} | |||
private void updateIndex(BuiltIndex<?> index) { | |||
boolean blueGreen = configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false); | |||
String indexName = index.getMainType().getIndex().getName(); | |||
if (blueGreen) { | |||
// SonarCloud | |||
if (migrationEsClient.getUpdatedIndices().contains(indexName)) { | |||
LOGGER.info("Resetting definition hash of Elasticsearch index [{}]", indexName); | |||
metadataIndex.setHash(index.getMainType().getIndex(), IndexDefinitionHash.of(index)); | |||
} else { | |||
throw new IllegalStateException("Blue/green deployment is not supported. Elasticsearch index [" + indexName + "] changed and needs to be dropped."); | |||
} | |||
} else { | |||
// SonarQube | |||
LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName); | |||
deleteIndex(indexName); | |||
createIndex(index, true); | |||
} | |||
LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName); | |||
deleteIndex(indexName); | |||
createIndex(index, true); | |||
} | |||
private boolean hasDefinitionChange(BuiltIndex<?> index) { |
@@ -19,7 +19,6 @@ | |||
*/ | |||
package org.sonar.server.es; | |||
import com.google.common.collect.Sets; | |||
import java.util.Map; | |||
import java.util.function.Consumer; | |||
import javax.annotation.CheckForNull; | |||
@@ -42,12 +41,8 @@ import org.sonar.server.es.metadata.MetadataIndexDefinition; | |||
import org.sonar.server.es.metadata.MetadataIndexImpl; | |||
import org.sonar.server.es.newindex.NewRegularIndex; | |||
import org.sonar.server.es.newindex.SettingsConfiguration; | |||
import org.sonar.server.platform.db.migration.es.MigrationEsClient; | |||
import static org.assertj.core.api.Assertions.assertThat; | |||
import static org.assertj.core.api.Assertions.assertThatThrownBy; | |||
import static org.mockito.Mockito.mock; | |||
import static org.mockito.Mockito.when; | |||
import static org.sonar.server.es.IndexType.main; | |||
import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder; | |||
@@ -66,7 +61,6 @@ public class IndexCreatorTest { | |||
private final MetadataIndex metadataIndex = new MetadataIndexImpl(es.client()); | |||
private final TestEsDbCompatibility esDbCompatibility = new TestEsDbCompatibility(); | |||
private final MapSettings settings = new MapSettings(); | |||
private final MigrationEsClient migrationEsClient = mock(MigrationEsClient.class); | |||
@Test | |||
public void create_index() { | |||
@@ -80,15 +74,6 @@ public class IndexCreatorTest { | |||
assertThat(mappings()).isNotEmpty(); | |||
} | |||
@Test | |||
public void creation_of_new_index_is_supported_in_blue_green_deployment() { | |||
enableBlueGreenDeployment(); | |||
run(new FakeIndexDefinition()); | |||
verifyFakeIndexV1(); | |||
} | |||
@Test | |||
public void recreate_index_on_definition_changes() { | |||
// v1 | |||
@@ -112,34 +97,6 @@ public class IndexCreatorTest { | |||
assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName()).id(id)).isExists()).isFalse(); | |||
} | |||
@Test | |||
public void fail_to_recreate_index_on_definition_changes_if_blue_green_deployment() { | |||
enableBlueGreenDeployment(); | |||
// v1 | |||
run(new FakeIndexDefinition()); | |||
// v2 | |||
FakeIndexDefinitionV2 definitionV2 = new FakeIndexDefinitionV2(); | |||
assertThatThrownBy(() -> run(definitionV2)) | |||
.isInstanceOf(IllegalStateException.class) | |||
.hasMessageContaining("Blue/green deployment is not supported. Elasticsearch index [fakes] changed and needs to be dropped."); | |||
} | |||
@Test | |||
public void reset_definition_hash_if_change_applied_during_migration_of_blue_green_deployment() { | |||
enableBlueGreenDeployment(); | |||
// v1 | |||
run(new FakeIndexDefinition()); | |||
// v2 | |||
when(migrationEsClient.getUpdatedIndices()).thenReturn(Sets.newHashSet(FakeIndexDefinition.INDEX_TYPE.getIndex().getName())); | |||
run(new FakeIndexDefinitionV2()); | |||
// index has not been dropped-and-recreated | |||
verifyFakeIndexV1(); | |||
} | |||
@Test | |||
public void mark_all_non_existing_index_types_as_uninitialized() { | |||
Index fakesIndex = Index.simple("fakes"); | |||
@@ -218,10 +175,6 @@ public class IndexCreatorTest { | |||
es.client().putSettings(new UpdateSettingsRequest().indices(mainType.getIndex().getName()).settings(builder.build())); | |||
} | |||
private void enableBlueGreenDeployment() { | |||
settings.setProperty("sonar.blueGreenEnabled", "true"); | |||
} | |||
private void testDeleteOnDbChange(String expectedLog, Consumer<TestEsDbCompatibility> afterFirstStart) { | |||
run(new FakeIndexDefinition()); | |||
assertThat(logTester.logs(LoggerLevel.INFO)) | |||
@@ -262,7 +215,7 @@ public class IndexCreatorTest { | |||
private IndexCreator run(IndexDefinition... definitions) { | |||
IndexDefinitions defs = new IndexDefinitions(definitions, new MapSettings().asConfig()); | |||
defs.start(); | |||
IndexCreator creator = new IndexCreator(es.client(), defs, metadataIndexDefinition, metadataIndex, migrationEsClient, esDbCompatibility, settings.asConfig()); | |||
IndexCreator creator = new IndexCreator(es.client(), defs, metadataIndexDefinition, metadataIndex, esDbCompatibility); | |||
creator.start(); | |||
return creator; | |||
} |
@@ -29,7 +29,6 @@ import org.sonar.api.utils.log.Loggers; | |||
import org.sonar.ce.queue.CeQueue; | |||
import org.sonar.db.DbClient; | |||
import org.sonar.db.DbSession; | |||
import org.sonar.process.ProcessProperties; | |||
/** | |||
* Cleans-up the Compute Engine queue. | |||
@@ -42,27 +41,21 @@ public class CeQueueCleaner implements Startable { | |||
private final DbClient dbClient; | |||
private final ServerUpgradeStatus serverUpgradeStatus; | |||
private final CeQueue queue; | |||
private final Configuration configuration; | |||
public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, CeQueue queue, Configuration configuration) { | |||
public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, CeQueue queue) { | |||
this.dbClient = dbClient; | |||
this.serverUpgradeStatus = serverUpgradeStatus; | |||
this.queue = queue; | |||
this.configuration = configuration; | |||
} | |||
@Override | |||
public void start() { | |||
if (serverUpgradeStatus.isUpgraded() && !isBlueGreenDeployment()) { | |||
if (serverUpgradeStatus.isUpgraded()) { | |||
cleanOnUpgrade(); | |||
} | |||
cleanUpTaskInputOrphans(); | |||
} | |||
private boolean isBlueGreenDeployment() { | |||
return configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false); | |||
} | |||
private void cleanOnUpgrade() { | |||
// we assume that pending tasks are not compatible with the new version | |||
// and can't be processed |
@@ -69,16 +69,6 @@ public class CeQueueCleanerTest { | |||
verify(queue).clear(); | |||
} | |||
@Test | |||
public void start_does_not_clear_queue_if_version_upgrade_but_blue_green_deployment() { | |||
when(serverUpgradeStatus.isUpgraded()).thenReturn(true); | |||
settings.setProperty(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey(), true); | |||
runCleaner(); | |||
verify(queue, never()).clear(); | |||
} | |||
@Test | |||
public void start_deletes_orphan_report_files() { | |||
// analysis reports are persisted but the associated | |||
@@ -114,7 +104,7 @@ public class CeQueueCleanerTest { | |||
} | |||
private void runCleaner() { | |||
CeQueueCleaner cleaner = new CeQueueCleaner(dbTester.getDbClient(), serverUpgradeStatus, queue, settings.asConfig()); | |||
CeQueueCleaner cleaner = new CeQueueCleaner(dbTester.getDbClient(), serverUpgradeStatus, queue); | |||
cleaner.start(); | |||
} | |||
} |
@@ -24,7 +24,6 @@ import org.sonar.core.extension.CoreExtensionsInstaller; | |||
import org.sonar.core.platform.PluginClassLoader; | |||
import org.sonar.core.platform.PluginClassloaderFactory; | |||
import org.sonar.core.platform.SpringComponentContainer; | |||
import org.sonar.server.es.MigrationEsClientImpl; | |||
import org.sonar.server.l18n.ServerI18n; | |||
import org.sonar.server.platform.DatabaseServerCompatibility; | |||
import org.sonar.server.platform.DefaultServerUpgradeStatus; | |||
@@ -59,7 +58,6 @@ public class PlatformLevel2 extends PlatformLevel { | |||
new MigrationConfigurationModule(), | |||
DatabaseVersion.class, | |||
DatabaseServerCompatibility.class, | |||
MigrationEsClientImpl.class, | |||
new StartupMetadataProvider(), | |||
DefaultServerUpgradeStatus.class, |
@@ -73,7 +73,7 @@ public class PlatformLevel2Test { | |||
verify(container).add(ServerPluginRepository.class); | |||
verify(container).add(DatabaseCharsetChecker.class); | |||
verify(container, times(22)).add(any()); | |||
verify(container, times(21)).add(any()); | |||
} | |||
@Test | |||
@@ -94,7 +94,7 @@ public class PlatformLevel2Test { | |||
verify(container).add(ServerPluginRepository.class); | |||
verify(container, never()).add(DatabaseCharsetChecker.class); | |||
verify(container, times(20)).add(any()); | |||
verify(container, times(19)).add(any()); | |||
} | |||