Browse Source

[NO-JIRA] Drop blue green deployment flag

tags/10.0.0.68432
Jacek Poreda 1 year ago
parent
commit
e5717b5f63
20 changed files with 25 additions and 544 deletions
  1. 0
    24
      server/sonar-db-dao/src/testFixtures/java/org/sonar/db/SQDatabase.java
  2. 1
    22
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/engine/MigrationEngineImpl.java
  3. 0
    47
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java
  4. 0
    24
      server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java
  5. 1
    37
      server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/engine/MigrationEngineImplTest.java
  6. 0
    3
      server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java
  7. 0
    77
      server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java
  8. 9
    15
      server/sonar-webserver-core/src/main/java/org/sonar/server/platform/DatabaseServerCompatibility.java
  9. 0
    4
      server/sonar-webserver-core/src/main/java/org/sonar/server/platform/DefaultServerUpgradeStatus.java
  10. 0
    3
      server/sonar-webserver-core/src/main/java/org/sonar/server/platform/db/migration/AutoDbMigration.java
  11. 0
    137
      server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java
  12. 4
    19
      server/sonar-webserver-core/src/test/java/org/sonar/server/platform/DatabaseServerCompatibilityTest.java
  13. 0
    13
      server/sonar-webserver-core/src/test/java/org/sonar/server/platform/DefaultServerUpgradeStatusTest.java
  14. 0
    24
      server/sonar-webserver-core/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationTest.java
  15. 4
    23
      server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java
  16. 1
    48
      server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java
  17. 2
    9
      server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/queue/CeQueueCleaner.java
  18. 1
    11
      server/sonar-webserver-webapi/src/test/java/org/sonar/server/ce/queue/CeQueueCleanerTest.java
  19. 0
    2
      server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java
  20. 2
    2
      server/sonar-webserver/src/test/java/org/sonar/server/platform/platformlevel/PlatformLevel2Test.java

+ 0
- 24
server/sonar-db-dao/src/testFixtures/java/org/sonar/db/SQDatabase.java View File

@@ -22,10 +22,7 @@ package org.sonar.db;
import java.io.PrintWriter;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.sql.DataSource;
import org.apache.commons.io.output.NullWriter;
import org.apache.ibatis.io.Resources;
@@ -45,7 +42,6 @@ import org.sonar.process.logging.LogbackHelper;
import org.sonar.server.platform.db.migration.MigrationConfigurationModule;
import org.sonar.server.platform.db.migration.engine.MigrationContainer;
import org.sonar.server.platform.db.migration.engine.MigrationContainerImpl;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;
import org.sonar.server.platform.db.migration.history.MigrationHistoryTableImpl;
import org.sonar.server.platform.db.migration.step.MigrationStep;
import org.sonar.server.platform.db.migration.step.MigrationStepExecutionException;
@@ -153,7 +149,6 @@ public class SQDatabase extends DefaultDatabase {
container.add(UuidFactoryFast.getInstance());
container.add(System2.INSTANCE);
container.add(MapSettings.class);
container.add(createMockMigrationEsClient());

container.startComponents();
MigrationContainer migrationContainer = new MigrationContainerImpl(container, H2StepExecutor.class);
@@ -163,25 +158,6 @@ public class SQDatabase extends DefaultDatabase {
executor.execute(migrationSteps.readAll());
}

private static MigrationEsClient createMockMigrationEsClient() {
return new MigrationEsClient() {
@Override
public void deleteIndexes(String name, String... otherNames) {
//No ES operation required for database tests
}

@Override
public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options) {
//No ES operation required for database tests
}

@Override
public Set<String> getUpdatedIndices() {
return Collections.emptySet();
}
};
}

private void createMigrationHistoryTable(NoopDatabase noopDatabase) {
new MigrationHistoryTableImpl(noopDatabase).start();
}

+ 1
- 22
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/engine/MigrationEngineImpl.java View File

@@ -21,36 +21,27 @@ package org.sonar.server.platform.db.migration.engine;

import java.util.List;
import java.util.Optional;
import org.sonar.api.config.Configuration;
import org.sonar.api.utils.AnnotationUtils;
import org.sonar.core.platform.SpringComponentContainer;
import org.sonar.process.ProcessProperties;
import org.sonar.server.platform.db.migration.SupportsBlueGreen;
import org.sonar.server.platform.db.migration.history.MigrationHistory;
import org.sonar.server.platform.db.migration.step.MigrationSteps;
import org.sonar.server.platform.db.migration.step.MigrationStepsExecutor;
import org.sonar.server.platform.db.migration.step.MigrationStepsExecutorImpl;
import org.sonar.server.platform.db.migration.step.RegisteredMigrationStep;

import static java.lang.String.format;

public class MigrationEngineImpl implements MigrationEngine {
private final MigrationHistory migrationHistory;
private final SpringComponentContainer serverContainer;
private final MigrationSteps migrationSteps;
private final Configuration configuration;

public MigrationEngineImpl(MigrationHistory migrationHistory, SpringComponentContainer serverContainer, MigrationSteps migrationSteps, Configuration configuration) {
public MigrationEngineImpl(MigrationHistory migrationHistory, SpringComponentContainer serverContainer, MigrationSteps migrationSteps) {
this.migrationHistory = migrationHistory;
this.serverContainer = serverContainer;
this.migrationSteps = migrationSteps;
this.configuration = configuration;
}

@Override
public void execute() {
MigrationContainer migrationContainer = new MigrationContainerImpl(serverContainer, MigrationStepsExecutorImpl.class);
boolean blueGreen = configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false);
try {
MigrationStepsExecutor stepsExecutor = migrationContainer.getComponentByType(MigrationStepsExecutor.class);
Optional<Long> lastMigrationNumber = migrationHistory.getLastMigrationNumber();
@@ -59,9 +50,6 @@ public class MigrationEngineImpl implements MigrationEngine {
.map(i -> migrationSteps.readFrom(i + 1))
.orElse(migrationSteps.readAll());

if (blueGreen) {
ensureSupportBlueGreen(steps);
}

stepsExecutor.execute(steps);

@@ -69,13 +57,4 @@ public class MigrationEngineImpl implements MigrationEngine {
migrationContainer.cleanup();
}
}

private static void ensureSupportBlueGreen(List<RegisteredMigrationStep> steps) {
for (RegisteredMigrationStep step : steps) {
if (AnnotationUtils.getAnnotation(step.getStepClass(), SupportsBlueGreen.class) == null) {
throw new IllegalStateException(format("All migrations canceled. #%d does not support blue/green deployment: %s",
step.getMigrationNumber(), step.getDescription()));
}
}
}
}

+ 0
- 47
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/MigrationEsClient.java View File

@@ -1,47 +0,0 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.platform.db.migration.es;

import java.util.Map;
import java.util.Set;

public interface MigrationEsClient {

/**
* This method is re-entrant and does not fail if indexName or otherIndexNames do not exist
*/
void deleteIndexes(String name, String... otherNames);

/**
* Adds a new mapping to an existing Elasticsearch index. Does nothing if index does not exist.
*
* @param index name of the index that the mapping is added to
* @param type document type in the index
* @param mappingName name of the new mapping
* @param mappingType type of the new mapping
* @param options additional options to be applied to the mapping
*/
void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options);

/**
* Returns the indices that have been touched by {@link #addMappingToExistingIndex(String, String, String, String, Map)}
*/
Set<String> getUpdatedIndices();
}

+ 0
- 24
server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/es/package-info.java View File

@@ -1,24 +0,0 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
@ParametersAreNonnullByDefault
package org.sonar.server.platform.db.migration.es;

import javax.annotation.ParametersAreNonnullByDefault;


+ 1
- 37
server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/engine/MigrationEngineImplTest.java View File

@@ -24,10 +24,8 @@ import java.util.List;
import java.util.Optional;
import org.junit.Before;
import org.junit.Test;
import org.sonar.api.config.internal.ConfigurationBridge;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.core.platform.SpringComponentContainer;
import org.sonar.process.ProcessProperties;
import org.sonar.server.platform.db.migration.SupportsBlueGreen;
import org.sonar.server.platform.db.migration.history.MigrationHistory;
import org.sonar.server.platform.db.migration.step.MigrationStep;
@@ -35,10 +33,8 @@ import org.sonar.server.platform.db.migration.step.MigrationSteps;
import org.sonar.server.platform.db.migration.step.MigrationStepsExecutor;
import org.sonar.server.platform.db.migration.step.RegisteredMigrationStep;

import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -50,7 +46,7 @@ public class MigrationEngineImplTest {
private final MigrationSteps migrationSteps = mock(MigrationSteps.class);
private final StepRegistry stepRegistry = new StepRegistry();
private final MapSettings settings = new MapSettings();
private final MigrationEngineImpl underTest = new MigrationEngineImpl(migrationHistory, serverContainer, migrationSteps, new ConfigurationBridge(settings));
private final MigrationEngineImpl underTest = new MigrationEngineImpl(migrationHistory, serverContainer, migrationSteps);

@Before
public void before() {
@@ -85,38 +81,6 @@ public class MigrationEngineImplTest {
assertThat(stepRegistry.stepRan).isTrue();
}

@Test
public void execute_steps_in_blue_green_mode() {
settings.setProperty(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey(), true);
when(migrationHistory.getLastMigrationNumber()).thenReturn(Optional.of(50L));
List<RegisteredMigrationStep> steps = singletonList(new RegisteredMigrationStep(1, "doo", TestBlueGreenMigrationStep.class));
when(migrationSteps.readFrom(51)).thenReturn(steps);
when(migrationSteps.readAll()).thenReturn(steps);

underTest.execute();

verify(migrationSteps).readFrom(51);
assertThat(stepRegistry.stepRan).isTrue();
}

@Test
public void fail_blue_green_execution_if_some_migrations_are_not_compatible() {
settings.setProperty(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey(), true);
when(migrationHistory.getLastMigrationNumber()).thenReturn(Optional.of(50L));
List<RegisteredMigrationStep> steps = asList(
new RegisteredMigrationStep(1, "foo", TestBlueGreenMigrationStep.class),
new RegisteredMigrationStep(2, "bar", TestMigrationStep.class));
when(migrationSteps.readFrom(51)).thenReturn(steps);

try {
underTest.execute();
fail();
} catch (IllegalStateException e) {
assertThat(e).hasMessage("All migrations canceled. #2 does not support blue/green deployment: bar");
assertThat(stepRegistry.stepRan).isFalse();
}
}

private static class NoOpExecutor implements MigrationStepsExecutor {
@Override
public void execute(List<RegisteredMigrationStep> steps) {

+ 0
- 3
server/sonar-process/src/main/java/org/sonar/process/ProcessProperties.java View File

@@ -182,9 +182,6 @@ public class ProcessProperties {
*/
ENABLE_STOP_COMMAND("sonar.enableStopCommand"),

// whether the blue/green deployment of server is enabled
BLUE_GREEN_ENABLED("sonar.blueGreenEnabled", DEFAULT_FALSE),

AUTO_DATABASE_UPGRADE("sonar.autoDatabaseUpgrade", DEFAULT_FALSE);

/**

+ 0
- 77
server/sonar-webserver-core/src/main/java/org/sonar/server/es/MigrationEsClientImpl.java View File

@@ -1,77 +0,0 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.es;

import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.stream.Stream;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.sonar.api.utils.log.Loggers;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;

public class MigrationEsClientImpl implements MigrationEsClient {
private final EsClient client;
private final Set<String> updatedIndices = new HashSet<>();

public MigrationEsClientImpl(EsClient client) {
this.client = client;
}

@Override
public void deleteIndexes(String name, String... otherNames) {
String[] indices = client.getIndex(new GetIndexRequest("_all")).getIndices();
Set<String> existingIndices = Arrays.stream(indices).collect(MoreCollectors.toSet());
Stream.concat(Stream.of(name), Arrays.stream(otherNames))
.distinct()
.filter(existingIndices::contains)
.forEach(this::deleteIndex);
}

@Override
public void addMappingToExistingIndex(String index, String type, String mappingName, String mappingType, Map<String, String> options) {
//TODO:: remove?
// String[] indices = client.getIndex(new GetIndexRequest(index)).getIndices();
// if (indices != null && indices.length == 1) {
// Loggers.get(getClass()).info("Add mapping [{}] to Elasticsearch index [{}]", mappingName, index);
// String mappingOptions = Stream.concat(Stream.of(Maps.immutableEntry("type", mappingType)), options.entrySet().stream())
// .map(e -> e.getKey() + "=" + e.getValue())
// .collect(Collectors.joining(","));
// client.putMapping(new PutMappingRequest(index)
// .source(mappingName, mappingOptions));
// updatedIndices.add(index);
// } else {
// throw new IllegalStateException("Expected only one index to be found, actual [" + String.join(",", indices) + "]");
// }
}

@Override
public Set<String> getUpdatedIndices() {
return updatedIndices;
}

private void deleteIndex(String index) {
Loggers.get(getClass()).info("Drop Elasticsearch index [{}]", index);
client.deleteIndex(new DeleteIndexRequest(index));
}
}

+ 9
- 15
server/sonar-webserver-core/src/main/java/org/sonar/server/platform/DatabaseServerCompatibility.java View File

@@ -21,11 +21,9 @@ package org.sonar.server.platform;

import java.util.Optional;
import org.sonar.api.Startable;
import org.sonar.api.config.Configuration;
import org.sonar.api.utils.MessageException;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.process.ProcessProperties;
import org.sonar.server.platform.db.migration.version.DatabaseVersion;

import static org.sonar.server.log.ServerProcessLogging.STARTUP_LOGGER_NAME;
@@ -35,11 +33,9 @@ public class DatabaseServerCompatibility implements Startable {
private static final String HIGHLIGHTER = "################################################################################";

private final DatabaseVersion version;
private final Configuration configuration;

public DatabaseServerCompatibility(DatabaseVersion version, Configuration configuration) {
public DatabaseServerCompatibility(DatabaseVersion version) {
this.version = version;
this.configuration = configuration;
}

@Override
@@ -54,16 +50,14 @@ public class DatabaseServerCompatibility implements Startable {
if (currentVersion.isPresent() && currentVersion.get() < DatabaseVersion.MIN_UPGRADE_VERSION) {
throw MessageException.of("The version of SonarQube is too old. Please upgrade to the Long Term Support version first.");
}
boolean blueGreen = configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false);
if (!blueGreen) {
String msg = "The database must be manually upgraded. Please backup the database and browse /setup. "
+ "For more information: https://docs.sonarqube.org/latest/setup/upgrading";
Loggers.get(DatabaseServerCompatibility.class).warn(msg);
Logger logger = Loggers.get(STARTUP_LOGGER_NAME);
logger.warn(HIGHLIGHTER);
logger.warn(msg);
logger.warn(HIGHLIGHTER);
}

String msg = "The database must be manually upgraded. Please backup the database and browse /setup. "
+ "For more information: https://docs.sonarqube.org/latest/setup/upgrading";
Loggers.get(DatabaseServerCompatibility.class).warn(msg);
Logger logger = Loggers.get(STARTUP_LOGGER_NAME);
logger.warn(HIGHLIGHTER);
logger.warn(msg);
logger.warn(HIGHLIGHTER);
}
}


+ 0
- 4
server/sonar-webserver-core/src/main/java/org/sonar/server/platform/DefaultServerUpgradeStatus.java View File

@@ -70,10 +70,6 @@ public class DefaultServerUpgradeStatus implements ServerUpgradeStatus, Startabl
return (int) initialDbVersion;
}

public boolean isBlueGreen() {
return configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false);
}

public boolean isAutoDbUpgrade() {
return configuration.getBoolean(ProcessProperties.Property.AUTO_DATABASE_UPGRADE.getKey()).orElse(false);
}

+ 0
- 3
server/sonar-webserver-core/src/main/java/org/sonar/server/platform/db/migration/AutoDbMigration.java View File

@@ -41,9 +41,6 @@ public class AutoDbMigration implements Startable {
} else if (serverUpgradeStatus.isUpgraded() && serverUpgradeStatus.isAutoDbUpgrade()) {
Loggers.get(getClass()).info("Automatically perform DB migration, as automatic database upgrade is enabled");
migrationEngine.execute();
} else if (serverUpgradeStatus.isUpgraded() && serverUpgradeStatus.isBlueGreen()) {
Loggers.get(getClass()).info("Automatically perform DB migration on blue/green deployment");
migrationEngine.execute();
}
}


+ 0
- 137
server/sonar-webserver-core/src/test/java/org/sonar/server/es/MigrationEsClientImplTest.java View File

@@ -1,137 +0,0 @@
/*
* SonarQube
* Copyright (C) 2009-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package org.sonar.server.es;

import com.google.common.collect.ImmutableMap;
import java.util.Iterator;
import java.util.Map;
import javax.annotation.CheckForNull;
import org.elasticsearch.client.indices.GetMappingsRequest;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.log.LogTester;
import org.sonar.api.utils.log.LoggerLevel;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;

@Ignore
public class MigrationEsClientImplTest {
@Rule
public LogTester logTester = new LogTester();
@Rule
public EsTester es = EsTester.createCustom(
new SimpleIndexDefinition("as"),
new SimpleIndexDefinition("bs"),
new SimpleIndexDefinition("cs"));

private MigrationEsClient underTest = new MigrationEsClientImpl(es.client());

@Test
public void delete_existing_index() {
underTest.deleteIndexes("as");

assertThat(loadExistingIndices())
.toIterable()
.doesNotContain("as")
.contains("bs", "cs");
assertThat(logTester.logs(LoggerLevel.INFO))
.contains("Drop Elasticsearch index [as]");
}

@Test
public void delete_index_that_does_not_exist() {
underTest.deleteIndexes("as", "xxx", "cs");

assertThat(loadExistingIndices())
.toIterable()
.doesNotContain("as", "cs")
.contains("bs");
assertThat(logTester.logs(LoggerLevel.INFO))
.contains("Drop Elasticsearch index [as]", "Drop Elasticsearch index [cs]")
.doesNotContain("Drop Elasticsearch index [xxx]");
}

@Test
public void addMappingToExistingIndex() {
Map<String, String> mappingOptions = ImmutableMap.of("norms", "false");
underTest.addMappingToExistingIndex("as", "s", "new_field", "keyword", mappingOptions);

assertThat(loadExistingIndices()).toIterable().contains("as");
Map<String, MappingMetadata> mappings = mappings();
MappingMetadata mapping = mappings.get("as");
assertThat(countMappingFields(mapping)).isOne();
assertThat(field(mapping, "new_field")).isNotNull();

assertThat(logTester.logs(LoggerLevel.INFO))
.contains("Add mapping [new_field] to Elasticsearch index [as]");
assertThat(underTest.getUpdatedIndices()).containsExactly("as");
}

@Test
public void shouldFailIfMoreThanOneIndexReturned() {
String indexPattern = "*s";
Map<String, String> mappingOptions = ImmutableMap.of("norms", "false");
assertThatThrownBy(() -> underTest.addMappingToExistingIndex(indexPattern, "s", "new_field", "keyword", mappingOptions))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("Expected only one index to be found, actual");
}

private Iterator<String> loadExistingIndices() {
return es.client().getMapping(new GetMappingsRequest()).mappings().keySet().iterator();
}

private Map<String, MappingMetadata> mappings() {
return es.client().getMapping(new GetMappingsRequest()).mappings();
}

@CheckForNull
@SuppressWarnings("unchecked")
private Map<String, Object> field(MappingMetadata mapping, String field) {
Map<String, Object> props = (Map<String, Object>) mapping.getSourceAsMap().get("properties");
return (Map<String, Object>) props.get(field);
}

private int countMappingFields(MappingMetadata mapping) {
return ((Map) mapping.getSourceAsMap().get("properties")).size();
}

private static class SimpleIndexDefinition implements IndexDefinition {
private final String indexName;

public SimpleIndexDefinition(String indexName) {
this.indexName = indexName;
}

@Override
public void define(IndexDefinitionContext context) {
IndexType.IndexMainType mainType = IndexType.main(Index.simple(indexName), indexName.substring(1));
context.create(
mainType.getIndex(),
newBuilder(new MapSettings().asConfig()).build())
.createTypeMapping(mainType);
}
}
}

+ 4
- 19
server/sonar-webserver-core/src/test/java/org/sonar/server/platform/DatabaseServerCompatibilityTest.java View File

@@ -37,14 +37,12 @@ public class DatabaseServerCompatibilityTest {

@Rule
public LogTester logTester = new LogTester();
private MapSettings settings = new MapSettings();

@Test
public void fail_if_requires_downgrade() {
DatabaseVersion version = mock(DatabaseVersion.class);
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_DOWNGRADE);
var config = settings.asConfig();
var compatibility = new DatabaseServerCompatibility(version, config);
var compatibility = new DatabaseServerCompatibility(version);
assertThatThrownBy(compatibility::start)
.isInstanceOf(MessageException.class)
.hasMessage("Database was upgraded to a more recent version of SonarQube. "
@@ -56,8 +54,7 @@ public class DatabaseServerCompatibilityTest {
DatabaseVersion version = mock(DatabaseVersion.class);
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_UPGRADE);
when(version.getVersion()).thenReturn(Optional.of(12L));
var config = settings.asConfig();
var compatibility = new DatabaseServerCompatibility(version, config);
var compatibility = new DatabaseServerCompatibility(version);
assertThatThrownBy(compatibility::start)
.isInstanceOf(MessageException.class)
.hasMessage("The version of SonarQube is too old. Please upgrade to the Long Term Support version first.");
@@ -68,7 +65,7 @@ public class DatabaseServerCompatibilityTest {
DatabaseVersion version = mock(DatabaseVersion.class);
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_UPGRADE);
when(version.getVersion()).thenReturn(Optional.of(DatabaseVersion.MIN_UPGRADE_VERSION));
new DatabaseServerCompatibility(version, settings.asConfig()).start();
new DatabaseServerCompatibility(version).start();

assertThat(logTester.logs()).hasSize(4);
assertThat(logTester.logs(LoggerLevel.WARN)).contains(
@@ -84,19 +81,7 @@ public class DatabaseServerCompatibilityTest {
public void do_nothing_if_up_to_date() {
DatabaseVersion version = mock(DatabaseVersion.class);
when(version.getStatus()).thenReturn(DatabaseVersion.Status.UP_TO_DATE);
new DatabaseServerCompatibility(version, settings.asConfig()).start();
new DatabaseServerCompatibility(version).start();
// no error
}

@Test
public void upgrade_automatically_if_blue_green_deployment() {
settings.setProperty("sonar.blueGreenEnabled", "true");
DatabaseVersion version = mock(DatabaseVersion.class);
when(version.getStatus()).thenReturn(DatabaseVersion.Status.REQUIRES_UPGRADE);
when(version.getVersion()).thenReturn(Optional.of(DatabaseVersion.MIN_UPGRADE_VERSION));

new DatabaseServerCompatibility(version, settings.asConfig()).start();

assertThat(logTester.logs(LoggerLevel.WARN)).isEmpty();
}
}

+ 0
- 13
server/sonar-webserver-core/src/test/java/org/sonar/server/platform/DefaultServerUpgradeStatusTest.java View File

@@ -77,19 +77,6 @@ public class DefaultServerUpgradeStatusTest {
assertThat(underTest.getInitialDbVersion()).isEqualTo((int) LAST_VERSION);
}

@Test
public void isBlueGreen() {
settings.clear();
assertThat(underTest.isBlueGreen()).isFalse();

settings.setProperty("sonar.blueGreenEnabled", true);
assertThat(underTest.isBlueGreen()).isTrue();

settings.setProperty("sonar.blueGreenEnabled", false);
assertThat(underTest.isBlueGreen()).isFalse();
}


@Test
public void isAutoDbUpgrade() {
settings.clear();

+ 0
- 24
server/sonar-webserver-core/src/test/java/org/sonar/server/platform/db/migration/AutoDbMigrationTest.java View File

@@ -88,30 +88,6 @@ public class AutoDbMigrationTest {
assertThat(logTester.logs(LoggerLevel.INFO)).isEmpty();
}

@Test
public void start_runs_MigrationEngine_if_blue_green_upgrade() {
mockFreshInstall(false);
when(serverUpgradeStatus.isUpgraded()).thenReturn(true);
when(serverUpgradeStatus.isBlueGreen()).thenReturn(true);

underTest.start();

verify(migrationEngine).execute();
assertThat(logTester.logs(LoggerLevel.INFO)).contains("Automatically perform DB migration on blue/green deployment");
}

@Test
public void start_does_nothing_if_blue_green_but_no_upgrade() {
mockFreshInstall(false);
when(serverUpgradeStatus.isUpgraded()).thenReturn(false);
when(serverUpgradeStatus.isBlueGreen()).thenReturn(true);

underTest.start();

verifyNoInteractions(migrationEngine);
assertThat(logTester.logs(LoggerLevel.INFO)).isEmpty();
}

@Test
public void start_runs_MigrationEngine_if_autoDbMigration_enabled() {
mockFreshInstall(false);

+ 4
- 23
server/sonar-webserver-es/src/main/java/org/sonar/server/es/IndexCreator.java View File

@@ -36,17 +36,14 @@ import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.sonar.api.Startable;
import org.sonar.api.config.Configuration;
import org.sonar.api.server.ServerSide;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
import org.sonar.process.ProcessProperties;
import org.sonar.server.es.metadata.EsDbCompatibility;
import org.sonar.server.es.metadata.MetadataIndex;
import org.sonar.server.es.metadata.MetadataIndexDefinition;
import org.sonar.server.es.newindex.BuiltIndex;
import org.sonar.server.es.newindex.NewIndex;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;

import static org.sonar.server.es.metadata.MetadataIndexDefinition.DESCRIPTOR;
import static org.sonar.server.es.metadata.MetadataIndexDefinition.TYPE_METADATA;
@@ -62,20 +59,16 @@ public class IndexCreator implements Startable {
private final MetadataIndexDefinition metadataIndexDefinition;
private final MetadataIndex metadataIndex;
private final EsClient client;
private final MigrationEsClient migrationEsClient;
private final IndexDefinitions definitions;
private final EsDbCompatibility esDbCompatibility;
private final Configuration configuration;

public IndexCreator(EsClient client, IndexDefinitions definitions, MetadataIndexDefinition metadataIndexDefinition,
MetadataIndex metadataIndex, MigrationEsClient migrationEsClient, EsDbCompatibility esDbCompatibility, Configuration configuration) {
MetadataIndex metadataIndex, EsDbCompatibility esDbCompatibility) {
this.client = client;
this.definitions = definitions;
this.metadataIndexDefinition = metadataIndexDefinition;
this.metadataIndex = metadataIndex;
this.migrationEsClient = migrationEsClient;
this.esDbCompatibility = esDbCompatibility;
this.configuration = configuration;
}

@Override
@@ -168,23 +161,11 @@ public class IndexCreator implements Startable {
}

private void updateIndex(BuiltIndex<?> index) {
boolean blueGreen = configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false);
String indexName = index.getMainType().getIndex().getName();

if (blueGreen) {
// SonarCloud
if (migrationEsClient.getUpdatedIndices().contains(indexName)) {
LOGGER.info("Resetting definition hash of Elasticsearch index [{}]", indexName);
metadataIndex.setHash(index.getMainType().getIndex(), IndexDefinitionHash.of(index));
} else {
throw new IllegalStateException("Blue/green deployment is not supported. Elasticsearch index [" + indexName + "] changed and needs to be dropped.");
}
} else {
// SonarQube
LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName);
deleteIndex(indexName);
createIndex(index, true);
}
LOGGER.info("Delete Elasticsearch index {} (structure changed)", indexName);
deleteIndex(indexName);
createIndex(index, true);
}

private boolean hasDefinitionChange(BuiltIndex<?> index) {

+ 1
- 48
server/sonar-webserver-es/src/test/java/org/sonar/server/es/IndexCreatorTest.java View File

@@ -19,7 +19,6 @@
*/
package org.sonar.server.es;

import com.google.common.collect.Sets;
import java.util.Map;
import java.util.function.Consumer;
import javax.annotation.CheckForNull;
@@ -42,12 +41,8 @@ import org.sonar.server.es.metadata.MetadataIndexDefinition;
import org.sonar.server.es.metadata.MetadataIndexImpl;
import org.sonar.server.es.newindex.NewRegularIndex;
import org.sonar.server.es.newindex.SettingsConfiguration;
import org.sonar.server.platform.db.migration.es.MigrationEsClient;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.sonar.server.es.IndexType.main;
import static org.sonar.server.es.newindex.SettingsConfiguration.newBuilder;

@@ -66,7 +61,6 @@ public class IndexCreatorTest {
private final MetadataIndex metadataIndex = new MetadataIndexImpl(es.client());
private final TestEsDbCompatibility esDbCompatibility = new TestEsDbCompatibility();
private final MapSettings settings = new MapSettings();
private final MigrationEsClient migrationEsClient = mock(MigrationEsClient.class);

@Test
public void create_index() {
@@ -80,15 +74,6 @@ public class IndexCreatorTest {
assertThat(mappings()).isNotEmpty();
}

@Test
public void creation_of_new_index_is_supported_in_blue_green_deployment() {
enableBlueGreenDeployment();

run(new FakeIndexDefinition());

verifyFakeIndexV1();
}

@Test
public void recreate_index_on_definition_changes() {
// v1
@@ -112,34 +97,6 @@ public class IndexCreatorTest {
assertThat(es.client().get(new GetRequest(fakeIndexType.getIndex().getName()).id(id)).isExists()).isFalse();
}

@Test
public void fail_to_recreate_index_on_definition_changes_if_blue_green_deployment() {
enableBlueGreenDeployment();

// v1
run(new FakeIndexDefinition());

// v2
FakeIndexDefinitionV2 definitionV2 = new FakeIndexDefinitionV2();
assertThatThrownBy(() -> run(definitionV2))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("Blue/green deployment is not supported. Elasticsearch index [fakes] changed and needs to be dropped.");
}

@Test
public void reset_definition_hash_if_change_applied_during_migration_of_blue_green_deployment() {
enableBlueGreenDeployment();

// v1
run(new FakeIndexDefinition());

// v2
when(migrationEsClient.getUpdatedIndices()).thenReturn(Sets.newHashSet(FakeIndexDefinition.INDEX_TYPE.getIndex().getName()));
run(new FakeIndexDefinitionV2());
// index has not been dropped-and-recreated
verifyFakeIndexV1();
}

@Test
public void mark_all_non_existing_index_types_as_uninitialized() {
Index fakesIndex = Index.simple("fakes");
@@ -218,10 +175,6 @@ public class IndexCreatorTest {
es.client().putSettings(new UpdateSettingsRequest().indices(mainType.getIndex().getName()).settings(builder.build()));
}

private void enableBlueGreenDeployment() {
settings.setProperty("sonar.blueGreenEnabled", "true");
}

private void testDeleteOnDbChange(String expectedLog, Consumer<TestEsDbCompatibility> afterFirstStart) {
run(new FakeIndexDefinition());
assertThat(logTester.logs(LoggerLevel.INFO))
@@ -262,7 +215,7 @@ public class IndexCreatorTest {
private IndexCreator run(IndexDefinition... definitions) {
IndexDefinitions defs = new IndexDefinitions(definitions, new MapSettings().asConfig());
defs.start();
IndexCreator creator = new IndexCreator(es.client(), defs, metadataIndexDefinition, metadataIndex, migrationEsClient, esDbCompatibility, settings.asConfig());
IndexCreator creator = new IndexCreator(es.client(), defs, metadataIndexDefinition, metadataIndex, esDbCompatibility);
creator.start();
return creator;
}

+ 2
- 9
server/sonar-webserver-webapi/src/main/java/org/sonar/server/ce/queue/CeQueueCleaner.java View File

@@ -29,7 +29,6 @@ import org.sonar.api.utils.log.Loggers;
import org.sonar.ce.queue.CeQueue;
import org.sonar.db.DbClient;
import org.sonar.db.DbSession;
import org.sonar.process.ProcessProperties;

/**
* Cleans-up the Compute Engine queue.
@@ -42,27 +41,21 @@ public class CeQueueCleaner implements Startable {
private final DbClient dbClient;
private final ServerUpgradeStatus serverUpgradeStatus;
private final CeQueue queue;
private final Configuration configuration;

public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, CeQueue queue, Configuration configuration) {
public CeQueueCleaner(DbClient dbClient, ServerUpgradeStatus serverUpgradeStatus, CeQueue queue) {
this.dbClient = dbClient;
this.serverUpgradeStatus = serverUpgradeStatus;
this.queue = queue;
this.configuration = configuration;
}

@Override
public void start() {
if (serverUpgradeStatus.isUpgraded() && !isBlueGreenDeployment()) {
if (serverUpgradeStatus.isUpgraded()) {
cleanOnUpgrade();
}
cleanUpTaskInputOrphans();
}

private boolean isBlueGreenDeployment() {
return configuration.getBoolean(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey()).orElse(false);
}

private void cleanOnUpgrade() {
// we assume that pending tasks are not compatible with the new version
// and can't be processed

+ 1
- 11
server/sonar-webserver-webapi/src/test/java/org/sonar/server/ce/queue/CeQueueCleanerTest.java View File

@@ -69,16 +69,6 @@ public class CeQueueCleanerTest {
verify(queue).clear();
}

@Test
public void start_does_not_clear_queue_if_version_upgrade_but_blue_green_deployment() {
when(serverUpgradeStatus.isUpgraded()).thenReturn(true);
settings.setProperty(ProcessProperties.Property.BLUE_GREEN_ENABLED.getKey(), true);

runCleaner();

verify(queue, never()).clear();
}

@Test
public void start_deletes_orphan_report_files() {
// analysis reports are persisted but the associated
@@ -114,7 +104,7 @@ public class CeQueueCleanerTest {
}

private void runCleaner() {
CeQueueCleaner cleaner = new CeQueueCleaner(dbTester.getDbClient(), serverUpgradeStatus, queue, settings.asConfig());
CeQueueCleaner cleaner = new CeQueueCleaner(dbTester.getDbClient(), serverUpgradeStatus, queue);
cleaner.start();
}
}

+ 0
- 2
server/sonar-webserver/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel2.java View File

@@ -24,7 +24,6 @@ import org.sonar.core.extension.CoreExtensionsInstaller;
import org.sonar.core.platform.PluginClassLoader;
import org.sonar.core.platform.PluginClassloaderFactory;
import org.sonar.core.platform.SpringComponentContainer;
import org.sonar.server.es.MigrationEsClientImpl;
import org.sonar.server.l18n.ServerI18n;
import org.sonar.server.platform.DatabaseServerCompatibility;
import org.sonar.server.platform.DefaultServerUpgradeStatus;
@@ -59,7 +58,6 @@ public class PlatformLevel2 extends PlatformLevel {
new MigrationConfigurationModule(),
DatabaseVersion.class,
DatabaseServerCompatibility.class,
MigrationEsClientImpl.class,

new StartupMetadataProvider(),
DefaultServerUpgradeStatus.class,

+ 2
- 2
server/sonar-webserver/src/test/java/org/sonar/server/platform/platformlevel/PlatformLevel2Test.java View File

@@ -73,7 +73,7 @@ public class PlatformLevel2Test {

verify(container).add(ServerPluginRepository.class);
verify(container).add(DatabaseCharsetChecker.class);
verify(container, times(22)).add(any());
verify(container, times(21)).add(any());
}

@Test
@@ -94,7 +94,7 @@ public class PlatformLevel2Test {

verify(container).add(ServerPluginRepository.class);
verify(container, never()).add(DatabaseCharsetChecker.class);
verify(container, times(20)).add(any());
verify(container, times(19)).add(any());
}



Loading…
Cancel
Save