diff options
10 files changed, 69 insertions, 105 deletions
diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index cf931dff496..7d2eb4dcefa 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -76,7 +76,7 @@ public class BulkIndexer { private final SizeHandler sizeHandler; public BulkIndexer(EsClient client, IndexType indexType, Size size) { - this(client, indexType, size, IndexingListener.NOOP); + this(client, indexType, size, IndexingListener.FAIL_ON_ERROR); } public BulkIndexer(EsClient client, IndexType indexType, Size size, IndexingListener indexingListener) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java b/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java index ce74bd9cdde..d4748a5652a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/IndexingListener.java @@ -27,7 +27,7 @@ public interface IndexingListener { void onFinish(IndexingResult result); - IndexingListener NOOP = new IndexingListener() { + IndexingListener FAIL_ON_ERROR = new IndexingListener() { @Override public void onSuccess(List<DocId> docIds) { // nothing to do @@ -35,7 +35,9 @@ public interface IndexingListener { @Override public void onFinish(IndexingResult result) { - // nothing to do + if (result.getFailures() > 0) { + throw new IllegalStateException("Indexation failures"); + } } }; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 63c08c3fb59..ec00ef43960 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -52,7 +52,6 @@ import org.sonar.server.permission.index.NeedAuthorizationIndexer; import static java.util.Collections.emptyList; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.sonar.server.es.DefaultIndexSettings.REFRESH_NONE; import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID; import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE; @@ -93,14 +92,14 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { @Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (IssueIterator issues = issueIteratorFactory.createForAll()) { - doIndex(issues, Size.LARGE, IndexingListener.NOOP); + doIndex(issues, Size.LARGE, IndexingListener.FAIL_ON_ERROR); } } @Override public void indexOnAnalysis(String branchUuid) { try (IssueIterator issues = issueIteratorFactory.createForProject(branchUuid)) { - doIndex(issues, Size.REGULAR, IndexingListener.NOOP); + doIndex(issues, Size.REGULAR, IndexingListener.FAIL_ON_ERROR); } } @@ -227,7 +226,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { return; } - BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.NOOP); + BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, issueKey, projectUuid)); bulkIndexer.stop(); @@ -235,7 +234,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer { @VisibleForTesting protected void index(Iterator<IssueDoc> issues) { - doIndex(issues, Size.LARGE, IndexingListener.NOOP); + doIndex(issues, Size.LARGE, IndexingListener.FAIL_ON_ERROR); } private void doIndex(Iterator<IssueDoc> issues, Size size, IndexingListener listener) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index daae14186da..f255de2b608 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -138,7 +138,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization try (DbSession dbSession = dbClient.openSession(false); ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { - BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.NOOP); + BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); while (rowIt.hasNext()) { ProjectMeasures doc = rowIt.next(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java index 0061346194e..b4d25dbc803 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java @@ -73,7 +73,7 @@ public class ActiveRuleIndexer implements ResilientIndexer { @Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { - BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.NOOP); + BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollAllForIndexing(dbSession, ar -> bulkIndexer.add(newIndexRequest(ar))); bulkIndexer.stop(); @@ -182,7 +182,7 @@ public class ActiveRuleIndexer implements ResilientIndexer { profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search); } else { - BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.NOOP); + BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i))); profileResult = bulkIndexer.stop(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java index c6ceb47cb8d..80828c4ac06 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java @@ -69,7 +69,7 @@ public class RuleIndexer implements ResilientIndexer { @Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { - BulkIndexer bulk = createBulkIndexer(Size.LARGE, IndexingListener.NOOP); + BulkIndexer bulk = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulk.start(); // index all definitions and system extensions diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 50d43af95af..f9d5b10c886 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -67,7 +67,7 @@ public class UserIndexer implements ResilientIndexer { ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create(); dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByLogin::put); - BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.NOOP); + BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.userDao().scrollAll(dbSession, // only index requests, no deletion requests. diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java index 4379bbb3480..0822c2731d5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/index/IssueIndexerTest.java @@ -141,10 +141,14 @@ public class IssueIndexerTest { es.lockWrites(INDEX_TYPE_ISSUE); db.issues().insertIssue(organization); - underTest.indexOnStartup(emptySet()); - - assertThatIndexHasSize(0); - assertThatEsQueueTableHasSize(0); + try { + // FIXME : test also message + expectedException.expect(IllegalStateException.class); + underTest.indexOnStartup(emptySet()); + } finally { + assertThatIndexHasSize(0); + assertThatEsQueueTableHasSize(0); + } } @Test @@ -187,10 +191,14 @@ public class IssueIndexerTest { es.lockWrites(INDEX_TYPE_ISSUE); IssueDto issue = db.issues().insertIssue(organization); - underTest.indexOnAnalysis(issue.getProjectUuid()); - - assertThatIndexHasSize(0); - assertThatEsQueueTableHasSize(0); + try { + // FIXME : test also message + expectedException.expect(IllegalStateException.class); + underTest.indexOnAnalysis(issue.getProjectUuid()); + } finally { + assertThatIndexHasSize(0); + assertThatEsQueueTableHasSize(0); + } } @Test @@ -416,10 +424,14 @@ public class IssueIndexerTest { addIssueToIndex("P1", "Issue1"); es.lockWrites(INDEX_TYPE_ISSUE); - underTest.deleteByKeys("P1", asList("Issue1")); - - assertThatIndexHasOnly("Issue1"); - assertThatEsQueueTableHasSize(0); + try { + // FIXME : test also message + expectedException.expect(IllegalStateException.class); + underTest.deleteByKeys("P1", asList("Issue1")); + } finally { + assertThatIndexHasOnly("Issue1"); + assertThatEsQueueTableHasSize(0); + } } @Test diff --git a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java index 60e297dbcae..cc1773a3f4a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/test/index/TestIndexerTest.java @@ -20,8 +20,6 @@ package org.sonar.server.test.index; import java.io.IOException; -import java.sql.SQLException; -import java.util.Collection; import java.util.List; import java.util.Map; import org.apache.commons.io.IOUtils; @@ -32,15 +30,10 @@ import org.junit.Test; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.utils.System2; import org.sonar.db.DbTester; -import org.sonar.db.es.EsQueueDto; import org.sonar.server.es.EsTester; -import org.sonar.server.es.IndexingResult; -import org.sonar.server.es.ProjectIndexer; import org.sonar.server.test.db.TestTesting; import static java.lang.String.format; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.spy; @@ -100,73 +93,6 @@ public class TestIndexerTest { assertThat(countDocuments()).isZero(); } - /** - * Indexing recovery is handled by Compute Engine, without using - * the table es_queue - */ - @Test - public void indexOnAnalysis_does_not_fail_on_errors_and_does_not_enable_recovery_mode() throws IOException, SQLException { - db.prepareDbUnit(getClass(), "db.xml"); - - es.lockWrites(INDEX_TYPE_TEST); - TestTesting.updateDataColumn(db.getSession(), "FILE_UUID", TestTesting.newRandomTests(3)); - - underTest.indexOnAnalysis("PROJECT_UUID"); - es.unlockWrites(INDEX_TYPE_TEST); - - assertThat(countDocuments()).isEqualTo(0); - assertThat(db.countRowsOfTable("es_queue")).isEqualTo(0); - } - - @Test - public void prepareForRecovery_must_be_empty_unless_cause_is_PROJECT_DELETION() { - db.prepareDbUnit(getClass(), "db.xml"); - assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_CREATION)) - .isEmpty(); - assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_KEY_UPDATE)) - .isEmpty(); - assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_TAGS_UPDATE)) - .isEmpty(); - assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PERMISSION_CHANGE)) - .isEmpty(); - - // Only deletion is resilient with recovery - assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_DELETION)) - .isNotEmpty(); - } - - @Test - public void errors_during_project_deletion_are_recovered() throws IOException, SQLException, InterruptedException { - // Create a project with 3 tests - db.prepareDbUnit(getClass(), "db.xml"); - TestTesting.updateDataColumn(db.getSession(), "FILE_UUID", TestTesting.newRandomTests(3)); - underTest.indexOnAnalysis("PROJECT_UUID"); //index(db.getSession(), items); - assertThat(countDocuments()).isEqualTo(3); - - // Now delete the files - es.lockWrites(INDEX_TYPE_TEST); - Collection<EsQueueDto> items = underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_DELETION); - db.commit(); - - underTest.deleteByFile("FILE_UUID"); - es.unlockWrites(INDEX_TYPE_TEST); - // Still 3 tests - assertThat(countDocuments()).isEqualTo(3); - - // Recover must delete the 3 tests - IndexingResult result = recover(); - assertThat(result.getTotal()).isEqualTo(3); - - assertThat(countDocuments()).isEqualTo(0); - } - - @Test - public void indexing_with_empty_esqueue_dto_does_nothing() { - assertThat(underTest.index(db.getSession(), emptyList())) - .extracting(IndexingResult::getTotal, IndexingResult::getFailures, IndexingResult::getSuccess) - .containsExactly(0L, 0L, 0L); - } - @Test public void delete_file_by_uuid() throws Exception { indexTest("P1", "F1", "T1", "U111"); @@ -202,9 +128,4 @@ public class TestIndexerTest { private long countDocuments() { return es.countDocuments(INDEX_TYPE_TEST); } - - private IndexingResult recover() { - Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), System.currentTimeMillis() + 10_000L, 10); - return underTest.index(db.getSession(), items); - } } diff --git a/tests/src/test/java/org/sonarqube/tests/analysis/AnalysisEsResilienceTest.java b/tests/src/test/java/org/sonarqube/tests/analysis/AnalysisEsResilienceTest.java index 6d6adeb2030..b522d311f74 100644 --- a/tests/src/test/java/org/sonarqube/tests/analysis/AnalysisEsResilienceTest.java +++ b/tests/src/test/java/org/sonarqube/tests/analysis/AnalysisEsResilienceTest.java @@ -22,6 +22,9 @@ package org.sonarqube.tests.analysis; import com.sonar.orchestrator.Orchestrator; import com.sonar.orchestrator.build.BuildResult; import com.sonar.orchestrator.build.SonarScanner; +import com.sonar.orchestrator.util.NetworkUtils; +import java.net.InetAddress; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -36,6 +39,7 @@ import org.sonarqube.tests.Tester; import org.sonarqube.ws.Issues; import org.sonarqube.ws.Organizations.Organization; import org.sonarqube.ws.QualityProfiles.CreateWsResponse.QualityProfile; +import org.sonarqube.ws.WsCe; import org.sonarqube.ws.WsProjects; import org.sonarqube.ws.WsUsers.CreateWsResponse.User; import org.sonarqube.ws.client.component.SuggestionsWsRequest; @@ -45,6 +49,7 @@ import util.ItUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; import static org.sonarqube.tests.Byteman.Process.CE; +import static org.sonarqube.ws.WsCe.TaskStatus.FAILED; import static util.ItUtils.projectDir; public class AnalysisEsResilienceTest { @@ -52,12 +57,14 @@ public class AnalysisEsResilienceTest { @ClassRule public static final Orchestrator orchestrator; private static final Byteman byteman; + private static final int esHttpPort = NetworkUtils.getNextAvailablePort(InetAddress.getLoopbackAddress()); static { byteman = new Byteman(Orchestrator.builderEnv(), CE); orchestrator = byteman .getOrchestratorBuilder() .addPlugin(ItUtils.xooPlugin()) + .setServerProperty("sonar.search.httpPort", "" + esHttpPort) .build(); } @@ -67,6 +74,9 @@ public class AnalysisEsResilienceTest { @After public void after() throws Exception { byteman.deactivateAllRules(); + for (String index : Arrays.asList("issues", "rules", "users", "components", "views", "tests", "projectmeasures")) { + tester.elasticsearch().unlockWrites(index); + } } @Test @@ -117,6 +127,27 @@ public class AnalysisEsResilienceTest { tuple(file3Key, "OPEN")); } + @Test + public void compute_engine_task_must_be_red_when_es_is_not_available() throws Exception { + Organization organization = tester.organizations().generate(); + User orgAdministrator = tester.users().generateAdministrator(organization); + WsProjects.CreateWsResponse.Project project = tester.projects().generate(organization); + String projectKey = project.getKey(); + String fileKey = projectKey + ":src/main/xoo/sample/Sample.xoo"; + + QualityProfile profile = tester.qProfiles().createXooProfile(organization); + tester.qProfiles() + .activateRule(profile, "xoo:OneIssuePerFile") + .assignQProfileToProject(profile, project); + + tester.elasticsearch().lockWrites("issues"); + + String analysisKey = executeAnalysis(projectKey, organization, orgAdministrator, "analysis/resilience/resilience-sample-v1"); + WsCe.TaskResponse task = tester.wsClient().ce().task(analysisKey); + + assertThat(task.getTask().getStatus()).isEqualTo(FAILED); + } + private List<Issues.Issue> searchIssues(String projectKey) { SearchWsRequest request = new SearchWsRequest() .setProjectKeys(Collections.singletonList(projectKey)); @@ -146,5 +177,4 @@ public class AnalysisEsResilienceTest { "sonar.password", orgAdministrator.getLogin())); return ItUtils.extractCeTaskId(buildResult); } - } |