private final SizeHandler sizeHandler;
public BulkIndexer(EsClient client, IndexType indexType, Size size) {
- this(client, indexType, size, IndexingListener.NOOP);
+ this(client, indexType, size, IndexingListener.FAIL_ON_ERROR);
}
public BulkIndexer(EsClient client, IndexType indexType, Size size, IndexingListener indexingListener) {
void onFinish(IndexingResult result);
- IndexingListener NOOP = new IndexingListener() {
+ IndexingListener FAIL_ON_ERROR = new IndexingListener() {
@Override
public void onSuccess(List<DocId> docIds) {
// nothing to do
@Override
public void onFinish(IndexingResult result) {
- // nothing to do
+ if (result.getFailures() > 0) {
+ throw new IllegalStateException("Indexation failures");
+ }
}
};
}
import static java.util.Collections.emptyList;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.sonar.server.es.DefaultIndexSettings.REFRESH_NONE;
import static org.sonar.server.issue.index.IssueIndexDefinition.FIELD_ISSUE_PROJECT_UUID;
import static org.sonar.server.issue.index.IssueIndexDefinition.INDEX_TYPE_ISSUE;
@Override
public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
try (IssueIterator issues = issueIteratorFactory.createForAll()) {
- doIndex(issues, Size.LARGE, IndexingListener.NOOP);
+ doIndex(issues, Size.LARGE, IndexingListener.FAIL_ON_ERROR);
}
}
@Override
public void indexOnAnalysis(String branchUuid) {
try (IssueIterator issues = issueIteratorFactory.createForProject(branchUuid)) {
- doIndex(issues, Size.REGULAR, IndexingListener.NOOP);
+ doIndex(issues, Size.REGULAR, IndexingListener.FAIL_ON_ERROR);
}
}
return;
}
- BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.NOOP);
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR);
bulkIndexer.start();
issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, issueKey, projectUuid));
bulkIndexer.stop();
@VisibleForTesting
protected void index(Iterator<IssueDoc> issues) {
- doIndex(issues, Size.LARGE, IndexingListener.NOOP);
+ doIndex(issues, Size.LARGE, IndexingListener.FAIL_ON_ERROR);
}
private void doIndex(Iterator<IssueDoc> issues, Size size, IndexingListener listener) {
try (DbSession dbSession = dbClient.openSession(false);
ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) {
- BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.NOOP);
+ BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.FAIL_ON_ERROR);
bulkIndexer.start();
while (rowIt.hasNext()) {
ProjectMeasures doc = rowIt.next();
@Override
public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
try (DbSession dbSession = dbClient.openSession(false)) {
- BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.NOOP);
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR);
bulkIndexer.start();
dbClient.activeRuleDao().scrollAllForIndexing(dbSession, ar -> bulkIndexer.add(newIndexRequest(ar)));
bulkIndexer.stop();
profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search);
} else {
- BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.NOOP);
+ BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR);
bulkIndexer.start();
dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i)));
profileResult = bulkIndexer.stop();
@Override
public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) {
try (DbSession dbSession = dbClient.openSession(false)) {
- BulkIndexer bulk = createBulkIndexer(Size.LARGE, IndexingListener.NOOP);
+ BulkIndexer bulk = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR);
bulk.start();
// index all definitions and system extensions
ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByLogin::put);
- BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.NOOP);
+ BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR);
bulkIndexer.start();
dbClient.userDao().scrollAll(dbSession,
// only index requests, no deletion requests.
es.lockWrites(INDEX_TYPE_ISSUE);
db.issues().insertIssue(organization);
- underTest.indexOnStartup(emptySet());
-
- assertThatIndexHasSize(0);
- assertThatEsQueueTableHasSize(0);
+ try {
+ // FIXME : test also message
+ expectedException.expect(IllegalStateException.class);
+ underTest.indexOnStartup(emptySet());
+ } finally {
+ assertThatIndexHasSize(0);
+ assertThatEsQueueTableHasSize(0);
+ }
}
@Test
es.lockWrites(INDEX_TYPE_ISSUE);
IssueDto issue = db.issues().insertIssue(organization);
- underTest.indexOnAnalysis(issue.getProjectUuid());
-
- assertThatIndexHasSize(0);
- assertThatEsQueueTableHasSize(0);
+ try {
+ // FIXME : test also message
+ expectedException.expect(IllegalStateException.class);
+ underTest.indexOnAnalysis(issue.getProjectUuid());
+ } finally {
+ assertThatIndexHasSize(0);
+ assertThatEsQueueTableHasSize(0);
+ }
}
@Test
addIssueToIndex("P1", "Issue1");
es.lockWrites(INDEX_TYPE_ISSUE);
- underTest.deleteByKeys("P1", asList("Issue1"));
-
- assertThatIndexHasOnly("Issue1");
- assertThatEsQueueTableHasSize(0);
+ try {
+ // FIXME : test also message
+ expectedException.expect(IllegalStateException.class);
+ underTest.deleteByKeys("P1", asList("Issue1"));
+ } finally {
+ assertThatIndexHasOnly("Issue1");
+ assertThatEsQueueTableHasSize(0);
+ }
}
@Test
package org.sonar.server.test.index;
import java.io.IOException;
-import java.sql.SQLException;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.System2;
import org.sonar.db.DbTester;
-import org.sonar.db.es.EsQueueDto;
import org.sonar.server.es.EsTester;
-import org.sonar.server.es.IndexingResult;
-import org.sonar.server.es.ProjectIndexer;
import org.sonar.server.test.db.TestTesting;
import static java.lang.String.format;
-import static java.util.Arrays.asList;
-import static java.util.Collections.emptyList;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.spy;
assertThat(countDocuments()).isZero();
}
- /**
- * Indexing recovery is handled by Compute Engine, without using
- * the table es_queue
- */
- @Test
- public void indexOnAnalysis_does_not_fail_on_errors_and_does_not_enable_recovery_mode() throws IOException, SQLException {
- db.prepareDbUnit(getClass(), "db.xml");
-
- es.lockWrites(INDEX_TYPE_TEST);
- TestTesting.updateDataColumn(db.getSession(), "FILE_UUID", TestTesting.newRandomTests(3));
-
- underTest.indexOnAnalysis("PROJECT_UUID");
- es.unlockWrites(INDEX_TYPE_TEST);
-
- assertThat(countDocuments()).isEqualTo(0);
- assertThat(db.countRowsOfTable("es_queue")).isEqualTo(0);
- }
-
- @Test
- public void prepareForRecovery_must_be_empty_unless_cause_is_PROJECT_DELETION() {
- db.prepareDbUnit(getClass(), "db.xml");
- assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_CREATION))
- .isEmpty();
- assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_KEY_UPDATE))
- .isEmpty();
- assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_TAGS_UPDATE))
- .isEmpty();
- assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PERMISSION_CHANGE))
- .isEmpty();
-
- // Only deletion is resilient with recovery
- assertThat(underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_DELETION))
- .isNotEmpty();
- }
-
- @Test
- public void errors_during_project_deletion_are_recovered() throws IOException, SQLException, InterruptedException {
- // Create a project with 3 tests
- db.prepareDbUnit(getClass(), "db.xml");
- TestTesting.updateDataColumn(db.getSession(), "FILE_UUID", TestTesting.newRandomTests(3));
- underTest.indexOnAnalysis("PROJECT_UUID"); //index(db.getSession(), items);
- assertThat(countDocuments()).isEqualTo(3);
-
- // Now delete the files
- es.lockWrites(INDEX_TYPE_TEST);
- Collection<EsQueueDto> items = underTest.prepareForRecovery(db.getSession(), asList("PROJECT_UUID"), ProjectIndexer.Cause.PROJECT_DELETION);
- db.commit();
-
- underTest.deleteByFile("FILE_UUID");
- es.unlockWrites(INDEX_TYPE_TEST);
- // Still 3 tests
- assertThat(countDocuments()).isEqualTo(3);
-
- // Recover must delete the 3 tests
- IndexingResult result = recover();
- assertThat(result.getTotal()).isEqualTo(3);
-
- assertThat(countDocuments()).isEqualTo(0);
- }
-
- @Test
- public void indexing_with_empty_esqueue_dto_does_nothing() {
- assertThat(underTest.index(db.getSession(), emptyList()))
- .extracting(IndexingResult::getTotal, IndexingResult::getFailures, IndexingResult::getSuccess)
- .containsExactly(0L, 0L, 0L);
- }
-
@Test
public void delete_file_by_uuid() throws Exception {
indexTest("P1", "F1", "T1", "U111");
private long countDocuments() {
return es.countDocuments(INDEX_TYPE_TEST);
}
-
- private IndexingResult recover() {
- Collection<EsQueueDto> items = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), System.currentTimeMillis() + 10_000L, 10);
- return underTest.index(db.getSession(), items);
- }
}
import com.sonar.orchestrator.Orchestrator;
import com.sonar.orchestrator.build.BuildResult;
import com.sonar.orchestrator.build.SonarScanner;
+import com.sonar.orchestrator.util.NetworkUtils;
+import java.net.InetAddress;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.sonarqube.ws.Issues;
import org.sonarqube.ws.Organizations.Organization;
import org.sonarqube.ws.QualityProfiles.CreateWsResponse.QualityProfile;
+import org.sonarqube.ws.WsCe;
import org.sonarqube.ws.WsProjects;
import org.sonarqube.ws.WsUsers.CreateWsResponse.User;
import org.sonarqube.ws.client.component.SuggestionsWsRequest;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.tuple;
import static org.sonarqube.tests.Byteman.Process.CE;
+import static org.sonarqube.ws.WsCe.TaskStatus.FAILED;
import static util.ItUtils.projectDir;
public class AnalysisEsResilienceTest {
@ClassRule
public static final Orchestrator orchestrator;
private static final Byteman byteman;
+ private static final int esHttpPort = NetworkUtils.getNextAvailablePort(InetAddress.getLoopbackAddress());
static {
byteman = new Byteman(Orchestrator.builderEnv(), CE);
orchestrator = byteman
.getOrchestratorBuilder()
.addPlugin(ItUtils.xooPlugin())
+ .setServerProperty("sonar.search.httpPort", "" + esHttpPort)
.build();
}
@After
public void after() throws Exception {
byteman.deactivateAllRules();
+ for (String index : Arrays.asList("issues", "rules", "users", "components", "views", "tests", "projectmeasures")) {
+ tester.elasticsearch().unlockWrites(index);
+ }
}
@Test
tuple(file3Key, "OPEN"));
}
+ @Test
+ public void compute_engine_task_must_be_red_when_es_is_not_available() throws Exception {
+ Organization organization = tester.organizations().generate();
+ User orgAdministrator = tester.users().generateAdministrator(organization);
+ WsProjects.CreateWsResponse.Project project = tester.projects().generate(organization);
+ String projectKey = project.getKey();
+ String fileKey = projectKey + ":src/main/xoo/sample/Sample.xoo";
+
+ QualityProfile profile = tester.qProfiles().createXooProfile(organization);
+ tester.qProfiles()
+ .activateRule(profile, "xoo:OneIssuePerFile")
+ .assignQProfileToProject(profile, project);
+
+ tester.elasticsearch().lockWrites("issues");
+
+ String analysisKey = executeAnalysis(projectKey, organization, orgAdministrator, "analysis/resilience/resilience-sample-v1");
+ WsCe.TaskResponse task = tester.wsClient().ce().task(analysisKey);
+
+ assertThat(task.getTask().getStatus()).isEqualTo(FAILED);
+ }
+
private List<Issues.Issue> searchIssues(String projectKey) {
SearchWsRequest request = new SearchWsRequest()
.setProjectKeys(Collections.singletonList(projectKey));
"sonar.password", orgAdministrator.getLogin()));
return ItUtils.extractCeTaskId(buildResult);
}
-
}
projects / sonarqube.git / commitdiff