From 75e926deef21688a74904b1feae2a687309a6ab3 Mon Sep 17 00:00:00 2001 From: Simon Brandhof Date: Thu, 6 Jul 2017 14:21:05 +0200 Subject: [PATCH] SONAR-9480 fix support of multiple docs to recover with same id --- .../sonar/xoo/rule/XooRulesDefinition.java | 6 + .../xoo/rule/XooRulesDefinitionTest.java | 2 +- .../main/java/org/sonar/db/es/EsQueueDto.java | 19 --- .../org/sonar/server/es/RecoveryIndexer.java | 13 +- .../server/es/ResiliencyIndexingListener.java | 7 +- .../org/sonar/server/rule/RuleUpdater.java | 3 +- .../server/rule/index/RuleExtensionDoc.java | 14 +- .../sonar/server/rule/index/RuleIndexer.java | 79 ++++++----- .../sonar/server/user/index/UserIndexer.java | 8 +- .../sonar/server/es/RecoveryIndexerTest.java | 44 ++++-- .../sonar/server/issue/ws/TagsActionTest.java | 8 +- .../server/rule/index/RuleIndexerTest.java | 2 +- .../server/rule/ws/SearchActionTest.java | 2 +- .../sonar/server/rule/ws/ShowActionTest.java | 2 +- .../sonar/server/rule/ws/TagsActionTest.java | 2 +- .../ws/client/rule/CreateWsRequest.java | 131 ++++++++++++++++++ .../ws/client/rule/RulesService.java | 14 ++ .../ws/client/rule/RulesServiceTest.java | 28 ++++ tests/resilience/rule_indexer.btm | 12 ++ .../java/org/sonarqube/tests/Byteman.java | 60 ++++++++ .../org/sonarqube/tests/Category5Suite.java | 2 + .../ActiveRuleEsResilienceTest.java | 35 ++--- .../tests/rule/RuleEsResilienceTest.java | 90 ++++++++++++ .../tests/user/UserEsResilienceTest.java | 31 +---- 24 files changed, 461 insertions(+), 153 deletions(-) create mode 100644 sonar-ws/src/main/java/org/sonarqube/ws/client/rule/CreateWsRequest.java create mode 100644 tests/resilience/rule_indexer.btm create mode 100644 tests/src/test/java/org/sonarqube/tests/Byteman.java create mode 100644 tests/src/test/java/org/sonarqube/tests/rule/RuleEsResilienceTest.java diff --git a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/XooRulesDefinition.java b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/XooRulesDefinition.java index 4dca671a89d..d4f46446a8e 100644 --- a/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/XooRulesDefinition.java +++ b/plugins/sonar-xoo-plugin/src/main/java/org/sonar/xoo/rule/XooRulesDefinition.java @@ -138,6 +138,12 @@ public class XooRulesDefinition implements RulesDefinition { .setGapDescription("A certified architect will need roughly half an hour to start working on removal of modules, " + "then it's about one hour per module."); + repo + .createRule("xoo-template") + .setTemplate(true) + .setName("Template of rule") + .setHtmlDescription("Template to be overridden by custom rules"); + repo.done(); } diff --git a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/XooRulesDefinitionTest.java b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/XooRulesDefinitionTest.java index fee5610822b..8f25da1ff79 100644 --- a/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/XooRulesDefinitionTest.java +++ b/plugins/sonar-xoo-plugin/src/test/java/org/sonar/xoo/rule/XooRulesDefinitionTest.java @@ -42,7 +42,7 @@ public class XooRulesDefinitionTest { assertThat(repo).isNotNull(); assertThat(repo.name()).isEqualTo("Xoo"); assertThat(repo.language()).isEqualTo("xoo"); - assertThat(repo.rules()).hasSize(17); + assertThat(repo.rules()).hasSize(18); RulesDefinition.Rule rule = repo.rule(OneIssuePerLineSensor.RULE_KEY); assertThat(rule.name()).isNotEmpty(); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java index 26f310d34ca..63751bf5de5 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java @@ -93,25 +93,6 @@ public final class EsQueueDto { return sb.toString(); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof EsQueueDto)) { - return false; - } - - EsQueueDto that = (EsQueueDto) o; - - return uuid.equals(that.uuid); - } - - @Override - public int hashCode() { - return uuid.hashCode(); - } - public static EsQueueDto create(Type docType, String docUuid) { return new EsQueueDto().setDocType(docType).setDocId(docUuid); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java index 985143d03b1..709f652e88c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java @@ -29,7 +29,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.commons.lang.math.RandomUtils; import org.sonar.api.Startable; -import org.sonar.api.config.Settings; +import org.sonar.api.config.Configuration; import org.sonar.api.utils.System2; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; @@ -63,7 +63,7 @@ public class RecoveryIndexer implements Startable { .setNameFormat("RecoveryIndexer-%d") .build()); private final System2 system2; - private final Settings settings; + private final Configuration config; private final DbClient dbClient; private final UserIndexer userIndexer; private final RuleIndexer ruleIndexer; @@ -71,10 +71,10 @@ public class RecoveryIndexer implements Startable { private final long minAgeInMs; private final long loopLimit; - public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, + public RecoveryIndexer(System2 system2, Configuration config, DbClient dbClient, UserIndexer userIndexer, RuleIndexer ruleIndexer, ActiveRuleIndexer activeRuleIndexer) { this.system2 = system2; - this.settings = settings; + this.config = config; this.dbClient = dbClient; this.userIndexer = userIndexer; this.ruleIndexer = ruleIndexer; @@ -161,10 +161,7 @@ public class RecoveryIndexer implements Startable { } private long getSetting(String key, long defaultValue) { - long val = settings.getLong(key); - if (val <= 0) { - val = defaultValue; - } + long val = config.getLong(key).orElse(defaultValue); LOGGER.debug(LOG_PREFIX + "{}={}", key, val); return val; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/ResiliencyIndexingListener.java b/server/sonar-server/src/main/java/org/sonar/server/es/ResiliencyIndexingListener.java index b9d1d27ff14..cc29e14a9f1 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/ResiliencyIndexingListener.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/ResiliencyIndexingListener.java @@ -19,8 +19,8 @@ */ package org.sonar.server.es; +import com.google.common.collect.Multimap; import java.util.Collection; -import java.util.Map; import java.util.Objects; import java.util.function.Function; import org.sonar.core.util.stream.MoreCollectors; @@ -28,8 +28,6 @@ import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.es.EsQueueDto; -import static java.util.stream.Collectors.toMap; - /** * Clean-up the db table es_queue when documents * are successfully indexed so that the recovery @@ -50,11 +48,12 @@ public class ResiliencyIndexingListener implements IndexingListener { @Override public void onSuccess(Collection docIds) { if (!docIds.isEmpty()) { - Map itemsById = items.stream().collect(toMap(EsQueueDto::getDocId, Function.identity())); + Multimap itemsById = items.stream().collect(MoreCollectors.index(EsQueueDto::getDocId, Function.identity())); Collection itemsToDelete = docIds .stream() .map(itemsById::get) + .flatMap(Collection::stream) .filter(Objects::nonNull) .collect(MoreCollectors.toArrayList(docIds.size())); dbClient.esQueueDao().delete(dbSession, itemsToDelete); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java b/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java index 70e11dee084..cf2bd389b2c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/RuleUpdater.java @@ -81,8 +81,7 @@ public class RuleUpdater { apply(update, rule, userSession); update(dbSession, rule); updateParameters(dbSession, organization, update, rule); - ruleIndexer.commitAndIndex(dbSession, rule.getKey()); - ruleIndexer.commitAndIndex(dbSession, organization, rule.getKey()); + ruleIndexer.commitAndIndex(dbSession, rule.getKey(), organization); return true; } diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java index 226e3cda99f..e0317b989c8 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleExtensionDoc.java @@ -26,7 +26,6 @@ import org.apache.commons.lang.builder.ReflectionToStringBuilder; import org.sonar.api.rule.RuleKey; import org.sonar.db.rule.RuleExtensionForIndexingDto; import org.sonar.db.rule.RuleForIndexingDto; -import org.sonar.db.rule.RuleMetadataDto; import org.sonar.server.es.BaseDoc; public class RuleExtensionDoc extends BaseDoc { @@ -41,7 +40,7 @@ public class RuleExtensionDoc extends BaseDoc { @Override public String getId() { - return getRuleKey() + "|" + getScope().getScope(); + return idOf(getRuleKey(), getScope()); } @Override @@ -81,13 +80,6 @@ public class RuleExtensionDoc extends BaseDoc { return this; } - public static RuleExtensionDoc of(RuleKey key, RuleExtensionScope scope, RuleMetadataDto ruleExtension) { - return new RuleExtensionDoc() - .setRuleKey(key) - .setScope(scope) - .setTags(ruleExtension.getTags()); - } - public static RuleExtensionDoc of(RuleForIndexingDto rule) { return new RuleExtensionDoc() .setRuleKey(rule.getRuleKey()) @@ -102,6 +94,10 @@ public class RuleExtensionDoc extends BaseDoc { .setTags(rule.getTagsAsSet()); } + public static String idOf(RuleKey ruleKey, RuleExtensionScope scope) { + return ruleKey + "|" + scope.getScope(); + } + @Override public String toString() { return ReflectionToStringBuilder.toString(this); diff --git a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java index 7c5db551866..eaf63b2f452 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/rule/index/RuleIndexer.java @@ -45,9 +45,10 @@ import org.sonar.server.es.ResilientIndexer; import org.sonar.server.es.StartupIndexer; import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; -import static org.sonar.core.util.stream.MoreCollectors.toHashSet; +import static org.sonar.core.util.stream.MoreCollectors.toArrayList; import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE; import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION; @@ -95,49 +96,41 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer { commitAndIndex(dbSession, singletonList(ruleKey)); } - public void commitAndIndex(DbSession dbSession, Collection ruleDtos) { - List items = ruleDtos.stream() - .map(key -> EsQueueDto.create(EsQueueDto.Type.RULE, key.toString())) + public void commitAndIndex(DbSession dbSession, Collection ruleKeys) { + List items = ruleKeys.stream() + .map(RuleIndexer::createQueueDtoForRule) .collect(MoreCollectors.toArrayList()); dbClient.esQueueDao().insert(dbSession, items); dbSession.commit(); - postCommit(dbSession, ruleDtos, items); + postCommit(dbSession, items); } - public void commitAndIndex(DbSession dbSession, OrganizationDto organizationDto, RuleKey ruleKey) { - List items = singletonList(EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, ruleKey + "|" + organizationDto.getUuid())); - + /** + * Commit a change on a rule and its extension on the given organization + */ + public void commitAndIndex(DbSession dbSession, RuleKey ruleKey, OrganizationDto organization) { + List items = asList(createQueueDtoForRule(ruleKey), createQueueDtoForRuleExtension(ruleKey, organization)); dbClient.esQueueDao().insert(dbSession, items); dbSession.commit(); - postCommit(dbSession, ruleKey, organizationDto, items); + postCommit(dbSession, items); } /** - * Entry point for Byteman tests. See directory tests/resilience. - * The parameter "ruleKeys" is used only by the Byteman script. + * This method is used by the Byteman script of integration tests. */ - private void postCommit(DbSession dbSession, Collection ruleKeys, Collection items) { - index(dbSession, items); - } - - private void postCommit(DbSession dbSession, RuleKey ruleKeys, OrganizationDto organizationDto, Collection items) { + private void postCommit(DbSession dbSession, List items) { index(dbSession, items); } @Override public IndexingResult index(DbSession dbSession, Collection items) { - if (items.isEmpty()) { - return new IndexingResult(); - } - IndexingResult result = new IndexingResult(); - - ListMultimap itemsByType = groupItemsByType(items); - - result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE))); - result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION))); - + if (!items.isEmpty()) { + ListMultimap itemsByType = groupItemsByType(items); + result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE))); + result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION))); + } return result; } @@ -145,28 +138,28 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); - Set rules = items + List ruleKeys = items .stream() .filter(i -> { requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing"); return i.getDocType() == EsQueueDto.Type.RULE; }) .map(i -> RuleKey.parse(i.getDocId())) - .collect(toHashSet(items.size())); + .collect(toArrayList(items.size())); - dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, rules, + dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleKeys, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. r -> { - rules.remove(r.getRuleKey()); bulkIndexer.add(newRuleDocIndexRequest(r)); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); + ruleKeys.remove(r.getRuleKey()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. - rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString())); - rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.toString())); + ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString(), r.toString())); + ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(r, RuleExtensionScope.system()), r.toString())); return bulkIndexer.stop(); } @@ -175,26 +168,30 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); - Set docIds = items + List docIds = items .stream() .filter(i -> { requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing"); return i.getDocType() == EsQueueDto.Type.RULE_EXTENSION; }) .map(RuleIndexer::explodeRuleExtensionDocId) - .collect(toHashSet(items.size())); + .collect(toArrayList(items.size())); dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. r -> { - docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey())); + RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()); + docIds.remove(docId); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. - docIds.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.getId())); + docIds.forEach(docId -> { + RuleKey ruleKey = RuleKey.of(docId.getRepositoryName(), docId.getRuleKey()); + bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, docId.getId(), ruleKey.toString()); + }); return bulkIndexer.stop(); } @@ -239,4 +236,14 @@ public class RuleIndexer implements StartupIndexer, ResilientIndexer { checkArgument(esQueueDto.getDocType() == EsQueueDto.Type.RULE_EXTENSION); return new RuleExtensionId(esQueueDto.getDocId()); } + + private static EsQueueDto createQueueDtoForRule(RuleKey ruleKey) { + return EsQueueDto.create(EsQueueDto.Type.RULE, ruleKey.toString(), null, ruleKey.toString()); + } + + private static EsQueueDto createQueueDtoForRuleExtension(RuleKey ruleKey, OrganizationDto organization) { + String docId = RuleExtensionDoc.idOf(ruleKey, RuleExtensionScope.organization(organization)); + return EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, docId, null, ruleKey.toString()); + } + } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 239a9bbcb60..eaede45f8aa 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -38,14 +38,14 @@ import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; import org.sonar.server.es.IndexType; import org.sonar.server.es.IndexingListener; +import org.sonar.server.es.IndexingResult; import org.sonar.server.es.ResiliencyIndexingListener; import org.sonar.server.es.ResilientIndexer; -import org.sonar.server.es.IndexingResult; import org.sonar.server.es.StartupIndexer; import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; -import static org.sonar.core.util.stream.MoreCollectors.toHashSet; +import static org.sonar.core.util.stream.MoreCollectors.toArrayList; import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER; public class UserIndexer implements StartupIndexer, ResilientIndexer { @@ -113,14 +113,14 @@ public class UserIndexer implements StartupIndexer, ResilientIndexer { if (items.isEmpty()) { return new IndexingResult(); } - Set logins = items + List logins = items .stream() .filter(i -> { requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing"); return i.getDocType() == EsQueueDto.Type.USER; }) .map(EsQueueDto::getDocId) - .collect(toHashSet(items.size())); + .collect(toArrayList(items.size())); ListMultimap organizationUuidsByLogin = ArrayListMultimap.create(); dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put); diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java index 0c8fa0f5f10..b3f42589d13 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java @@ -33,7 +33,6 @@ import org.junit.Test; import org.junit.rules.DisableOnDebug; import org.junit.rules.TestRule; import org.junit.rules.Timeout; -import org.sonar.api.config.Settings; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.utils.internal.TestSystem2; import org.sonar.api.utils.log.LogTester; @@ -49,6 +48,7 @@ import org.sonar.server.rule.index.RuleIndexer; import org.sonar.server.user.index.UserIndexDefinition; import org.sonar.server.user.index.UserIndexer; +import static java.util.Arrays.asList; import static java.util.stream.IntStream.rangeClosed; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doAnswer; @@ -72,7 +72,7 @@ public class RecoveryIndexerTest { @Rule public final LogTester logTester = new LogTester().setLevel(TRACE); @Rule - public TestRule safeguard = new DisableOnDebug(new Timeout(60, TimeUnit.SECONDS)); + public TestRule safeguard = new DisableOnDebug(Timeout.builder().withTimeout(60, TimeUnit.SECONDS).withLookingForStuckThread(true).build()); private UserIndexer mockedUserIndexer = mock(UserIndexer.class); private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class); @@ -101,10 +101,10 @@ public class RecoveryIndexerTest { @Test public void start_triggers_recovery_run_at_fixed_rate() throws Exception { - Settings settings = new MapSettings() + MapSettings settings = new MapSettings() .setProperty("sonar.search.recovery.initialDelayInMs", "0") .setProperty("sonar.search.recovery.delayInMs", "1"); - underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer, mockedActiveRuleIndexer)); + underTest = spy(new RecoveryIndexer(system2, settings.asConfig(), db.getDbClient(), mockedUserIndexer, mockedRuleIndexer, mockedActiveRuleIndexer)); AtomicInteger calls = new AtomicInteger(0); doAnswer(invocation -> { calls.incrementAndGet(); @@ -242,7 +242,7 @@ public class RecoveryIndexerTest { // 10 docs to process, by groups of 3. // The first group successfully recovers only 1 docs --> above 30% of failures --> stop run PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(1); - Settings settings = new MapSettings() + MapSettings settings = new MapSettings() .setProperty("sonar.search.recovery.loopLimit", "3"); underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings); underTest.recover(); @@ -262,7 +262,7 @@ public class RecoveryIndexerTest { // 10 docs to process, by groups of 5. // Each group successfully recovers 4 docs --> below 30% of failures --> continue run PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(4, 4, 2); - Settings settings = new MapSettings() + MapSettings settings = new MapSettings() .setProperty("sonar.search.recovery.loopLimit", "5"); underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings); underTest.recover(); @@ -287,6 +287,29 @@ public class RecoveryIndexerTest { assertThatQueueHasSize(1); } + @Test + public void recover_multiple_times_the_same_document() { + UserDto user = db.users().insertUser(); + EsQueueDto item1 = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin()); + EsQueueDto item2 = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin()); + EsQueueDto item3 = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin()); + db.getDbClient().esQueueDao().insert(db.getSession(), asList(item1, item2, item3)); + db.commit(); + + ProxyUserIndexer userIndexer = new ProxyUserIndexer(); + advanceInTime(); + underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer); + underTest.recover(); + + assertThatQueueHasSize(0); + assertThat(userIndexer.called) + .extracting(EsQueueDto::getUuid) + .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid(), item3.getUuid()); + + assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 3 USER"); + assertThatLogsContain(INFO, "Elasticsearch recovery - 3 documents processed [0 failures]"); + } + private class ProxyUserIndexer extends UserIndexer { private final List called = new ArrayList<>(); @@ -363,7 +386,8 @@ public class RecoveryIndexerTest { List filteredItems = items.stream().filter( i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList()); IndexingResult result = super.index(dbSession, filteredItems); - if (items.contains(failing)) { + if (result.getTotal() == items.size() - 1) { + // the failing item was in the items list result.incrementRequests(); } @@ -425,15 +449,15 @@ public class RecoveryIndexerTest { } private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer) { - Settings settings = new MapSettings() + MapSettings settings = new MapSettings() .setProperty("sonar.search.recovery.initialDelayInMs", "0") .setProperty("sonar.search.recovery.delayInMs", "1") .setProperty("sonar.search.recovery.minAgeInMs", "1"); return newRecoveryIndexer(userIndexer, ruleIndexer, settings); } - private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, Settings settings) { - return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer, mockedActiveRuleIndexer); + private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, MapSettings settings) { + return new RecoveryIndexer(system2, settings.asConfig(), db.getDbClient(), userIndexer, ruleIndexer, mockedActiveRuleIndexer); } private EsQueueDto createUnindexedUser() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java index b32d3e58cd1..356bfbf4ad8 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/issue/ws/TagsActionTest.java @@ -96,12 +96,12 @@ public class TagsActionTest { RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag1")); ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag2")); - ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization); RuleDefinitionDto r2 = dbTester.rules().insert(setSystemTags("tag3")); ruleIndexer.commitAndIndex(dbTester.getSession(), r2.getKey()); dbTester.rules().insertOrUpdateMetadata(r2, organization, setTags("tag4", "tag5")); - ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r2.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), r2.getKey(), organization); String result = tester.newRequest() .setParam("organization", organization.getKey()) @@ -118,7 +118,7 @@ public class TagsActionTest { RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag6")); ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag7")); - ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization); String result = tester.newRequest() .setParam("organization", organization.getKey()) @@ -180,7 +180,7 @@ public class TagsActionTest { RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("cwe")); ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("security")); - ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization); String result = tester.newRequest() .setParam("organization", organization.getKey()) diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java index 95a47a88e51..aaf4de5a8f5 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/index/RuleIndexerTest.java @@ -102,7 +102,7 @@ public class RuleIndexerTest { underTest.commitAndIndex(dbTester.getSession(), rule.getKey()); OrganizationDto organization = dbTester.organizations().insert(); dbTester.rules().insertOrUpdateMetadata(rule, organization, m -> m.setTags(ImmutableSet.of("bla"))); - underTest.commitAndIndex(dbTester.getSession(), organization, rule.getKey()); + underTest.commitAndIndex(dbTester.getSession(), rule.getKey(), organization); RuleExtensionDoc doc = new RuleExtensionDoc() .setRuleKey(rule.getKey()) diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java index e7dd9c99e82..ba57d07a228 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/SearchActionTest.java @@ -738,7 +738,7 @@ public class SearchActionTest { @SafeVarargs private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer... populaters) { RuleMetadataDto metadata = db.rules().insertOrUpdateMetadata(rule, organization, populaters); - ruleIndexer.commitAndIndex(db.getSession(), organization, rule.getKey()); + ruleIndexer.commitAndIndex(db.getSession(), rule.getKey(), organization); return metadata; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java index df4b13920f0..7859a14ac68 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/ShowActionTest.java @@ -236,7 +236,7 @@ public class ShowActionTest { @SafeVarargs private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer... populaters) { RuleMetadataDto metadata = dbTester.rules().insertOrUpdateMetadata(rule, organization, populaters); - ruleIndexer.commitAndIndex(dbTester.getSession(), organization, rule.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), rule.getKey(), organization); return metadata; } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java index a75116f809b..d8e56b62b29 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/rule/ws/TagsActionTest.java @@ -110,7 +110,7 @@ public class TagsActionTest { RuleDefinitionDto r = dbTester.rules().insert(setSystemTags()); ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey()); dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag")); - ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey()); + ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization); String result = tester.newRequest().setParam("organization", organization.getKey()).execute().getInput(); assertJson(result).isSimilarTo("{\"tags\":[\"tag\"]}"); diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/rule/CreateWsRequest.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/rule/CreateWsRequest.java new file mode 100644 index 00000000000..9f6c2956d4a --- /dev/null +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/rule/CreateWsRequest.java @@ -0,0 +1,131 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.ws.client.rule; + +public class CreateWsRequest { + + private String customKey; + private String markdownDescription; + private String name; + private String params; + private Boolean preventReactivation; + private String severity; + private String status; + private String templateKey; + + private CreateWsRequest(Builder builder) { + this.customKey = builder.customKey; + this.markdownDescription = builder.markdownDescription; + this.name = builder.name; + this.params = builder.params; + this.preventReactivation = builder.preventReactivation; + this.severity = builder.severity; + this.status = builder.status; + this.templateKey = builder.templateKey; + } + + public String getCustomKey() { + return customKey; + } + + public String getMarkdownDescription() { + return markdownDescription; + } + + public String getName() { + return name; + } + + public String getParams() { + return params; + } + + public Boolean getPreventReactivation() { + return preventReactivation; + } + + public String getSeverity() { + return severity; + } + + public String getStatus() { + return status; + } + + public String getTemplateKey() { + return templateKey; + } + + public static class Builder { + private String customKey; + private String markdownDescription; + private String name; + private String params; + private Boolean preventReactivation; + private String severity; + private String status; + private String templateKey; + + public Builder setCustomKey(String customKey) { + this.customKey = customKey; + return this; + } + + public Builder setMarkdownDescription(String markdownDescription) { + this.markdownDescription = markdownDescription; + return this; + } + + public Builder setName(String name) { + this.name = name; + return this; + } + + public Builder setParams(String params) { + this.params = params; + return this; + } + + public Builder setPreventReactivation(Boolean preventReactivation) { + this.preventReactivation = preventReactivation; + return this; + } + + public Builder setSeverity(String severity) { + this.severity = severity; + return this; + } + + public Builder setStatus(String status) { + this.status = status; + return this; + } + + public Builder setTemplateKey(String templateKey) { + this.templateKey = templateKey; + return this; + } + + public CreateWsRequest build() { + return new CreateWsRequest(this); + } + } + +} diff --git a/sonar-ws/src/main/java/org/sonarqube/ws/client/rule/RulesService.java b/sonar-ws/src/main/java/org/sonarqube/ws/client/rule/RulesService.java index 870fd294660..e744ff84ea5 100644 --- a/sonar-ws/src/main/java/org/sonarqube/ws/client/rule/RulesService.java +++ b/sonar-ws/src/main/java/org/sonarqube/ws/client/rule/RulesService.java @@ -24,6 +24,7 @@ import org.sonarqube.ws.Rules; import org.sonarqube.ws.Rules.SearchResponse; import org.sonarqube.ws.client.BaseService; import org.sonarqube.ws.client.GetRequest; +import org.sonarqube.ws.client.PostRequest; import org.sonarqube.ws.client.WsConnector; import static org.sonarqube.ws.client.rule.RulesWsParameters.PARAM_ACTIVATION; @@ -82,4 +83,17 @@ public class RulesService extends BaseService { .setParam("key", key); return call(request, Rules.ShowResponse.parser()); } + + public void create(CreateWsRequest request) { + PostRequest httpRequest = new PostRequest(path("create")); + httpRequest.setParam("custom_key", request.getCustomKey()); + httpRequest.setParam("markdown_description", request.getMarkdownDescription()); + httpRequest.setParam("name", request.getName()); + httpRequest.setParam("params", request.getParams()); + httpRequest.setParam("prevent_reactivation", request.getPreventReactivation()); + httpRequest.setParam("severity", request.getSeverity()); + httpRequest.setParam("status", request.getStatus()); + httpRequest.setParam("template_key", request.getTemplateKey()); + call(httpRequest); + } } diff --git a/sonar-ws/src/test/java/org/sonarqube/ws/client/rule/RulesServiceTest.java b/sonar-ws/src/test/java/org/sonarqube/ws/client/rule/RulesServiceTest.java index ae38a45e491..45248faa16b 100644 --- a/sonar-ws/src/test/java/org/sonarqube/ws/client/rule/RulesServiceTest.java +++ b/sonar-ws/src/test/java/org/sonarqube/ws/client/rule/RulesServiceTest.java @@ -26,6 +26,7 @@ import org.junit.Test; import org.sonarqube.ws.Rules; import org.sonarqube.ws.Rules.SearchResponse; import org.sonarqube.ws.client.GetRequest; +import org.sonarqube.ws.client.PostRequest; import org.sonarqube.ws.client.ServiceTester; import org.sonarqube.ws.client.WsConnector; @@ -153,4 +154,31 @@ public class RulesServiceTest { .hasParam("key", "the-rule/key") .andNoOtherParam(); } + + @Test + public void test_create() { + underTest.create(new CreateWsRequest.Builder() + .setTemplateKey("the-template-key") + .setCustomKey("the-custom-key") + .setSeverity("BLOCKER") + .setParams("the-params") + .setPreventReactivation(true) + .setMarkdownDescription("the-desc") + .setStatus("BETA") + .setName("the-name") + .build()); + + PostRequest postRequest = serviceTester.getPostRequest(); + serviceTester.assertThat(postRequest) + .hasPath("create") + .hasParam("template_key", "the-template-key") + .hasParam("custom_key", "the-custom-key") + .hasParam("severity", "BLOCKER") + .hasParam("params", "the-params") + .hasParam("prevent_reactivation", "true") + .hasParam("markdown_description", "the-desc") + .hasParam("status", "BETA") + .hasParam("name", "the-name") + .andNoOtherParam(); + } } diff --git a/tests/resilience/rule_indexer.btm b/tests/resilience/rule_indexer.btm new file mode 100644 index 00000000000..8de28f81435 --- /dev/null +++ b/tests/resilience/rule_indexer.btm @@ -0,0 +1,12 @@ +# sonar.web.javaAdditionalOpts=-javaagent:/path/to/byteman-3.0.10/lib/byteman.jar=script:/path/to/active_rule_indexer.btm,boot:/path/to/byteman-3.0.10/lib/byteman.jar +# sonar.search.recovery.delayInMs=10000 +# sonar.search.recovery.minAgeInMs=30000 + +RULE make indexing of custom rules silently fail +CLASS RuleIndexer +METHOD postCommit +COMPILE +AT ENTRY +IF TRUE +DO RETURN +ENDRULE diff --git a/tests/src/test/java/org/sonarqube/tests/Byteman.java b/tests/src/test/java/org/sonarqube/tests/Byteman.java new file mode 100644 index 00000000000..ccb025e34f9 --- /dev/null +++ b/tests/src/test/java/org/sonarqube/tests/Byteman.java @@ -0,0 +1,60 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package org.sonarqube.tests; + +import com.sonar.orchestrator.OrchestratorBuilder; +import java.io.File; + +import static java.lang.String.format; + +/** + * byteman.jboss.org is used by resilience tests in order to + * change behavior of server at runtime. + */ +public class Byteman { + + public static OrchestratorBuilder enableScript(OrchestratorBuilder builder, String filename) { + String jar = findBytemanJar(); + builder + .setServerProperty("sonar.web.javaAdditionalOpts", + format("-javaagent:%s=script:%s,boot:%s", jar, findBytemanScript(filename), jar)) + .setServerProperty("sonar.search.recovery.delayInMs", "1000") + .setServerProperty("sonar.search.recovery.minAgeInMs", "3000"); + return builder; + } + + private static String findBytemanJar() { + // see pom.xml, Maven copies and renames the artifact. + File jar = new File("target/byteman.jar"); + if (!jar.exists()) { + throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' once in directory tests/."); + } + return jar.getAbsolutePath(); + } + + private static String findBytemanScript(String filename) { + // see pom.xml, Maven copies and renames the artifact. + File script = new File( filename); + if (!script.exists()) { + throw new IllegalStateException("Can't find " + script); + } + return script.getAbsolutePath(); + } +} diff --git a/tests/src/test/java/org/sonarqube/tests/Category5Suite.java b/tests/src/test/java/org/sonarqube/tests/Category5Suite.java index 49bc73e4c44..bb3ca95a281 100644 --- a/tests/src/test/java/org/sonarqube/tests/Category5Suite.java +++ b/tests/src/test/java/org/sonarqube/tests/Category5Suite.java @@ -23,6 +23,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Suite; import org.sonarqube.tests.qualityProfile.ActiveRuleEsResilienceTest; import org.sonarqube.tests.qualityProfile.BuiltInQualityProfilesNotificationTest; +import org.sonarqube.tests.rule.RuleEsResilienceTest; import org.sonarqube.tests.serverSystem.ClusterTest; import org.sonarqube.tests.serverSystem.RestartTest; import org.sonarqube.tests.serverSystem.ServerSystemRestartingOrchestrator; @@ -53,6 +54,7 @@ import org.sonarqube.tests.user.UserEsResilienceTest; OnboardingTest.class, BuiltInQualityProfilesNotificationTest.class, ActiveRuleEsResilienceTest.class, + RuleEsResilienceTest.class, UserEsResilienceTest.class }) public class Category5Suite { diff --git a/tests/src/test/java/org/sonarqube/tests/qualityProfile/ActiveRuleEsResilienceTest.java b/tests/src/test/java/org/sonarqube/tests/qualityProfile/ActiveRuleEsResilienceTest.java index 6203a647054..cfa5f160423 100644 --- a/tests/src/test/java/org/sonarqube/tests/qualityProfile/ActiveRuleEsResilienceTest.java +++ b/tests/src/test/java/org/sonarqube/tests/qualityProfile/ActiveRuleEsResilienceTest.java @@ -20,7 +20,6 @@ package org.sonarqube.tests.qualityProfile; import com.sonar.orchestrator.Orchestrator; -import java.io.File; import java.util.concurrent.TimeUnit; import org.junit.ClassRule; import org.junit.Rule; @@ -28,26 +27,26 @@ import org.junit.Test; import org.junit.rules.DisableOnDebug; import org.junit.rules.TestRule; import org.junit.rules.Timeout; +import org.sonarqube.tests.Byteman; import org.sonarqube.tests.Tester; import org.sonarqube.ws.Organizations; import org.sonarqube.ws.QualityProfiles; import org.sonarqube.ws.client.rule.SearchWsRequest; import util.ItUtils; -import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; public class ActiveRuleEsResilienceTest { private static final String RULE_ONE_BUG_PER_LINE = "xoo:OneBugIssuePerLine"; @ClassRule - public static final Orchestrator orchestrator = Orchestrator.builderEnv() - .setServerProperty("sonar.web.javaAdditionalOpts", - format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar())) - .setServerProperty("sonar.search.recovery.delayInMs", "500") - .setServerProperty("sonar.search.recovery.minAgeInMs", "3000") - .addPlugin(ItUtils.xooPlugin()) - .build(); + public static final Orchestrator orchestrator; + + static { + orchestrator = Byteman.enableScript(Orchestrator.builderEnv(), "resilience/active_rule_indexer.btm") + .addPlugin(ItUtils.xooPlugin()) + .build(); + } @Rule public TestRule timeout = new DisableOnDebug(Timeout.builder() @@ -86,22 +85,4 @@ public class ActiveRuleEsResilienceTest { SearchWsRequest request = new SearchWsRequest().setActivation(true).setQProfile(profile.getKey()); return tester.wsClient().rules().search(request).getRulesCount(); } - - private static String findBytemanJar() { - // see pom.xml, Maven copies and renames the artifact. - File jar = new File("target/byteman.jar"); - if (!jar.exists()) { - throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once."); - } - return jar.getAbsolutePath(); - } - - private static String findBytemanScript() { - // see pom.xml, Maven copies and renames the artifact. - File script = new File("resilience/active_rule_indexer.btm"); - if (!script.exists()) { - throw new IllegalStateException("Can't find " + script); - } - return script.getAbsolutePath(); - } } diff --git a/tests/src/test/java/org/sonarqube/tests/rule/RuleEsResilienceTest.java b/tests/src/test/java/org/sonarqube/tests/rule/RuleEsResilienceTest.java new file mode 100644 index 00000000000..a9f606fb320 --- /dev/null +++ b/tests/src/test/java/org/sonarqube/tests/rule/RuleEsResilienceTest.java @@ -0,0 +1,90 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.tests.rule; + +import com.sonar.orchestrator.Orchestrator; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; +import org.sonarqube.tests.Byteman; +import org.sonarqube.tests.Tester; +import org.sonarqube.ws.client.rule.CreateWsRequest; +import org.sonarqube.ws.client.rule.SearchWsRequest; +import util.ItUtils; + +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; + +public class RuleEsResilienceTest { + + @ClassRule + public static final Orchestrator orchestrator; + + static { + orchestrator = Byteman.enableScript(Orchestrator.builderEnv(), "resilience/rule_indexer.btm") + .addPlugin(ItUtils.xooPlugin()) + .build(); + } + + @Rule + public TestRule timeout = new DisableOnDebug(Timeout.builder() + .withLookingForStuckThread(true) + .withTimeout(60L, TimeUnit.SECONDS) + .build()); + + @Rule + public Tester tester = new Tester(orchestrator) + // custom rules are not supported when organizations are enabled + .disableOrganizations(); + + @Test + public void creation_of_custom_rule_is_resilient_to_elasticsearch_errors() throws Exception { + CreateWsRequest request = new CreateWsRequest.Builder() + .setCustomKey("my_custom_rule") + .setName("My custom rule") + .setTemplateKey("xoo:xoo-template") + .setMarkdownDescription("The *initial* rule") + .setSeverity("MAJOR") + .build(); + tester.wsClient().rules().create(request); + + // rule exists in db but is not indexed. Search returns no results. + assertThat(nameFoundInSearch("initial rule")).isFalse(); + + // rule already exists in db, can't be created twice + ItUtils.expectHttpError(400, () -> tester.wsClient().rules().create(request)); + + while (!nameFoundInSearch("initial rule")) { + // rule is indexed by the recovery daemon, which runs every 3 seconds + Thread.sleep(500L); + } + } + + private boolean nameFoundInSearch(String query) { + SearchWsRequest request = new SearchWsRequest() + .setQuery(query) + .setRepositories(singletonList("xoo")); + return tester.wsClient().rules().search(request).getRulesCount() > 0; + } +} diff --git a/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java b/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java index 6232cf1c2d5..b98661e8fe4 100644 --- a/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java +++ b/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java @@ -20,7 +20,6 @@ package org.sonarqube.tests.user; import com.sonar.orchestrator.Orchestrator; -import java.io.File; import java.util.concurrent.TimeUnit; import org.junit.ClassRule; import org.junit.Rule; @@ -28,24 +27,23 @@ import org.junit.Test; import org.junit.rules.DisableOnDebug; import org.junit.rules.TestRule; import org.junit.rules.Timeout; +import org.sonarqube.tests.Byteman; import org.sonarqube.tests.Tester; import org.sonarqube.ws.WsUsers.CreateWsResponse.User; import org.sonarqube.ws.client.user.SearchRequest; import org.sonarqube.ws.client.user.UpdateRequest; -import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static util.ItUtils.expectHttpError; public class UserEsResilienceTest { @ClassRule - public static final Orchestrator orchestrator = Orchestrator.builderEnv() - .setServerProperty("sonar.web.javaAdditionalOpts", - format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar())) - .setServerProperty("sonar.search.recovery.delayInMs", "1000") - .setServerProperty("sonar.search.recovery.minAgeInMs", "3000") - .build(); + public static final Orchestrator orchestrator; + + static { + orchestrator = Byteman.enableScript(Orchestrator.builderEnv(), "resilience/user_indexer.btm").build(); + } @Rule public TestRule timeout = new DisableOnDebug(Timeout.builder() @@ -123,21 +121,4 @@ public class UserEsResilienceTest { return tester.users().service().search(SearchRequest.builder().setQuery(name).build()).getUsersCount() == 1L; } - private static String findBytemanJar() { - // see pom.xml, Maven copies and renames the artifact. - File jar = new File("target/byteman.jar"); - if (!jar.exists()) { - throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once."); - } - return jar.getAbsolutePath(); - } - - private static String findBytemanScript() { - // see pom.xml, Maven copies and renames the artifact. - File script = new File("resilience/user_indexer.btm"); - if (!script.exists()) { - throw new IllegalStateException("Can't find " + script); - } - return script.getAbsolutePath(); - } } -- 2.39.5