.setGapDescription("A certified architect will need roughly half an hour to start working on removal of modules, " +
"then it's about one hour per module.");
+ repo
+ .createRule("xoo-template")
+ .setTemplate(true)
+ .setName("Template of rule")
+ .setHtmlDescription("Template to be overridden by custom rules");
+
repo.done();
}
assertThat(repo).isNotNull();
assertThat(repo.name()).isEqualTo("Xoo");
assertThat(repo.language()).isEqualTo("xoo");
- assertThat(repo.rules()).hasSize(17);
+ assertThat(repo.rules()).hasSize(18);
RulesDefinition.Rule rule = repo.rule(OneIssuePerLineSensor.RULE_KEY);
assertThat(rule.name()).isNotEmpty();
return sb.toString();
}
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof EsQueueDto)) {
- return false;
- }
-
- EsQueueDto that = (EsQueueDto) o;
-
- return uuid.equals(that.uuid);
- }
-
- @Override
- public int hashCode() {
- return uuid.hashCode();
- }
-
public static EsQueueDto create(Type docType, String docUuid) {
return new EsQueueDto().setDocType(docType).setDocId(docUuid);
}
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.math.RandomUtils;
import org.sonar.api.Startable;
-import org.sonar.api.config.Settings;
+import org.sonar.api.config.Configuration;
import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
.setNameFormat("RecoveryIndexer-%d")
.build());
private final System2 system2;
- private final Settings settings;
+ private final Configuration config;
private final DbClient dbClient;
private final UserIndexer userIndexer;
private final RuleIndexer ruleIndexer;
private final long minAgeInMs;
private final long loopLimit;
- public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient,
+ public RecoveryIndexer(System2 system2, Configuration config, DbClient dbClient,
UserIndexer userIndexer, RuleIndexer ruleIndexer, ActiveRuleIndexer activeRuleIndexer) {
this.system2 = system2;
- this.settings = settings;
+ this.config = config;
this.dbClient = dbClient;
this.userIndexer = userIndexer;
this.ruleIndexer = ruleIndexer;
}
private long getSetting(String key, long defaultValue) {
- long val = settings.getLong(key);
- if (val <= 0) {
- val = defaultValue;
- }
+ long val = config.getLong(key).orElse(defaultValue);
LOGGER.debug(LOG_PREFIX + "{}={}", key, val);
return val;
}
*/
package org.sonar.server.es;
+import com.google.common.collect.Multimap;
import java.util.Collection;
-import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import org.sonar.core.util.stream.MoreCollectors;
import org.sonar.db.DbSession;
import org.sonar.db.es.EsQueueDto;
-import static java.util.stream.Collectors.toMap;
-
/**
* Clean-up the db table es_queue when documents
* are successfully indexed so that the recovery
@Override
public void onSuccess(Collection<String> docIds) {
if (!docIds.isEmpty()) {
- Map<String, EsQueueDto> itemsById = items.stream().collect(toMap(EsQueueDto::getDocId, Function.identity()));
+ Multimap<String, EsQueueDto> itemsById = items.stream().collect(MoreCollectors.index(EsQueueDto::getDocId, Function.identity()));
Collection<EsQueueDto> itemsToDelete = docIds
.stream()
.map(itemsById::get)
+ .flatMap(Collection::stream)
.filter(Objects::nonNull)
.collect(MoreCollectors.toArrayList(docIds.size()));
dbClient.esQueueDao().delete(dbSession, itemsToDelete);
apply(update, rule, userSession);
update(dbSession, rule);
updateParameters(dbSession, organization, update, rule);
- ruleIndexer.commitAndIndex(dbSession, rule.getKey());
- ruleIndexer.commitAndIndex(dbSession, organization, rule.getKey());
+ ruleIndexer.commitAndIndex(dbSession, rule.getKey(), organization);
return true;
}
import org.sonar.api.rule.RuleKey;
import org.sonar.db.rule.RuleExtensionForIndexingDto;
import org.sonar.db.rule.RuleForIndexingDto;
-import org.sonar.db.rule.RuleMetadataDto;
import org.sonar.server.es.BaseDoc;
public class RuleExtensionDoc extends BaseDoc {
@Override
public String getId() {
- return getRuleKey() + "|" + getScope().getScope();
+ return idOf(getRuleKey(), getScope());
}
@Override
return this;
}
- public static RuleExtensionDoc of(RuleKey key, RuleExtensionScope scope, RuleMetadataDto ruleExtension) {
- return new RuleExtensionDoc()
- .setRuleKey(key)
- .setScope(scope)
- .setTags(ruleExtension.getTags());
- }
-
public static RuleExtensionDoc of(RuleForIndexingDto rule) {
return new RuleExtensionDoc()
.setRuleKey(rule.getRuleKey())
.setTags(rule.getTagsAsSet());
}
+ public static String idOf(RuleKey ruleKey, RuleExtensionScope scope) {
+ return ruleKey + "|" + scope.getScope();
+ }
+
@Override
public String toString() {
return ReflectionToStringBuilder.toString(this);
import org.sonar.server.es.StartupIndexer;
import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE;
import static org.sonar.server.rule.index.RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION;
commitAndIndex(dbSession, singletonList(ruleKey));
}
- public void commitAndIndex(DbSession dbSession, Collection<RuleKey> ruleDtos) {
- List<EsQueueDto> items = ruleDtos.stream()
- .map(key -> EsQueueDto.create(EsQueueDto.Type.RULE, key.toString()))
+ public void commitAndIndex(DbSession dbSession, Collection<RuleKey> ruleKeys) {
+ List<EsQueueDto> items = ruleKeys.stream()
+ .map(RuleIndexer::createQueueDtoForRule)
.collect(MoreCollectors.toArrayList());
dbClient.esQueueDao().insert(dbSession, items);
dbSession.commit();
- postCommit(dbSession, ruleDtos, items);
+ postCommit(dbSession, items);
}
- public void commitAndIndex(DbSession dbSession, OrganizationDto organizationDto, RuleKey ruleKey) {
- List<EsQueueDto> items = singletonList(EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, ruleKey + "|" + organizationDto.getUuid()));
-
+ /**
+ * Commit a change on a rule and its extension on the given organization
+ */
+ public void commitAndIndex(DbSession dbSession, RuleKey ruleKey, OrganizationDto organization) {
+ List<EsQueueDto> items = asList(createQueueDtoForRule(ruleKey), createQueueDtoForRuleExtension(ruleKey, organization));
dbClient.esQueueDao().insert(dbSession, items);
dbSession.commit();
- postCommit(dbSession, ruleKey, organizationDto, items);
+ postCommit(dbSession, items);
}
/**
- * Entry point for Byteman tests. See directory tests/resilience.
- * The parameter "ruleKeys" is used only by the Byteman script.
+ * This method is used by the Byteman script of integration tests.
*/
- private void postCommit(DbSession dbSession, Collection<RuleKey> ruleKeys, Collection<EsQueueDto> items) {
- index(dbSession, items);
- }
-
- private void postCommit(DbSession dbSession, RuleKey ruleKeys, OrganizationDto organizationDto, Collection<EsQueueDto> items) {
+ private void postCommit(DbSession dbSession, List<EsQueueDto> items) {
index(dbSession, items);
}
@Override
public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
- if (items.isEmpty()) {
- return new IndexingResult();
- }
-
IndexingResult result = new IndexingResult();
-
- ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
-
- result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE)));
- result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION)));
-
+ if (!items.isEmpty()) {
+ ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items);
+ result.add(doIndexRules(dbSession, itemsByType.get(EsQueueDto.Type.RULE)));
+ result.add(doIndexRuleExtensions(dbSession, itemsByType.get(EsQueueDto.Type.RULE_EXTENSION)));
+ }
return result;
}
BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
bulkIndexer.start();
- Set<RuleKey> rules = items
+ List<RuleKey> ruleKeys = items
.stream()
.filter(i -> {
requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
return i.getDocType() == EsQueueDto.Type.RULE;
})
.map(i -> RuleKey.parse(i.getDocId()))
- .collect(toHashSet(items.size()));
+ .collect(toArrayList(items.size()));
- dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, rules,
+ dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleKeys,
// only index requests, no deletion requests.
// Deactivated users are not deleted but updated.
r -> {
- rules.remove(r.getRuleKey());
bulkIndexer.add(newRuleDocIndexRequest(r));
bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
+ ruleKeys.remove(r.getRuleKey());
});
// the remaining items reference rows that don't exist in db. They must
// be deleted from index.
- rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString()));
- rules.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.toString()));
+ ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE, r.toString(), r.toString()));
+ ruleKeys.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(r, RuleExtensionScope.system()), r.toString()));
return bulkIndexer.stop();
}
BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new ResiliencyIndexingListener(dbClient, dbSession, items));
bulkIndexer.start();
- Set<RuleExtensionId> docIds = items
+ List<RuleExtensionId> docIds = items
.stream()
.filter(i -> {
requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
return i.getDocType() == EsQueueDto.Type.RULE_EXTENSION;
})
.map(RuleIndexer::explodeRuleExtensionDocId)
- .collect(toHashSet(items.size()));
+ .collect(toArrayList(items.size()));
dbClient.ruleDao().scrollIndexingRuleExtensionsByIds(dbSession, docIds,
// only index requests, no deletion requests.
// Deactivated users are not deleted but updated.
r -> {
- docIds.remove(new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey()));
+ RuleExtensionId docId = new RuleExtensionId(r.getOrganizationUuid(), r.getPluginName(), r.getPluginRuleKey());
+ docIds.remove(docId);
bulkIndexer.add(newRuleExtensionDocIndexRequest(r));
});
// the remaining items reference rows that don't exist in db. They must
// be deleted from index.
- docIds.forEach(r -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, r.getId()));
+ docIds.forEach(docId -> {
+ RuleKey ruleKey = RuleKey.of(docId.getRepositoryName(), docId.getRuleKey());
+ bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_RULE_EXTENSION, docId.getId(), ruleKey.toString());
+ });
return bulkIndexer.stop();
}
checkArgument(esQueueDto.getDocType() == EsQueueDto.Type.RULE_EXTENSION);
return new RuleExtensionId(esQueueDto.getDocId());
}
+
+ private static EsQueueDto createQueueDtoForRule(RuleKey ruleKey) {
+ return EsQueueDto.create(EsQueueDto.Type.RULE, ruleKey.toString(), null, ruleKey.toString());
+ }
+
+ private static EsQueueDto createQueueDtoForRuleExtension(RuleKey ruleKey, OrganizationDto organization) {
+ String docId = RuleExtensionDoc.idOf(ruleKey, RuleExtensionScope.organization(organization));
+ return EsQueueDto.create(EsQueueDto.Type.RULE_EXTENSION, docId, null, ruleKey.toString());
+ }
+
}
import org.sonar.server.es.EsClient;
import org.sonar.server.es.IndexType;
import org.sonar.server.es.IndexingListener;
+import org.sonar.server.es.IndexingResult;
import org.sonar.server.es.ResiliencyIndexingListener;
import org.sonar.server.es.ResilientIndexer;
-import org.sonar.server.es.IndexingResult;
import org.sonar.server.es.StartupIndexer;
import static java.util.Collections.singletonList;
import static java.util.Objects.requireNonNull;
-import static org.sonar.core.util.stream.MoreCollectors.toHashSet;
+import static org.sonar.core.util.stream.MoreCollectors.toArrayList;
import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER;
public class UserIndexer implements StartupIndexer, ResilientIndexer {
if (items.isEmpty()) {
return new IndexingResult();
}
- Set<String> logins = items
+ List<String> logins = items
.stream()
.filter(i -> {
requireNonNull(i.getDocId(), () -> "BUG - " + i + " has not been persisted before indexing");
return i.getDocType() == EsQueueDto.Type.USER;
})
.map(EsQueueDto::getDocId)
- .collect(toHashSet(items.size()));
+ .collect(toArrayList(items.size()));
ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create();
dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put);
import org.junit.rules.DisableOnDebug;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
-import org.sonar.api.config.Settings;
import org.sonar.api.config.internal.MapSettings;
import org.sonar.api.utils.internal.TestSystem2;
import org.sonar.api.utils.log.LogTester;
import org.sonar.server.user.index.UserIndexDefinition;
import org.sonar.server.user.index.UserIndexer;
+import static java.util.Arrays.asList;
import static java.util.stream.IntStream.rangeClosed;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.doAnswer;
@Rule
public final LogTester logTester = new LogTester().setLevel(TRACE);
@Rule
- public TestRule safeguard = new DisableOnDebug(new Timeout(60, TimeUnit.SECONDS));
+ public TestRule safeguard = new DisableOnDebug(Timeout.builder().withTimeout(60, TimeUnit.SECONDS).withLookingForStuckThread(true).build());
private UserIndexer mockedUserIndexer = mock(UserIndexer.class);
private RuleIndexer mockedRuleIndexer = mock(RuleIndexer.class);
@Test
public void start_triggers_recovery_run_at_fixed_rate() throws Exception {
- Settings settings = new MapSettings()
+ MapSettings settings = new MapSettings()
.setProperty("sonar.search.recovery.initialDelayInMs", "0")
.setProperty("sonar.search.recovery.delayInMs", "1");
- underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mockedUserIndexer, mockedRuleIndexer, mockedActiveRuleIndexer));
+ underTest = spy(new RecoveryIndexer(system2, settings.asConfig(), db.getDbClient(), mockedUserIndexer, mockedRuleIndexer, mockedActiveRuleIndexer));
AtomicInteger calls = new AtomicInteger(0);
doAnswer(invocation -> {
calls.incrementAndGet();
// 10 docs to process, by groups of 3.
// The first group successfully recovers only 1 docs --> above 30% of failures --> stop run
PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(1);
- Settings settings = new MapSettings()
+ MapSettings settings = new MapSettings()
.setProperty("sonar.search.recovery.loopLimit", "3");
underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings);
underTest.recover();
// 10 docs to process, by groups of 5.
// Each group successfully recovers 4 docs --> below 30% of failures --> continue run
PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(4, 4, 2);
- Settings settings = new MapSettings()
+ MapSettings settings = new MapSettings()
.setProperty("sonar.search.recovery.loopLimit", "5");
underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, mockedRuleIndexer, settings);
underTest.recover();
assertThatQueueHasSize(1);
}
+ @Test
+ public void recover_multiple_times_the_same_document() {
+ UserDto user = db.users().insertUser();
+ EsQueueDto item1 = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin());
+ EsQueueDto item2 = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin());
+ EsQueueDto item3 = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin());
+ db.getDbClient().esQueueDao().insert(db.getSession(), asList(item1, item2, item3));
+ db.commit();
+
+ ProxyUserIndexer userIndexer = new ProxyUserIndexer();
+ advanceInTime();
+ underTest = newRecoveryIndexer(userIndexer, mockedRuleIndexer);
+ underTest.recover();
+
+ assertThatQueueHasSize(0);
+ assertThat(userIndexer.called)
+ .extracting(EsQueueDto::getUuid)
+ .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid(), item3.getUuid());
+
+ assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 3 USER");
+ assertThatLogsContain(INFO, "Elasticsearch recovery - 3 documents processed [0 failures]");
+ }
+
private class ProxyUserIndexer extends UserIndexer {
private final List<EsQueueDto> called = new ArrayList<>();
List<EsQueueDto> filteredItems = items.stream().filter(
i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList());
IndexingResult result = super.index(dbSession, filteredItems);
- if (items.contains(failing)) {
+ if (result.getTotal() == items.size() - 1) {
+ // the failing item was in the items list
result.incrementRequests();
}
}
private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer) {
- Settings settings = new MapSettings()
+ MapSettings settings = new MapSettings()
.setProperty("sonar.search.recovery.initialDelayInMs", "0")
.setProperty("sonar.search.recovery.delayInMs", "1")
.setProperty("sonar.search.recovery.minAgeInMs", "1");
return newRecoveryIndexer(userIndexer, ruleIndexer, settings);
}
- private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, Settings settings) {
- return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer, ruleIndexer, mockedActiveRuleIndexer);
+ private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, RuleIndexer ruleIndexer, MapSettings settings) {
+ return new RecoveryIndexer(system2, settings.asConfig(), db.getDbClient(), userIndexer, ruleIndexer, mockedActiveRuleIndexer);
}
private EsQueueDto createUnindexedUser() {
RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag1"));
ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag2"));
- ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
+ ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization);
RuleDefinitionDto r2 = dbTester.rules().insert(setSystemTags("tag3"));
ruleIndexer.commitAndIndex(dbTester.getSession(), r2.getKey());
dbTester.rules().insertOrUpdateMetadata(r2, organization, setTags("tag4", "tag5"));
- ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r2.getKey());
+ ruleIndexer.commitAndIndex(dbTester.getSession(), r2.getKey(), organization);
String result = tester.newRequest()
.setParam("organization", organization.getKey())
RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("tag6"));
ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag7"));
- ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
+ ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization);
String result = tester.newRequest()
.setParam("organization", organization.getKey())
RuleDefinitionDto r = dbTester.rules().insert(setSystemTags("cwe"));
ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("security"));
- ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
+ ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization);
String result = tester.newRequest()
.setParam("organization", organization.getKey())
underTest.commitAndIndex(dbTester.getSession(), rule.getKey());
OrganizationDto organization = dbTester.organizations().insert();
dbTester.rules().insertOrUpdateMetadata(rule, organization, m -> m.setTags(ImmutableSet.of("bla")));
- underTest.commitAndIndex(dbTester.getSession(), organization, rule.getKey());
+ underTest.commitAndIndex(dbTester.getSession(), rule.getKey(), organization);
RuleExtensionDoc doc = new RuleExtensionDoc()
.setRuleKey(rule.getKey())
@SafeVarargs
private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer<RuleMetadataDto>... populaters) {
RuleMetadataDto metadata = db.rules().insertOrUpdateMetadata(rule, organization, populaters);
- ruleIndexer.commitAndIndex(db.getSession(), organization, rule.getKey());
+ ruleIndexer.commitAndIndex(db.getSession(), rule.getKey(), organization);
return metadata;
}
@SafeVarargs
private final RuleMetadataDto insertMetadata(OrganizationDto organization, RuleDefinitionDto rule, Consumer<RuleMetadataDto>... populaters) {
RuleMetadataDto metadata = dbTester.rules().insertOrUpdateMetadata(rule, organization, populaters);
- ruleIndexer.commitAndIndex(dbTester.getSession(), organization, rule.getKey());
+ ruleIndexer.commitAndIndex(dbTester.getSession(), rule.getKey(), organization);
return metadata;
}
}
RuleDefinitionDto r = dbTester.rules().insert(setSystemTags());
ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey());
dbTester.rules().insertOrUpdateMetadata(r, organization, setTags("tag"));
- ruleIndexer.commitAndIndex(dbTester.getSession(), organization, r.getKey());
+ ruleIndexer.commitAndIndex(dbTester.getSession(), r.getKey(), organization);
String result = tester.newRequest().setParam("organization", organization.getKey()).execute().getInput();
assertJson(result).isSimilarTo("{\"tags\":[\"tag\"]}");
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonarqube.ws.client.rule;
+
+public class CreateWsRequest {
+
+ private String customKey;
+ private String markdownDescription;
+ private String name;
+ private String params;
+ private Boolean preventReactivation;
+ private String severity;
+ private String status;
+ private String templateKey;
+
+ private CreateWsRequest(Builder builder) {
+ this.customKey = builder.customKey;
+ this.markdownDescription = builder.markdownDescription;
+ this.name = builder.name;
+ this.params = builder.params;
+ this.preventReactivation = builder.preventReactivation;
+ this.severity = builder.severity;
+ this.status = builder.status;
+ this.templateKey = builder.templateKey;
+ }
+
+ public String getCustomKey() {
+ return customKey;
+ }
+
+ public String getMarkdownDescription() {
+ return markdownDescription;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getParams() {
+ return params;
+ }
+
+ public Boolean getPreventReactivation() {
+ return preventReactivation;
+ }
+
+ public String getSeverity() {
+ return severity;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public String getTemplateKey() {
+ return templateKey;
+ }
+
+ public static class Builder {
+ private String customKey;
+ private String markdownDescription;
+ private String name;
+ private String params;
+ private Boolean preventReactivation;
+ private String severity;
+ private String status;
+ private String templateKey;
+
+ public Builder setCustomKey(String customKey) {
+ this.customKey = customKey;
+ return this;
+ }
+
+ public Builder setMarkdownDescription(String markdownDescription) {
+ this.markdownDescription = markdownDescription;
+ return this;
+ }
+
+ public Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder setParams(String params) {
+ this.params = params;
+ return this;
+ }
+
+ public Builder setPreventReactivation(Boolean preventReactivation) {
+ this.preventReactivation = preventReactivation;
+ return this;
+ }
+
+ public Builder setSeverity(String severity) {
+ this.severity = severity;
+ return this;
+ }
+
+ public Builder setStatus(String status) {
+ this.status = status;
+ return this;
+ }
+
+ public Builder setTemplateKey(String templateKey) {
+ this.templateKey = templateKey;
+ return this;
+ }
+
+ public CreateWsRequest build() {
+ return new CreateWsRequest(this);
+ }
+ }
+
+}
import org.sonarqube.ws.Rules.SearchResponse;
import org.sonarqube.ws.client.BaseService;
import org.sonarqube.ws.client.GetRequest;
+import org.sonarqube.ws.client.PostRequest;
import org.sonarqube.ws.client.WsConnector;
import static org.sonarqube.ws.client.rule.RulesWsParameters.PARAM_ACTIVATION;
.setParam("key", key);
return call(request, Rules.ShowResponse.parser());
}
+
+ public void create(CreateWsRequest request) {
+ PostRequest httpRequest = new PostRequest(path("create"));
+ httpRequest.setParam("custom_key", request.getCustomKey());
+ httpRequest.setParam("markdown_description", request.getMarkdownDescription());
+ httpRequest.setParam("name", request.getName());
+ httpRequest.setParam("params", request.getParams());
+ httpRequest.setParam("prevent_reactivation", request.getPreventReactivation());
+ httpRequest.setParam("severity", request.getSeverity());
+ httpRequest.setParam("status", request.getStatus());
+ httpRequest.setParam("template_key", request.getTemplateKey());
+ call(httpRequest);
+ }
}
import org.sonarqube.ws.Rules;
import org.sonarqube.ws.Rules.SearchResponse;
import org.sonarqube.ws.client.GetRequest;
+import org.sonarqube.ws.client.PostRequest;
import org.sonarqube.ws.client.ServiceTester;
import org.sonarqube.ws.client.WsConnector;
.hasParam("key", "the-rule/key")
.andNoOtherParam();
}
+
+ @Test
+ public void test_create() {
+ underTest.create(new CreateWsRequest.Builder()
+ .setTemplateKey("the-template-key")
+ .setCustomKey("the-custom-key")
+ .setSeverity("BLOCKER")
+ .setParams("the-params")
+ .setPreventReactivation(true)
+ .setMarkdownDescription("the-desc")
+ .setStatus("BETA")
+ .setName("the-name")
+ .build());
+
+ PostRequest postRequest = serviceTester.getPostRequest();
+ serviceTester.assertThat(postRequest)
+ .hasPath("create")
+ .hasParam("template_key", "the-template-key")
+ .hasParam("custom_key", "the-custom-key")
+ .hasParam("severity", "BLOCKER")
+ .hasParam("params", "the-params")
+ .hasParam("prevent_reactivation", "true")
+ .hasParam("markdown_description", "the-desc")
+ .hasParam("status", "BETA")
+ .hasParam("name", "the-name")
+ .andNoOtherParam();
+ }
}
--- /dev/null
+# sonar.web.javaAdditionalOpts=-javaagent:/path/to/byteman-3.0.10/lib/byteman.jar=script:/path/to/active_rule_indexer.btm,boot:/path/to/byteman-3.0.10/lib/byteman.jar
+# sonar.search.recovery.delayInMs=10000
+# sonar.search.recovery.minAgeInMs=30000
+
+RULE make indexing of custom rules silently fail
+CLASS RuleIndexer
+METHOD postCommit
+COMPILE
+AT ENTRY
+IF TRUE
+DO RETURN
+ENDRULE
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+ package org.sonarqube.tests;
+
+import com.sonar.orchestrator.OrchestratorBuilder;
+import java.io.File;
+
+import static java.lang.String.format;
+
+/**
+ * byteman.jboss.org is used by resilience tests in order to
+ * change behavior of server at runtime.
+ */
+public class Byteman {
+
+ public static OrchestratorBuilder enableScript(OrchestratorBuilder builder, String filename) {
+ String jar = findBytemanJar();
+ builder
+ .setServerProperty("sonar.web.javaAdditionalOpts",
+ format("-javaagent:%s=script:%s,boot:%s", jar, findBytemanScript(filename), jar))
+ .setServerProperty("sonar.search.recovery.delayInMs", "1000")
+ .setServerProperty("sonar.search.recovery.minAgeInMs", "3000");
+ return builder;
+ }
+
+ private static String findBytemanJar() {
+ // see pom.xml, Maven copies and renames the artifact.
+ File jar = new File("target/byteman.jar");
+ if (!jar.exists()) {
+ throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' once in directory tests/.");
+ }
+ return jar.getAbsolutePath();
+ }
+
+ private static String findBytemanScript(String filename) {
+ // see pom.xml, Maven copies and renames the artifact.
+ File script = new File( filename);
+ if (!script.exists()) {
+ throw new IllegalStateException("Can't find " + script);
+ }
+ return script.getAbsolutePath();
+ }
+}
import org.junit.runners.Suite;
import org.sonarqube.tests.qualityProfile.ActiveRuleEsResilienceTest;
import org.sonarqube.tests.qualityProfile.BuiltInQualityProfilesNotificationTest;
+import org.sonarqube.tests.rule.RuleEsResilienceTest;
import org.sonarqube.tests.serverSystem.ClusterTest;
import org.sonarqube.tests.serverSystem.RestartTest;
import org.sonarqube.tests.serverSystem.ServerSystemRestartingOrchestrator;
OnboardingTest.class,
BuiltInQualityProfilesNotificationTest.class,
ActiveRuleEsResilienceTest.class,
+ RuleEsResilienceTest.class,
UserEsResilienceTest.class
})
public class Category5Suite {
package org.sonarqube.tests.qualityProfile;
import com.sonar.orchestrator.Orchestrator;
-import java.io.File;
import java.util.concurrent.TimeUnit;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.rules.DisableOnDebug;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
+import org.sonarqube.tests.Byteman;
import org.sonarqube.tests.Tester;
import org.sonarqube.ws.Organizations;
import org.sonarqube.ws.QualityProfiles;
import org.sonarqube.ws.client.rule.SearchWsRequest;
import util.ItUtils;
-import static java.lang.String.format;
import static org.assertj.core.api.Assertions.assertThat;
public class ActiveRuleEsResilienceTest {
private static final String RULE_ONE_BUG_PER_LINE = "xoo:OneBugIssuePerLine";
@ClassRule
- public static final Orchestrator orchestrator = Orchestrator.builderEnv()
- .setServerProperty("sonar.web.javaAdditionalOpts",
- format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar()))
- .setServerProperty("sonar.search.recovery.delayInMs", "500")
- .setServerProperty("sonar.search.recovery.minAgeInMs", "3000")
- .addPlugin(ItUtils.xooPlugin())
- .build();
+ public static final Orchestrator orchestrator;
+
+ static {
+ orchestrator = Byteman.enableScript(Orchestrator.builderEnv(), "resilience/active_rule_indexer.btm")
+ .addPlugin(ItUtils.xooPlugin())
+ .build();
+ }
@Rule
public TestRule timeout = new DisableOnDebug(Timeout.builder()
SearchWsRequest request = new SearchWsRequest().setActivation(true).setQProfile(profile.getKey());
return tester.wsClient().rules().search(request).getRulesCount();
}
-
- private static String findBytemanJar() {
- // see pom.xml, Maven copies and renames the artifact.
- File jar = new File("target/byteman.jar");
- if (!jar.exists()) {
- throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once.");
- }
- return jar.getAbsolutePath();
- }
-
- private static String findBytemanScript() {
- // see pom.xml, Maven copies and renames the artifact.
- File script = new File("resilience/active_rule_indexer.btm");
- if (!script.exists()) {
- throw new IllegalStateException("Can't find " + script);
- }
- return script.getAbsolutePath();
- }
}
--- /dev/null
+/*
+ * SonarQube
+ * Copyright (C) 2009-2017 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonarqube.tests.rule;
+
+import com.sonar.orchestrator.Orchestrator;
+import java.util.concurrent.TimeUnit;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.DisableOnDebug;
+import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
+import org.sonarqube.tests.Byteman;
+import org.sonarqube.tests.Tester;
+import org.sonarqube.ws.client.rule.CreateWsRequest;
+import org.sonarqube.ws.client.rule.SearchWsRequest;
+import util.ItUtils;
+
+import static java.util.Collections.singletonList;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class RuleEsResilienceTest {
+
+ @ClassRule
+ public static final Orchestrator orchestrator;
+
+ static {
+ orchestrator = Byteman.enableScript(Orchestrator.builderEnv(), "resilience/rule_indexer.btm")
+ .addPlugin(ItUtils.xooPlugin())
+ .build();
+ }
+
+ @Rule
+ public TestRule timeout = new DisableOnDebug(Timeout.builder()
+ .withLookingForStuckThread(true)
+ .withTimeout(60L, TimeUnit.SECONDS)
+ .build());
+
+ @Rule
+ public Tester tester = new Tester(orchestrator)
+ // custom rules are not supported when organizations are enabled
+ .disableOrganizations();
+
+ @Test
+ public void creation_of_custom_rule_is_resilient_to_elasticsearch_errors() throws Exception {
+ CreateWsRequest request = new CreateWsRequest.Builder()
+ .setCustomKey("my_custom_rule")
+ .setName("My custom rule")
+ .setTemplateKey("xoo:xoo-template")
+ .setMarkdownDescription("The *initial* rule")
+ .setSeverity("MAJOR")
+ .build();
+ tester.wsClient().rules().create(request);
+
+ // rule exists in db but is not indexed. Search returns no results.
+ assertThat(nameFoundInSearch("initial rule")).isFalse();
+
+ // rule already exists in db, can't be created twice
+ ItUtils.expectHttpError(400, () -> tester.wsClient().rules().create(request));
+
+ while (!nameFoundInSearch("initial rule")) {
+ // rule is indexed by the recovery daemon, which runs every 3 seconds
+ Thread.sleep(500L);
+ }
+ }
+
+ private boolean nameFoundInSearch(String query) {
+ SearchWsRequest request = new SearchWsRequest()
+ .setQuery(query)
+ .setRepositories(singletonList("xoo"));
+ return tester.wsClient().rules().search(request).getRulesCount() > 0;
+ }
+}
package org.sonarqube.tests.user;
import com.sonar.orchestrator.Orchestrator;
-import java.io.File;
import java.util.concurrent.TimeUnit;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.rules.DisableOnDebug;
import org.junit.rules.TestRule;
import org.junit.rules.Timeout;
+import org.sonarqube.tests.Byteman;
import org.sonarqube.tests.Tester;
import org.sonarqube.ws.WsUsers.CreateWsResponse.User;
import org.sonarqube.ws.client.user.SearchRequest;
import org.sonarqube.ws.client.user.UpdateRequest;
-import static java.lang.String.format;
import static org.assertj.core.api.Assertions.assertThat;
import static util.ItUtils.expectHttpError;
public class UserEsResilienceTest {
@ClassRule
- public static final Orchestrator orchestrator = Orchestrator.builderEnv()
- .setServerProperty("sonar.web.javaAdditionalOpts",
- format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar()))
- .setServerProperty("sonar.search.recovery.delayInMs", "1000")
- .setServerProperty("sonar.search.recovery.minAgeInMs", "3000")
- .build();
+ public static final Orchestrator orchestrator;
+
+ static {
+ orchestrator = Byteman.enableScript(Orchestrator.builderEnv(), "resilience/user_indexer.btm").build();
+ }
@Rule
public TestRule timeout = new DisableOnDebug(Timeout.builder()
return tester.users().service().search(SearchRequest.builder().setQuery(name).build()).getUsersCount() == 1L;
}
- private static String findBytemanJar() {
- // see pom.xml, Maven copies and renames the artifact.
- File jar = new File("target/byteman.jar");
- if (!jar.exists()) {
- throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once.");
- }
- return jar.getAbsolutePath();
- }
-
- private static String findBytemanScript() {
- // see pom.xml, Maven copies and renames the artifact.
- File script = new File("resilience/user_indexer.btm");
- if (!script.exists()) {
- throw new IllegalStateException("Can't find " + script);
- }
- return script.getAbsolutePath();
- }
}
projects / sonarqube.git / commitdiff