diff options
86 files changed, 2229 insertions, 897 deletions
diff --git a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java index ca5b507a24b..9b009c3452f 100644 --- a/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java +++ b/server/sonar-ce/src/test/java/org/sonar/ce/container/ComputeEngineContainerImplTest.java @@ -135,7 +135,7 @@ public class ComputeEngineContainerImplTest { assertThat(picoContainer.getParent().getParent().getParent().getComponentAdapters()).hasSize( COMPONENTS_IN_LEVEL_1_AT_CONSTRUCTION + 24 // level 1 - + 46 // content of DaoModule + + 47 // content of DaoModule + 3 // content of EsSearchModule + 58 // content of CorePropertyDefinitions ); diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java b/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java index 6e8f7baa74f..09482d16d00 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/DatabaseUtils.java @@ -237,21 +237,6 @@ public class DatabaseUtils { } /** - * @deprecated replaced by Commons Land {@code StringUtils.repeat(sql, separator, count) - */ - @Deprecated - public static String repeatCondition(String sql, int count, String separator) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < count; i++) { - sb.append(sql); - if (i < count - 1) { - sb.append(" ").append(separator).append(" "); - } - } - return sb.toString(); - } - - /** * Logback does not log exceptions associated to {@link java.sql.SQLException#getNextException()}. * See http://jira.qos.ch/browse/LOGBACK-775 */ diff --git a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java index c86a1b6a92c..bff4cb2eb80 100644 --- a/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java +++ b/server/sonar-db-core/src/main/java/org/sonar/db/version/SqTables.java @@ -54,6 +54,7 @@ public final class SqTables { "ce_scanner_context", "default_qprofiles", "duplications_index", + "es_queue", "events", "file_sources", "groups", diff --git a/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl b/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl index 5901161d0d1..202db3d5e1f 100644 --- a/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl +++ b/server/sonar-db-core/src/main/resources/org/sonar/db/version/schema-h2.ddl @@ -658,3 +658,12 @@ CREATE TABLE "WEBHOOK_DELIVERIES" ( CREATE UNIQUE INDEX "PK_WEBHOOK_DELIVERIES" ON "WEBHOOK_DELIVERIES" ("UUID"); CREATE INDEX "COMPONENT_UUID" ON "WEBHOOK_DELIVERIES" ("COMPONENT_UUID"); CREATE INDEX "CE_TASK_UUID" ON "WEBHOOK_DELIVERIES" ("CE_TASK_UUID"); + +CREATE TABLE "ES_QUEUE" ( + "UUID" VARCHAR(40) NOT NULL PRIMARY KEY, + "DOC_TYPE" VARCHAR(40) NOT NULL, + "DOC_UUID" VARCHAR(255) NOT NULL, + "CREATED_AT" BIGINT NOT NULL +); +CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID"); +CREATE INDEX "ES_QUEUE_CREATED_AT" ON "ES_QUEUE" ("CREATED_AT"); diff --git a/server/sonar-db-core/src/test/java/org/sonar/db/DatabaseUtilsTest.java b/server/sonar-db-core/src/test/java/org/sonar/db/DatabaseUtilsTest.java index 88dcd5e5a2e..49060d8bf60 100644 --- a/server/sonar-db-core/src/test/java/org/sonar/db/DatabaseUtilsTest.java +++ b/server/sonar-db-core/src/test/java/org/sonar/db/DatabaseUtilsTest.java @@ -245,12 +245,6 @@ public class DatabaseUtilsTest { } @Test - public void repeatCondition() { - assertThat(DatabaseUtils.repeatCondition("uuid=?", 1, "or")).isEqualTo("uuid=?"); - assertThat(DatabaseUtils.repeatCondition("uuid=?", 3, "or")).isEqualTo("uuid=? or uuid=? or uuid=?"); - } - - @Test public void executeLargeInputs() { List<Integer> inputs = newArrayList(); List<String> expectedOutputs = newArrayList(); diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java index d6ad5559e52..9ee4a9586cf 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DaoModule.java @@ -35,6 +35,7 @@ import org.sonar.db.duplication.DuplicationDao; import org.sonar.db.event.EventDao; import org.sonar.db.issue.IssueChangeDao; import org.sonar.db.issue.IssueDao; +import org.sonar.db.es.EsQueueDao; import org.sonar.db.loadedtemplate.LoadedTemplateDao; import org.sonar.db.measure.MeasureDao; import org.sonar.db.measure.custom.CustomMeasureDao; @@ -86,6 +87,7 @@ public class DaoModule extends Module { CustomMeasureDao.class, DefaultQProfileDao.class, DuplicationDao.class, + EsQueueDao.class, EventDao.class, FileSourceDao.class, GroupDao.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java index f2eb34c6bb1..6eb7c664a25 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/DbClient.java @@ -34,6 +34,7 @@ import org.sonar.db.duplication.DuplicationDao; import org.sonar.db.event.EventDao; import org.sonar.db.issue.IssueChangeDao; import org.sonar.db.issue.IssueDao; +import org.sonar.db.es.EsQueueDao; import org.sonar.db.loadedtemplate.LoadedTemplateDao; import org.sonar.db.measure.MeasureDao; import org.sonar.db.measure.custom.CustomMeasureDao; @@ -118,6 +119,7 @@ public class DbClient { private final UserPermissionDao userPermissionDao; private final WebhookDeliveryDao webhookDeliveryDao; private final DefaultQProfileDao defaultQProfileDao; + private final EsQueueDao esQueueDao; public DbClient(Database database, MyBatis myBatis, Dao... daos) { this.database = database; @@ -173,6 +175,7 @@ public class DbClient { userPermissionDao = getDao(map, UserPermissionDao.class); webhookDeliveryDao = getDao(map, WebhookDeliveryDao.class); defaultQProfileDao = getDao(map, DefaultQProfileDao.class); + esQueueDao = getDao(map, EsQueueDao.class); } public DbSession openSession(boolean batch) { @@ -367,6 +370,10 @@ public class DbClient { return defaultQProfileDao; } + public EsQueueDao esQueueDao() { + return esQueueDao; + } + protected <K extends Dao> K getDao(Map<Class, Dao> map, Class<K> clazz) { return (K) map.get(clazz); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java index e99aec847ea..044086a79b3 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/MyBatis.java @@ -55,6 +55,7 @@ import org.sonar.db.issue.IssueChangeDto; import org.sonar.db.issue.IssueChangeMapper; import org.sonar.db.issue.IssueDto; import org.sonar.db.issue.IssueMapper; +import org.sonar.db.es.EsQueueMapper; import org.sonar.db.loadedtemplate.LoadedTemplateDto; import org.sonar.db.loadedtemplate.LoadedTemplateMapper; import org.sonar.db.measure.MeasureDto; @@ -197,6 +198,7 @@ public class MyBatis implements Startable { CustomMeasureMapper.class, DefaultQProfileMapper.class, DuplicationMapper.class, + EsQueueMapper.class, EventMapper.class, FileSourceMapper.class, GroupMapper.class, diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDao.java new file mode 100644 index 00000000000..c47e087047d --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDao.java @@ -0,0 +1,79 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.es; + +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import org.sonar.api.utils.System2; +import org.sonar.core.util.UuidFactory; +import org.sonar.db.Dao; +import org.sonar.db.DbSession; + +import static java.util.Collections.singletonList; +import static org.sonar.core.util.stream.MoreCollectors.toArrayList; +import static org.sonar.db.DatabaseUtils.executeLargeUpdates; + +public class EsQueueDao implements Dao { + + private final System2 system2; + private final UuidFactory uuidFactory; + + public EsQueueDao(System2 system2, UuidFactory uuidFactory) { + this.system2 = system2; + this.uuidFactory = uuidFactory; + } + + public EsQueueDto insert(DbSession dbSession, EsQueueDto item) { + insert(dbSession, singletonList(item)); + return item; + } + + public Collection<EsQueueDto> insert(DbSession dbSession, Collection<EsQueueDto> items) { + long now = system2.now(); + EsQueueMapper mapper = mapper(dbSession); + items.forEach(item -> { + item.setUuid(uuidFactory.create()); + mapper.insert(item, now); + }); + return items; + } + + public void delete(DbSession dbSession, EsQueueDto item) { + delete(dbSession, singletonList(item)); + } + + public void delete(DbSession dbSession, Collection<EsQueueDto> items) { + EsQueueMapper mapper = mapper(dbSession); + List<String> uuids = items.stream() + .map(EsQueueDto::getUuid) + .filter(Objects::nonNull) + .collect(toArrayList(items.size())); + executeLargeUpdates(uuids, mapper::delete); + } + + public Collection<EsQueueDto> selectForRecovery(DbSession dbSession, long beforeDate, long limit) { + return mapper(dbSession).selectForRecovery(beforeDate, limit); + } + + private static EsQueueMapper mapper(DbSession dbSession) { + return dbSession.getMapper(EsQueueMapper.class); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java new file mode 100644 index 00000000000..feb3148fa57 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueDto.java @@ -0,0 +1,72 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.es; + +public final class EsQueueDto { + + public enum Type { + USER + } + + private String uuid; + private Type docType; + private String docUuid; + + public String getUuid() { + return uuid; + } + + EsQueueDto setUuid(String uuid) { + this.uuid = uuid; + return this; + } + + public Type getDocType() { + return docType; + } + + private EsQueueDto setDocType(Type t) { + this.docType = t; + return this; + } + + public String getDocUuid() { + return docUuid; + } + + private EsQueueDto setDocUuid(String s) { + this.docUuid = s; + return this; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("EsQueueDto{"); + sb.append("uuid='").append(uuid).append('\''); + sb.append(", docType=").append(docType); + sb.append(", docUuid='").append(docUuid).append('\''); + sb.append('}'); + return sb.toString(); + } + + public static EsQueueDto create(Type docType, String docUuid) { + return new EsQueueDto().setDocType(docType).setDocUuid(docUuid); + } +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueMapper.java new file mode 100644 index 00000000000..04837587182 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/EsQueueMapper.java @@ -0,0 +1,33 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.es; + +import java.util.Collection; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface EsQueueMapper { + + void insert(@Param("dto") EsQueueDto dto, @Param("now") long now); + + void delete(@Param("uuids") List<String> uuids); + + Collection<EsQueueDto> selectForRecovery(@Param("beforeDate") long beforeDate, @Param("limit") long limit); +} diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/es/package-info.java b/server/sonar-db-dao/src/main/java/org/sonar/db/es/package-info.java new file mode 100644 index 00000000000..5b01ba2c8f2 --- /dev/null +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/es/package-info.java @@ -0,0 +1,24 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.db.es; + +import javax.annotation.ParametersAreNonnullByDefault; + diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/organization/OrganizationMemberDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/organization/OrganizationMemberDao.java index abc23c06245..4b0c91bcf32 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/organization/OrganizationMemberDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/organization/OrganizationMemberDao.java @@ -20,6 +20,7 @@ package org.sonar.db.organization; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; @@ -70,7 +71,7 @@ public class OrganizationMemberDao implements Dao { * * @param loginOrganizationConsumer {@link BiConsumer}<String,String> (login, organization uuid) */ - public void selectForUserIndexing(DbSession dbSession, List<String> logins, BiConsumer<String, String> loginOrganizationConsumer) { + public void selectForUserIndexing(DbSession dbSession, Collection<String> logins, BiConsumer<String, String> loginOrganizationConsumer) { executeLargeInputsWithoutOutput(logins, list -> mapper(dbSession).selectForIndexing(list) .forEach(row -> loginOrganizationConsumer.accept(row.get("login"), row.get("organizationUuid")))); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/property/PropertiesDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/property/PropertiesDao.java index b0156a8c79a..966b5a564b0 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/property/PropertiesDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/property/PropertiesDao.java @@ -33,12 +33,12 @@ import javax.annotation.Nullable; import org.sonar.api.resources.Scopes; import org.sonar.api.utils.System2; import org.sonar.db.Dao; -import org.sonar.db.DatabaseUtils; import org.sonar.db.DbSession; import org.sonar.db.MyBatis; import org.sonar.db.WildcardPosition; import static com.google.common.base.Preconditions.checkArgument; +import static org.apache.commons.lang.StringUtils.repeat; import static org.sonar.db.DaoDatabaseUtils.buildLikeValue; import static org.sonar.db.DatabaseUtils.executeLargeInputs; import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput; @@ -89,7 +89,7 @@ public class PropertiesDao implements Dao { String sql = "SELECT count(1) FROM properties pp " + "left outer join projects pj on pp.resource_id = pj.id " + "where pp.user_id is not null and (pp.resource_id is null or pj.uuid=?) " + - "and (" + DatabaseUtils.repeatCondition("pp.prop_key like ?", dispatcherKeys.size(), "or") + ")"; + "and (" + repeat("pp.prop_key like ?", " or ", dispatcherKeys.size()) + ")"; PreparedStatement res = connection.prepareStatement(sql); res.setString(1, projectUuid); int index = 2; @@ -282,7 +282,7 @@ public class PropertiesDao implements Dao { executeLargeInputsWithoutOutput(ids, list -> getMapper(dbSession).deleteByIds(list)); } - public void deleteByKeyAndValue(DbSession dbSession, String key, String value){ + public void deleteByKeyAndValue(DbSession dbSession, String key, String value) { getMapper(dbSession).deleteByKeyAndValue(key, value); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java index ecc767b006f..ef7c49d07de 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDao.java @@ -19,13 +19,15 @@ */ package org.sonar.db.user; -import com.google.common.base.Function; -import com.google.common.base.Predicates; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import org.sonar.api.user.UserQuery; @@ -34,8 +36,8 @@ import org.sonar.db.Dao; import org.sonar.db.DbSession; import org.sonar.db.RowNotFoundException; -import static com.google.common.collect.FluentIterable.from; import static org.sonar.db.DatabaseUtils.executeLargeInputs; +import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput; public class UserDao implements Dao { @@ -82,7 +84,10 @@ public class UserDao implements Dao { */ public List<UserDto> selectByOrderedLogins(DbSession session, Collection<String> logins) { List<UserDto> unordered = selectByLogins(session, logins); - return from(logins).transform(new LoginToUser(unordered)).filter(Predicates.notNull()).toList(); + return logins.stream() + .map(new LoginToUser(unordered)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } public List<UserDto> selectUsers(DbSession dbSession, UserQuery query) { @@ -94,12 +99,17 @@ public class UserDao implements Dao { } public UserDto insert(DbSession session, UserDto dto) { - mapper(session).insert(dto); + long now = system2.now(); + mapper(session).insert(dto, now); + dto.setCreatedAt(now); + dto.setUpdatedAt(now); return dto; } public UserDto update(DbSession session, UserDto dto) { - mapper(session).update(dto); + long now = system2.now(); + mapper(session).update(dto, now); + dto.setUpdatedAt(now); return dto; } @@ -107,8 +117,8 @@ public class UserDao implements Dao { mapper(session).setRoot(login, root, system2.now()); } - public void deactivateUserById(DbSession dbSession, int userId) { - mapper(dbSession).deactivateUser(userId, system2.now()); + public void deactivateUser(DbSession dbSession, UserDto user) { + mapper(dbSession).deactivateUser(user.getLogin(), system2.now()); } @CheckForNull @@ -140,6 +150,22 @@ public class UserDao implements Dao { return mapper(dbSession).countByEmail(email.toLowerCase(Locale.ENGLISH)) > 0; } + public void scrollByLogins(DbSession dbSession, Collection<String> logins, Consumer<UserDto> consumer) { + UserMapper mapper = mapper(dbSession); + + executeLargeInputsWithoutOutput(logins, + pageOfLogins -> mapper + .selectByLogins(pageOfLogins) + .forEach(consumer)); + } + + public void scrollAll(DbSession dbSession, Consumer<UserDto> consumer) { + mapper(dbSession).scrollAll(context -> { + UserDto user = (UserDto) context.getResultObject(); + consumer.accept(user); + }); + } + private static UserMapper mapper(DbSession session) { return session.getMapper(UserMapper.class); } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDto.java b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDto.java index e85f6ed8785..fc3dd43df6f 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDto.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserDto.java @@ -189,7 +189,7 @@ public class UserDto { return createdAt; } - public UserDto setCreatedAt(Long createdAt) { + UserDto setCreatedAt(long createdAt) { this.createdAt = createdAt; return this; } @@ -198,7 +198,7 @@ public class UserDto { return updatedAt; } - public UserDto setUpdatedAt(Long updatedAt) { + UserDto setUpdatedAt(long updatedAt) { this.updatedAt = updatedAt; return this; } diff --git a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java index 17c8cc0bbec..e2938d4a776 100644 --- a/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java +++ b/server/sonar-db-dao/src/main/java/org/sonar/db/user/UserMapper.java @@ -22,6 +22,7 @@ package org.sonar.db.user; import java.util.List; import javax.annotation.CheckForNull; import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.session.ResultHandler; import org.sonar.api.user.UserQuery; public interface UserMapper { @@ -51,6 +52,8 @@ public interface UserMapper { List<UserDto> selectByIds(@Param("ids") List<Integer> ids); + void scrollAll(ResultHandler handler); + long countByEmail(String email); /** @@ -58,14 +61,12 @@ public interface UserMapper { */ long countRootUsersButLogin(@Param("login") String login); - void insert(UserDto userDto); + void insert(@Param("user") UserDto userDto, @Param("now") long now); - void update(UserDto userDto); + void update(@Param("user") UserDto userDto, @Param("now") long now); void setRoot(@Param("login") String login, @Param("root") boolean root, @Param("now") long now); - void deleteOrganisationMembership(int userId); - - void deactivateUser(@Param("id") int userId, @Param("now") long now); + void deactivateUser(@Param("login") String login, @Param("now") long now); } diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml new file mode 100644 index 00000000000..08b3761380c --- /dev/null +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/es/EsQueueMapper.xml @@ -0,0 +1,67 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> + +<mapper namespace="org.sonar.db.es.EsQueueMapper"> + + <sql id="esQueueColumns"> + uuid, + doc_type as docType, + doc_uuid as docUuid, + created_at as createdAt + </sql> + + <insert id="insert" parameterType="map" useGeneratedKeys="false"> + insert into es_queue ( + uuid, + doc_type, + doc_uuid, + created_at + ) values ( + #{dto.uuid, jdbcType=VARCHAR}, + #{dto.docType, jdbcType=VARCHAR}, + #{dto.docUuid, jdbcType=VARCHAR}, + #{now, jdbcType=BIGINT} + ) + </insert> + + <delete id="delete" parameterType="string"> + delete from es_queue + where uuid in + <foreach item="uuid" collection="uuids" open="(" separator="," close=")"> + #{uuid, jdbcType=VARCHAR} + </foreach> + </delete> + + <select id="selectForRecovery" parameterType="map" resultType="org.sonar.db.es.EsQueueDto"> + select <include refid="esQueueColumns" /> + from es_queue + where + created_at <= #{beforeDate, jdbcType=BIGINT} + order by created_at desc + limit #{limit, jdbcType=INTEGER} + </select> + + <select id="selectForRecovery" parameterType="map" resultType="org.sonar.db.es.EsQueueDto" databaseId="oracle"> + select * from ( + select rownum as rn, t.* from ( + select <include refid="esQueueColumns" /> + from es_queue + where + created_at <= #{beforeDate, jdbcType=BIGINT} + order by created_at desc + ) t + ) t + where + t.rn <= #{limit, jdbcType=INTEGER} + </select> + + <select id="selectForRecovery" parameterType="map" resultType="org.sonar.db.es.EsQueueDto" databaseId="mssql"> + select top(#{limit, jdbcType=INTEGER}) <include refid="esQueueColumns" /> + from es_queue + where + created_at <= #{beforeDate, jdbcType=BIGINT} + order by created_at desc + </select> + +</mapper> + diff --git a/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml b/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml index ef9d08398df..9dddabd501d 100644 --- a/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml +++ b/server/sonar-db-dao/src/main/resources/org/sonar/db/user/UserMapper.xml @@ -62,6 +62,11 @@ </foreach> </select> + <select id="scrollAll" resultType="User" fetchSize="${_scrollFetchSize}" resultSetType="FORWARD_ONLY"> + select <include refid="userColumns"/> + from users u + </select> + <select id="selectByIds" parameterType="string" resultType="User"> SELECT <include refid="userColumns"/> @@ -116,35 +121,31 @@ and u.login <> #{login} </select> - <delete id="deleteOrganisationMembership" parameterType="int"> - DELETE FROM organization_members WHERE user_id=#{id,jdbcType=BIGINT} - </delete> - <update id="deactivateUser" parameterType="map"> - UPDATE users SET - active=${_false}, - email=null, - scm_accounts=null, - external_identity=null, - external_identity_provider=null, - salt=null, - crypted_password=null, - updated_at=#{now,jdbcType=BIGINT} - WHERE - id=#{id,jdbcType=INTEGER} + update users set + active = ${_false}, + email = null, + scm_accounts = null, + external_identity = null, + external_identity_provider = null, + salt = null, + crypted_password = null, + updated_at = #{now, jdbcType=BIGINT} + where + login = #{login, jdbcType=VARCHAR} </update> <update id="setRoot"> update users set - is_root = #{root,jdbcType=BOOLEAN}, - updated_at=#{now,jdbcType=BIGINT} + is_root = #{root, jdbcType=BOOLEAN}, + updated_at = #{now, jdbcType=BIGINT} where - login = #{login,jdbcType=VARCHAR} + login = #{login, jdbcType=VARCHAR} and active = ${_true} </update> - <insert id="insert" parameterType="User" keyColumn="id" useGeneratedKeys="true" keyProperty="id"> - INSERT INTO users ( + <insert id="insert" parameterType="map" keyColumn="id" useGeneratedKeys="true" keyProperty="user.id"> + insert into users ( login, name, email, @@ -159,40 +160,39 @@ onboarded, created_at, updated_at - ) - VALUES ( - #{login,jdbcType=VARCHAR}, - #{name,jdbcType=VARCHAR}, - #{email,jdbcType=VARCHAR}, - #{active,jdbcType=BOOLEAN}, - #{scmAccounts,jdbcType=VARCHAR}, - #{externalIdentity,jdbcType=VARCHAR}, - #{externalIdentityProvider,jdbcType=VARCHAR}, - #{local,jdbcType=BOOLEAN}, - #{salt,jdbcType=VARCHAR}, - #{cryptedPassword,jdbcType=VARCHAR}, - #{root,jdbcType=BOOLEAN}, - #{onboarded,jdbcType=BOOLEAN}, - #{createdAt,jdbcType=BIGINT}, - #{updatedAt,jdbcType=BIGINT} + ) values ( + #{user.login,jdbcType=VARCHAR}, + #{user.name,jdbcType=VARCHAR}, + #{user.email,jdbcType=VARCHAR}, + #{user.active,jdbcType=BOOLEAN}, + #{user.scmAccounts,jdbcType=VARCHAR}, + #{user.externalIdentity,jdbcType=VARCHAR}, + #{user.externalIdentityProvider,jdbcType=VARCHAR}, + #{user.local,jdbcType=BOOLEAN}, + #{user.salt,jdbcType=VARCHAR}, + #{user.cryptedPassword,jdbcType=VARCHAR}, + #{user.root,jdbcType=BOOLEAN}, + #{user.onboarded,jdbcType=BOOLEAN}, + #{now,jdbcType=BIGINT}, + #{now,jdbcType=BIGINT} ) </insert> - <insert id="update" parameterType="User" useGeneratedKeys="false"> - UPDATE users set - name=#{name,jdbcType=VARCHAR}, - email=#{email,jdbcType=VARCHAR}, - active=#{active,jdbcType=BOOLEAN}, - scm_accounts=#{scmAccounts,jdbcType=VARCHAR}, - external_identity=#{externalIdentity,jdbcType=VARCHAR}, - external_identity_provider=#{externalIdentityProvider,jdbcType=VARCHAR}, - user_local=#{local,jdbcType=BOOLEAN}, - onboarded=#{onboarded,jdbcType=BOOLEAN}, - salt=#{salt,jdbcType=VARCHAR}, - crypted_password=#{cryptedPassword,jdbcType=BIGINT}, - updated_at=#{updatedAt,jdbcType=BIGINT} - WHERE - login = #{login,jdbcType=VARCHAR} - </insert> + <update id="update" parameterType="map"> + update users set + name = #{user.name, jdbcType=VARCHAR}, + email = #{user.email, jdbcType=VARCHAR}, + active = #{user.active, jdbcType=BOOLEAN}, + scm_accounts = #{user.scmAccounts, jdbcType=VARCHAR}, + external_identity = #{user.externalIdentity, jdbcType=VARCHAR}, + external_identity_provider = #{user.externalIdentityProvider, jdbcType=VARCHAR}, + user_local = #{user.local, jdbcType=BOOLEAN}, + onboarded = #{user.onboarded, jdbcType=BOOLEAN}, + salt = #{user.salt, jdbcType=VARCHAR}, + crypted_password = #{user.cryptedPassword, jdbcType=BIGINT}, + updated_at = #{now, jdbcType=BIGINT} + where + login = #{user.login, jdbcType=VARCHAR} + </update> </mapper> diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/DaoModuleTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/DaoModuleTest.java index 4e2c7cb812e..2ffc454abd9 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/DaoModuleTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/DaoModuleTest.java @@ -29,6 +29,6 @@ public class DaoModuleTest { public void verify_count_of_added_components() { ComponentContainer container = new ComponentContainer(); new DaoModule().configure(container); - assertThat(container.size()).isEqualTo(2 + 46); + assertThat(container.size()).isEqualTo(2 + 47); } } diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/es/EsQueueDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/es/EsQueueDaoTest.java new file mode 100644 index 00000000000..373a88b2013 --- /dev/null +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/es/EsQueueDaoTest.java @@ -0,0 +1,129 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.db.es; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.stream.IntStream; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.api.utils.internal.TestSystem2; +import org.sonar.core.util.UuidFactoryFast; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EsQueueDaoTest { + + private static final int LIMIT = 10; + private static TestSystem2 system2 = new TestSystem2().setNow(1_000); + + @Rule + public DbTester dbTester = DbTester.create(system2); + + private DbSession dbSession = dbTester.getSession(); + private EsQueueDao underTest = dbTester.getDbClient().esQueueDao(); + + @Test + public void insert_data() { + int nbOfInsert = 10 + new Random().nextInt(20); + List<EsQueueDto> esQueueDtos = new ArrayList<>(); + IntStream.rangeClosed(1, nbOfInsert).forEach( + i -> esQueueDtos.add(EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())) + ); + underTest.insert(dbSession, esQueueDtos); + + assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(nbOfInsert); + } + + @Test + public void delete_unknown_EsQueueDto_does_not_throw_exception() { + int nbOfInsert = 10 + new Random().nextInt(20); + List<EsQueueDto> esQueueDtos = new ArrayList<>(); + IntStream.rangeClosed(1, nbOfInsert).forEach( + i -> esQueueDtos.add(EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())) + ); + underTest.insert(dbSession, esQueueDtos); + + underTest.delete(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())); + + assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(nbOfInsert); + } + + @Test + public void delete_EsQueueDto_does_not_throw_exception() { + int nbOfInsert = 10 + new Random().nextInt(20); + List<EsQueueDto> esQueueDtos = new ArrayList<>(); + IntStream.rangeClosed(1, nbOfInsert).forEach( + i -> esQueueDtos.add(EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())) + ); + underTest.insert(dbSession, esQueueDtos); + assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(nbOfInsert); + + underTest.delete(dbSession, esQueueDtos); + + assertThat(dbTester.countSql(dbSession, "select count(*) from es_queue")).isEqualTo(0); + } + + @Test + public void selectForRecovery_must_return_limit_when_there_are_more_rows() { + system2.setNow(1_000L); + EsQueueDto i1 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())); + system2.setNow(1_001L); + EsQueueDto i2 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())); + system2.setNow(1_002L); + EsQueueDto i3 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())); + + assertThat(underTest.selectForRecovery(dbSession, 2_000, 1)) + .extracting(EsQueueDto::getUuid) + .containsExactly(i3.getUuid()); + + assertThat(underTest.selectForRecovery(dbSession, 2_000, 2)) + .extracting(EsQueueDto::getUuid) + .containsExactly(i3.getUuid(), i2.getUuid()); + + assertThat(underTest.selectForRecovery(dbSession, 2_000, 10)) + .extracting(EsQueueDto::getUuid) + .containsExactly(i3.getUuid(), i2.getUuid(), i1.getUuid()); + } + + @Test + public void selectForRecovery_returns_ordered_rows_created_before_date() { + system2.setNow(1_000L); + EsQueueDto i1 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())); + system2.setNow(1_001L); + EsQueueDto i2 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())); + system2.setNow(1_002L); + EsQueueDto i3 = underTest.insert(dbSession, EsQueueDto.create(EsQueueDto.Type.USER, UuidFactoryFast.getInstance().create())); + + assertThat(underTest.selectForRecovery(dbSession, 999, LIMIT)).hasSize(0); + assertThat(underTest.selectForRecovery(dbSession, 1_000, LIMIT)) + .extracting(EsQueueDto::getUuid) + .containsExactly(i1.getUuid()); + assertThat(underTest.selectForRecovery(dbSession, 1_001, LIMIT)) + .extracting(EsQueueDto::getUuid) + .containsExactly(i2.getUuid(), i1.getUuid()); + assertThat(underTest.selectForRecovery(dbSession, 2_000, LIMIT)) + .extracting(EsQueueDto::getUuid) + .containsExactly(i3.getUuid(), i2.getUuid(), i1.getUuid()); + } +} diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/user/RootFlagAssertions.java b/server/sonar-db-dao/src/test/java/org/sonar/db/user/RootFlagAssertions.java index 42c78876717..0885c813365 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/user/RootFlagAssertions.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/user/RootFlagAssertions.java @@ -31,10 +31,6 @@ public class RootFlagAssertions { this.db = db; } - public void verifyUnchanged(UserDto user) { - verify(user, user.isRoot(), user.getUpdatedAt()); - } - public void verify(UserDto userDto, boolean root, long updatedAt) { Map<String, Object> row = db.selectFirst("select is_root as \"isRoot\", updated_at as \"updatedAt\" from users where login = '" + userDto.getLogin() + "'"); Object isRoot = row.get("isRoot"); diff --git a/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java b/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java index ef3d48316b9..773c0eb1fa4 100644 --- a/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java +++ b/server/sonar-db-dao/src/test/java/org/sonar/db/user/UserDaoTest.java @@ -19,6 +19,7 @@ */ package org.sonar.db.user; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.junit.Before; @@ -28,6 +29,7 @@ import org.junit.rules.ExpectedException; import org.sonar.api.user.UserQuery; import org.sonar.api.utils.DateUtils; import org.sonar.api.utils.System2; +import org.sonar.db.DatabaseUtils; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; @@ -37,6 +39,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static org.apache.commons.lang.RandomStringUtils.randomAlphanumeric; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.groups.Tuple.tuple; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -58,7 +61,7 @@ public class UserDaoTest { private UserDao underTest = db.getDbClient().userDao(); @Before - public void setUp() throws Exception { + public void setUp() { when(system2.now()).thenReturn(NOW); } @@ -337,23 +340,19 @@ public class UserDaoTest { assertThat(user.getExternalIdentityProvider()).isEqualTo("github"); assertThat(user.isLocal()).isTrue(); assertThat(user.isRoot()).isFalse(); - assertThat(user.getCreatedAt()).isEqualTo(date); - assertThat(user.getUpdatedAt()).isEqualTo(date); } @Test public void update_user() { - UserDto existingUser = db.users().insertUser(user -> user + UserDto user = db.users().insertUser(u -> u .setLogin("john") .setName("John") .setEmail("jo@hn.com") - .setCreatedAt(1418215735482L) - .setUpdatedAt(1418215735482L) .setActive(true) .setLocal(true) .setOnboarded(false)); - UserDto userDto = new UserDto() + UserDto userUpdate = new UserDto() .setId(1) .setLogin("john") .setName("John Doo") @@ -365,38 +364,34 @@ public class UserDaoTest { .setCryptedPassword("abcde") .setExternalIdentity("johngithub") .setExternalIdentityProvider("github") - .setLocal(false) - .setUpdatedAt(1500000000000L); - underTest.update(db.getSession(), userDto); - db.getSession().commit(); - - UserDto user = underTest.selectUserById(db.getSession(), existingUser.getId()); - assertThat(user).isNotNull(); - assertThat(user.getId()).isEqualTo(existingUser.getId()); - assertThat(user.getLogin()).isEqualTo("john"); - assertThat(user.getName()).isEqualTo("John Doo"); - assertThat(user.getEmail()).isEqualTo("jodoo@hn.com"); - assertThat(user.isActive()).isFalse(); - assertThat(user.isOnboarded()).isTrue(); - assertThat(user.getScmAccounts()).isEqualTo(",jo.hn,john2,johndoo,"); - assertThat(user.getSalt()).isEqualTo("12345"); - assertThat(user.getCryptedPassword()).isEqualTo("abcde"); - assertThat(user.getExternalIdentity()).isEqualTo("johngithub"); - assertThat(user.getExternalIdentityProvider()).isEqualTo("github"); - assertThat(user.isLocal()).isFalse(); - assertThat(user.isRoot()).isFalse(); - assertThat(user.getCreatedAt()).isEqualTo(1418215735482L); - assertThat(user.getUpdatedAt()).isEqualTo(1500000000000L); - } - - @Test - public void deactivate_user() throws Exception { - UserDto user = newActiveUser(); + .setLocal(false); + underTest.update(db.getSession(), userUpdate); + + UserDto reloaded = underTest.selectByLogin(db.getSession(), user.getLogin()); + assertThat(reloaded).isNotNull(); + assertThat(reloaded.getId()).isEqualTo(user.getId()); + assertThat(reloaded.getLogin()).isEqualTo(user.getLogin()); + assertThat(reloaded.getName()).isEqualTo("John Doo"); + assertThat(reloaded.getEmail()).isEqualTo("jodoo@hn.com"); + assertThat(reloaded.isActive()).isFalse(); + assertThat(reloaded.isOnboarded()).isTrue(); + assertThat(reloaded.getScmAccounts()).isEqualTo(",jo.hn,john2,johndoo,"); + assertThat(reloaded.getSalt()).isEqualTo("12345"); + assertThat(reloaded.getCryptedPassword()).isEqualTo("abcde"); + assertThat(reloaded.getExternalIdentity()).isEqualTo("johngithub"); + assertThat(reloaded.getExternalIdentityProvider()).isEqualTo("github"); + assertThat(reloaded.isLocal()).isFalse(); + assertThat(reloaded.isRoot()).isFalse(); + } + + @Test + public void deactivate_user() { + UserDto user = insertActiveUser(); insertUserGroup(user); - UserDto otherUser = newActiveUser(); + UserDto otherUser = insertActiveUser(); session.commit(); - underTest.deactivateUserById(session, user.getId()); + underTest.deactivateUser(session, user); UserDto userReloaded = underTest.selectUserById(session, user.getId()); assertThat(userReloaded.isActive()).isFalse(); @@ -413,7 +408,7 @@ public class UserDaoTest { @Test public void does_not_fail_to_deactivate_missing_user() { - underTest.deactivateUserById(session, 123); + underTest.deactivateUser(session, UserTesting.newUserDto()); } @Test @@ -425,13 +420,11 @@ public class UserDaoTest { .setActive(true) .setScmAccounts("\nma\nmarius33\n") .setSalt("79bd6a8e79fb8c76ac8b121cc7e8e11ad1af8365") - .setCryptedPassword("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg") - .setCreatedAt(1418215735482L) - .setUpdatedAt(1418215735485L)); - UserDto user2 = db.users().insertUser(user -> user.setLogin("sbrandhof")); + .setCryptedPassword("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg")); + UserDto user2 = db.users().insertUser(); underTest.setRoot(session, user2.getLogin(), true); - UserDto dto = underTest.selectOrFailByLogin(session, "marius"); + UserDto dto = underTest.selectOrFailByLogin(session, user1.getLogin()); assertThat(dto.getId()).isEqualTo(user1.getId()); assertThat(dto.getLogin()).isEqualTo("marius"); assertThat(dto.getName()).isEqualTo("Marius"); @@ -441,10 +434,10 @@ public class UserDaoTest { assertThat(dto.getSalt()).isEqualTo("79bd6a8e79fb8c76ac8b121cc7e8e11ad1af8365"); assertThat(dto.getCryptedPassword()).isEqualTo("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg"); assertThat(dto.isRoot()).isFalse(); - assertThat(dto.getCreatedAt()).isEqualTo(1418215735482L); - assertThat(dto.getUpdatedAt()).isEqualTo(1418215735485L); + assertThat(dto.getCreatedAt()).isEqualTo(user1.getCreatedAt()); + assertThat(dto.getUpdatedAt()).isEqualTo(user1.getUpdatedAt()); - dto = underTest.selectOrFailByLogin(session, "sbrandhof"); + dto = underTest.selectOrFailByLogin(session, user2.getLogin()); assertThat(dto.isRoot()).isTrue(); } @@ -490,8 +483,8 @@ public class UserDaoTest { } @Test - public void exists_by_email() throws Exception { - UserDto activeUser = newActiveUser(); + public void exists_by_email() { + UserDto activeUser = insertActiveUser(); UserDto disableUser = insertUser(false); assertThat(underTest.doesEmailExist(session, activeUser.getEmail())).isTrue(); @@ -507,8 +500,8 @@ public class UserDaoTest { @Test public void setRoot_set_root_flag_of_specified_user_to_specified_value_and_updates_udpateAt() { - String login = newActiveUser().getLogin(); - UserDto otherUser = newActiveUser(); + String login = insertActiveUser().getLogin(); + UserDto otherUser = insertActiveUser(); assertThat(underTest.selectByLogin(session, login).isRoot()).isEqualTo(false); assertThat(underTest.selectByLogin(session, otherUser.getLogin()).isRoot()).isEqualTo(false); @@ -550,7 +543,7 @@ public class UserDaoTest { assertThat(underTest.selectByLogin(session, nonRootInactiveUser).isRoot()).isFalse(); // create inactive root user - UserDto rootUser = newActiveUser(); + UserDto rootUser = insertActiveUser(); commit(() -> underTest.setRoot(session, rootUser.getLogin(), true)); rootUser.setActive(false); commit(() -> underTest.update(session, rootUser)); @@ -562,12 +555,52 @@ public class UserDaoTest { assertThat(underTest.selectByLogin(session, inactiveRootUser.getLogin()).isRoot()).isTrue(); } + @Test + public void scrollByLogins() { + UserDto u1 = insertUser(true); + UserDto u2 = insertUser(false); + UserDto u3 = insertUser(false); + + List<UserDto> result = new ArrayList<>(); + underTest.scrollByLogins(db.getSession(), asList(u2.getLogin(), u3.getLogin(), "does not exist"), result::add); + + assertThat(result).extracting(UserDto::getLogin, UserDto::getName) + .containsExactlyInAnyOrder(tuple(u2.getLogin(), u2.getName()), tuple(u3.getLogin(), u3.getName())); + } + + @Test + public void scrollByLogins_scrolls_by_pages_of_1000_logins() { + List<String> logins = new ArrayList<>(); + for (int i = 0; i < DatabaseUtils.PARTITION_SIZE_FOR_ORACLE + 10; i++) { + logins.add(insertUser(true).getLogin()); + } + + List<UserDto> result = new ArrayList<>(); + underTest.scrollByLogins(db.getSession(), logins, result::add); + + assertThat(result) + .extracting(UserDto::getLogin) + .containsExactlyInAnyOrder(logins.toArray(new String[0])); + } + + @Test + public void scrollAll() { + UserDto u1 = insertUser(true); + UserDto u2 = insertUser(false); + + List<UserDto> result = new ArrayList<>(); + underTest.scrollAll(db.getSession(), result::add); + + assertThat(result).extracting(UserDto::getLogin, UserDto::getName) + .containsExactlyInAnyOrder(tuple(u1.getLogin(), u1.getName()), tuple(u2.getLogin(), u2.getName())); + } + private void commit(Runnable runnable) { runnable.run(); session.commit(); } - private UserDto newActiveUser() { + private UserDto insertActiveUser() { return insertUser(true); } diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v62/UpdateQualityGateConditionsOnCoverage.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v62/UpdateQualityGateConditionsOnCoverage.java index 72b4b0d322e..bbb3ffe1933 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v62/UpdateQualityGateConditionsOnCoverage.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v62/UpdateQualityGateConditionsOnCoverage.java @@ -39,9 +39,9 @@ import org.sonar.server.platform.db.migration.step.Select; import org.sonar.server.platform.db.migration.step.Upsert; import static java.util.Objects.requireNonNull; +import static org.apache.commons.lang.StringUtils.repeat; import static org.sonar.core.util.stream.MoreCollectors.index; import static org.sonar.core.util.stream.MoreCollectors.uniqueIndex; -import static org.sonar.db.DatabaseUtils.repeatCondition; /** * Migrate every quality gates that have conditions on related coverage metrics. @@ -144,7 +144,7 @@ public class UpdateQualityGateConditionsOnCoverage extends DataChange { try { Select select = context.prepareSelect("select qgc.id, qgc.metric_id " + "from quality_gate_conditions qgc " + - "where qgc.qgate_id=? and qgc.metric_id in (" + repeatCondition("?", metricIds.size(), ",") + ")") + "where qgc.qgate_id=? and qgc.metric_id in (" + repeat("?", " , ", metricIds.size()) + ")") .setLong(1, qualityGateId); for (int i = 0; i < metricIds.size(); i++) { select.setLong(i + 2, metricIds.get(i)); @@ -192,7 +192,7 @@ public class UpdateQualityGateConditionsOnCoverage extends DataChange { metricKeys.addAll(COVERAGE_METRIC_KEYS.stream().map(metricKey -> IT_PREFIX + metricKey).collect(Collectors.toList())); metricKeys.addAll(COVERAGE_METRIC_KEYS.stream().map(metricKey -> OVERALL_PREFIX + metricKey).collect(Collectors.toList())); metricKeys.addAll(metricKeys.stream().map(metricKey -> NEW_PREFIX + metricKey).collect(Collectors.toList())); - Select select = context.prepareSelect("select id, name from metrics where name in (" + repeatCondition("?", metricKeys.size(), ",") + ")"); + Select select = context.prepareSelect("select id, name from metrics where name in (" + repeat("?", ",", metricKeys.size()) + ")"); for (int i = 0; i < metricKeys.size(); i++) { select.setString(i + 1, metricKeys.get(i)); } diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAt.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAt.java new file mode 100644 index 00000000000..4be823c7ed0 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAt.java @@ -0,0 +1,48 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.platform.db.migration.version.v65; + +import java.sql.SQLException; +import org.sonar.db.Database; +import org.sonar.server.platform.db.migration.def.BigIntegerColumnDef; +import org.sonar.server.platform.db.migration.sql.CreateIndexBuilder; +import org.sonar.server.platform.db.migration.step.DdlChange; + +public class AddIndexOnEsQueueCreatedAt extends DdlChange { + + public AddIndexOnEsQueueCreatedAt(Database db) { + super(db); + } + + @Override + public void execute(Context context) throws SQLException { + BigIntegerColumnDef column = BigIntegerColumnDef.newBigIntegerColumnDefBuilder() + .setColumnName("created_at") + .setIsNullable(false) + .build(); + + context.execute(new CreateIndexBuilder(getDialect()) + .setName("es_queue_created_at") + .setTable("es_queue") + .addColumn(column) + .build()); + } +} diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java new file mode 100644 index 00000000000..a8e8bc70bc9 --- /dev/null +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTable.java @@ -0,0 +1,63 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.platform.db.migration.version.v65; + +import java.sql.SQLException; +import org.sonar.db.Database; +import org.sonar.server.platform.db.migration.def.BigIntegerColumnDef; +import org.sonar.server.platform.db.migration.def.VarcharColumnDef; +import org.sonar.server.platform.db.migration.sql.CreateTableBuilder; +import org.sonar.server.platform.db.migration.step.DdlChange; + +public class CreateEsQueueTable extends DdlChange { + + private static final String TABLE_NAME = "es_queue"; + + public CreateEsQueueTable(Database db) { + super(db); + } + + @Override + public void execute(Context context) throws SQLException { + context.execute(new CreateTableBuilder(getDialect(), TABLE_NAME) + .addPkColumn(VarcharColumnDef.newVarcharColumnDefBuilder() + .setColumnName("uuid") + .setIsNullable(false) + .setLimit(40) + .build()) + .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder() + .setColumnName("doc_type") + .setIsNullable(false) + .setLimit(40) + .build()) + .addColumn(VarcharColumnDef.newVarcharColumnDefBuilder() + .setColumnName("doc_uuid") + .setIsNullable(false) + .setLimit(255) + .build()) + .addColumn(BigIntegerColumnDef.newBigIntegerColumnDefBuilder() + .setColumnName("created_at") + .setIsNullable(false) + .build()) + .build() + ); + } +} diff --git a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65.java b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65.java index 5dd66b02490..66788574f1a 100644 --- a/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65.java +++ b/server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65.java @@ -60,6 +60,8 @@ public class DbVersion65 implements DbVersion { .add(1730, "Add USERS.ONBOARDED", AddUsersOnboarded.class) .add(1731, "Populate USERS.ONBOARDED", PopulateUsersOnboarded.class) .add(1732, "Make USERS.ONBOARDED not nullable", MakeUsersOnboardedNotNullable.class) + .add(1733, "Create table es_queue", CreateEsQueueTable.class) + .add(1734, "Add index on es_queue.created_at", AddIndexOnEsQueueCreatedAt.class) ; } } diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest.java new file mode 100644 index 00000000000..824b656aaae --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest.java @@ -0,0 +1,41 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.platform.db.migration.version.v65; + +import java.sql.SQLException; +import org.junit.Rule; +import org.junit.Test; +import org.sonar.db.CoreDbTester; +import org.sonar.server.platform.db.migration.step.DdlChange; + +public class AddIndexOnEsQueueCreatedAtTest { + + @Rule + public CoreDbTester db = CoreDbTester.createForSchema(AddIndexOnEsQueueCreatedAtTest.class, "initial.sql"); + + private DdlChange underTest = new AddIndexOnEsQueueCreatedAt(db.database()); + + @Test + public void add_index() throws SQLException { + underTest.execute(); + + db.assertIndex("es_queue", "es_queue_created_at", "created_at"); + } +} diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java new file mode 100644 index 00000000000..23a950006f5 --- /dev/null +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest.java @@ -0,0 +1,54 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.platform.db.migration.version.v65; + +import java.sql.SQLException; +import java.sql.Types; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.sonar.db.CoreDbTester; + +import static org.assertj.core.api.Assertions.assertThat; + +public class CreateEsQueueTableTest { + + private static final String TABLE = "es_queue"; + + @Rule + public final CoreDbTester db = CoreDbTester.createForSchema(CreateEsQueueTableTest.class, "empty.sql"); + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private CreateEsQueueTable underTest = new CreateEsQueueTable(db.database()); + + @Test + public void creates_table_on_empty_db() throws SQLException { + underTest.execute(); + + assertThat(db.countRowsOfTable(TABLE)).isEqualTo(0); + + db.assertColumnDefinition(TABLE, "uuid", Types.VARCHAR, 40, false); + db.assertColumnDefinition(TABLE, "doc_type", Types.VARCHAR, 40, false); + db.assertColumnDefinition(TABLE, "doc_uuid", Types.VARCHAR, 255, false); + db.assertColumnDefinition(TABLE, "created_at", Types.BIGINT, null, false); + } +} diff --git a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65Test.java b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65Test.java index 068b8a76a41..61243077c0a 100644 --- a/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65Test.java +++ b/server/sonar-db-migration/src/test/java/org/sonar/server/platform/db/migration/version/v65/DbVersion65Test.java @@ -35,6 +35,6 @@ public class DbVersion65Test { @Test public void verify_migration_count() { - verifyMigrationCount(underTest, 33); + verifyMigrationCount(underTest, 35); } } diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest/initial.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest/initial.sql new file mode 100644 index 00000000000..a6a15d71fa2 --- /dev/null +++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/AddIndexOnEsQueueCreatedAtTest/initial.sql @@ -0,0 +1,7 @@ +CREATE TABLE "ES_QUEUE" ( + "UUID" VARCHAR(40) NOT NULL PRIMARY KEY, + "DOC_TYPE" VARCHAR(40) NOT NULL, + "DOC_UUID" VARCHAR(255) NOT NULL, + "CREATED_AT" BIGINT NOT NULL +); +CREATE UNIQUE INDEX "PK_ES_QUEUE" ON "ES_QUEUE" ("UUID"); diff --git a/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest/empty.sql b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest/empty.sql new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/server/sonar-db-migration/src/test/resources/org/sonar/server/platform/db/migration/version/v65/CreateEsQueueTableTest/empty.sql diff --git a/server/sonar-server/src/main/java/org/sonar/server/authentication/UserIdentityAuthenticator.java b/server/sonar-server/src/main/java/org/sonar/server/authentication/UserIdentityAuthenticator.java index 5b7ecb54220..396cea50b03 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/authentication/UserIdentityAuthenticator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/authentication/UserIdentityAuthenticator.java @@ -86,21 +86,21 @@ public class UserIdentityAuthenticator { } } - private UserDto registerNewUser(DbSession dbSession, UserIdentity user, IdentityProvider provider, AuthenticationEvent.Source source) { + private UserDto registerNewUser(DbSession dbSession, UserIdentity identity, IdentityProvider provider, AuthenticationEvent.Source source) { if (!provider.allowsUsersToSignUp()) { throw AuthenticationException.newBuilder() .setSource(source) - .setLogin(user.getLogin()) + .setLogin(identity.getLogin()) .setMessage(format("User signup disabled for provider '%s'", provider.getKey())) .setPublicMessage(format("'%s' users are not allowed to sign up", provider.getKey())) .build(); } - String email = user.getEmail(); + String email = identity.getEmail(); if (email != null && dbClient.userDao().doesEmailExist(dbSession, email)) { throw AuthenticationException.newBuilder() .setSource(source) - .setLogin(user.getLogin()) + .setLogin(identity.getLogin()) .setMessage(format("Email '%s' is already used", email)) .setPublicMessage(format( "You can't sign up because email '%s' is already used by an existing user. This means that you probably already registered with another account.", @@ -108,25 +108,22 @@ public class UserIdentityAuthenticator { .build(); } - String userLogin = user.getLogin(); - userUpdater.create(dbSession, NewUser.builder() + String userLogin = identity.getLogin(); + return userUpdater.createAndCommit(dbSession, NewUser.builder() .setLogin(userLogin) - .setEmail(user.getEmail()) - .setName(user.getName()) - .setExternalIdentity(new ExternalIdentity(provider.getKey(), user.getProviderLogin())) - .build()); - UserDto newUser = dbClient.userDao().selectOrFailByLogin(dbSession, userLogin); - syncGroups(dbSession, user, newUser); - return newUser; + .setEmail(identity.getEmail()) + .setName(identity.getName()) + .setExternalIdentity(new ExternalIdentity(provider.getKey(), identity.getProviderLogin())) + .build(), u -> syncGroups(dbSession, identity, u)); } - private void registerExistingUser(DbSession dbSession, UserDto userDto, UserIdentity user, IdentityProvider provider) { - userUpdater.update(dbSession, UpdateUser.create(userDto.getLogin()) - .setEmail(user.getEmail()) - .setName(user.getName()) - .setExternalIdentity(new ExternalIdentity(provider.getKey(), user.getProviderLogin())) - .setPassword(null)); - syncGroups(dbSession, user, userDto); + private void registerExistingUser(DbSession dbSession, UserDto userDto, UserIdentity identity, IdentityProvider provider) { + UpdateUser update = UpdateUser.create(userDto.getLogin()) + .setEmail(identity.getEmail()) + .setName(identity.getName()) + .setExternalIdentity(new ExternalIdentity(provider.getKey(), identity.getProviderLogin())) + .setPassword(null); + userUpdater.updateAndCommit(dbSession, update, u -> syncGroups(dbSession, identity, u)); } private void syncGroups(DbSession dbSession, UserIdentity userIdentity, UserDto userDto) { @@ -149,8 +146,6 @@ public class UserIdentityAuthenticator { addGroups(dbSession, userDto, groupsToAdd, groupsByName); removeGroups(dbSession, userDto, groupsToRemove, groupsByName); - - dbSession.commit(); } private void addGroups(DbSession dbSession, UserDto userDto, Collection<String> groupsToAdd, Map<String, GroupDto> groupsByName) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java index 429ae9dedf6..d845806e06b 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/component/index/ComponentIndexer.java @@ -59,7 +59,7 @@ public class ComponentIndexer implements ProjectIndexer, NeedAuthorizationIndexe } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { doIndexByProjectUuid(null, Size.LARGE); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java deleted file mode 100644 index f83fa993468..00000000000 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BaseIndexer.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2017 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.es; - -import com.google.common.base.Throwables; -import com.google.common.util.concurrent.Uninterruptibles; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import org.picocontainer.Startable; -import org.sonar.api.utils.System2; - -public abstract class BaseIndexer implements Startable { - - private final System2 system2; - private final ThreadPoolExecutor executor; - private final IndexType indexType; - protected final EsClient esClient; - private final String dateFieldName; - private volatile long lastUpdatedAt = -1L; - - protected BaseIndexer(System2 system2, EsClient client, long threadKeepAliveSeconds, IndexType indexType, - String dateFieldName) { - this.system2 = system2; - this.indexType = indexType; - this.dateFieldName = dateFieldName; - this.esClient = client; - this.executor = new ThreadPoolExecutor(0, 1, - threadKeepAliveSeconds, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); - } - - public void index(final IndexerTask task) { - final long requestedAt = system2.now(); - Future submit = executor.submit(() -> { - if (lastUpdatedAt == -1L) { - lastUpdatedAt = esClient.getMaxFieldValue(indexType, dateFieldName); - } - if (requestedAt > lastUpdatedAt) { - long l = task.index(lastUpdatedAt); - // l can be 0 if no documents were indexed - lastUpdatedAt = Math.max(l, lastUpdatedAt); - } - }); - try { - Uninterruptibles.getUninterruptibly(submit); - } catch (ExecutionException e) { - Throwables.propagate(e); - } - } - - public void index() { - index(this::doIndex); - } - - protected abstract long doIndex(long lastUpdatedAt); - - @Override - public void start() { - // nothing to do at startup - } - - @Override - public void stop() { - executor.shutdown(); - } - - @FunctionalInterface - public interface IndexerTask { - long index(long lastUpdatedAt); - } - -} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java index 3d5b5439495..8aaff572899 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/BulkIndexer.java @@ -20,10 +20,15 @@ package org.sonar.server.es; import com.google.common.annotations.VisibleForTesting; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nullable; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; @@ -43,12 +48,15 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHitField; import org.elasticsearch.search.sort.SortOrder; -import org.picocontainer.Startable; import org.sonar.api.utils.log.Logger; import org.sonar.api.utils.log.Loggers; import org.sonar.core.util.ProgressLogger; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.es.EsQueueDto; import static java.lang.String.format; +import static java.util.stream.Collectors.toList; /** * Helper to bulk requests in an efficient way : @@ -57,7 +65,7 @@ import static java.lang.String.format; * <li>on large table indexing, replicas and automatic refresh can be temporarily disabled</li> * </ul> */ -public class BulkIndexer implements Startable { +public class BulkIndexer { private static final Logger LOGGER = Loggers.get(BulkIndexer.class); private static final ByteSizeValue FLUSH_BYTE_SIZE = new ByteSizeValue(1, ByteSizeUnit.MB); @@ -69,13 +77,22 @@ public class BulkIndexer implements Startable { private final String indexName; private final BulkProcessor bulkProcessor; private final AtomicLong counter = new AtomicLong(0L); + private final AtomicLong successCounter = new AtomicLong(0L); private final SizeHandler sizeHandler; + private final BulkProcessorListener bulkProcessorListener; + @Nullable + private DbClient dbClient; + @Nullable + private DbSession dbSession; + private Collection<EsQueueDto> esQueueDtos; public BulkIndexer(EsClient client, String indexName, Size size) { + this.dbClient = null; this.client = client; this.indexName = indexName; this.sizeHandler = size.createHandler(Runtime2.INSTANCE); - this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), new BulkProcessorListener()) + this.bulkProcessorListener = new BulkProcessorListener(); + this.bulkProcessor = BulkProcessor.builder(client.nativeClient(), bulkProcessorListener) .setBackoffPolicy(BackoffPolicy.exponentialBackoff()) .setBulkSize(FLUSH_BYTE_SIZE) .setBulkActions(FLUSH_ACTIONS) @@ -83,22 +100,36 @@ public class BulkIndexer implements Startable { .build(); } - @Override public void start() { sizeHandler.beforeStart(this); counter.set(0L); + successCounter.set(0L); } - @Override - public void stop() { + public void start(DbSession dbSession, DbClient dbClient, Collection<EsQueueDto> esQueueDtos) { + this.dbClient = dbClient; + this.dbSession = dbSession; + this.esQueueDtos = esQueueDtos; + sizeHandler.beforeStart(this); + counter.set(0L); + successCounter.set(0L); + } + + /** + * @return the number of documents successfully indexed + */ + public long stop() { try { - bulkProcessor.awaitClose(10, TimeUnit.MINUTES); + bulkProcessor.awaitClose(1, TimeUnit.MINUTES); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new IllegalStateException("Elasticsearch bulk requests still being executed after 10 minutes", e); + throw new IllegalStateException("Elasticsearch bulk requests still being executed after 1 minute", e); + } finally { + dbSession = null; } client.prepareRefresh(indexName).get(); sizeHandler.afterStop(this); + return successCounter.get(); } public void add(ActionRequest<?> request) { @@ -161,7 +192,6 @@ public class BulkIndexer implements Startable { } private final class BulkProcessorListener implements Listener { - @Override public void beforeBulk(long executionId, BulkRequest request) { // no action required @@ -174,14 +204,31 @@ public class BulkIndexer implements Startable { for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage()); + } else { + successCounter.incrementAndGet(); } } + + deleteSuccessfulItems(response); } @Override public void afterBulk(long executionId, BulkRequest req, Throwable e) { LOGGER.error("Fail to execute bulk index request: " + req, e); } + + private void deleteSuccessfulItems(BulkResponse bulkResponse) { + if (esQueueDtos != null) { + List<EsQueueDto> itemsToDelete = Arrays.stream(bulkResponse.getItems()) + .filter(b -> !b.isFailed()) + .map(b -> esQueueDtos.stream().filter(t -> b.getId().equals(t.getDocUuid())).findFirst().orElse(null)) + .filter(Objects::nonNull) + .collect(toList()); + + dbClient.esQueueDao().delete(dbSession, itemsToDelete); + dbSession.commit(); + } + } } public enum Size { @@ -293,5 +340,4 @@ public class BulkIndexer implements Startable { req.get(); } } - } diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java new file mode 100644 index 00000000000..c7ffdb9b2d6 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/RecoveryIndexer.java @@ -0,0 +1,161 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package org.sonar.server.es; + +import com.google.common.collect.ListMultimap; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang.math.RandomUtils; +import org.sonar.api.Startable; +import org.sonar.api.config.Settings; +import org.sonar.api.utils.System2; +import org.sonar.api.utils.log.Logger; +import org.sonar.api.utils.log.Loggers; +import org.sonar.api.utils.log.Profiler; +import org.sonar.core.util.stream.MoreCollectors; +import org.sonar.db.DbClient; +import org.sonar.db.DbSession; +import org.sonar.db.es.EsQueueDto; +import org.sonar.server.user.index.UserIndexer; + +import static java.lang.String.format; + +public class RecoveryIndexer implements Startable { + + private static final Logger LOGGER = Loggers.get(RecoveryIndexer.class); + private static final String LOG_PREFIX = "Elasticsearch recovery - "; + private static final String PROPERTY_INITIAL_DELAY = "sonar.search.recovery.initialDelayInMs"; + private static final String PROPERTY_DELAY = "sonar.search.recovery.delayInMs"; + private static final String PROPERTY_MIN_AGE = "sonar.search.recovery.minAgeInMs"; + private static final String PROPERTY_LOOP_LIMIT = "sonar.search.recovery.loopLimit"; + private static final long DEFAULT_DELAY_IN_MS = 5L * 60 * 1000; + private static final long DEFAULT_MIN_AGE_IN_MS = 5L * 60 * 1000; + private static final int DEFAULT_LOOP_LIMIT = 10_000; + private static final double CIRCUIT_BREAKER_IN_PERCENT = 0.3; + + private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, + new ThreadFactoryBuilder() + .setPriority(Thread.MIN_PRIORITY) + .setNameFormat("RecoveryIndexer-%d") + .build()); + private final System2 system2; + private final Settings settings; + private final DbClient dbClient; + private final UserIndexer userIndexer; + private final long minAgeInMs; + private final long loopLimit; + + public RecoveryIndexer(System2 system2, Settings settings, DbClient dbClient, UserIndexer userIndexer) { + this.system2 = system2; + this.settings = settings; + this.dbClient = dbClient; + this.userIndexer = userIndexer; + this.minAgeInMs = getSetting(PROPERTY_MIN_AGE, DEFAULT_MIN_AGE_IN_MS); + this.loopLimit = getSetting(PROPERTY_LOOP_LIMIT, DEFAULT_LOOP_LIMIT); + } + + @Override + public void start() { + long delayInMs = getSetting(PROPERTY_DELAY, DEFAULT_DELAY_IN_MS); + + // in the cluster mode, avoid (but not prevent!) simultaneous executions of recovery + // indexers so that a document is not handled multiple times. + long initialDelayInMs = getSetting(PROPERTY_INITIAL_DELAY, RandomUtils.nextInt(1 + (int) (delayInMs / 2))); + + executorService.scheduleAtFixedRate( + this::recover, + initialDelayInMs, + delayInMs, + TimeUnit.MILLISECONDS); + } + + @Override + public void stop() { + try { + executorService.shutdown(); + executorService.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOGGER.error(LOG_PREFIX + "Unable to stop recovery indexer in timely fashion", e); + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + + void recover() { + try (DbSession dbSession = dbClient.openSession(false)) { + Profiler profiler = Profiler.create(LOGGER).start(); + long beforeDate = system2.now() - minAgeInMs; + long total = 0L; + long totalSuccess = 0L; + + Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); + while (!items.isEmpty()) { + total += items.size(); + long loopSuccess = 0L; + + ListMultimap<EsQueueDto.Type, EsQueueDto> itemsByType = groupItemsByType(items); + for (Map.Entry<EsQueueDto.Type, Collection<EsQueueDto>> entry : itemsByType.asMap().entrySet()) { + loopSuccess += doIndex(dbSession, entry.getKey(), entry.getValue()); + } + + totalSuccess += loopSuccess; + if (1.0d * (items.size() - loopSuccess) / items.size() >= CIRCUIT_BREAKER_IN_PERCENT) { + LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", items.size() - loopSuccess, items.size()); + break; + } + items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); + } + if (total > 0L) { + profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", total, total - totalSuccess)); + } + } catch (Throwable t) { + LOGGER.error(LOG_PREFIX + "fail to recover documents", t); + } + } + + private long doIndex(DbSession dbSession, EsQueueDto.Type type, Collection<EsQueueDto> typeItems) { + LOGGER.trace(LOG_PREFIX + "processing {} {}", typeItems.size(), type); + switch (type) { + case USER: + return userIndexer.index(dbSession, typeItems); + default: + LOGGER.error(LOG_PREFIX + "ignore {} documents with unsupported type {}", typeItems.size(), type); + return 0; + } + } + + private static ListMultimap<EsQueueDto.Type, EsQueueDto> groupItemsByType(Collection<EsQueueDto> items) { + return items.stream().collect(MoreCollectors.index(EsQueueDto::getDocType)); + } + + private long getSetting(String key, long defaultValue) { + long val = settings.getLong(key); + if (val <= 0) { + val = defaultValue; + } + LOGGER.debug(LOG_PREFIX + "{}={}", key, val); + return val; + } +} diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/StartupIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/es/StartupIndexer.java index d007e402713..a1fef5fabf6 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/es/StartupIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/es/StartupIndexer.java @@ -28,9 +28,9 @@ public interface StartupIndexer { /** * This reindexing method will only be called on startup, and only, - * if there is at least one empty types. + * if there is at least one uninitialized type. */ - void indexOnStartup(Set<IndexType> emptyIndexTypes); + void indexOnStartup(Set<IndexType> uninitializedIndexTypes); Set<IndexType> getIndexTypes(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/es/queue/package-info.java b/server/sonar-server/src/main/java/org/sonar/server/es/queue/package-info.java new file mode 100644 index 00000000000..89f951738d6 --- /dev/null +++ b/server/sonar-server/src/main/java/org/sonar/server/es/queue/package-info.java @@ -0,0 +1,23 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +@ParametersAreNonnullByDefault +package org.sonar.server.es.queue; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java index 4eaa6523a30..5dbf88356bb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/issue/index/IssueIndexer.java @@ -69,7 +69,7 @@ public class IssueIndexer implements ProjectIndexer, NeedAuthorizationIndexer, S } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { doIndex(createBulkIndexer(Size.LARGE), (String) null); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java index e7851052f49..84d52030769 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/measure/index/ProjectMeasuresIndexer.java @@ -59,7 +59,7 @@ public class ProjectMeasuresIndexer implements ProjectIndexer, NeedAuthorization } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { doIndex(createBulkIndexer(Size.LARGE), (String) null); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/organization/OrganizationCreationImpl.java b/server/sonar-server/src/main/java/org/sonar/server/organization/OrganizationCreationImpl.java index 25a4939746b..010d6dfafad 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/organization/OrganizationCreationImpl.java +++ b/server/sonar-server/src/main/java/org/sonar/server/organization/OrganizationCreationImpl.java @@ -106,11 +106,10 @@ public class OrganizationCreationImpl implements OrganizationCreation { addCurrentUserToGroup(dbSession, ownerGroup, userCreator.getId()); addCurrentUserToGroup(dbSession, defaultGroup, userCreator.getId()); - dbSession.commit(); batchDbSession.commit(); // Elasticsearch is updated when DB session is committed - userIndexer.index(userCreator.getLogin()); + userIndexer.commitAndIndex(dbSession, userCreator); return organization; } @@ -144,11 +143,10 @@ public class OrganizationCreationImpl implements OrganizationCreation { insertQualityProfiles(dbSession, batchDbSession, organization); addCurrentUserToGroup(dbSession, defaultGroup, newUser.getId()); - dbSession.commit(); batchDbSession.commit(); // Elasticsearch is updated when DB session is committed - userIndexer.index(newUser.getLogin()); + userIndexer.commitAndIndex(dbSession, newUser); return Optional.of(organization); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/AddMemberAction.java b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/AddMemberAction.java index 7c6c1242ded..535ace66812 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/AddMemberAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/AddMemberAction.java @@ -118,8 +118,7 @@ public class AddMemberAction implements OrganizationsWsAction { .setUserId(user.getId())); dbClient.userGroupDao().insert(dbSession, new UserGroupDto().setGroupId(defaultGroupFinder.findDefaultGroup(dbSession, organization.getUuid()).getId()).setUserId(user.getId())); - dbSession.commit(); - userIndexer.index(user.getLogin()); + userIndexer.commitAndIndex(dbSession, user); } private AddMemberWsResponse buildResponse(UserDto user, int groups) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/DeleteAction.java b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/DeleteAction.java index bd99338c81f..f3c4f596446 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/DeleteAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/DeleteAction.java @@ -19,6 +19,7 @@ */ package org.sonar.server.organization.ws; +import java.util.Collection; import java.util.List; import org.sonar.api.server.ws.Request; import org.sonar.api.server.ws.Response; @@ -138,11 +139,10 @@ public class DeleteAction implements OrganizationsWsAction { } private void deleteOrganization(DbSession dbSession, OrganizationDto organization) { - List<String> logins = dbClient.organizationMemberDao().selectLoginsByOrganizationUuid(dbSession, organization.getUuid()); + Collection<String> logins = dbClient.organizationMemberDao().selectLoginsByOrganizationUuid(dbSession, organization.getUuid()); dbClient.organizationMemberDao().deleteByOrganizationUuid(dbSession, organization.getUuid()); dbClient.organizationDao().deleteByUuid(dbSession, organization.getUuid()); - dbSession.commit(); - userIndexer.index(logins); + userIndexer.commitAndIndexByLogins(dbSession, logins); } private static void preventDeletionOfDefaultOrganization(String key, DefaultOrganization defaultOrganization) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/RemoveMemberAction.java b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/RemoveMemberAction.java index 5c81f8b17bb..a927bdfcbd5 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/organization/ws/RemoveMemberAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/organization/ws/RemoveMemberAction.java @@ -103,8 +103,7 @@ public class RemoveMemberAction implements OrganizationsWsAction { dbClient.propertiesDao().deleteByOrganizationAndMatchingLogin(dbSession, organizationUuid, user.getLogin(), singletonList(DEFAULT_ISSUE_ASSIGNEE)); dbClient.organizationMemberDao().delete(dbSession, organizationUuid, userId); - dbSession.commit(); - userIndexer.index(user.getLogin()); + userIndexer.commitAndIndex(dbSession, user); } private void ensureLastAdminIsNotRemoved(DbSession dbSession, OrganizationDto organizationDto, UserDto user) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java index 41dd5b64ba4..8c7ad3e6693 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexer.java @@ -80,9 +80,9 @@ public class PermissionIndexer implements ProjectIndexer, StartupIndexer { } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { List<Dto> authorizations = getAllAuthorizations(); - Stream<AuthorizationScope> scopes = getScopes(emptyIndexTypes); + Stream<AuthorizationScope> scopes = getScopes(uninitializedIndexTypes); index(authorizations, scopes, Size.LARGE); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java index 81c98dc39c8..55d575a8269 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java +++ b/server/sonar-server/src/main/java/org/sonar/server/permission/index/PermissionIndexerDao.java @@ -32,8 +32,8 @@ import org.apache.commons.lang.StringUtils; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import static org.apache.commons.lang.StringUtils.repeat; import static org.sonar.db.DatabaseUtils.executeLargeInputs; -import static org.sonar.db.DatabaseUtils.repeatCondition; /** * No streaming because of union of joins -> no need to use ResultSetIterator @@ -200,7 +200,7 @@ public class PermissionIndexerDao { if (projectUuids.isEmpty()) { sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", ""); } else { - sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", " AND (" + repeatCondition("projects.uuid = ?", projectUuids.size(), "OR") + ")"); + sql = StringUtils.replace(SQL_TEMPLATE, "{projectsCondition}", " AND (" + repeat("projects.uuid = ?", " or ", projectUuids.size()) + ")"); } PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); int index = 1; diff --git a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java index ac0aa79be26..434eab1d84e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java +++ b/server/sonar-server/src/main/java/org/sonar/server/platform/platformlevel/PlatformLevel4.java @@ -53,6 +53,7 @@ import org.sonar.server.duplication.ws.ShowResponseBuilder; import org.sonar.server.email.ws.EmailsWsModule; import org.sonar.server.es.IndexCreator; import org.sonar.server.es.IndexDefinitions; +import org.sonar.server.es.RecoveryIndexer; import org.sonar.server.event.NewAlerts; import org.sonar.server.favorite.FavoriteModule; import org.sonar.server.issue.AddTagsAction; @@ -520,7 +521,9 @@ public class PlatformLevel4 extends PlatformLevel { WebhooksWsModule.class, // Http Request ID - HttpRequestIdModule.class); + HttpRequestIdModule.class, + + RecoveryIndexer.class); addAll(level4AddedComponents); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/property/InternalProperties.java b/server/sonar-server/src/main/java/org/sonar/server/property/InternalProperties.java index 1d84ad9441a..42925dbb097 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/property/InternalProperties.java +++ b/server/sonar-server/src/main/java/org/sonar/server/property/InternalProperties.java @@ -34,6 +34,8 @@ public interface InternalProperties { String ORGANIZATION_ENABLED = "organization.enabled"; + String ES_INDEX_INITIALIZING_PREFIX = "es.initializing."; + /** * Read the value of the specified property. * diff --git a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java index 1cd6a8f8c63..f61b6089a68 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/qualityprofile/index/ActiveRuleIndexer.java @@ -58,7 +58,7 @@ public class ActiveRuleIndexer implements StartupIndexer { } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { ActiveRuleIterator dbCursor = activeRuleIteratorFactory.createForAll(dbSession); scrollDbAndIndex(dbCursor, Size.LARGE); diff --git a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java index 4e0eccebfcb..30919221b0a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/test/index/TestIndexer.java @@ -77,7 +77,7 @@ public class TestIndexer implements ProjectIndexer, StartupIndexer { } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { doIndex(null, Size.LARGE); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/UserUpdater.java b/server/sonar-server/src/main/java/org/sonar/server/user/UserUpdater.java index b92fae91e36..b29394d3845 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/UserUpdater.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/UserUpdater.java @@ -22,16 +22,17 @@ package org.sonar.server.user; import com.google.common.base.Joiner; import com.google.common.base.Strings; import java.security.SecureRandom; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Random; +import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.commons.codec.digest.DigestUtils; import org.sonar.api.config.Configuration; import org.sonar.api.platform.NewUserHandler; import org.sonar.api.server.ServerSide; -import org.sonar.api.utils.System2; import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; @@ -73,19 +74,17 @@ public class UserUpdater { private final NewUserNotifier newUserNotifier; private final DbClient dbClient; private final UserIndexer userIndexer; - private final System2 system2; private final OrganizationFlags organizationFlags; private final DefaultOrganizationProvider defaultOrganizationProvider; private final OrganizationCreation organizationCreation; private final DefaultGroupFinder defaultGroupFinder; private final Configuration config; - public UserUpdater(NewUserNotifier newUserNotifier, DbClient dbClient, UserIndexer userIndexer, System2 system2, OrganizationFlags organizationFlags, + public UserUpdater(NewUserNotifier newUserNotifier, DbClient dbClient, UserIndexer userIndexer, OrganizationFlags organizationFlags, DefaultOrganizationProvider defaultOrganizationProvider, OrganizationCreation organizationCreation, DefaultGroupFinder defaultGroupFinder, Configuration config) { this.newUserNotifier = newUserNotifier; this.dbClient = dbClient; this.userIndexer = userIndexer; - this.system2 = system2; this.organizationFlags = organizationFlags; this.defaultOrganizationProvider = defaultOrganizationProvider; this.organizationCreation = organizationCreation; @@ -93,14 +92,17 @@ public class UserUpdater { this.config = config; } - public UserDto create(DbSession dbSession, NewUser newUser) { + public UserDto createAndCommit(DbSession dbSession, NewUser newUser, Consumer<UserDto> beforeCommit) { String login = newUser.login(); UserDto userDto = dbClient.userDao().selectByLogin(dbSession, newUser.login()); if (userDto == null) { - userDto = saveUser(dbSession, createNewUserDto(dbSession, newUser)); + userDto = saveUser(dbSession, createDto(dbSession, newUser)); } else { reactivateUser(dbSession, userDto, login, newUser); } + beforeCommit.accept(userDto); + userIndexer.commitAndIndex(dbSession, userDto); + notifyNewUser(userDto.getLogin(), userDto.getName(), newUser.email()); return userDto; } @@ -120,26 +122,31 @@ public class UserUpdater { // Hack to allow to change the password of the user existingUser.setLocal(true); setOnboarded(existingUser); - updateUserDto(dbSession, updateUser, existingUser); + updateDto(dbSession, updateUser, existingUser); updateUser(dbSession, existingUser); addUserToDefaultOrganizationAndDefaultGroup(dbSession, existingUser); - dbSession.commit(); } - public void update(DbSession dbSession, UpdateUser updateUser) { - UserDto user = dbClient.userDao().selectByLogin(dbSession, updateUser.login()); - checkFound(user, "User with login '%s' has not been found", updateUser.login()); - boolean isUserUpdated = updateUserDto(dbSession, updateUser, user); - if (!isUserUpdated) { - return; + public void updateAndCommit(DbSession dbSession, UpdateUser updateUser, Consumer<UserDto> beforeCommit) { + UserDto dto = dbClient.userDao().selectByLogin(dbSession, updateUser.login()); + checkFound(dto, "User with login '%s' has not been found", updateUser.login()); + boolean isUserUpdated = updateDto(dbSession, updateUser, dto); + if (isUserUpdated) { + // at least one change. Database must be updated and Elasticsearch re-indexed + updateUser(dbSession, dto); + beforeCommit.accept(dto); + userIndexer.commitAndIndex(dbSession, dto); + notifyNewUser(dto.getLogin(), dto.getName(), dto.getEmail()); + } else { + // no changes but still execute the consumer + beforeCommit.accept(dto); + dbSession.commit(); } - updateUser(dbSession, user); - notifyNewUser(user.getLogin(), user.getName(), user.getEmail()); } - private UserDto createNewUserDto(DbSession dbSession, NewUser newUser) { + private UserDto createDto(DbSession dbSession, NewUser newUser) { UserDto userDto = new UserDto(); - List<String> messages = newArrayList(); + List<String> messages = new ArrayList<>(); String login = newUser.login(); if (validateLoginFormat(login, messages)) { @@ -158,7 +165,7 @@ public class UserUpdater { String password = newUser.password(); if (password != null && validatePasswords(password, messages)) { - setEncryptedPassWord(password, userDto); + setEncryptedPassword(password, userDto); } List<String> scmAccounts = sanitizeScmAccounts(newUser.scmAccounts()); @@ -173,13 +180,13 @@ public class UserUpdater { return userDto; } - private boolean updateUserDto(DbSession dbSession, UpdateUser updateUser, UserDto userDto) { + private boolean updateDto(DbSession dbSession, UpdateUser update, UserDto dto) { List<String> messages = newArrayList(); - boolean changed = updateName(updateUser, userDto, messages); - changed |= updateEmail(updateUser, userDto, messages); - changed |= updateExternalIdentity(updateUser, userDto); - changed |= updatePassword(updateUser, userDto, messages); - changed |= updateScmAccounts(dbSession, updateUser, userDto, messages); + boolean changed = updateName(update, dto, messages); + changed |= updateEmail(update, dto, messages); + changed |= updateExternalIdentity(update, dto); + changed |= updatePassword(update, dto, messages); + changed |= updateScmAccounts(dbSession, update, dto, messages); checkRequest(messages.isEmpty(), messages); return changed; } @@ -216,7 +223,7 @@ public class UserUpdater { private static boolean updatePassword(UpdateUser updateUser, UserDto userDto, List<String> messages) { String password = updateUser.password(); if (!updateUser.isExternalIdentityChanged() && updateUser.isPasswordChanged() && validatePasswords(password, messages) && checkPasswordChangeAllowed(userDto, messages)) { - setEncryptedPassWord(password, userDto); + setEncryptedPassword(password, userDto); return true; } return false; @@ -355,25 +362,19 @@ public class UserUpdater { } private UserDto saveUser(DbSession dbSession, UserDto userDto) { - long now = system2.now(); - userDto.setActive(true).setCreatedAt(now).setUpdatedAt(now); + userDto.setActive(true); UserDto res = dbClient.userDao().insert(dbSession, userDto); addUserToDefaultOrganizationAndDefaultGroup(dbSession, userDto); organizationCreation.createForUser(dbSession, userDto); - dbSession.commit(); - userIndexer.index(userDto.getLogin()); return res; } - private void updateUser(DbSession dbSession, UserDto userDto) { - long now = system2.now(); - userDto.setActive(true).setUpdatedAt(now); - dbClient.userDao().update(dbSession, userDto); - dbSession.commit(); - userIndexer.index(userDto.getLogin()); + private void updateUser(DbSession dbSession, UserDto dto) { + dto.setActive(true); + dbClient.userDao().update(dbSession, dto); } - private static void setEncryptedPassWord(String password, UserDto userDto) { + private static void setEncryptedPassword(String password, UserDto userDto) { Random random = new SecureRandom(); byte[] salt = new byte[32]; random.nextBytes(salt); @@ -382,7 +383,7 @@ public class UserUpdater { userDto.setCryptedPassword(encryptPassword(password, saltHex)); } - private void notifyNewUser(String login, String name, String email) { + private void notifyNewUser(String login, String name, @Nullable String email) { newUserNotifier.onNewUser(NewUserHandler.Context.builder() .setLogin(login) .setName(name) diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java index 1b7591d0f55..153619e2d3f 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserDoc.java @@ -20,7 +20,6 @@ package org.sonar.server.user.index; import com.google.common.collect.Maps; -import java.util.Date; import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -83,14 +82,6 @@ public class UserDoc extends BaseDoc implements User { return getField(FIELD_ORGANIZATION_UUIDS); } - public long createdAt() { - return getFieldAsDate(UserIndexDefinition.FIELD_CREATED_AT).getTime(); - } - - public long updatedAt() { - return getFieldAsDate(UserIndexDefinition.FIELD_UPDATED_AT).getTime(); - } - public UserDoc setLogin(@Nullable String s) { setField(UserIndexDefinition.FIELD_LOGIN, s); return this; @@ -120,14 +111,4 @@ public class UserDoc extends BaseDoc implements User { setField(FIELD_ORGANIZATION_UUIDS, organizationUuids); return this; } - - public UserDoc setCreatedAt(long l) { - setField(UserIndexDefinition.FIELD_CREATED_AT, new Date(l)); - return this; - } - - public UserDoc setUpdatedAt(long l) { - setField(UserIndexDefinition.FIELD_UPDATED_AT, new Date(l)); - return this; - } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java index d64f2fd8145..4178d607ead 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexDefinition.java @@ -36,8 +36,6 @@ public class UserIndexDefinition implements IndexDefinition { public static final String FIELD_LOGIN = "login"; public static final String FIELD_NAME = "name"; public static final String FIELD_EMAIL = "email"; - public static final String FIELD_CREATED_AT = "createdAt"; - public static final String FIELD_UPDATED_AT = "updatedAt"; public static final String FIELD_ACTIVE = "active"; public static final String FIELD_SCM_ACCOUNTS = "scmAccounts"; public static final String FIELD_ORGANIZATION_UUIDS = "organizationUuids"; @@ -59,8 +57,6 @@ public class UserIndexDefinition implements IndexDefinition { mapping.stringFieldBuilder(FIELD_LOGIN).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build(); mapping.stringFieldBuilder(FIELD_NAME).addSubFields(USER_SEARCH_GRAMS_ANALYZER).build(); mapping.stringFieldBuilder(FIELD_EMAIL).addSubFields(USER_SEARCH_GRAMS_ANALYZER, SORTABLE_ANALYZER).build(); - mapping.createDateTimeField(FIELD_CREATED_AT); - mapping.createDateTimeField(FIELD_UPDATED_AT); mapping.createBooleanField(FIELD_ACTIVE); mapping.stringFieldBuilder(FIELD_SCM_ACCOUNTS).disableNorms().addSubFields(SORTABLE_ANALYZER).build(); mapping.stringFieldBuilder(FIELD_ORGANIZATION_UUIDS).disableNorms().build(); diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java index 65241119240..6f99292e7cb 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserIndexer.java @@ -19,14 +19,20 @@ */ package org.sonar.server.user.index; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableSet; -import java.util.Iterator; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Maps; +import java.util.Collection; import java.util.List; import java.util.Set; -import javax.annotation.Nullable; import org.elasticsearch.action.index.IndexRequest; +import org.sonar.core.util.stream.MoreCollectors; import org.sonar.db.DbClient; import org.sonar.db.DbSession; +import org.sonar.db.es.EsQueueDto; +import org.sonar.db.user.UserDto; import org.sonar.server.es.BulkIndexer; import org.sonar.server.es.BulkIndexer.Size; import org.sonar.server.es.EsClient; @@ -35,7 +41,7 @@ import org.sonar.server.es.StartupIndexer; import static java.util.Collections.singletonList; import static java.util.Objects.requireNonNull; -import static org.sonar.db.DatabaseUtils.executeLargeInputsWithoutOutput; +import static org.sonar.core.util.stream.MoreCollectors.toHashSet; import static org.sonar.server.user.index.UserIndexDefinition.INDEX_TYPE_USER; public class UserIndexer implements StartupIndexer { @@ -54,56 +60,98 @@ public class UserIndexer implements StartupIndexer { } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { - doIndex(newBulkIndexer(Size.LARGE), null); + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { + try (DbSession dbSession = dbClient.openSession(false)) { + ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create(); + dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByLogin::put); + + BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE); + bulkIndexer.start(); + dbClient.userDao().scrollAll(dbSession, + // only index requests, no deletion requests. + // Deactivated users are not deleted but updated. + u -> bulkIndexer.add(newIndexRequest(u, organizationUuidsByLogin))); + bulkIndexer.stop(); + } } - public void index(String login) { - requireNonNull(login); - doIndex(newBulkIndexer(Size.REGULAR), singletonList(login)); + public void commitAndIndex(DbSession dbSession, UserDto user) { + commitAndIndexByLogins(dbSession, singletonList(user.getLogin())); } - public void index(List<String> logins) { - requireNonNull(logins); - if (logins.isEmpty()) { - return; - } - - doIndex(newBulkIndexer(Size.REGULAR), logins); + public void commitAndIndex(DbSession dbSession, Collection<UserDto> users) { + commitAndIndexByLogins(dbSession, Collections2.transform(users, UserDto::getLogin)); } - private void doIndex(BulkIndexer bulk, @Nullable List<String> logins) { - try (DbSession dbSession = dbClient.openSession(false)) { - if (logins == null) { - processLogins(bulk, dbSession, null); - } else { - executeLargeInputsWithoutOutput(logins, l -> processLogins(bulk, dbSession, l)); - } - } + public void commitAndIndexByLogins(DbSession dbSession, Collection<String> logins) { + List<EsQueueDto> items = logins.stream() + .map(l -> EsQueueDto.create(EsQueueDto.Type.USER, l)) + .collect(MoreCollectors.toArrayList()); + + dbClient.esQueueDao().insert(dbSession, items); + dbSession.commit(); + postCommit(dbSession, logins, items); } - private void processLogins(BulkIndexer bulk, DbSession dbSession, @Nullable List<String> logins) { - try (UserResultSetIterator rowIt = UserResultSetIterator.create(dbClient, dbSession, logins)) { - processResultSet(bulk, rowIt); - } + /** + * Entry point for Byteman tests. See directory tests/resilience. + * The parameter "logins" is used only by the Byteman script. + */ + private void postCommit(DbSession dbSession, Collection<String> logins, Collection<EsQueueDto> items) { + index(dbSession, items); } - private static void processResultSet(BulkIndexer bulk, Iterator<UserDoc> users) { - bulk.start(); - while (users.hasNext()) { - UserDoc user = users.next(); - bulk.add(newIndexRequest(user)); + /** + * @return the number of items that have been successfully indexed + */ + public long index(DbSession dbSession, Collection<EsQueueDto> items) { + if (items.isEmpty()) { + return 0L; } - bulk.stop(); + Set<String> logins = items + .stream() + .filter(i -> { + requireNonNull(i.getDocUuid(), () -> "BUG - " + i + " has not been persisted before indexing"); + return true; + }) + .map(EsQueueDto::getDocUuid) + .collect(toHashSet(items.size())); + + ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create(); + dbClient.organizationMemberDao().selectForUserIndexing(dbSession, logins, organizationUuidsByLogin::put); + + BulkIndexer bulkIndexer = newBulkIndexer(Size.REGULAR); + bulkIndexer.start(dbSession, dbClient, items); + dbClient.userDao().scrollByLogins(dbSession, logins, + // only index requests, no deletion requests. + // Deactivated users are not deleted but updated. + u -> { + logins.remove(u.getLogin()); + bulkIndexer.add(newIndexRequest(u, organizationUuidsByLogin)); + }); + + // the remaining logins reference rows that don't exist in db. They must + // be deleted from index. + logins.forEach(l -> bulkIndexer.addDeletion(UserIndexDefinition.INDEX_TYPE_USER, l)); + return bulkIndexer.stop(); } private BulkIndexer newBulkIndexer(Size bulkSize) { return new BulkIndexer(esClient, UserIndexDefinition.INDEX_TYPE_USER.getIndex(), bulkSize); } - private static IndexRequest newIndexRequest(UserDoc user) { - return new IndexRequest(UserIndexDefinition.INDEX_TYPE_USER.getIndex(), UserIndexDefinition.INDEX_TYPE_USER.getType(), user.login()) - .source(user.getFields()); + private static IndexRequest newIndexRequest(UserDto user, ListMultimap<String, String> organizationUuidsByLogins) { + UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(8)); + // all the keys must be present, even if value is null + doc.setLogin(user.getLogin()); + doc.setName(user.getName()); + doc.setEmail(user.getEmail()); + doc.setActive(user.isActive()); + doc.setScmAccounts(UserDto.decodeScmAccounts(user.getScmAccounts())); + doc.setOrganizationUuids(organizationUuidsByLogins.get(user.getLogin())); + + return new IndexRequest(UserIndexDefinition.INDEX_TYPE_USER.getIndex(), UserIndexDefinition.INDEX_TYPE_USER.getType()) + .id(doc.getId()) + .source(doc.getFields()); } - } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java index dc1a707310d..3bdf905f729 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/index/UserResultSetIterator.java @@ -19,22 +19,25 @@ */ package org.sonar.server.user.index; -import com.google.common.base.Joiner; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.lang.StringUtils; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.ResultSetIterator; +import org.sonar.db.es.EsQueueDto; import org.sonar.db.user.UserDto; +import static org.apache.commons.lang.StringUtils.repeat; +import static org.sonar.core.util.stream.MoreCollectors.toArrayList; + /** * Scrolls over table USERS and reads documents to populate the user index */ @@ -52,8 +55,6 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> { }; private static final String SQL_ALL = "select " + StringUtils.join(FIELDS, ",") + " from users u "; - private static final String LOGIN_FILTER = "u.login=?"; - private static final Joiner OR_JOINER = Joiner.on(" or "); private final ListMultimap<String, String> organizationUuidsByLogins; @@ -62,16 +63,25 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> { this.organizationUuidsByLogins = organizationUuidsByLogins; } - static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable List<String> logins) { + static UserResultSetIterator create(DbClient dbClient, DbSession session, @Nullable Collection<EsQueueDto> esQueueDtos) { try { - String sql = createSql(logins); + String sql = SQL_ALL; + List<String> logins = null; + if (esQueueDtos != null) { + logins = esQueueDtos.stream() + .filter(i -> i.getDocType() == EsQueueDto.Type.USER) + .map(EsQueueDto::getDocUuid).collect(toArrayList()); + sql += "where (" + repeat("u.login=?", " or ", logins.size()) + ")"; + } + PreparedStatement stmt = dbClient.getMyBatis().newScrollingSelectStatement(session, sql); setParameters(stmt, logins); ListMultimap<String, String> organizationUuidsByLogin = ArrayListMultimap.create(); - if (logins == null) { + if (esQueueDtos == null) { dbClient.organizationMemberDao().selectAllForUserIndexing(session, organizationUuidsByLogin::put); } else { + dbClient.organizationMemberDao().selectForUserIndexing(session, logins, organizationUuidsByLogin::put); } @@ -81,35 +91,21 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> { } } - private static String createSql(@Nullable List<String> logins) { - if (logins == null) { - return SQL_ALL; - } - - List<String> sqlLogins = logins.stream() - .map(l -> LOGIN_FILTER) - .collect(Collectors.toList()); - - String sql = SQL_ALL; - sql += " WHERE "; - sql += "(" + OR_JOINER.join(sqlLogins) + ")"; - - return sql; - } - - private static void setParameters(PreparedStatement stmt, @Nullable List<String> logins) throws SQLException { + private static void setParameters(PreparedStatement stmt, @Nullable Collection<String> logins) throws SQLException { if (logins == null) { return; } - for (int i = 0; i < logins.size(); i++) { - stmt.setString(i + 1, logins.get(i)); + int paramIndex = 1; + for (String login : logins) { + stmt.setString(paramIndex, login); + paramIndex++; } } @Override protected UserDoc read(ResultSet rs) throws SQLException { - UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(8)); + UserDoc doc = new UserDoc(Maps.newHashMapWithExpectedSize(6)); String login = rs.getString(1); @@ -119,10 +115,7 @@ class UserResultSetIterator extends ResultSetIterator<UserDoc> { doc.setEmail(rs.getString(3)); doc.setActive(rs.getBoolean(4)); doc.setScmAccounts(UserDto.decodeScmAccounts(rs.getString(5))); - doc.setCreatedAt(rs.getLong(6)); - doc.setUpdatedAt(rs.getLong(7)); doc.setOrganizationUuids(organizationUuidsByLogins.get(login)); return doc; } - } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/ws/ChangePasswordAction.java b/server/sonar-server/src/main/java/org/sonar/server/user/ws/ChangePasswordAction.java index bd973387892..5faca4c8879 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/ws/ChangePasswordAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/ws/ChangePasswordAction.java @@ -90,7 +90,7 @@ public class ChangePasswordAction implements UsersWsAction { String password = request.mandatoryParam(PARAM_PASSWORD); UpdateUser updateUser = UpdateUser.create(login).setPassword(password); - userUpdater.update(dbSession, updateUser); + userUpdater.updateAndCommit(dbSession, updateUser, u -> {}); } response.noContent(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/ws/CreateAction.java b/server/sonar-server/src/main/java/org/sonar/server/user/ws/CreateAction.java index d0d43c426e7..7feb6dc48d9 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/ws/CreateAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/ws/CreateAction.java @@ -127,9 +127,8 @@ public class CreateAction implements UsersWsAction { if (!request.isLocal()) { newUser.setExternalIdentity(new ExternalIdentity(SQ_AUTHORITY, request.getLogin())); } - UserDto userDto = userUpdater.create(dbSession, newUser.build()); - dbSession.commit(); - return buildResponse(userDto); + UserDto createdUser = userUpdater.createAndCommit(dbSession, newUser.build(), u -> {}); + return buildResponse(createdUser); } } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/ws/DeactivateAction.java b/server/sonar-server/src/main/java/org/sonar/server/user/ws/DeactivateAction.java index 1419e4abebc..94ec4c6926a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/ws/DeactivateAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/ws/DeactivateAction.java @@ -101,11 +101,10 @@ public class DeactivateAction implements UsersWsAction { dbClient.userPermissionDao().deleteByUserId(dbSession, userId); dbClient.permissionTemplateDao().deleteUserPermissionsByUserId(dbSession, userId); dbClient.organizationMemberDao().deleteByUserId(dbSession, userId); - dbClient.userDao().deactivateUserById(dbSession, userId); - dbSession.commit(); + dbClient.userDao().deactivateUser(dbSession, user); + userIndexer.commitAndIndex(dbSession, user); } - userIndexer.index(login); writeResponse(response, login); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/ws/SkipOnboardingTutorialAction.java b/server/sonar-server/src/main/java/org/sonar/server/user/ws/SkipOnboardingTutorialAction.java index 003208121d2..2caddc3379e 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/ws/SkipOnboardingTutorialAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/ws/SkipOnboardingTutorialAction.java @@ -35,12 +35,10 @@ public class SkipOnboardingTutorialAction implements UsersWsAction { private final UserSession userSession; private final DbClient dbClient; - private final System2 system2; - public SkipOnboardingTutorialAction(UserSession userSession, DbClient dbClient, System2 system2) { + public SkipOnboardingTutorialAction(UserSession userSession, DbClient dbClient) { this.userSession = userSession; this.dbClient = dbClient; - this.system2 = system2; } @Override @@ -63,7 +61,8 @@ public class SkipOnboardingTutorialAction implements UsersWsAction { checkState(userDto != null, "User login '%s' cannot be found", userLogin); if (!userDto.isOnboarded()) { userDto.setOnboarded(true); - userDto.setUpdatedAt(system2.now()); + // no need to update Elasticsearch, the field onBoarded + // is not indexed dbClient.userDao().update(dbSession, userDto); dbSession.commit(); } diff --git a/server/sonar-server/src/main/java/org/sonar/server/user/ws/UpdateAction.java b/server/sonar-server/src/main/java/org/sonar/server/user/ws/UpdateAction.java index b9292bd4f73..dc50009b16a 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/user/ws/UpdateAction.java +++ b/server/sonar-server/src/main/java/org/sonar/server/user/ws/UpdateAction.java @@ -118,7 +118,7 @@ public class UpdateAction implements UsersWsAction { if (!request.getScmAccounts().isEmpty()) { updateUser.setScmAccounts(request.getScmAccounts()); } - userUpdater.update(dbSession, updateUser); + userUpdater.updateAndCommit(dbSession, updateUser, u -> {}); } private void writeUser(DbSession dbSession, Response response, String login) { diff --git a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java index 14b5ee7d892..2c92637976c 100644 --- a/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java +++ b/server/sonar-server/src/main/java/org/sonar/server/view/index/ViewIndexer.java @@ -53,7 +53,7 @@ public class ViewIndexer implements StartupIndexer { } @Override - public void indexOnStartup(Set<IndexType> emptyIndexTypes) { + public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { Map<String, String> viewAndProjectViewUuidMap = newHashMap(); for (UuidWithProjectUuidDto uuidWithProjectUuidDto : dbClient.componentDao().selectAllViewsAndSubViews(dbSession)) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/authentication/SsoAuthenticatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/authentication/SsoAuthenticatorTest.java index 5875442d41d..6094bac37d4 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/authentication/SsoAuthenticatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/authentication/SsoAuthenticatorTest.java @@ -40,12 +40,14 @@ import org.sonar.db.user.GroupDto; import org.sonar.db.user.UserDto; import org.sonar.server.authentication.event.AuthenticationEvent; import org.sonar.server.authentication.event.AuthenticationEvent.Source; +import org.sonar.server.es.EsTester; import org.sonar.server.organization.DefaultOrganizationProvider; import org.sonar.server.organization.OrganizationCreation; import org.sonar.server.organization.TestDefaultOrganizationProvider; import org.sonar.server.organization.TestOrganizationFlags; import org.sonar.server.user.NewUserNotifier; import org.sonar.server.user.UserUpdater; +import org.sonar.server.user.index.UserIndexDefinition; import org.sonar.server.user.index.UserIndexer; import org.sonar.server.usergroups.DefaultGroupFinder; @@ -65,11 +67,14 @@ import static org.sonar.server.authentication.event.AuthenticationExceptionMatch public class SsoAuthenticatorTest { + private MapSettings settings = new MapSettings(); + @Rule public ExpectedException expectedException = none(); - @Rule public DbTester db = DbTester.create(new AlwaysIncreasingSystem2()); + @Rule + public EsTester es = new EsTester(new UserIndexDefinition(settings.asConfig())); private static final String DEFAULT_LOGIN = "john"; private static final String DEFAULT_NAME = "John"; @@ -93,14 +98,14 @@ public class SsoAuthenticatorTest { private GroupDto sonarUsers; private System2 system2 = mock(System2.class); - private MapSettings settings = new MapSettings(); private OrganizationCreation organizationCreation = mock(OrganizationCreation.class); private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone(); + private UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client()); private UserIdentityAuthenticator userIdentityAuthenticator = new UserIdentityAuthenticator( db.getDbClient(), - new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), mock(UserIndexer.class), System2.INSTANCE, organizationFlags, defaultOrganizationProvider, organizationCreation, + new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), userIndexer, organizationFlags, defaultOrganizationProvider, organizationCreation, new DefaultGroupFinder(db.getDbClient()), settings.asConfig()), defaultOrganizationProvider, organizationFlags, new DefaultGroupFinder(db.getDbClient())); diff --git a/server/sonar-server/src/test/java/org/sonar/server/authentication/UserIdentityAuthenticatorTest.java b/server/sonar-server/src/test/java/org/sonar/server/authentication/UserIdentityAuthenticatorTest.java index 40638401375..1371534a98f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/authentication/UserIdentityAuthenticatorTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/authentication/UserIdentityAuthenticatorTest.java @@ -35,12 +35,14 @@ import org.sonar.db.user.GroupDto; import org.sonar.db.user.UserDto; import org.sonar.server.authentication.event.AuthenticationEvent.Method; import org.sonar.server.authentication.event.AuthenticationEvent.Source; +import org.sonar.server.es.EsTester; import org.sonar.server.organization.DefaultOrganizationProvider; import org.sonar.server.organization.OrganizationCreation; import org.sonar.server.organization.TestDefaultOrganizationProvider; import org.sonar.server.organization.TestOrganizationFlags; import org.sonar.server.user.NewUserNotifier; import org.sonar.server.user.UserUpdater; +import org.sonar.server.user.index.UserIndexDefinition; import org.sonar.server.user.index.UserIndexer; import org.sonar.server.usergroups.DefaultGroupFinder; @@ -69,32 +71,33 @@ public class UserIdentityAuthenticatorTest { .setEnabled(true) .setAllowsUsersToSignUp(true); + private MapSettings settings = new MapSettings(); + @Rule public ExpectedException thrown = ExpectedException.none(); - @Rule public DbTester db = DbTester.create(new AlwaysIncreasingSystem2()); - + @Rule + public EsTester es = new EsTester(new UserIndexDefinition(settings.asConfig())); + private UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client()); private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); private OrganizationCreation organizationCreation = mock(OrganizationCreation.class); private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone(); - private MapSettings settings = new MapSettings(); - private UserUpdater userUpdater = new UserUpdater( mock(NewUserNotifier.class), db.getDbClient(), - mock(UserIndexer.class), - System2.INSTANCE, + userIndexer, organizationFlags, defaultOrganizationProvider, organizationCreation, new DefaultGroupFinder(db.getDbClient()), settings.asConfig()); + private UserIdentityAuthenticator underTest = new UserIdentityAuthenticator(db.getDbClient(), userUpdater, defaultOrganizationProvider, organizationFlags, new DefaultGroupFinder(db.getDbClient())); @Test - public void authenticate_new_user() throws Exception { + public void authenticate_new_user() { organizationFlags.setEnabled(true); underTest.authenticate(USER_IDENTITY, IDENTITY_PROVIDER, Source.realm(Method.BASIC, IDENTITY_PROVIDER.getName())); @@ -111,7 +114,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void authenticate_new_user_with_groups() throws Exception { + public void authenticate_new_user_with_groups() { organizationFlags.setEnabled(true); GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1"); GroupDto group2 = db.users().insertGroup(db.getDefaultOrganization(), "group2"); @@ -123,7 +126,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void authenticate_new_user_and_force_default_group_when_organizations_are_disabled() throws Exception { + public void authenticate_new_user_and_force_default_group_when_organizations_are_disabled() { organizationFlags.setEnabled(false); UserDto user = db.users().insertUser(); GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1"); @@ -137,7 +140,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void does_not_force_default_group_when_authenticating_new_user_if_organizations_are_enabled() throws Exception { + public void does_not_force_default_group_when_authenticating_new_user_if_organizations_are_enabled() { organizationFlags.setEnabled(true); UserDto user = db.users().insertUser(); GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1"); @@ -171,7 +174,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void authenticate_existing_user() throws Exception { + public void authenticate_existing_user() { db.users().insertUser(newUserDto() .setLogin(USER_LOGIN) .setActive(true) @@ -192,7 +195,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void authenticate_existing_disabled_user() throws Exception { + public void authenticate_existing_disabled_user() { organizationFlags.setEnabled(true); db.users().insertUser(newUserDto() .setLogin(USER_LOGIN) @@ -214,7 +217,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void authenticate_existing_user_and_add_new_groups() throws Exception { + public void authenticate_existing_user_and_add_new_groups() { organizationFlags.setEnabled(true); UserDto user = db.users().insertUser(newUserDto() .setLogin(USER_LOGIN) @@ -229,7 +232,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void authenticate_existing_user_and_remove_groups() throws Exception { + public void authenticate_existing_user_and_remove_groups() { organizationFlags.setEnabled(true); UserDto user = db.users().insertUser(newUserDto() .setLogin(USER_LOGIN) @@ -246,7 +249,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void authenticate_existing_user_and_remove_all_groups_expect_default_when_organizations_are_disabled() throws Exception { + public void authenticate_existing_user_and_remove_all_groups_expect_default_when_organizations_are_disabled() { organizationFlags.setEnabled(false); UserDto user = db.users().insertUser(); GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1"); @@ -262,7 +265,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void does_not_force_default_group_when_authenticating_existing_user_when_organizations_are_enabled() throws Exception { + public void does_not_force_default_group_when_authenticating_existing_user_when_organizations_are_enabled() { organizationFlags.setEnabled(true); UserDto user = db.users().insertUser(); GroupDto group1 = db.users().insertGroup(db.getDefaultOrganization(), "group1"); @@ -276,7 +279,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void ignore_groups_on_non_default_organizations() throws Exception { + public void ignore_groups_on_non_default_organizations() { organizationFlags.setEnabled(true); OrganizationDto org = db.organizations().insert(); UserDto user = db.users().insertUser(newUserDto() @@ -299,7 +302,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void fail_to_authenticate_new_user_when_allow_users_to_signup_is_false() throws Exception { + public void fail_to_authenticate_new_user_when_allow_users_to_signup_is_false() { TestIdentityProvider identityProvider = new TestIdentityProvider() .setKey("github") .setName("Github") @@ -313,7 +316,7 @@ public class UserIdentityAuthenticatorTest { } @Test - public void fail_to_authenticate_new_user_when_email_already_exists() throws Exception { + public void fail_to_authenticate_new_user_when_email_already_exists() { db.users().insertUser(newUserDto() .setLogin("Existing user with same email") .setActive(true) diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java index 00a6028c46b..a1c5519384a 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/es/BulkIndexerTest.java @@ -20,13 +20,21 @@ package org.sonar.server.es; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.IntStream; +import org.apache.commons.lang.math.RandomUtils; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; +import org.sonar.api.utils.internal.TestSystem2; +import org.sonar.db.DbTester; +import org.sonar.db.es.EsQueueDto; import org.sonar.server.es.BulkIndexer.Size; import static org.assertj.core.api.Assertions.assertThat; @@ -35,8 +43,12 @@ import static org.sonar.server.es.FakeIndexDefinition.INDEX_TYPE_FAKE; public class BulkIndexerTest { + private TestSystem2 testSystem2 = new TestSystem2().setNow(1_000L); + @Rule public EsTester esTester = new EsTester(new FakeIndexDefinition().setReplicas(1)); + @Rule + public DbTester dbTester = DbTester.create(testSystem2); @Test public void index_nothing() { @@ -102,6 +114,42 @@ public class BulkIndexerTest { assertThat(count()).isEqualTo(removeFrom); } + @Test + @Ignore + public void when_index_is_done_EsQueues_must_be_deleted() { + BulkIndexer indexer = new BulkIndexer(esTester.client(), INDEX, Size.REGULAR); + int nbOfDelete = 10 + RandomUtils.nextInt(10); + int nbOfInsert = 10 + RandomUtils.nextInt(10); + int nbOfDocumentNotToBeDeleted = 10 + RandomUtils.nextInt(10); + Collection<EsQueueDto> esQueueDtos = new ArrayList<>(); + + // Those documents must be kept + FakeDoc[] docs = new FakeDoc[nbOfDocumentNotToBeDeleted]; + for (int i = 1; i <= nbOfDocumentNotToBeDeleted; i++) { + docs[i] = FakeIndexDefinition.newDoc(-i); + } + esTester.putDocuments(INDEX_TYPE_FAKE, docs); + + // Create nbOfDelete documents to be deleted + docs = new FakeDoc[nbOfDelete]; + for (int i = 1; i <= nbOfDelete; i++) { + docs[i] = FakeIndexDefinition.newDoc(i); + } + esTester.putDocuments(INDEX_TYPE_FAKE, docs); + assertThat(count()).isEqualTo(nbOfDelete + nbOfDocumentNotToBeDeleted); + + indexer.start(dbTester.getSession(), dbTester.getDbClient(), esQueueDtos); + // Create nbOfDelete for old Documents + IntStream.rangeClosed(1, nbOfDelete).forEach( + i -> indexer.addDeletion(INDEX_TYPE_FAKE, "" + i)); + // Create nbOfInsert for new Documents + IntStream.rangeClosed(nbOfDelete + 1, nbOfInsert).forEach( + i -> indexer.add(newIndexRequest(i))); + indexer.stop(); + + assertThat(count()).isEqualTo(nbOfInsert + nbOfDocumentNotToBeDeleted); + } + private long count() { return esTester.countDocuments("fakes", "fake"); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java new file mode 100644 index 00000000000..bb43319c9e4 --- /dev/null +++ b/server/sonar-server/src/test/java/org/sonar/server/es/RecoveryIndexerTest.java @@ -0,0 +1,394 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonar.server.es; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; +import org.sonar.api.config.Settings; +import org.sonar.api.config.internal.MapSettings; +import org.sonar.api.utils.internal.TestSystem2; +import org.sonar.api.utils.log.LogTester; +import org.sonar.api.utils.log.LoggerLevel; +import org.sonar.db.DbSession; +import org.sonar.db.DbTester; +import org.sonar.db.es.EsQueueDto; +import org.sonar.db.user.UserDto; +import org.sonar.server.user.index.UserIndexDefinition; +import org.sonar.server.user.index.UserIndexer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.sonar.api.utils.log.LoggerLevel.ERROR; +import static org.sonar.api.utils.log.LoggerLevel.INFO; +import static org.sonar.api.utils.log.LoggerLevel.TRACE; +import static org.sonar.core.util.stream.MoreCollectors.toArrayList; + +public class RecoveryIndexerTest { + + private static final long PAST = 1_000L; + private TestSystem2 system2 = new TestSystem2().setNow(PAST); + + @Rule + public final EsTester es = new EsTester(new UserIndexDefinition(new MapSettings().asConfig())); + @Rule + public final DbTester db = DbTester.create(system2); + @Rule + public final LogTester logTester = new LogTester().setLevel(TRACE); + @Rule + public TestRule safeguard = new Timeout(60, TimeUnit.SECONDS); + + private RecoveryIndexer underTest; + + @After + public void tearDown() { + if (underTest != null) { + underTest.stop(); + } + } + + @Test + public void display_default_configuration_at_startup() { + UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client()); + underTest = newRecoveryIndexer(userIndexer, new MapSettings()); + + underTest.start(); + + assertThat(logTester.logs(LoggerLevel.DEBUG)).contains( + "Elasticsearch recovery - sonar.search.recovery.delayInMs=300000", + "Elasticsearch recovery - sonar.search.recovery.minAgeInMs=300000"); + } + + @Test + public void start_triggers_recovery_run_at_fixed_rate() throws Exception { + Settings settings = new MapSettings() + .setProperty("sonar.search.recovery.initialDelayInMs", "0") + .setProperty("sonar.search.recovery.delayInMs", "1"); + underTest = spy(new RecoveryIndexer(system2, settings, db.getDbClient(), mock(UserIndexer.class))); + AtomicInteger calls = new AtomicInteger(0); + doAnswer(invocation -> { + calls.incrementAndGet(); + return null; + }).when(underTest).recover(); + + underTest.start(); + + // wait for 2 runs + while (calls.get() < 2) { + Thread.sleep(1L); + } + } + + @Test + public void successfully_index_old_records() { + EsQueueDto item1 = createUnindexedUser(); + EsQueueDto item2 = createUnindexedUser(); + + ProxyUserIndexer userIndexer = new ProxyUserIndexer(); + advanceInTime(); + underTest = newRecoveryIndexer(userIndexer); + underTest.recover(); + + assertThatQueueHasSize(0); + assertThat(userIndexer.called) + .extracting(EsQueueDto::getUuid) + .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid()); + + assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 2 USER"); + assertThatLogsContain(INFO, "Elasticsearch recovery - 2 documents processed [0 failures]"); + } + + @Test + public void recent_records_are_not_recovered() { + createUnindexedUser(); + createUnindexedUser(); + + ProxyUserIndexer userIndexer = new ProxyUserIndexer(); + // do not advance in time + underTest = newRecoveryIndexer(userIndexer); + underTest.recover(); + + assertThatQueueHasSize(2); + assertThat(userIndexer.called).isEmpty(); + + assertThatLogsDoNotContain(TRACE, "Elasticsearch recovery - processing 2 USER"); + assertThatLogsDoNotContain(INFO, "documents processed"); + } + + @Test + public void do_nothing_if_queue_is_empty() { + underTest = newRecoveryIndexer(); + + underTest.recover(); + + assertThatNoLogsFromRecovery(INFO); + assertThatNoLogsFromRecovery(ERROR); + assertThatQueueHasSize(0); + } + + @Test + public void log_exception_on_recovery_failure() { + createUnindexedUser(); + FailingOnceUserIndexer failingOnceUserIndexer = new FailingOnceUserIndexer(); + advanceInTime(); + + underTest = newRecoveryIndexer(failingOnceUserIndexer); + underTest.recover(); + + // No rows treated + assertThatQueueHasSize(1); + assertThatLogsContain(ERROR, "Elasticsearch recovery - fail to recover documents"); + } + + @Test + public void scheduler_is_not_stopped_on_failures() throws Exception { + createUnindexedUser(); + advanceInTime(); + FailingUserIndexer userIndexer = new FailingUserIndexer(); + + underTest = newRecoveryIndexer(userIndexer); + underTest.start(); + + // all runs fail, but they are still scheduled + // -> waiting for 2 runs + while (userIndexer.called.size() < 2) { + Thread.sleep(1L); + } + } + + @Test + public void recovery_retries_on_next_run_if_failure() throws Exception { + createUnindexedUser(); + advanceInTime(); + FailingOnceUserIndexer userIndexer = new FailingOnceUserIndexer(); + + underTest = newRecoveryIndexer(userIndexer); + underTest.start(); + + // first run fails, second run succeeds + userIndexer.counter.await(30, TimeUnit.SECONDS); + + // First we expecting an exception at first run + // Then the second run must have treated all records + assertThatLogsContain(ERROR, "Elasticsearch recovery - fail to recover documents"); + assertThatQueueHasSize(0); + } + + @Test + public void stop_run_if_too_many_failures() throws Exception { + IntStream.range(0, 10).forEach(i -> createUnindexedUser()); + advanceInTime(); + + // 10 docs to process, by groups of 3. + // The first group successfully recovers only 1 docs --> above 30% of failures --> stop run + PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(1); + Settings settings = new MapSettings() + .setProperty("sonar.search.recovery.loopLimit", "3"); + underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings); + underTest.recover(); + + assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [2/3 documents], waiting for next run"); + assertThatQueueHasSize(9); + + // The indexer must have been called once and only once. + assertThat(failingAboveRatioUserIndexer.called).hasSize(3); + } + + @Test + public void do_not_stop_run_if_success_rate_is_greater_than_ratio() throws Exception { + IntStream.range(0, 10).forEach(i -> createUnindexedUser()); + advanceInTime(); + + // 10 docs to process, by groups of 5. + // Each group successfully recovers 4 docs --> below 30% of failures --> continue run + PartiallyFailingUserIndexer failingAboveRatioUserIndexer = new PartiallyFailingUserIndexer(4, 4, 2); + Settings settings = new MapSettings() + .setProperty("sonar.search.recovery.loopLimit", "5"); + underTest = newRecoveryIndexer(failingAboveRatioUserIndexer, settings); + underTest.recover(); + + assertThatLogsDoNotContain(ERROR, "too many failures"); + assertThatQueueHasSize(0); + assertThat(failingAboveRatioUserIndexer.indexed).hasSize(10); + assertThat(failingAboveRatioUserIndexer.called).hasSize(10 + 2 /* retries */); + } + + @Test + public void failing_always_on_same_document_does_not_generate_infinite_loop() { + EsQueueDto buggy = createUnindexedUser(); + IntStream.range(0, 10).forEach(i -> createUnindexedUser()); + advanceInTime(); + + FailingAlwaysOnSameElementIndexer indexer = new FailingAlwaysOnSameElementIndexer(buggy); + underTest = newRecoveryIndexer(indexer); + underTest.recover(); + + assertThatLogsContain(ERROR, "Elasticsearch recovery - too many failures [1/1 documents], waiting for next run"); + assertThatQueueHasSize(1); + } + + private class ProxyUserIndexer extends UserIndexer { + private final List<EsQueueDto> called = new ArrayList<>(); + + ProxyUserIndexer() { + super(db.getDbClient(), es.client()); + } + + @Override + public long index(DbSession dbSession, Collection<EsQueueDto> items) { + called.addAll(items); + return super.index(dbSession, items); + } + } + + private class FailingUserIndexer extends UserIndexer { + private final List<EsQueueDto> called = new ArrayList<>(); + + FailingUserIndexer() { + super(db.getDbClient(), es.client()); + } + + @Override + public long index(DbSession dbSession, Collection<EsQueueDto> items) { + called.addAll(items); + throw new RuntimeException("boom"); + } + + } + + private class FailingOnceUserIndexer extends UserIndexer { + private final CountDownLatch counter = new CountDownLatch(2); + + FailingOnceUserIndexer() { + super(db.getDbClient(), es.client()); + } + + @Override + public long index(DbSession dbSession, Collection<EsQueueDto> items) { + try { + if (counter.getCount() == 2) { + throw new RuntimeException("boom"); + } + return super.index(dbSession, items); + } finally { + counter.countDown(); + } + } + } + + private class FailingAlwaysOnSameElementIndexer extends UserIndexer { + private final EsQueueDto failing; + + FailingAlwaysOnSameElementIndexer(EsQueueDto failing) { + super(db.getDbClient(), es.client()); + this.failing = failing; + } + + @Override + public long index(DbSession dbSession, Collection<EsQueueDto> items) { + List<EsQueueDto> filteredItems = items.stream().filter( + i -> !i.getUuid().equals(failing.getUuid())).collect(toArrayList()); + return super.index(dbSession, filteredItems); + } + } + + private class PartiallyFailingUserIndexer extends UserIndexer { + private final List<EsQueueDto> called = new ArrayList<>(); + private final List<EsQueueDto> indexed = new ArrayList<>(); + private final Iterator<Integer> successfulReturns; + + PartiallyFailingUserIndexer(int... successfulReturns) { + super(db.getDbClient(), es.client()); + this.successfulReturns = IntStream.of(successfulReturns).iterator(); + } + + @Override + public long index(DbSession dbSession, Collection<EsQueueDto> items) { + System.out.println("called with " + items.size()); + called.addAll(items); + int success = successfulReturns.next(); + items.stream().limit(success).forEach(i -> { + System.out.println(" + success"); + db.getDbClient().esQueueDao().delete(dbSession, i); + indexed.add(i); + }); + dbSession.commit(); + return success; + } + } + + private void advanceInTime() { + system2.setNow(system2.now() + 100_000_000L); + } + + private void assertThatLogsContain(LoggerLevel loggerLevel, String message) { + assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains(message)).isNotEmpty(); + } + + private void assertThatLogsDoNotContain(LoggerLevel loggerLevel, String message) { + assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains(message)).isEmpty(); + } + + private void assertThatNoLogsFromRecovery(LoggerLevel loggerLevel) { + assertThat(logTester.logs(loggerLevel)).filteredOn(m -> m.contains("Elasticsearch recovery - ")).isEmpty(); + } + + private void assertThatQueueHasSize(int number) { + assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isEqualTo(number); + } + + private RecoveryIndexer newRecoveryIndexer() { + UserIndexer userIndexer = new UserIndexer(db.getDbClient(), es.client()); + return newRecoveryIndexer(userIndexer); + } + + private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer) { + Settings settings = new MapSettings() + .setProperty("sonar.search.recovery.initialDelayInMs", "0") + .setProperty("sonar.search.recovery.delayInMs", "1") + .setProperty("sonar.search.recovery.minAgeInMs", "1"); + return newRecoveryIndexer(userIndexer, settings); + } + + private RecoveryIndexer newRecoveryIndexer(UserIndexer userIndexer, Settings settings) { + return new RecoveryIndexer(system2, settings, db.getDbClient(), userIndexer); + } + + private EsQueueDto createUnindexedUser() { + UserDto user = db.users().insertUser(); + EsQueueDto item = EsQueueDto.create(EsQueueDto.Type.USER, user.getLogin()); + db.getDbClient().esQueueDao().insert(db.getSession(), item); + db.commit(); + + return item; + } +} diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/OrganizationCreationImplTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/OrganizationCreationImplTest.java index 44e487ffc9f..6ba745d2b4d 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/OrganizationCreationImplTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/OrganizationCreationImplTest.java @@ -20,6 +20,7 @@ package org.sonar.server.organization; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; import org.apache.commons.lang.RandomStringUtils; @@ -113,7 +114,7 @@ public class OrganizationCreationImplTest { @Before public void setUp() { someUser = db.users().insertUser(); - userIndexer.index(someUser.getLogin()); + userIndexer.indexOnStartup(new HashSet<>()); } @Test @@ -263,10 +264,8 @@ public class OrganizationCreationImplTest { @Test public void create_add_current_user_as_member_of_organization() throws OrganizationCreation.KeyConflictException { UserDto user = db.users().insertUser(); - userIndexer.index(user.getLogin()); - builtInQProfileRepositoryRule.initialize(); - userIndexer.index(someUser.getLogin()); + userIndexer.commitAndIndex(db.getSession(), someUser); OrganizationDto result = underTest.create(dbSession, someUser, FULL_POPULATED_NEW_ORGANIZATION); diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java index 7d7d33f10d9..e387274390f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/CreateActionTest.java @@ -21,6 +21,7 @@ package org.sonar.server.organization.ws; import java.io.IOException; import java.net.URISyntaxException; +import java.util.HashSet; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -112,7 +113,7 @@ public class CreateActionTest { @Before public void setUp() { user = dbTester.users().insertUser(); - userIndexer.index(user.getLogin()); + userIndexer.indexOnStartup(new HashSet<>()); userSession.logIn(user); } diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/DeleteActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/DeleteActionTest.java index 903c7fcd16a..117e38fe396 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/DeleteActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/DeleteActionTest.java @@ -20,7 +20,6 @@ package org.sonar.server.organization.ws; -import java.util.Arrays; import java.util.List; import org.junit.Rule; import org.junit.Test; @@ -59,6 +58,7 @@ import org.sonar.server.user.index.UserQuery; import org.sonar.server.ws.WsActionTester; import static com.google.common.collect.ImmutableList.of; +import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; @@ -305,7 +305,7 @@ public class DeleteActionTest { db.organizations().addMember(org, user1); db.organizations().addMember(otherOrg, user1); db.organizations().addMember(org, user2); - userIndexer.index(Arrays.asList(user1.getLogin(), user2.getLogin())); + userIndexer.commitAndIndex(db.getSession(), asList(user1, user2)); logInAsAdministrator(org); sendRequest(org); diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/RemoveMemberActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/RemoveMemberActionTest.java index c925621cab5..fe6a84f5132 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/RemoveMemberActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/RemoveMemberActionTest.java @@ -20,6 +20,7 @@ package org.sonar.server.organization.ws; +import java.util.HashSet; import javax.annotation.Nullable; import org.junit.Before; import org.junit.Rule; @@ -54,6 +55,7 @@ import org.sonar.server.ws.TestResponse; import org.sonar.server.ws.WsActionTester; import static java.net.HttpURLConnection.HTTP_NO_CONTENT; +import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.groups.Tuple.tuple; import static org.sonar.api.CoreProperties.DEFAULT_ISSUE_ASSIGNEE; @@ -93,11 +95,11 @@ public class RemoveMemberActionTest { user = db.users().insertUser(); db.organizations().addMember(organization, user); - userIndexer.index(user.getLogin()); UserDto adminUser = db.users().insertAdminByUserPermission(organization); db.organizations().addMember(organization, adminUser); - userIndexer.index(adminUser.getLogin()); + + userIndexer.indexOnStartup(new HashSet<>()); } @Test @@ -317,10 +319,9 @@ public class RemoveMemberActionTest { OrganizationDto anotherOrganization = db.organizations().insert(); UserDto admin1 = db.users().insertAdminByUserPermission(anotherOrganization); db.organizations().addMember(anotherOrganization, admin1); - userIndexer.index(admin1.getLogin()); UserDto admin2 = db.users().insertAdminByUserPermission(anotherOrganization); db.organizations().addMember(anotherOrganization, admin2); - userIndexer.index(admin2.getLogin()); + userIndexer.commitAndIndex(db.getSession(), asList(admin1, admin2)); call(anotherOrganization.getKey(), admin1.getLogin()); diff --git a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/SearchMembersActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/SearchMembersActionTest.java index db4e01c6fbb..ecf87cde61f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/organization/ws/SearchMembersActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/organization/ws/SearchMembersActionTest.java @@ -127,7 +127,7 @@ public class SearchMembersActionTest { public void return_avatar() { UserDto user = db.users().insertUser(u -> u.setEmail("email@domain.com")); db.organizations().addMember(db.getDefaultOrganization(), user); - indexer.index(user.getLogin()); + indexer.commitAndIndex(db.getSession(), user); SearchMembersWsResponse result = call(); @@ -195,7 +195,6 @@ public class SearchMembersActionTest { IntStream.range(0, 10).forEach(i -> { UserDto userDto = db.users().insertUser(user -> user.setName("USER_" + i)); db.organizations().addMember(db.getDefaultOrganization(), userDto); - indexer.index(userDto.getLogin()); }); indexAllUsers(); request.setPage(2).setPageSize(3); @@ -214,7 +213,6 @@ public class SearchMembersActionTest { IntStream.range(0, 10).forEach(i -> { UserDto userDto = db.users().insertUser(user -> user.setName("USER_" + i)); db.organizations().addMember(db.getDefaultOrganization(), userDto); - indexer.index(userDto.getLogin()); }); indexAllUsers(); request.setQuery("_9"); @@ -229,7 +227,6 @@ public class SearchMembersActionTest { IntStream.range(0, 10).forEach(i -> { UserDto userDto = db.users().insertUser(user -> user.setLogin("USER_" + i)); db.organizations().addMember(db.getDefaultOrganization(), userDto); - indexer.index(userDto.getLogin()); }); indexAllUsers(); request.setQuery("_9"); @@ -246,7 +243,6 @@ public class SearchMembersActionTest { .setLogin("L" + i) .setEmail("USER_" + i + "@email.com")); db.organizations().addMember(db.getDefaultOrganization(), userDto); - indexer.index(userDto.getLogin()); }); indexAllUsers(); request.setQuery("_9"); diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ServerUserSessionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ServerUserSessionTest.java index f8a2f50108b..ed73fca1371 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ServerUserSessionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ServerUserSessionTest.java @@ -484,7 +484,7 @@ public class ServerUserSessionTest { session.checkIsSystemAdministrator(); - db.getDbClient().userDao().deactivateUserById(db.getSession(), user.getId()); + db.getDbClient().userDao().deactivateUser(db.getSession(), user); db.commit(); // should fail but succeeds because flag is kept in cache diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterTest.java index 0a3f827f0a9..18db33a72dc 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/UserUpdaterTest.java @@ -31,7 +31,7 @@ import org.mockito.ArgumentCaptor; import org.sonar.api.config.internal.MapSettings; import org.sonar.api.platform.NewUserHandler; import org.sonar.api.utils.System2; -import org.sonar.api.utils.internal.TestSystem2; +import org.sonar.api.utils.internal.AlwaysIncreasingSystem2; import org.sonar.db.DbClient; import org.sonar.db.DbSession; import org.sonar.db.DbTester; @@ -49,7 +49,6 @@ import org.sonar.server.user.index.UserIndexDefinition; import org.sonar.server.user.index.UserIndexer; import org.sonar.server.usergroups.DefaultGroupFinder; -import static com.google.common.collect.Lists.newArrayList; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; @@ -67,11 +66,9 @@ import static org.sonar.server.user.ExternalIdentity.SQ_AUTHORITY; public class UserUpdaterTest { - private static final long NOW = 1418215735482L; - private static final long PAST = 1000000000000L; private static final String DEFAULT_LOGIN = "marius"; - private System2 system2 = new TestSystem2().setNow(NOW); + private System2 system2 = new AlwaysIncreasingSystem2(); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -91,20 +88,21 @@ public class UserUpdaterTest { private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone(); private MapSettings settings = new MapSettings(); - private UserUpdater underTest = new UserUpdater(newUserNotifier, dbClient, userIndexer, system2, organizationFlags, defaultOrganizationProvider, organizationCreation, + private UserUpdater underTest = new UserUpdater(newUserNotifier, dbClient, userIndexer, organizationFlags, defaultOrganizationProvider, organizationCreation, new DefaultGroupFinder(dbClient), settings.asConfig()); @Test public void create_user() { createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder() + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("PASSWORD") .setScmAccounts(ImmutableList.of("u1", "u_1", "User 1")) - .build()); + .build(), u -> { + }); assertThat(dto.getId()).isNotNull(); assertThat(dto.getLogin()).isEqualTo("user"); @@ -116,8 +114,9 @@ public class UserUpdaterTest { assertThat(dto.getSalt()).isNotNull(); assertThat(dto.getCryptedPassword()).isNotNull(); - assertThat(dto.getCreatedAt()).isEqualTo(1418215735482L); - assertThat(dto.getUpdatedAt()).isEqualTo(1418215735482L); + assertThat(dto.getCreatedAt()) + .isPositive() + .isEqualTo(dto.getUpdatedAt()); assertThat(dbClient.userDao().selectByLogin(session, "user").getId()).isEqualTo(dto.getId()); List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER); @@ -133,10 +132,11 @@ public class UserUpdaterTest { public void create_user_with_minimum_fields() { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("us") .setName("User") - .build()); + .build(), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, "us"); assertThat(dto.getId()).isNotNull(); @@ -151,11 +151,12 @@ public class UserUpdaterTest { public void create_user_with_sq_authority_when_no_authority_set() throws Exception { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setPassword("password") - .build()); + .build(), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, "user"); assertThat(dto.getExternalIdentity()).isEqualTo("user"); @@ -167,11 +168,12 @@ public class UserUpdaterTest { public void create_user_with_identity_provider() throws Exception { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setExternalIdentity(new ExternalIdentity("github", "github-user")) - .build()); + .build(), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, "user"); assertThat(dto.isLocal()).isFalse(); @@ -185,11 +187,12 @@ public class UserUpdaterTest { public void create_user_with_sonarqube_external_identity() throws Exception { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setExternalIdentity(new ExternalIdentity(SQ_AUTHORITY, "user")) - .build()); + .build(), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, "user"); assertThat(dto.isLocal()).isFalse(); @@ -203,12 +206,13 @@ public class UserUpdaterTest { public void create_user_with_scm_accounts_containing_blank_or_null_entries() { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setPassword("password") - .setScmAccounts(newArrayList("u1", "", null)) - .build()); + .setScmAccounts(asList("u1", "", null)) + .build(), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, "user").getScmAccountsAsList()).containsOnly("u1"); } @@ -217,12 +221,13 @@ public class UserUpdaterTest { public void create_user_with_scm_accounts_containing_one_blank_entry() { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setPassword("password") - .setScmAccounts(newArrayList("")) - .build()); + .setScmAccounts(asList("")) + .build(), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, "user").getScmAccounts()).isNull(); } @@ -231,12 +236,13 @@ public class UserUpdaterTest { public void create_user_with_scm_accounts_containing_duplications() { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setPassword("password") - .setScmAccounts(newArrayList("u1", "u1")) - .build()); + .setScmAccounts(asList("u1", "u1")) + .build(), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, "user").getScmAccountsAsList()).containsOnly("u1"); } @@ -246,10 +252,11 @@ public class UserUpdaterTest { createDefaultGroup(); settings.setProperty(ONBOARDING_TUTORIAL_SHOW_TO_NEW_USERS, false); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") - .build()); + .build(), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, "user").isOnboarded()).isTrue(); } @@ -259,10 +266,11 @@ public class UserUpdaterTest { createDefaultGroup(); settings.setProperty(ONBOARDING_TUTORIAL_SHOW_TO_NEW_USERS, true); - UserDto user = underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") - .build()); + .build(), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, "user").isOnboarded()).isFalse(); } @@ -272,12 +280,13 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login can't be empty"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(null) .setName("Marius") .setEmail("marius@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); } @Test @@ -285,12 +294,13 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Use only letters, numbers, and .-_@ please."); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("/marius/") .setName("Marius") .setEmail("marius@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); } @Test @@ -298,12 +308,13 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Use only letters, numbers, and .-_@ please."); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("mari us") .setName("Marius") .setEmail("marius@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); } @Test @@ -311,12 +322,13 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login is too short (minimum is 2 characters)"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("m") .setName("Marius") .setEmail("marius@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); } @Test @@ -324,12 +336,13 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login is too long (maximum is 255 characters)"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(Strings.repeat("m", 256)) .setName("Marius") .setEmail("marius@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); } @Test @@ -337,12 +350,13 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Name can't be empty"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName(null) .setEmail("marius@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); } @Test @@ -350,12 +364,13 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Name is too long (maximum is 200 characters)"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName(Strings.repeat("m", 201)) .setEmail("marius@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); } @Test @@ -363,23 +378,25 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Email is too long (maximum is 100 characters)"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius") .setEmail(Strings.repeat("m", 101)) .setPassword("password") - .build()); + .build(), u -> { + }); } @Test public void fail_to_create_user_with_many_errors() { try { - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("") .setName("") .setEmail("marius@mail.com") .setPassword("") - .build()); + .build(), u -> { + }); fail(); } catch (BadRequestException e) { assertThat(e.errors()).hasSize(3); @@ -393,13 +410,14 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("The scm account 'jo' is already used by user(s) : 'John (john)'"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius") .setEmail("marius@mail.com") .setPassword("password") - .setScmAccounts(newArrayList("jo")) - .build()); + .setScmAccounts(asList("jo")) + .build(), u -> { + }); } @Test @@ -410,13 +428,14 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("The scm account 'john@email.com' is already used by user(s) : 'John (john), Technical account (technical-account)'"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius") .setEmail("marius@mail.com") .setPassword("password") - .setScmAccounts(newArrayList("john@email.com")) - .build()); + .setScmAccounts(asList("john@email.com")) + .build(), u -> { + }); } @Test @@ -424,13 +443,14 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login and email are automatically considered as SCM accounts"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList(DEFAULT_LOGIN)) - .build()); + .setScmAccounts(asList(DEFAULT_LOGIN)) + .build(), u -> { + }); } @Test @@ -438,26 +458,28 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login and email are automatically considered as SCM accounts"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList("marius2@mail.com")) - .build()); + .setScmAccounts(asList("marius2@mail.com")) + .build(), u -> { + }); } @Test public void notify_new_user() { createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("password") - .setScmAccounts(newArrayList("u1", "u_1")) - .build()); + .setScmAccounts(asList("u1", "u_1")) + .build(), u -> { + }); verify(newUserNotifier).onNewUser(newUserHandler.capture()); assertThat(newUserHandler.getValue().getLogin()).isEqualTo("user"); @@ -470,12 +492,13 @@ public class UserUpdaterTest { organizationFlags.setEnabled(false); GroupDto defaultGroup = createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); Multimap<String, String> groups = dbClient.groupMembershipDao().selectGroupsByLogins(session, asList("user")); assertThat(groups.get("user")).containsOnly(defaultGroup.getName()); @@ -486,12 +509,13 @@ public class UserUpdaterTest { organizationFlags.setEnabled(true); createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("password") - .build()); + .build(), u -> { + }); Multimap<String, String> groups = dbClient.groupMembershipDao().selectGroupsByLogins(session, asList("user")); assertThat(groups.get("user")).isEmpty(); @@ -502,25 +526,27 @@ public class UserUpdaterTest { expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Default group cannot be found"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("password") - .setScmAccounts(newArrayList("u1", "u_1")) - .build()); + .setScmAccounts(asList("u1", "u_1")) + .build(), u -> { + }); } @Test public void create_personal_organization_when_creating_user() { createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder() + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("PASSWORD") - .build()); + .build(), u -> { + }); verify(organizationCreation).createForUser(any(DbSession.class), eq(dto)); } @@ -530,12 +556,13 @@ public class UserUpdaterTest { organizationFlags.setEnabled(false); createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder() + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("PASSWORD") - .build()); + .build(), u -> { + }); assertThat(dbClient.organizationMemberDao().select(db.getSession(), defaultOrganizationProvider.get().getUuid(), dto.getId())).isPresent(); } @@ -545,30 +572,30 @@ public class UserUpdaterTest { organizationFlags.setEnabled(true); createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder() + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin("user") .setName("User") .setEmail("user@mail.com") .setPassword("PASSWORD") - .build()); + .build(), u -> { + }); assertThat(dbClient.organizationMemberDao().select(db.getSession(), defaultOrganizationProvider.get().getUuid(), dto.getId())).isNotPresent(); } @Test public void reactivate_user_when_creating_user_with_existing_login() { - db.users().insertUser(newDisabledUser(DEFAULT_LOGIN) - .setLocal(false) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + UserDto user = db.users().insertUser(newDisabledUser(DEFAULT_LOGIN) + .setLocal(false)); createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder() + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .build()); + .build(), u -> { + }); session.commit(); assertThat(dto.isActive()).isTrue(); @@ -579,26 +606,25 @@ public class UserUpdaterTest { assertThat(dto.getSalt()).isNotNull().isNotEqualTo("79bd6a8e79fb8c76ac8b121cc7e8e11ad1af8365"); assertThat(dto.getCryptedPassword()).isNotNull().isNotEqualTo("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg"); - assertThat(dto.getCreatedAt()).isEqualTo(PAST); - assertThat(dto.getUpdatedAt()).isEqualTo(NOW); + assertThat(dto.getCreatedAt()).isEqualTo(user.getCreatedAt()); + assertThat(dto.getUpdatedAt()).isGreaterThan(user.getCreatedAt()); assertThat(dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN).isActive()).isTrue(); } @Test public void reactivate_user_not_having_password() { - db.users().insertUser(newDisabledUser("marius").setName("Marius").setEmail("marius@lesbronzes.fr") + UserDto user = db.users().insertUser(newDisabledUser("marius").setName("Marius").setEmail("marius@lesbronzes.fr") .setSalt(null) - .setCryptedPassword(null) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setCryptedPassword(null)); createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder() + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") - .build()); + .build(), u -> { + }); session.commit(); assertThat(dto.isActive()).isTrue(); @@ -608,23 +634,22 @@ public class UserUpdaterTest { assertThat(dto.getSalt()).isNull(); assertThat(dto.getCryptedPassword()).isNull(); - assertThat(dto.getCreatedAt()).isEqualTo(PAST); - assertThat(dto.getUpdatedAt()).isEqualTo(NOW); + assertThat(dto.getCreatedAt()).isEqualTo(user.getCreatedAt()); + assertThat(dto.getUpdatedAt()).isGreaterThan(user.getCreatedAt()); } @Test public void update_external_provider_when_reactivating_user() { db.users().insertUser(newDisabledUser(DEFAULT_LOGIN) - .setLocal(true) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setLocal(true)); createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setExternalIdentity(new ExternalIdentity("github", "john")) - .build()); + .build(), u -> { + }); session.commit(); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); @@ -635,40 +660,38 @@ public class UserUpdaterTest { @Test public void fail_to_reactivate_user_if_not_disabled() { - db.users().insertUser(newLocalUser("marius", "Marius", "marius@lesbronzes.fr") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + db.users().insertUser(newLocalUser("marius", "Marius", "marius@lesbronzes.fr")); createDefaultGroup(); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("An active user with login 'marius' already exists"); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .build()); + .build(), u -> { + }); } @Test public void associate_default_groups_when_reactivating_user_and_organizations_are_disabled() { organizationFlags.setEnabled(false); UserDto userDto = db.users().insertUser(newDisabledUser(DEFAULT_LOGIN) - .setLocal(true) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setLocal(true)); db.organizations().insertForUuid("org1"); GroupDto groupDto = db.users().insertGroup(GroupTesting.newGroupDto().setName("sonar-devs").setOrganizationUuid("org1")); db.users().insertMember(groupDto, userDto); GroupDto defaultGroup = createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .build()); + .build(), u -> { + }); session.commit(); Multimap<String, String> groups = dbClient.groupMembershipDao().selectGroupsByLogins(session, asList(DEFAULT_LOGIN)); @@ -679,20 +702,19 @@ public class UserUpdaterTest { public void does_not_associate_default_groups_when_reactivating_user_and_organizations_are_enabled() { organizationFlags.setEnabled(true); UserDto userDto = db.users().insertUser(newDisabledUser(DEFAULT_LOGIN) - .setLocal(true) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setLocal(true)); db.organizations().insertForUuid("org1"); GroupDto groupDto = db.users().insertGroup(GroupTesting.newGroupDto().setName("sonar-devs").setOrganizationUuid("org1")); db.users().insertMember(groupDto, userDto); GroupDto defaultGroup = createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .build()); + .build(), u -> { + }); session.commit(); Multimap<String, String> groups = dbClient.groupMembershipDao().selectGroupsByLogins(session, asList(DEFAULT_LOGIN)); @@ -705,7 +727,8 @@ public class UserUpdaterTest { db.users().insertUser(newDisabledUser(DEFAULT_LOGIN)); createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder().setLogin(DEFAULT_LOGIN).setName("Name").build()); + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder().setLogin(DEFAULT_LOGIN).setName("Name").build(), u -> { + }); session.commit(); assertThat(dbClient.organizationMemberDao().select(db.getSession(), defaultOrganizationProvider.get().getUuid(), dto.getId())).isPresent(); @@ -717,7 +740,8 @@ public class UserUpdaterTest { db.users().insertUser(newDisabledUser(DEFAULT_LOGIN)); createDefaultGroup(); - UserDto dto = underTest.create(db.getSession(), NewUser.builder().setLogin(DEFAULT_LOGIN).setName("Name").build()); + UserDto dto = underTest.createAndCommit(db.getSession(), NewUser.builder().setLogin(DEFAULT_LOGIN).setName("Name").build(), u -> { + }); session.commit(); assertThat(dbClient.organizationMemberDao().select(db.getSession(), defaultOrganizationProvider.get().getUuid(), dto.getId())).isNotPresent(); @@ -731,10 +755,11 @@ public class UserUpdaterTest { .setOnboarded(false)); createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(user.getLogin()) .setName("name") - .build()); + .build(), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, user.getLogin()).isOnboarded()).isTrue(); } @@ -747,10 +772,11 @@ public class UserUpdaterTest { .setOnboarded(true)); createDefaultGroup(); - underTest.create(db.getSession(), NewUser.builder() + underTest.createAndCommit(db.getSession(), NewUser.builder() .setLogin(user.getLogin()) .setName("name") - .build()); + .build(), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, user.getLogin()).isOnboarded()).isFalse(); } @@ -760,17 +786,15 @@ public class UserUpdaterTest { UserDto user = db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@email.com") .setScmAccounts(asList("ma", "marius33")) .setSalt("79bd6a8e79fb8c76ac8b121cc7e8e11ad1af8365") - .setCryptedPassword("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setCryptedPassword("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList("ma2"))); - session.commit(); + .setScmAccounts(asList("ma2")), u -> { + }); UserDto updatedUser = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(updatedUser.isActive()).isTrue(); @@ -780,8 +804,8 @@ public class UserUpdaterTest { assertThat(updatedUser.getSalt()).isNotEqualTo(user.getSalt()); assertThat(updatedUser.getCryptedPassword()).isNotEqualTo(user.getCryptedPassword()); - assertThat(updatedUser.getCreatedAt()).isEqualTo(PAST); - assertThat(updatedUser.getUpdatedAt()).isEqualTo(NOW); + assertThat(updatedUser.getCreatedAt()).isEqualTo(user.getCreatedAt()); + assertThat(updatedUser.getUpdatedAt()).isGreaterThan(user.getCreatedAt()); List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER); assertThat(indexUsers).hasSize(1); @@ -794,37 +818,33 @@ public class UserUpdaterTest { @Test public void update_user_external_identity_when_user_was_not_local() { - db.users().insertUser(UserTesting.newExternalUser(DEFAULT_LOGIN, "Marius", "marius@email.com") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + UserDto user = db.users().insertUser(UserTesting.newExternalUser(DEFAULT_LOGIN, "Marius", "marius@email.com")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@email.com") .setPassword(null) - .setExternalIdentity(new ExternalIdentity("github", "john"))); - session.commit(); + .setExternalIdentity(new ExternalIdentity("github", "john")), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getExternalIdentity()).isEqualTo("john"); assertThat(dto.getExternalIdentityProvider()).isEqualTo("github"); - assertThat(dto.getUpdatedAt()).isEqualTo(NOW); + assertThat(dto.getUpdatedAt()).isGreaterThan(user.getCreatedAt()); } @Test public void update_user_external_identity_when_user_was_local() { - db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@email.com") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + UserDto user = db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@email.com")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@email.com") .setPassword(null) - .setExternalIdentity(new ExternalIdentity("github", "john"))); - session.commit(); + .setExternalIdentity(new ExternalIdentity("github", "john")), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getExternalIdentity()).isEqualTo("john"); @@ -832,7 +852,7 @@ public class UserUpdaterTest { // Password must be removed assertThat(dto.getCryptedPassword()).isNull(); assertThat(dto.getSalt()).isNull(); - assertThat(dto.getUpdatedAt()).isEqualTo(NOW); + assertThat(dto.getUpdatedAt()).isGreaterThan(user.getCreatedAt()); } @Test @@ -840,17 +860,15 @@ public class UserUpdaterTest { UserDto user = db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") .setScmAccounts(asList("ma", "marius33")) .setSalt("salt") - .setCryptedPassword("crypted password") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setCryptedPassword("crypted password")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList("ma2"))); - session.commit(); + .setScmAccounts(asList("ma2")), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.isActive()).isTrue(); @@ -860,8 +878,8 @@ public class UserUpdaterTest { assertThat(dto.getSalt()).isNotEqualTo(user.getSalt()); assertThat(dto.getCryptedPassword()).isNotEqualTo(user.getCryptedPassword()); - assertThat(dto.getCreatedAt()).isEqualTo(PAST); - assertThat(dto.getUpdatedAt()).isEqualTo(NOW); + assertThat(dto.getCreatedAt()).isEqualTo(user.getCreatedAt()); + assertThat(dto.getUpdatedAt()).isGreaterThan(user.getUpdatedAt()); List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER); assertThat(indexUsers).hasSize(1); @@ -875,17 +893,15 @@ public class UserUpdaterTest { @Test public void update_user_with_scm_accounts_containing_blank_entry() { db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") - .setScmAccounts(asList("ma", "marius33")) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setScmAccounts(asList("ma", "marius33"))); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList("ma2", "", null))); - session.commit(); + .setScmAccounts(asList("ma2", "", null)), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getScmAccountsAsList()).containsOnly("ma2"); @@ -896,14 +912,12 @@ public class UserUpdaterTest { db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") .setScmAccounts(asList("ma", "marius33")) .setSalt("salt") - .setCryptedPassword("crypted password") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setCryptedPassword("crypted password")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) - .setName("Marius2")); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) + .setName("Marius2"), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getName()).isEqualTo("Marius2"); @@ -920,14 +934,12 @@ public class UserUpdaterTest { db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") .setScmAccounts(asList("ma", "marius33")) .setSalt("salt") - .setCryptedPassword("crypted password") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setCryptedPassword("crypted password")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) - .setEmail("marius2@mail.com")); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) + .setEmail("marius2@mail.com"), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getEmail()).isEqualTo("marius2@mail.com"); @@ -944,14 +956,12 @@ public class UserUpdaterTest { db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") .setScmAccounts(asList("ma", "marius33")) .setSalt("salt") - .setCryptedPassword("crypted password") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setCryptedPassword("crypted password")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) - .setScmAccounts(newArrayList("ma2"))); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) + .setScmAccounts(asList("ma2")), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getScmAccountsAsList()).containsOnly("ma2"); @@ -966,14 +976,12 @@ public class UserUpdaterTest { @Test public void update_scm_accounts_with_same_values() { db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") - .setScmAccounts(asList("ma", "marius33")) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setScmAccounts(asList("ma", "marius33"))); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) - .setScmAccounts(newArrayList("ma", "marius33"))); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) + .setScmAccounts(asList("ma", "marius33")), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getScmAccountsAsList()).containsOnly("ma", "marius33"); @@ -982,14 +990,12 @@ public class UserUpdaterTest { @Test public void remove_scm_accounts() { db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") - .setScmAccounts(asList("ma", "marius33")) - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setScmAccounts(asList("ma", "marius33"))); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) - .setScmAccounts(null)); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) + .setScmAccounts(null), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getScmAccounts()).isNull(); @@ -1000,14 +1006,12 @@ public class UserUpdaterTest { db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@lesbronzes.fr") .setScmAccounts(asList("ma", "marius33")) .setSalt("salt") - .setCryptedPassword("crypted password") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setCryptedPassword("crypted password")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) - .setPassword("password2")); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) + .setPassword("password2"), u -> { + }); UserDto dto = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN); assertThat(dto.getSalt()).isNotEqualTo("salt"); @@ -1023,34 +1027,30 @@ public class UserUpdaterTest { public void update_only_external_identity_id() { db.users().insertUser(UserTesting.newExternalUser(DEFAULT_LOGIN, "Marius", "marius@email.com") .setExternalIdentity("john") - .setExternalIdentityProvider("github") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setExternalIdentityProvider("github")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN).setExternalIdentity(new ExternalIdentity("github", "john.smith"))); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN).setExternalIdentity(new ExternalIdentity("github", "john.smith")), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN)) - .extracting(UserDto::getExternalIdentity, UserDto::getExternalIdentityProvider, UserDto::getUpdatedAt) - .containsOnly("john.smith", "github", NOW); + .extracting(UserDto::getExternalIdentity, UserDto::getExternalIdentityProvider) + .containsOnly("john.smith", "github"); } @Test public void update_only_external_identity_provider() { db.users().insertUser(UserTesting.newExternalUser(DEFAULT_LOGIN, "Marius", "marius@email.com") .setExternalIdentity("john") - .setExternalIdentityProvider("github") - .setCreatedAt(PAST) - .setUpdatedAt(PAST)); + .setExternalIdentityProvider("github")); createDefaultGroup(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN).setExternalIdentity(new ExternalIdentity("bitbucket", "john"))); - session.commit(); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN).setExternalIdentity(new ExternalIdentity("bitbucket", "john")), u -> { + }); assertThat(dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN)) - .extracting(UserDto::getExternalIdentity, UserDto::getExternalIdentityProvider, UserDto::getUpdatedAt) - .containsOnly("john", "bitbucket", NOW); + .extracting(UserDto::getExternalIdentity, UserDto::getExternalIdentityProvider) + .containsOnly("john", "bitbucket"); } @Test @@ -1058,20 +1058,18 @@ public class UserUpdaterTest { UserDto user = UserTesting.newExternalUser(DEFAULT_LOGIN, "Marius", "marius@email.com") .setExternalIdentity("john") .setExternalIdentityProvider("github") - .setScmAccounts(asList("ma1", "ma2")) - .setCreatedAt(PAST) - .setUpdatedAt(PAST); + .setScmAccounts(asList("ma1", "ma2")); db.users().insertUser(user); createDefaultGroup(); - underTest.update(session, UpdateUser.create(user.getLogin()) + underTest.updateAndCommit(session, UpdateUser.create(user.getLogin()) .setName(user.getName()) .setEmail(user.getEmail()) .setScmAccounts(user.getScmAccountsAsList()) - .setExternalIdentity(new ExternalIdentity(user.getExternalIdentityProvider(), user.getExternalIdentity()))); - session.commit(); + .setExternalIdentity(new ExternalIdentity(user.getExternalIdentityProvider(), user.getExternalIdentity())), u -> { + }); - assertThat(dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN).getUpdatedAt()).isEqualTo(PAST); + assertThat(dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN).getUpdatedAt()).isEqualTo(user.getUpdatedAt()); } @Test @@ -1079,20 +1077,18 @@ public class UserUpdaterTest { UserDto user = UserTesting.newExternalUser(DEFAULT_LOGIN, "Marius", "marius@email.com") .setExternalIdentity("john") .setExternalIdentityProvider("github") - .setScmAccounts(asList("ma1", "ma2")) - .setCreatedAt(PAST) - .setUpdatedAt(PAST); + .setScmAccounts(asList("ma1", "ma2")); db.users().insertUser(user); createDefaultGroup(); - underTest.update(session, UpdateUser.create(user.getLogin()) + underTest.updateAndCommit(session, UpdateUser.create(user.getLogin()) .setName(user.getName()) .setEmail(user.getEmail()) .setScmAccounts(asList("ma2", "ma1")) - .setExternalIdentity(new ExternalIdentity(user.getExternalIdentityProvider(), user.getExternalIdentity()))); - session.commit(); + .setExternalIdentity(new ExternalIdentity(user.getExternalIdentityProvider(), user.getExternalIdentity())), u -> { + }); - assertThat(dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN).getUpdatedAt()).isEqualTo(PAST); + assertThat(dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN).getUpdatedAt()).isEqualTo(user.getUpdatedAt()); } @Test @@ -1102,7 +1098,8 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Password can't be empty"); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN).setPassword(null)); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN).setPassword(null), u -> { + }); } @Test @@ -1114,7 +1111,8 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Password cannot be changed when external authentication is used"); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN).setPassword("password2")); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN).setPassword("password2"), u -> { + }); } @Test @@ -1123,12 +1121,12 @@ public class UserUpdaterTest { GroupDto defaultGroup = createDefaultGroup(); // Existing user, he has no group, and should not be associated to the default one - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList("ma2"))); - session.commit(); + .setScmAccounts(asList("ma2")), u -> { + }); Multimap<String, String> groups = dbClient.groupMembershipDao().selectGroupsByLogins(session, asList(DEFAULT_LOGIN)); assertThat(groups.get(DEFAULT_LOGIN).stream().anyMatch(g -> g.equals(defaultGroup.getName()))).isFalse(); @@ -1144,12 +1142,12 @@ public class UserUpdaterTest { Multimap<String, String> groups = dbClient.groupMembershipDao().selectGroupsByLogins(session, asList(DEFAULT_LOGIN)); assertThat(groups.get(DEFAULT_LOGIN).stream().anyMatch(g -> g.equals(defaultGroup.getName()))).as("Current user groups : %s", groups.get(defaultGroup.getName())).isTrue(); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList("ma2"))); - session.commit(); + .setScmAccounts(asList("ma2")), u -> { + }); // Nothing as changed groups = dbClient.groupMembershipDao().selectGroupsByLogins(session, asList(DEFAULT_LOGIN)); @@ -1165,11 +1163,12 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("The scm account 'jo' is already used by user(s) : 'John (john)'"); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setName("Marius2") .setEmail("marius2@mail.com") .setPassword("password2") - .setScmAccounts(newArrayList("jo"))); + .setScmAccounts(asList("jo")), u -> { + }); } @Test @@ -1179,7 +1178,8 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login and email are automatically considered as SCM accounts"); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN).setScmAccounts(newArrayList(DEFAULT_LOGIN))); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN).setScmAccounts(asList(DEFAULT_LOGIN)), u -> { + }); } @Test @@ -1189,7 +1189,8 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login and email are automatically considered as SCM accounts"); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN).setScmAccounts(newArrayList("marius@lesbronzes.fr"))); + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN).setScmAccounts(asList("marius@lesbronzes.fr")), u -> { + }); } @Test @@ -1199,9 +1200,10 @@ public class UserUpdaterTest { expectedException.expect(BadRequestException.class); expectedException.expectMessage("Login and email are automatically considered as SCM accounts"); - underTest.update(session, UpdateUser.create(DEFAULT_LOGIN) + underTest.updateAndCommit(session, UpdateUser.create(DEFAULT_LOGIN) .setEmail("marius@newmail.com") - .setScmAccounts(newArrayList("marius@newmail.com"))); + .setScmAccounts(asList("marius@newmail.com")), u -> { + }); } private GroupDto createDefaultGroup() { diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java index 66ba400145a..f7026e2958f 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexTest.java @@ -67,8 +67,6 @@ public class UserIndexTest { assertThat(userDoc.email()).isEqualTo(user1.email()); assertThat(userDoc.active()).isTrue(); assertThat(userDoc.scmAccounts()).isEqualTo(user1.scmAccounts()); - assertThat(userDoc.createdAt()).isEqualTo(user1.createdAt()); - assertThat(userDoc.updatedAt()).isEqualTo(user1.updatedAt()); assertThat(underTest.getNullableByLogin("")).isNull(); assertThat(underTest.getNullableByLogin("unknown")).isNull(); @@ -202,9 +200,7 @@ public class UserIndexTest { .setName(login.toUpperCase(Locale.ENGLISH)) .setEmail(login + "@mail.com") .setActive(true) - .setScmAccounts(scmAccounts) - .setCreatedAt(DATE_1) - .setUpdatedAt(DATE_2); + .setScmAccounts(scmAccounts); } private static UserDoc newUser(String login, String email, List<String> scmAccounts) { @@ -213,8 +209,6 @@ public class UserIndexTest { .setName(login.toUpperCase(Locale.ENGLISH)) .setEmail(email) .setActive(true) - .setScmAccounts(scmAccounts) - .setCreatedAt(DATE_1) - .setUpdatedAt(DATE_2); + .setScmAccounts(scmAccounts); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java index 66fb5c2ceae..3f4738b1aae 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserIndexerTest.java @@ -19,6 +19,7 @@ */ package org.sonar.server.user.index; +import java.util.HashSet; import java.util.List; import org.junit.Rule; import org.junit.Test; @@ -26,6 +27,7 @@ import org.sonar.api.config.internal.MapSettings; import org.sonar.api.utils.System2; import org.sonar.db.DbTester; import org.sonar.db.user.UserDto; +import org.sonar.db.user.UserTesting; import org.sonar.server.es.EsTester; import static java.util.Arrays.asList; @@ -45,53 +47,34 @@ public class UserIndexerTest { @Test public void index_nothing_on_startup() { - underTest.indexOnStartup(null); + underTest.indexOnStartup(new HashSet<>()); assertThat(es.countDocuments(UserIndexDefinition.INDEX_TYPE_USER)).isEqualTo(0L); } @Test - public void index_everything_on_startup() { - db.users().insertUser(user -> user - .setLogin("user1") - .setName("User1") - .setEmail("user1@mail.com") - .setActive(true) - .setScmAccounts(asList("user_1", "u1")) - .setCreatedAt(1500000000000L) - .setUpdatedAt(1500000000000L)); - - underTest.indexOnStartup(null); + public void indexOnStartup_adds_all_users_to_index() { + UserDto user = db.users().insertUser(u -> u + .setScmAccounts(asList("user_1", "u1"))); - List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); - assertThat(docs).hasSize(1); - UserDoc doc = docs.get(0); - assertThat(doc.login()).isEqualTo("user1"); - assertThat(doc.name()).isEqualTo("User1"); - assertThat(doc.email()).isEqualTo("user1@mail.com"); - assertThat(doc.active()).isTrue(); - assertThat(doc.scmAccounts()).containsOnly("user_1", "u1"); - assertThat(doc.createdAt()).isEqualTo(1500000000000L); - assertThat(doc.updatedAt()).isEqualTo(1500000000000L); - } - - @Test - public void index_single_user_on_startup() { - UserDto user = db.users().insertUser(); - - underTest.indexOnStartup(null); + underTest.indexOnStartup(new HashSet<>()); List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); assertThat(docs).hasSize(1); - assertThat(docs).extracting(UserDoc::login).containsExactly(user.getLogin()); + UserDoc doc = docs.get(0); + assertThat(doc.login()).isEqualTo(user.getLogin()); + assertThat(doc.name()).isEqualTo(user.getName()); + assertThat(doc.email()).isEqualTo(user.getEmail()); + assertThat(doc.active()).isEqualTo(user.isActive()); + assertThat(doc.scmAccounts()).isEqualTo(user.getScmAccountsAsList()); } @Test - public void index_single_user() { + public void commitAndIndex_single_user() { UserDto user = db.users().insertUser(); UserDto anotherUser = db.users().insertUser(); - underTest.index(user.getLogin()); + underTest.commitAndIndex(db.getSession(), user); List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); assertThat(docs).hasSize(1); @@ -101,14 +84,14 @@ public class UserIndexerTest { } @Test - public void index_several_users() { - UserDto user = db.users().insertUser(); - UserDto anotherUser = db.users().insertUser(); + public void commitAndIndex_multiple_users() { + UserDto user1 = db.getDbClient().userDao().insert(db.getSession(), UserTesting.newUserDto()); + UserDto user2 = db.getDbClient().userDao().insert(db.getSession(), UserTesting.newUserDto()); - underTest.index(asList(user.getLogin(), anotherUser.getLogin())); + underTest.commitAndIndex(db.getSession(), asList(user1, user2)); List<UserDoc> docs = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER, UserDoc.class); - assertThat(docs).hasSize(2); - assertThat(docs).extracting(UserDoc::login).containsOnly(user.getLogin(), anotherUser.getLogin()); + assertThat(docs).extracting(UserDoc::login).containsExactlyInAnyOrder(user1.getLogin(), user2.getLogin()); + assertThat(db.countRowsOfTable(db.getSession(), "users")).isEqualTo(2); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserResultSetIteratorTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/index/UserResultSetIteratorTest.java deleted file mode 100644 index 852a8f75f45..00000000000 --- a/server/sonar-server/src/test/java/org/sonar/server/user/index/UserResultSetIteratorTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SonarQube - * Copyright (C) 2009-2017 SonarSource SA - * mailto:info AT sonarsource DOT com - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.sonar.server.user.index; - -import com.google.common.collect.Maps; -import java.util.Arrays; -import java.util.Map; -import org.junit.Rule; -import org.junit.Test; -import org.sonar.db.DbTester; -import org.sonar.db.organization.OrganizationDto; -import org.sonar.db.user.UserDto; - -import static org.assertj.core.api.Assertions.assertThat; - -public class UserResultSetIteratorTest { - - @Rule - public DbTester db = DbTester.create(); - - @Test - public void iterator_over_users() { - UserDto userDto1 = db.users().insertUser(u -> u - .setName("User1") - .setLogin("user1") - .setEmail("user1@mail.com") - .setScmAccounts(Arrays.asList("user_1", "u1")) - .setCreatedAt(1_500_000_000_000L) - .setUpdatedAt(1_500_000_000_000L)); - UserDto userDto2 = db.users().insertUser(u -> u - .setName("User2") - .setLogin("user2") - .setEmail("user2@mail.com") - .setScmAccounts(Arrays.asList("user,2", "user_2")) - .setCreatedAt(1_500_000_000_000L) - .setUpdatedAt(1_500_000_000_000L)); - UserDto inactiveUser = db.users().insertUser(u -> u - .setName("User3") - .setLogin("user3") - .setEmail(null) - .setActive(false) - .setScmAccounts((String) null) - .setCreatedAt(1_500_000_000_000L) - .setUpdatedAt(1_550_000_000_000L)); - OrganizationDto org1 = db.organizations().insertForUuid("ORG_1"); - OrganizationDto org2 = db.organizations().insertForUuid("ORG_2"); - db.organizations().addMember(org1, userDto1); - db.organizations().addMember(org1, userDto2); - db.organizations().addMember(org2, userDto1); - - UserResultSetIterator it = UserResultSetIterator.create(db.getDbClient(), db.getSession(), null); - Map<String, UserDoc> usersByLogin = Maps.uniqueIndex(it, UserDoc::login); - it.close(); - - assertThat(usersByLogin).hasSize(3); - - UserDoc user1 = usersByLogin.get("user1"); - assertThat(user1.name()).isEqualTo("User1"); - assertThat(user1.email()).isEqualTo("user1@mail.com"); - assertThat(user1.active()).isTrue(); - assertThat(user1.scmAccounts()).containsOnly("user_1", "u1"); - assertThat(user1.createdAt()).isEqualTo(1_500_000_000_000L); - assertThat(user1.updatedAt()).isEqualTo(1_500_000_000_000L); - assertThat(user1.organizationUuids()).containsOnly("ORG_1", "ORG_2"); - - UserDoc user2 = usersByLogin.get("user2"); - assertThat(user2.name()).isEqualTo("User2"); - assertThat(user2.email()).isEqualTo("user2@mail.com"); - assertThat(user2.active()).isTrue(); - assertThat(user2.scmAccounts()).containsOnly("user,2", "user_2"); - assertThat(user2.createdAt()).isEqualTo(1_500_000_000_000L); - assertThat(user2.updatedAt()).isEqualTo(1_500_000_000_000L); - assertThat(user2.organizationUuids()).containsOnly("ORG_1"); - - UserDoc user3 = usersByLogin.get("user3"); - assertThat(user3.name()).isEqualTo("User3"); - assertThat(user3.email()).isNull(); - assertThat(user3.active()).isFalse(); - assertThat(user3.scmAccounts()).isEmpty(); - assertThat(user3.createdAt()).isEqualTo(1500000000000L); - assertThat(user3.updatedAt()).isEqualTo(1550000000000L); - assertThat(user3.organizationUuids()).isEmpty(); - } -} diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/ChangePasswordActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/ChangePasswordActionTest.java index 6a09f91f7e8..3ff528cf1b6 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/ChangePasswordActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/ChangePasswordActionTest.java @@ -24,7 +24,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.sonar.api.config.internal.MapSettings; -import org.sonar.api.utils.System2; import org.sonar.db.DbTester; import org.sonar.server.es.EsTester; import org.sonar.server.exceptions.BadRequestException; @@ -60,7 +59,6 @@ public class ChangePasswordActionTest { private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone(); private UserUpdater userUpdater = new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), new UserIndexer(db.getDbClient(), esTester.client()), - System2.INSTANCE, organizationFlags, TestDefaultOrganizationProvider.from(db), mock(OrganizationCreation.class), @@ -159,13 +157,15 @@ public class ChangePasswordActionTest { public void fail_to_update_password_on_external_auth() throws Exception { userSessionRule.logIn().setSystemAdministrator(); - userUpdater.create(db.getSession(), NewUser.builder() + NewUser newUser = NewUser.builder() .setEmail("john@email.com") .setLogin("john") .setName("John") .setScmAccounts(newArrayList("jn")) .setExternalIdentity(new ExternalIdentity("gihhub", "john")) - .build()); + .build(); + userUpdater.createAndCommit(db.getSession(), newUser, u -> { + }); expectedException.expect(BadRequestException.class); tester.newPostRequest("api/users", "change_password") @@ -175,12 +175,13 @@ public class ChangePasswordActionTest { } private void createUser() { - userUpdater.create(db.getSession(), NewUser.builder() + userUpdater.createAndCommit(db.getSession(), NewUser.builder() .setEmail("john@email.com") .setLogin("john") .setName("John") .setScmAccounts(newArrayList("jn")) .setPassword("Valar Dohaeris") - .build()); + .build(), u -> { + }); } } diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java index d95b91cc0b2..704dc379de7 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/CreateActionTest.java @@ -19,6 +19,7 @@ */ package org.sonar.server.user.ws; +import java.util.HashSet; import java.util.Optional; import org.junit.Before; import org.junit.Rule; @@ -81,11 +82,10 @@ public class CreateActionTest { private GroupDto defaultGroupInDefaultOrg; private DefaultOrganizationProvider defaultOrganizationProvider = TestDefaultOrganizationProvider.from(db); private TestOrganizationFlags organizationFlags = TestOrganizationFlags.standalone(); - private OrganizationCreation organizationCreation = mock(OrganizationCreation.class); private WsActionTester tester = new WsActionTester(new CreateAction( db.getDbClient(), - new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), userIndexer, system2, organizationFlags, defaultOrganizationProvider, + new UserUpdater(mock(NewUserNotifier.class), db.getDbClient(), userIndexer, organizationFlags, defaultOrganizationProvider, organizationCreation, new DefaultGroupFinder(db.getDbClient()), settings.asConfig()), userSessionRule)); @@ -234,7 +234,7 @@ public class CreateActionTest { logInAsSystemAdministrator(); db.users().insertUser(newUserDto("john", "John", "john@email.com").setActive(false)); - userIndexer.index("john"); + userIndexer.indexOnStartup(new HashSet<>()); call(CreateRequest.builder() .setLogin("john") diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java index a0c8afd3350..9b8346c9de9 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/DeactivateActionTest.java @@ -334,14 +334,10 @@ public class DeactivateActionTest { } private UserDto insertUser(UserDto user) { - user - .setCreatedAt(system2.now()) - .setUpdatedAt(system2.now()); dbClient.userDao().insert(dbSession, user); dbClient.userTokenDao().insert(dbSession, newUserToken().setLogin(user.getLogin())); dbClient.propertiesDao().saveProperty(dbSession, new PropertyDto().setUserId(user.getId()).setKey("foo").setValue("bar")); - dbSession.commit(); - userIndexer.index(user.getLogin()); + userIndexer.commitAndIndex(dbSession, user); return user; } diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/SearchActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/SearchActionTest.java index 6c44ded965a..2001d838249 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/SearchActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/SearchActionTest.java @@ -19,7 +19,7 @@ */ package org.sonar.server.user.ws; -import com.google.common.collect.Lists; +import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; import org.junit.Rule; @@ -281,8 +281,7 @@ public class SearchActionTest { } private List<UserDto> injectUsers(int numberOfUsers) throws Exception { - List<UserDto> userDtos = Lists.newArrayList(); - long createdAt = System.currentTimeMillis(); + List<UserDto> userDtos = new ArrayList<>(); GroupDto group1 = db.users().insertGroup(newGroupDto().setName("sonar-users")); GroupDto group2 = db.users().insertGroup(newGroupDto().setName("sonar-admins")); for (int index = 0; index < numberOfUsers; index++) { @@ -293,15 +292,13 @@ public class SearchActionTest { UserDto userDto = dbClient.userDao().insert(dbSession, new UserDto() .setActive(true) - .setCreatedAt(createdAt) .setEmail(email) .setLogin(login) .setName(name) .setScmAccounts(scmAccounts) .setLocal(true) .setExternalIdentity(login) - .setExternalIdentityProvider("sonarqube") - .setUpdatedAt(createdAt)); + .setExternalIdentityProvider("sonarqube")); userDtos.add(userDto); for (int tokenIndex = 0; tokenIndex < index; tokenIndex++) { diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/SkipOnboardingTutorialActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/SkipOnboardingTutorialActionTest.java index df97cd664de..620287f5534 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/SkipOnboardingTutorialActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/SkipOnboardingTutorialActionTest.java @@ -23,7 +23,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.sonar.api.server.ws.WebService; -import org.sonar.api.utils.internal.TestSystem2; import org.sonar.db.DbTester; import org.sonar.db.user.UserDto; import org.sonar.server.exceptions.UnauthorizedException; @@ -35,9 +34,6 @@ import static org.assertj.core.api.Assertions.assertThat; public class SkipOnboardingTutorialActionTest { - private final static long PAST = 100_000_000_000L; - private final static long NOW = 500_000_000_000L; - @Rule public UserSessionRule userSession = UserSessionRule.standalone(); @@ -47,36 +43,31 @@ public class SkipOnboardingTutorialActionTest { @Rule public ExpectedException expectedException = ExpectedException.none(); - private TestSystem2 system2 = new TestSystem2().setNow(NOW); - - private WsActionTester ws = new WsActionTester(new SkipOnboardingTutorialAction(userSession, db.getDbClient(), system2)); + private WsActionTester ws = new WsActionTester(new SkipOnboardingTutorialAction(userSession, db.getDbClient())); @Test public void mark_user_as_onboarded() { UserDto user = db.users().insertUser(u -> u - .setOnboarded(false) - .setUpdatedAt(PAST)); + .setOnboarded(false)); userSession.logIn(user); call(); UserDto userDto = selectUser(user.getLogin()); assertThat(userDto.isOnboarded()).isEqualTo(true); - assertThat(userDto.getUpdatedAt()).isEqualTo(NOW); } @Test public void does_nothing_if_user_already_onboarded() { UserDto user = db.users().insertUser(u -> u - .setOnboarded(true) - .setUpdatedAt(PAST)); + .setOnboarded(true)); userSession.logIn(user); call(); UserDto userDto = selectUser(user.getLogin()); assertThat(userDto.isOnboarded()).isEqualTo(true); - assertThat(userDto.getUpdatedAt()).isEqualTo(PAST); + assertThat(userDto.getUpdatedAt()).isEqualTo(user.getUpdatedAt()); } @Test @@ -112,7 +103,6 @@ public class SkipOnboardingTutorialActionTest { @Test public void test_definition() { - WsActionTester ws = new WsActionTester(new SkipOnboardingTutorialAction(userSession, db.getDbClient(), system2)); WebService.Action def = ws.getDef(); assertThat(def.isPost()).isTrue(); assertThat(def.isInternal()).isTrue(); diff --git a/server/sonar-server/src/test/java/org/sonar/server/user/ws/UpdateActionTest.java b/server/sonar-server/src/test/java/org/sonar/server/user/ws/UpdateActionTest.java index 4747ce234a9..bce95d9b62c 100644 --- a/server/sonar-server/src/test/java/org/sonar/server/user/ws/UpdateActionTest.java +++ b/server/sonar-server/src/test/java/org/sonar/server/user/ws/UpdateActionTest.java @@ -75,7 +75,7 @@ public class UpdateActionTest { dbTester.users().insertDefaultGroup(dbTester.getDefaultOrganization(), "sonar-users"); userIndexer = new UserIndexer(dbClient, esTester.client()); tester = new WsTester(new UsersWs(new UpdateAction( - new UserUpdater(mock(NewUserNotifier.class), dbClient, userIndexer, system2, organizationFlags, defaultOrganizationProvider, ORGANIZATION_CREATION_NOT_USED_FOR_UPDATE, + new UserUpdater(mock(NewUserNotifier.class), dbClient, userIndexer, organizationFlags, defaultOrganizationProvider, ORGANIZATION_CREATION_NOT_USED_FOR_UPDATE, new DefaultGroupFinder(dbTester.getDbClient()), settings.asConfig()), userSessionRule, new UserJsonWriter(userSessionRule), dbClient))); @@ -226,7 +226,6 @@ public class UpdateActionTest { .setExternalIdentity("jo") .setExternalIdentityProvider("sonarqube"); dbClient.userDao().insert(session, userDto); - session.commit(); - userIndexer.index(userDto.getLogin()); + userIndexer.commitAndIndex(session, userDto); } } diff --git a/tests/pom.xml b/tests/pom.xml index 2409cdfda33..4ac41b49d2f 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -157,6 +157,34 @@ </includes> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-byteman-for-resilience-tests</id> + <phase>generate-test-resources</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.jboss.byteman</groupId> + <artifactId>byteman</artifactId> + <version>3.0.10</version> + <overWrite>false</overWrite> + <destFileName>byteman.jar</destFileName> + </artifactItem> + </artifactItems> + <outputDirectory>${project.basedir}/target</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + </configuration> + </execution> + </executions> + </plugin> + </plugins> </build> diff --git a/tests/resilience/user_indexer.btm b/tests/resilience/user_indexer.btm new file mode 100644 index 00000000000..a80c2910d32 --- /dev/null +++ b/tests/resilience/user_indexer.btm @@ -0,0 +1,23 @@ +# sonar.web.javaAdditionalOpts=-javaagent:/path/to/byteman-3.0.10/lib/byteman.jar=script:/path/to/user_indexer.btm,boot:/path/to/byteman-3.0.10/lib/byteman.jar +# sonar.search.recovery.delayInMs=10000 +# sonar.search.recovery.minAgeInMs=30000 + +RULE make indexing of users silently fail +CLASS org.sonar.server.user.index.UserIndexer +METHOD postCommit +COMPILE +AT ENTRY +BIND logins:Collection = $logins +IF logins.contains("error") +DO RETURN +ENDRULE + +RULE make indexing of users fail +CLASS org.sonar.server.user.index.UserIndexer +METHOD postCommit +COMPILE +AT ENTRY +BIND logins:Collection = $logins +IF logins.contains("crash") +DO THROW new RuntimeException("Fail to index users to Elasticsearch because a user 'crash' has been given") +ENDRULE diff --git a/tests/src/test/java/org/sonarqube/tests/Category5Suite.java b/tests/src/test/java/org/sonarqube/tests/Category5Suite.java index 5335ef37b3b..63b7c1f1628 100644 --- a/tests/src/test/java/org/sonarqube/tests/Category5Suite.java +++ b/tests/src/test/java/org/sonarqube/tests/Category5Suite.java @@ -31,6 +31,7 @@ import org.sonarqube.tests.updateCenter.UpdateCenterTest; import org.sonarqube.tests.user.OnboardingTest; import org.sonarqube.tests.user.RealmAuthenticationTest; import org.sonarqube.tests.user.SsoAuthenticationTest; +import org.sonarqube.tests.user.UserEsResilienceTest; /** * This suite is reserved to the tests that start their own instance of Orchestrator. @@ -49,7 +50,8 @@ import org.sonarqube.tests.user.SsoAuthenticationTest; RealmAuthenticationTest.class, SsoAuthenticationTest.class, OnboardingTest.class, - BuiltInQualityProfilesNotificationTest.class + BuiltInQualityProfilesNotificationTest.class, + UserEsResilienceTest.class }) public class Category5Suite { diff --git a/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java b/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java new file mode 100644 index 00000000000..10c3a441a53 --- /dev/null +++ b/tests/src/test/java/org/sonarqube/tests/user/UserEsResilienceTest.java @@ -0,0 +1,144 @@ +/* + * SonarQube + * Copyright (C) 2009-2017 SonarSource SA + * mailto:info AT sonarsource DOT com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package org.sonarqube.tests.user; + +import com.sonar.orchestrator.Orchestrator; +import java.io.File; +import java.util.concurrent.TimeUnit; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.DisableOnDebug; +import org.junit.rules.TestRule; +import org.junit.rules.Timeout; +import org.sonarqube.tests.Tester; +import org.sonarqube.ws.WsUsers.CreateWsResponse.User; +import org.sonarqube.ws.client.user.SearchRequest; +import org.sonarqube.ws.client.user.UpdateRequest; + +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; +import static util.ItUtils.expectHttpError; + +public class UserEsResilienceTest { + + @ClassRule + public static final Orchestrator orchestrator = Orchestrator.builderEnv() + .setServerProperty("sonar.web.javaAdditionalOpts", + format("-javaagent:%s=script:%s,boot:%s", findBytemanJar(), findBytemanScript(), findBytemanJar())) + .setServerProperty("sonar.search.recovery.delayInMs", "1000") + .setServerProperty("sonar.search.recovery.minAgeInMs", "3000") + .build(); + + @Rule + public TestRule timeout = new DisableOnDebug(Timeout.builder() + .withLookingForStuckThread(true) + .withTimeout(60L, TimeUnit.SECONDS) + .build()); + + @Rule + public Tester tester = new Tester(orchestrator); + + @Test + public void creation_and_update_of_user_are_resilient_to_indexing_errors() throws Exception { + String login = "error"; + + // creation of user succeeds but index is not up-to-date (indexing + // failures are not propagated to web services) + User user = tester.users().generate(u -> u.setLogin(login)); + + // user exists in db, it can't be created again. + // However he's not indexed. + expectHttpError(400, "An active user with login '" + login + "' already exists", + () -> tester.users().generate(u -> u.setLogin(login))); + assertThat(isReturnedInSearch(user.getLogin())).isFalse(); + + while (!isReturnedInSearch(user.getLogin())) { + // user is indexed by the recovery daemon, which runs every 5 seconds + Thread.sleep(1_000L); + } + + // update the name. Db operation succeeds but not ES indexing. + // Renaming is not propagated to index as long as recovery does not + // run. + String newName = "renamed"; + tester.users().service().update(UpdateRequest.builder().setLogin(login).setName(newName).build()); + assertThat(isReturnedInSearch(newName)).isFalse(); + + while (!isReturnedInSearch(newName)) { + // user is indexed by the recovery daemon, which runs every 5 seconds + Thread.sleep(1_000L); + } + } + + @Test + public void creation_and_update_of_user_are_resilient_to_indexing_crash() throws Exception { + String login = "crash"; + + // creation of user succeeds but index is not up-to-date (indexing + // crashes are not propagated to web services) + expectHttpError(500, () -> tester.users().generate(u -> u.setLogin(login))); + + // user exists in db, it can't be created again. + // However he's not indexed. + expectHttpError(400, "An active user with login '" + login + "' already exists", + () -> tester.users().generate(u -> u.setLogin(login))); + assertThat(isReturnedInSearch(login)).isFalse(); + + while (!isReturnedInSearch(login)) { + // user is indexed by the recovery daemon, which runs every 5 seconds + Thread.sleep(1_000L); + } + + // update the name. Db operation succeeds but ES indexing crashes. + // Renaming is not propagated to index as long as recovery does not + // run. + String newName = "renamed"; + expectHttpError(500, () -> tester.users().service().update(UpdateRequest.builder().setLogin(login).setName(newName).build())); + assertThat(isReturnedInSearch(newName)).isFalse(); + + while (!isReturnedInSearch(newName)) { + // user is indexed by the recovery daemon, which runs every 5 seconds + Thread.sleep(1_000L); + } + } + + private boolean isReturnedInSearch(String name) { + return tester.users().service().search(SearchRequest.builder().setQuery(name).build()).getUsersCount() == 1L; + } + + private static String findBytemanJar() { + // see pom.xml, Maven copies and renames the artifact. + File jar = new File("target/byteman.jar"); + if (!jar.exists()) { + throw new IllegalStateException("Can't find " + jar + ". Please execute 'mvn generate-test-resources' on integration tests once."); + } + return jar.getAbsolutePath(); + } + + private static String findBytemanScript() { + // see pom.xml, Maven copies and renames the artifact. + File script = new File("resilience/user_indexer.btm"); + if (!script.exists()) { + throw new IllegalStateException("Can't find " + script); + } + return script.getAbsolutePath(); + } +} |